From b3cb8f7ab24224adb0e130212df2bbcbd0fc7e76 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 12 Aug 2026 07:19:56 +0800 Subject: [PATCH] fix(qa): preserve nested tool success evidence (#122189) Record successful logical target tools at the attempt owner so Code Mode exec receipts retain the nested tool evidence needed by release qualification. Co-authored-by: Peter Steinberger --- ...model-switch-tool-continuity-proof.test.ts | 15 ++ ...omplete-turn.settled-tool-recovery.test.ts | 4 + .../run.overflow-compaction.fixture.ts | 6 +- .../run/attempt-execution-settle.test.ts | 165 ++++++++++++++++-- .../run/attempt-phase-lifecycle.test.ts | 1 + .../run/attempt-result.test.ts | 9 + .../run/attempt-result.ts | 12 +- .../run/attempt-settle.ts | 10 +- .../run/attempt-stream-settle.ts | 10 ++ .../run/settled-turn-finalization.ts | 20 ++- .../run/terminal-preparation.test.ts | 15 +- .../run/terminal-preparation.ts | 12 +- 12 files changed, 242 insertions(+), 37 deletions(-) diff --git a/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts index 97592e540ddc..f880c27bef76 100644 --- a/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts @@ -141,6 +141,21 @@ describe("model-switch tool continuity terminal evidence", () => { ); }); + it("accepts a logical read appended after the physical Code Mode exec", async () => { + const { result } = await runToolContinuity(["exec", "read"]); + + expect(result.status).toBe("pass"); + expect(result.modelSwitchEvidence).toMatchObject({ + alternate: { runId: "run-2", successfulToolNames: ["exec", "read"] }, + }); + }); + + it("rejects a bare successful Code Mode exec without logical read evidence", async () => { + await expect(runToolContinuity(["exec"])).rejects.toThrow( + "alternate-model run did not return exact owned successful read evidence", + ); + }); + it("does not let a successful prior-run read satisfy the alternate run", async () => { await expect(runToolContinuity([])).rejects.toThrow( "alternate-model run did not return exact owned successful read evidence", diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts index 7641614b6abe..b3170e6e6e5a 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts @@ -442,6 +442,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { { toolName: "cron" }, { toolName: "sessions_spawn" }, ], + successfulNestedToolNames: ["read"], acceptedSessionSpawns, successfulCronAdds: 1, itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, @@ -484,6 +485,9 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { codeModeEngaged: true, assistantTurns: 2, bridgeCalls: { search: 1, describe: 2, call: 3 }, + terminalReceipt: { + successfulToolNames: ["read"], + }, }); const secondCall = runAttemptCall(1); expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts index c3d87d495d6b..d639644df0b7 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts @@ -4,8 +4,8 @@ import type { ContextEngineSessionTarget } from "../../context-engine/types.js"; import { normalizeAgentRunAttemptTerminal } from "../agent-run-terminal-outcome.js"; import { isAgentToolReplaySafe } from "../tool-replay-safety.js"; +import type { EmbeddedRunAttemptWithReceiptEvidence } from "./run/attempt-result.js"; import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js"; -import type { EmbeddedRunAttemptResult } from "./run/types.js"; const DEFAULT_OVERFLOW_ERROR_MESSAGE = "request_too_large: Request size exceeds model context window"; @@ -38,7 +38,7 @@ export function makeCompactionSuccess(params: { }; } -type AttemptResultOverrides = Partial & +type AttemptResultOverrides = Partial & Parameters[0]; function resolveFixtureTerminal(overrides: AttemptResultOverrides) { @@ -47,7 +47,7 @@ function resolveFixtureTerminal(overrides: AttemptResultOverrides) { export function makeAttemptResult( overrides: AttemptResultOverrides = {}, -): EmbeddedRunAttemptResult { +): EmbeddedRunAttemptWithReceiptEvidence { const toolMetas = (overrides.toolMetas ?? []).map((entry) => Object.assign({}, entry, { replaySafe: entry.replaySafe ?? isAgentToolReplaySafe({ name: entry.toolName }), diff --git a/src/agents/embedded-agent-runner/run/attempt-execution-settle.test.ts b/src/agents/embedded-agent-runner/run/attempt-execution-settle.test.ts index ee61ac4c8893..94bfb75c73f1 100644 --- a/src/agents/embedded-agent-runner/run/attempt-execution-settle.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-execution-settle.test.ts @@ -1,4 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createTestAdmittedRunContext } from "../../admitted-run-context.test-support.js"; +import { createUsageAccumulator } from "../usage-accumulator.js"; const mocks = vi.hoisted(() => ({ clearActiveEmbeddedRun: vi.fn(), @@ -15,9 +17,14 @@ const mocks = vi.hoisted(() => ({ vi.mock("../logger.js", () => ({ log: { debug: mocks.logDebug, error: mocks.logError, warn: mocks.logWarn }, })); -vi.mock("../../subagents/registry/subagent-registry.js", () => ({ - settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns, -})); +vi.mock("../../subagents/registry/subagent-registry.js", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns, + }; +}); vi.mock("../runs.js", () => ({ clearActiveEmbeddedRun: mocks.clearActiveEmbeddedRun })); vi.mock("./attempt-prompt-phase.js", () => ({ runEmbeddedAttemptPromptPhase: mocks.runPrompt, @@ -38,6 +45,8 @@ vi.mock("./attempt-stream-settle.js", () => ({ import { SESSIONS_YIELD_ABORT_REASON } from "./attempt-sessions-yield.js"; import { runEmbeddedAttemptSettledPhase } from "./attempt-settle.js"; +import { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; +import { prepareEmbeddedRunTerminal } from "./terminal-preparation.js"; type SettledInput = Parameters[0]; @@ -46,14 +55,68 @@ function createFixture() { const queueHandle = { kind: "embedded", runId: "run-1" }; const unsubscribe = vi.fn(() => order.push("unsubscribe")); const waitForPendingEvents = vi.fn(async () => undefined); - const subscription = { unsubscribe, waitForPendingEvents }; + const subscription = { + assistantTexts: [], + didSendDeterministicApprovalPrompt: vi.fn(() => false), + didSendViaMessagingTool: vi.fn(() => false), + getAcceptedSessionSpawns: vi.fn(() => []), + getAssistantTurnCount: vi.fn(() => 1), + getCompactionCount: vi.fn(() => 0), + getCurrentAttemptAssistant: vi.fn(() => undefined), + getHeartbeatToolResponse: vi.fn(() => undefined), + getItemLifecycle: vi.fn(() => ({ startedCount: 0, completedCount: 0, activeCount: 0 })), + getLastAssistantTextMessageIndex: vi.fn(() => undefined), + getLastAssistantUsage: vi.fn(() => undefined), + getLastCompactionTokensAfter: vi.fn(() => undefined), + getLastToolError: vi.fn(() => undefined), + getLatestMcpAppChannelView: vi.fn(() => undefined), + getLatestMcpConnectAction: vi.fn(() => undefined), + getMessagingToolSentMediaUrls: vi.fn(() => []), + getMessagingToolSentTargets: vi.fn(() => []), + getMessagingToolSentTexts: vi.fn(() => []), + getMessagingToolSourceReplyPayloads: vi.fn(() => []), + getPendingToolMediaReply: vi.fn(() => undefined), + getReplayState: vi.fn(() => ({ replayInvalid: false, hadPotentialSideEffects: false })), + getSuccessfulCronAdds: vi.fn(() => []), + getUsageTotals: vi.fn(() => ({ input: 1, output: 2, total: 3 })), + getVisibleBlockReplyCount: vi.fn(() => 0), + hasToolMediaBlockReply: vi.fn(() => false), + isCompactionInFlight: vi.fn(() => false), + setTerminalLifecycleMeta: vi.fn(), + toolMetas: [{ toolName: "exec", isError: false }], + unsubscribe, + waitForCompactionRetry: vi.fn(async () => undefined), + waitForPendingEvents, + }; const detachBackend = vi.fn(() => order.push("detach-backend")); const clearTimers = vi.fn(() => order.push("clear-timers")); const getBeforeAgentFinalizeRevisionReason = vi.fn(() => "revision"); const getBeforeAgentFinalizeRevisionEntryId = vi.fn(() => undefined); const promptActiveSession = vi.fn(async () => undefined); + const messages = [ + { + role: "assistant", + content: [{ type: "text", text: "done" }], + api: "openai-responses", + provider: "openai", + model: "model", + usage: { + input: 1, + output: 2, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 3, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 100, + }, + ]; const activeSession = { - agent: { state: { messages: [] } }, + agent: { state: { messages } }, + isCompacting: false, + isStreaming: false, + messages, sessionId: "active-session", getActiveToolNames: vi.fn(() => ["read"]), }; @@ -61,9 +124,9 @@ function createFixture() { kind: "session-manager", buildSessionContext: vi.fn(() => ({ messages: [] })), }; - const hookRunner = { kind: "hook-runner" }; - const cacheTrace = { kind: "cache-trace" }; - const trajectoryRecorder = { kind: "trajectory" }; + const hookRunner = { hasHooks: vi.fn(() => false) }; + const cacheTrace = { recordStage: vi.fn() }; + const trajectoryRecorder = { recordEvent: vi.fn(), flush: vi.fn(async () => undefined) }; const toolResultPromptProjectionState = { kind: "tool-result-projection" }; const sessionPromptState = { toolResults: toolResultPromptProjectionState }; const sessionRuntimeState = { @@ -144,11 +207,19 @@ function createFixture() { }; const input = { attempt: { + admittedRunContext: createTestAdmittedRunContext("run-1"), + config: {}, + model: { api: "openai-responses" }, + modelId: "model", + promptCacheKey: undefined, + provider: "openai", replyOperation: { detachBackend }, runId: "run-1", sessionFile: "/tmp/session.jsonl", sessionId: "session-1", sessionKey: "agent:main", + trigger: "user", + workspaceDir: "/workspace", }, agentDir: "/agent", isRawModelRun: false, @@ -156,7 +227,7 @@ function createFixture() { runAbortController: new AbortController(), prepared: { bootstrap: { - bootstrapPromptWarning: undefined, + bootstrapPromptWarning: {}, shouldRecordCompletedBootstrapTurn: false, }, bundleTools: { @@ -168,7 +239,7 @@ function createFixture() { runtimeInfo: { model: { id: "model" } }, systemPromptReport: { chars: 13 }, }, - toolBase: { toolSearchTargetTranscriptProjections: new Map() }, + toolBase: { toolSearchTargetTranscriptProjections: [] }, toolCatalog: { effectiveTools: [{ name: "read" }], emptyExplicitToolAllowlistError: undefined, @@ -176,7 +247,7 @@ function createFixture() { }, }, sessionLock: { - withOwnedTranscriptWrite: vi.fn(), + withOwnedTranscriptWrite: vi.fn(async (operation: () => unknown) => await operation()), }, setup: { effectiveFsWorkspaceOnly: false, @@ -327,6 +398,78 @@ describe("runEmbeddedAttemptSettledPhase", () => { ); }); + it("carries a successful hidden target through settlement into the terminal receipt", async () => { + const fixture = createFixture(); + fixture.input.prepared.toolBase.toolSearchTargetTranscriptProjections.push( + { + parentToolCallId: "outer-exec", + toolCallId: "tool_search_code:outer-exec:read:1", + toolName: "read", + input: { path: "qa/scenarios/index.yaml" }, + result: { + content: [{ type: "text", text: "QA scenario pack mission" }], + details: {}, + }, + isError: false, + }, + { + parentToolCallId: "outer-exec", + toolCallId: "tool_search_code:outer-exec:write:2", + toolName: "write", + input: { path: "qa/scenarios/index.yaml", content: "invalid" }, + result: { + content: [{ type: "text", text: "write failed" }], + details: {}, + }, + isError: true, + }, + ); + const actualStreamSettle = await vi.importActual( + "./attempt-stream-settle.js", + ); + const actualAttemptResult = + await vi.importActual("./attempt-result.js"); + mocks.settleStream.mockImplementationOnce(actualStreamSettle.settleEmbeddedAttemptStream); + mocks.completeResult.mockImplementationOnce(actualAttemptResult.completeEmbeddedAttemptResult); + + const attempt = await runEmbeddedAttemptSettledPhase(fixture.input); + const prepared = prepareEmbeddedRunTerminal({ + runParams: { + admittedRunContext: createTestAdmittedRunContext("run-1"), + sessionId: "session-1", + runId: "run-1", + workspaceDir: "/workspace", + prompt: "read the QA scenario index", + trigger: "user", + timeoutMs: 60_000, + }, + attempt, + currentAttemptCompletedAssistant: attempt.currentAttemptCompletedAssistant, + provider: "openai", + model: "model", + activeErrorContext: { provider: "openai", model: "model" }, + authProfileStore: { version: 1, profiles: {} }, + sessionIdUsed: attempt.sessionIdUsed, + sessionFileUsed: attempt.sessionFileUsed, + outerContextTokenMeta: {}, + usageAccumulator: createUsageAccumulator(), + contextRecoveryState: createEmbeddedRunContextRecoveryState(), + resolvedToolResultFormat: "markdown", + terminalState: { + outcome: { reason: "completed", status: "ok", stopReason: "stop" }, + signalOwnedInterruption: false, + }, + }); + + expect( + ( + prepared.agentMeta as { + terminalReceipt?: { successfulToolNames?: string[] }; + } + ).terminalReceipt?.successfulToolNames, + ).toEqual(["exec", "read"]); + }); + it("preserves a prompt failure while still completing stream cleanup", async () => { const fixture = createFixture(); const failure = new Error("prompt failed"); diff --git a/src/agents/embedded-agent-runner/run/attempt-phase-lifecycle.test.ts b/src/agents/embedded-agent-runner/run/attempt-phase-lifecycle.test.ts index a051955d4923..14d634cf57c9 100644 --- a/src/agents/embedded-agent-runner/run/attempt-phase-lifecycle.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-phase-lifecycle.test.ts @@ -292,6 +292,7 @@ describe("embedded attempt phase lifecycle state", () => { expect(result.lastAssistant).toBe(modelAssistant); expect(result.currentAttemptAssistant).toBe(modelAssistant); expect(result.currentAttemptCompletedAssistant).toEqual(modelAssistant); + expect(result.successfulNestedToolNames).toEqual([]); expect(result.messagesSnapshot).toHaveLength(5); expect(result.messagesSnapshot.at(-2)).toMatchObject({ role: "assistant", diff --git a/src/agents/embedded-agent-runner/run/attempt-result.test.ts b/src/agents/embedded-agent-runner/run/attempt-result.test.ts index 7c0d2b0f5207..3b923bda0cc3 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { completeEmbeddedAttemptResult, createMcpAttemptCarryover } from "./attempt-result.js"; function completeResult(params?: { + successfulNestedToolNames?: string[]; latestMcpAppChannelView?: { viewId: string }; clientToolCallSlots?: Array<{ toolCallId: string; @@ -59,6 +60,7 @@ function completeResult(params?: { terminal: { kind: "ok" }, sessionIdUsed: "session-1", messagesSnapshot: [], + successfulNestedToolNames: params?.successfulNestedToolNames, yieldDetected: false, didDeliverSourceReplyViaMessageTool: false, diagnosticTrace: { traceId: "trace-1", spanId: "span-1" }, @@ -156,6 +158,13 @@ describe("attempt result projection", () => { ]); }); + it("projects successful nested tool names from settled attempt state", () => { + expect( + completeResult({ successfulNestedToolNames: ["read", "memory_search"] }) + .successfulNestedToolNames, + ).toEqual(["read", "memory_search"]); + }); + it("projects pending media and voice fields", () => { expect(completeResult().toolMediaUrls).toBeUndefined(); expect(completeResult({ pendingToolMediaReply: { mediaUrls: [" "] } }).toolMediaUrls).toEqual([ diff --git a/src/agents/embedded-agent-runner/run/attempt-result.ts b/src/agents/embedded-agent-runner/run/attempt-result.ts index 7601879f431c..4d376d6a1218 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.ts @@ -49,6 +49,10 @@ export function createMcpAttemptCarryover() { }; } +export type EmbeddedRunAttemptWithReceiptEvidence = EmbeddedRunAttemptResult & { + successfulNestedToolNames?: string[]; +}; + export type EmbeddedAttemptClientToolCallSlot = { toolCallId: string; name: string; @@ -57,7 +61,7 @@ export type EmbeddedAttemptClientToolCallSlot = { }; type EmbeddedAttemptResultState = Pick< - EmbeddedRunAttemptResult, + EmbeddedRunAttemptWithReceiptEvidence, | "terminal" | "preflightRecovery" | "sessionIdUsed" @@ -69,6 +73,7 @@ type EmbeddedAttemptResultState = Pick< | "lastAssistant" | "currentAttemptAssistant" | "currentAttemptCompletedAssistant" + | "successfulNestedToolNames" | "attemptUsage" | "promptCache" | "contextBudgetStatus" @@ -158,7 +163,7 @@ function hasVisiblePendingToolMediaReply( /** Runs output hooks, classifies terminal effects, and returns the finalized attempt result. */ export function completeEmbeddedAttemptResult( input: CompleteEmbeddedAttemptResultInput, -): EmbeddedRunAttemptResult { +): EmbeddedRunAttemptWithReceiptEvidence { const { attempt, state, subscription } = input; const terminal = projectAgentRunAttemptTerminal(state.terminal); const { @@ -389,7 +394,7 @@ export function completeEmbeddedAttemptResult( terminal: state.terminal, }, }); - const result: EmbeddedRunAttemptResult = { + const result: EmbeddedRunAttemptWithReceiptEvidence = { ...state, replayMetadata, currentAttemptReplayMetadata, @@ -403,6 +408,7 @@ export function completeEmbeddedAttemptResult( latestMcpConnectAction: getLatestMcpConnectAction(), lastAssistantTextMessageIndex: getLastAssistantTextMessageIndex(), toolMetas: toolMetasNormalized, + successfulNestedToolNames: state.successfulNestedToolNames, acceptedSessionSpawns, lastToolError, didSendViaMessagingTool: didSendViaMessagingTool(), diff --git a/src/agents/embedded-agent-runner/run/attempt-settle.ts b/src/agents/embedded-agent-runner/run/attempt-settle.ts index 0b30cb9fa2ac..49ed91c0348e 100644 --- a/src/agents/embedded-agent-runner/run/attempt-settle.ts +++ b/src/agents/embedded-agent-runner/run/attempt-settle.ts @@ -23,7 +23,10 @@ import type { import { completeEmbeddedAttemptAfterTurn } from "./attempt-finalize.js"; import type { prepareEmbeddedAttemptHistory } from "./attempt-history.js"; import { runEmbeddedAttemptPromptPhase } from "./attempt-prompt-phase.js"; -import { completeEmbeddedAttemptResult } from "./attempt-result.js"; +import { + completeEmbeddedAttemptResult, + type EmbeddedRunAttemptWithReceiptEvidence, +} from "./attempt-result.js"; import type { prepareEmbeddedAttemptStream } from "./attempt-stream-prepare.js"; import { settleEmbeddedAttemptStream } from "./attempt-stream-settle.js"; import type { installEmbeddedAttemptStreamGuards } from "./attempt-stream.js"; @@ -107,7 +110,7 @@ export async function runEmbeddedAttemptSettledPhase( getRepairedRejectedThinkingReplay: () => boolean; preparedStreamRuntime: PreparedStreamRuntime; }, -): Promise { +): Promise { const { attempt, state } = input; const { bootstrap, bundleTools, sessionRuntime, systemPrompt, toolBase, toolCatalog } = input.prepared; @@ -173,6 +176,7 @@ export async function runEmbeddedAttemptSettledPhase( let lastAssistant: AssistantMessage | undefined; let currentAttemptAssistant: EmbeddedRunAttemptResult["currentAttemptAssistant"]; let currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"]; + let successfulNestedToolNames: EmbeddedRunAttemptWithReceiptEvidence["successfulNestedToolNames"]; let attemptUsage: NormalizedUsage | undefined; let cacheBreak: PromptCacheBreak | null = null; let contextBudgetStatus: EmbeddedRunAttemptResult["contextBudgetStatus"]; @@ -447,6 +451,7 @@ export async function runEmbeddedAttemptSettledPhase( lastAssistant = settledStream.lastAssistant; currentAttemptAssistant = settledStream.currentAttemptAssistant; currentAttemptCompletedAssistant = settledStream.currentAttemptCompletedAssistant; + successfulNestedToolNames = settledStream.successfulNestedToolNames; attemptUsage = settledStream.attemptUsage; cacheBreak = settledStream.cacheBreak; sessionRuntimeState.promptCache = settledStream.promptCache; @@ -530,6 +535,7 @@ export async function runEmbeddedAttemptSettledPhase( lastAssistant, currentAttemptAssistant, currentAttemptCompletedAssistant, + successfulNestedToolNames, attemptUsage, promptCache: sessionRuntimeState.promptCache, contextBudgetStatus, diff --git a/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts b/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts index a98489a2ff30..cc07d4af66a7 100644 --- a/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts +++ b/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts @@ -92,6 +92,7 @@ type StreamSettleResult = { lastAssistant: EmbeddedRunAttemptResult["lastAssistant"]; currentAttemptAssistant: EmbeddedRunAttemptResult["currentAttemptAssistant"]; currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"]; + successfulNestedToolNames: string[]; attemptUsage: EmbeddedRunAttemptResult["attemptUsage"]; cacheBreak: PromptCacheBreak | null; lastCallUsage: NormalizedUsage | undefined; @@ -426,6 +427,15 @@ export async function settleEmbeddedAttemptStream(input: { lastAssistant, currentAttemptAssistant, currentAttemptCompletedAssistant, + successfulNestedToolNames: [ + ...new Set( + input.toolSearchTargetTranscriptProjections + // Receipt evidence admits only projections explicitly recorded as successful. + .filter((projection) => Object.is(projection.isError, false)) + .map((projection) => projection.toolName.trim()) + .filter(Boolean), + ), + ], attemptUsage, cacheBreak, lastCallUsage, diff --git a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts index 6ff959b92648..3aea8ff8d73c 100644 --- a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts +++ b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts @@ -9,6 +9,7 @@ import { mergeAttemptRunStatsIntoAccumulator, mergeUsageIntoAccumulator, } from "../usage-accumulator.js"; +import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js"; import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js"; import { withEmbeddedRunLaneProgressHeartbeat } from "./lane-runtime.js"; import { @@ -20,7 +21,7 @@ import { copyAttemptDeliveryState, resolveSettledTurnFinalizationRequest, } from "./terminal-resolution.js"; -import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js"; +import type { EmbeddedRunAttemptParams } from "./types.js"; type TerminalPreparationInput = Parameters[0]; type TerminalPreparationBase = Omit< @@ -35,9 +36,9 @@ type TerminalPreparationBase = Omit< export async function prepareTerminalWithSettledTurnFinalization(input: { initial: { - attempt: EmbeddedRunAttemptResult; - attemptAssistant: EmbeddedRunAttemptResult["lastAssistant"]; - currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"]; + attempt: EmbeddedRunAttemptWithReceiptEvidence; + attemptAssistant: EmbeddedRunAttemptWithReceiptEvidence["lastAssistant"]; + currentAttemptCompletedAssistant: EmbeddedRunAttemptWithReceiptEvidence["currentAttemptCompletedAssistant"]; sessionIdUsed: string; sessionFileUsed?: string; terminalState: EmbeddedRunTerminalState; @@ -181,12 +182,12 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { async function runPreparedSettledTurnFinalization(input: { attempt: EmbeddedRunAttemptParams; - settledAttempt: EmbeddedRunAttemptResult; + settledAttempt: EmbeddedRunAttemptWithReceiptEvidence; harness: AgentHarness; prompt: string; noteLaneTaskProgress: () => void; }): Promise< - | { outcome: "answered"; attempt: EmbeddedRunAttemptResult } + | { outcome: "answered"; attempt: EmbeddedRunAttemptWithReceiptEvidence } | { outcome: "empty"; result: AgentHarnessSettledTurnFinalizationResult; @@ -222,13 +223,13 @@ async function runPreparedSettledTurnFinalization(input: { function buildSettledTurnFinalizationAttemptResult(input: { result: AgentHarnessSettledTurnFinalizationResult; - settledAttempt: EmbeddedRunAttemptResult; + settledAttempt: EmbeddedRunAttemptWithReceiptEvidence; prompt: string; agentHarnessId?: string; -}): EmbeddedRunAttemptResult { +}): EmbeddedRunAttemptWithReceiptEvidence { const { result, settledAttempt } = input; const text = resolveSettledTurnFinalizationText(result); - // Finalization replaces terminal ownership, not facts from already-settled tools. + // Finalization replaces terminal ownership, not host-private facts from settled tools. // Keep those facts while replay, abort, and lifecycle state remain finalizer-local. return { terminal: { kind: "ok" }, @@ -249,6 +250,7 @@ function buildSettledTurnFinalizationAttemptResult(input: { currentAttemptAssistant: result.assistant, currentAttemptCompletedAssistant: result.assistant, toolMetas: settledAttempt.toolMetas, + successfulNestedToolNames: settledAttempt.successfulNestedToolNames, hasToolMediaBlockReply: false, cloudCodeAssistFormatError: false, attemptUsage: result.usage, diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts index 4dc216f871d7..89888064dd20 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts @@ -2,8 +2,8 @@ import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { describe, expect, it, vi } from "vitest"; import { createTestAdmittedRunContext } from "../../admitted-run-context.test-support.js"; import { createUsageAccumulator } from "../usage-accumulator.js"; +import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js"; import { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; -import type { EmbeddedRunAttemptResult } from "./types.js"; vi.mock("./payloads.js", () => ({ buildEmbeddedRunPayloads: () => [], @@ -43,8 +43,8 @@ function assistantMessage(stopReason: AssistantMessage["stopReason"] = "stop"): } function attemptResult( - overrides: Partial = {}, -): EmbeddedRunAttemptResult { + overrides: Partial = {}, +): EmbeddedRunAttemptWithReceiptEvidence { const assistant = assistantMessage("error"); return { terminal: { kind: "ok" }, @@ -114,7 +114,7 @@ describe("prepareEmbeddedRunTerminal", () => { describe("prepareEmbeddedRunTerminal run stats", () => { type StatsInput = { - attempt?: Partial & { + attempt?: Partial & { terminalTurnId?: string; }; assistantTurns?: number; @@ -257,12 +257,13 @@ describe("prepareEmbeddedRunTerminal run stats", () => { attempt: { terminalTurnId: "turn-7", toolMetas: [ - { toolName: "started" }, + { toolName: "exec", isError: false }, { toolName: "unknown" }, { toolName: "write", isError: true }, { toolName: "read", isError: false }, - { toolName: "read", isError: false }, + { toolName: "exec", isError: false }, ], + successfulNestedToolNames: ["read", "zeta", "alpha", "Zeta", " exec ", "alpha", " "], }, }); @@ -278,7 +279,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => { model: "cost-model-rerouted", responseModel: "cost-model-rerouted", }, - successfulToolNames: ["read"], + successfulToolNames: ["exec", "read", "Zeta", "alpha", "zeta"], rerouted: true, }); expect( diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.ts index 5ad6443f73f5..a44bc5eb267b 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.ts @@ -9,6 +9,7 @@ import type { NormalizedUsage, UsageLike } from "../../usage.js"; import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js"; import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; import type { UsageAccumulator } from "../usage-accumulator.js"; +import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js"; import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; import { buildUsageAgentMetaFields, @@ -25,11 +26,10 @@ import { type EmbeddedRunTerminalState, } from "./terminal-outcome.js"; import { mergeAttemptToolMediaPayloads } from "./tool-media-payloads.js"; -import type { EmbeddedRunAttemptResult } from "./types.js"; export function prepareEmbeddedRunTerminal(input: { runParams: RunEmbeddedAgentParams; - attempt: EmbeddedRunAttemptResult; + attempt: EmbeddedRunAttemptWithReceiptEvidence; currentAttemptCompletedAssistant?: AssistantMessage; provider: string; providerOwner?: PreparedProviderFailoverOwner; @@ -143,6 +143,14 @@ export function prepareEmbeddedRunTerminal(input: { .filter(Boolean), ), ]; + const missingNestedToolNames = [ + ...new Set( + (attempt.successfulNestedToolNames ?? []).map((name) => name.trim()).filter(Boolean), + ), + ] + .filter((name) => !successfulToolNames.includes(name)) + .toSorted(); + successfulToolNames.push(...missingNestedToolNames); Object.assign(agentMeta, { terminalReceipt: { runId: runParams.runId,