From 5a795f4ddacc1e416102ce16fd9e5e53dbc2f9f2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 7 Aug 2026 12:05:32 -0700 Subject: [PATCH] fix(qa): require a fresh reply and real tool use after switching models (#119662) * fix(qa): require a fresh reply after switching models * fix(qa): prove successful tool use after model switching * fix(qa): wait for the new persisted model-switch tool result * fix(qa): authenticate canonical model-switch attempt evidence * fix(ai): preserve effective response model evidence Punchcard-Session: golden-valley-workshop-br * fix(agent): publish run-owned terminal receipts Punchcard-Session: golden-valley-workshop-br * fix(qa): require run-owned model-switch evidence Punchcard-Session: golden-valley-workshop-br * fix(agent): record explicit tool completion outcomes Punchcard-Session: golden-valley-workshop-br * fix(agent): exclude unavailable approvals from receipts Punchcard-Session: golden-valley-workshop-br * fix(agent): derive receipt visibility from terminal reply Punchcard-Session: golden-valley-workshop-br * fix(qa): bind model-switch continuity to terminal reply Punchcard-Session: golden-valley-workshop-br * fix(qa): project Crabline Telegram visible text Punchcard-Session: golden-valley-workshop-br * fix(qa): record run-owned delivery evidence Punchcard-Session: golden-valley-workshop-br * fix(qa): bind primary model-switch delivery Punchcard-Session: golden-valley-workshop-br --------- Co-authored-by: Vincent Koc --- .../app-server/event-projector-assistant.ts | 10 +- .../event-projector-tool-progress.ts | 15 +- .../event-projector.assistant.test.ts | 22 ++ .../event-projector.dynamic-tools.test.ts | 2 + ...vent-projector.native-finalization.test.ts | 29 +++ .../codex/src/app-server/event-projector.ts | 10 +- extensions/copilot/src/attempt.test.ts | 2 +- extensions/copilot/src/event-bridge.test.ts | 6 +- extensions/copilot/src/event-bridge.ts | 2 +- extensions/qa-lab/src/runtime-parity.test.ts | 13 ++ extensions/qa-lab/src/runtime-parity.ts | 5 + ...talog-model-switch-follow-up-proof.test.ts | 211 ++++++++++++++++++ ...model-switch-tool-continuity-proof.test.ts | 193 ++++++++++++++++ extensions/qa-lab/src/scenario-flow-runner.ts | 6 +- .../qa-lab/src/scenario-runtime-api.test.ts | 1 + extensions/qa-lab/src/scenario-runtime-api.ts | 1 + .../src/suite-runtime-agent-process.test.ts | 6 +- .../qa-lab/src/suite-runtime-agent-process.ts | 11 + .../qa-lab/src/suite-runtime-flow.test.ts | 29 ++- extensions/qa-lab/src/suite-runtime-flow.ts | 11 + extensions/qa-lab/src/suite.ts | 1 + .../providers/openai-responses-shared.test.ts | 22 ++ ...enai-responses-stream-terminal-internal.ts | 2 + .../models/model-switch-follow-up.yaml | 76 ++++--- .../models/model-switch-tool-continuity.yaml | 73 +++--- .../agent-command.live-model-switch.test.ts | 35 +++ src/agents/agent-run-terminal-delivery.ts | 30 +++ src/agents/agent-run-terminal-receipt.ts | 27 +++ src/agents/command/lifecycle.test.ts | 72 +++++- src/agents/command/lifecycle.ts | 14 +- src/agents/command/post-run.ts | 13 +- .../embedded-agent-runner/run-entry.test.ts | 30 ++- src/agents/embedded-agent-runner/run-entry.ts | 16 +- .../run/attempt-result.test.ts | 9 +- .../run/attempt-result.ts | 6 +- .../run/terminal-preparation.test.ts | 41 +++- .../run/terminal-preparation.ts | 29 ++- ...ded-agent-subscribe.handlers.tools.test.ts | 102 ++++++++- ...embedded-agent-subscribe.handlers.tools.ts | 2 +- ...embedded-agent-subscribe.handlers.types.ts | 2 +- src/gateway/server-methods/agent-job.ts | 23 ++ .../server-methods/agent-wait-dedupe.test.ts | 24 +- src/gateway/server-methods/agent-wait.ts | 2 + 43 files changed, 1124 insertions(+), 112 deletions(-) create mode 100644 extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts create mode 100644 extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts create mode 100644 src/agents/agent-run-terminal-delivery.ts create mode 100644 src/agents/agent-run-terminal-receipt.ts diff --git a/extensions/codex/src/app-server/event-projector-assistant.ts b/extensions/codex/src/app-server/event-projector-assistant.ts index 19ae5b4a05f1..327656123a31 100644 --- a/extensions/codex/src/app-server/event-projector-assistant.ts +++ b/extensions/codex/src/app-server/event-projector-assistant.ts @@ -36,6 +36,7 @@ export class CodexAssistantProjection { // would drop legitimate verbatim answers ("reply with exactly the command output"). private readonly rawPromotedAssistantItemIds = new Set(); private assistantStarted = false; + private responseModel: string | undefined; private streamedPartialAssistantItemId: string | undefined; private streamedPartialAssistantItemReplaceable = false; @@ -85,6 +86,12 @@ export class CodexAssistantProjection { ); } + handleNotification(method: string, params: JsonObject): void { + if (method === "model/rerouted") { + this.responseModel = readString(params, "toModel") ?? this.responseModel; + } + } + async handleAssistantDelta(params: JsonObject): Promise { const itemId = readString(params, "itemId") ?? "assistant"; const delta = readString(params, "delta") ?? ""; @@ -397,7 +404,8 @@ export class CodexAssistantProjection { } createAssistantMessage(text: string, options: AssistantMessageOptions): AssistantMessage { - return buildAssistantMessage(this.params, text, options); + const message = buildAssistantMessage(this.params, text, options); + return this.responseModel ? { ...message, responseModel: this.responseModel } : message; } createAssistantMirrorMessage(title: string, text: string): AssistantMessage { diff --git a/extensions/codex/src/app-server/event-projector-tool-progress.ts b/extensions/codex/src/app-server/event-projector-tool-progress.ts index a6dba9601e2d..5bf2c22ee898 100644 --- a/extensions/codex/src/app-server/event-projector-tool-progress.ts +++ b/extensions/codex/src/app-server/event-projector-tool-progress.ts @@ -7,6 +7,7 @@ import { import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import type { EmbeddedRunAttemptResult } from "./attempt-terminal.js"; import { + auditNativeToolTerminalStatus, isMutatingNativeToolItem, isNonSuccessItemStatus, isSideEffectingNativeToolItem, @@ -167,7 +168,7 @@ export class CodexToolProgressProjection { toolName: existing?.toolName ?? params.tool, ...(existing?.meta ? { meta: existing.meta } : {}), ...(params.asyncStarted === true ? { asyncStarted: true } : {}), - ...(!params.success ? { isError: true } : {}), + isError: !params.success, }); if (params.terminalResolution) { this.lastNativeToolError = params.terminalResolution.lastToolError; @@ -347,13 +348,21 @@ export class CodexToolProgressProjection { return; } const meta = itemMeta(item, this.toolProgressDetailMode()); - const status = itemStatus(item); const existing = this.metas.get(item.id); + const terminalStatus = auditNativeToolTerminalStatus(item); + const isError = + typeof existing?.isError === "boolean" + ? existing.isError + : terminalStatus === "completed" + ? false + : terminalStatus === "failed" || terminalStatus === "blocked" + ? true + : undefined; this.metas.set(item.id, { toolName, ...(meta ? { meta } : {}), ...(existing?.asyncStarted ? { asyncStarted: true } : {}), - ...(status !== "running" && isNonSuccessItemStatus(status) ? { isError: true } : {}), + ...(isError === undefined ? {} : { isError }), }); } diff --git a/extensions/codex/src/app-server/event-projector.assistant.test.ts b/extensions/codex/src/app-server/event-projector.assistant.test.ts index abf414f40cf4..3d6221375480 100644 --- a/extensions/codex/src/app-server/event-projector.assistant.test.ts +++ b/extensions/codex/src/app-server/event-projector.assistant.test.ts @@ -86,6 +86,28 @@ describe("CodexAppServerEventProjector assistant projection", () => { expect(result.replayMetadata.replaySafe).toBe(true); }); + it("projects a current-turn model reroute onto the terminal assistant", async () => { + const projector = await createProjector(); + await projector.handleNotification( + forCurrentTurn("model/rerouted", { + fromModel: "gpt-5.4-codex", + toModel: "gpt-5.4-codex-mini", + reason: "high_risk_cyber_activity", + }), + ); + await projector.handleNotification( + turnCompleted([{ type: "agentMessage", id: "msg-rerouted", text: "done" }]), + ); + + const result = projector.buildResult(buildEmptyToolTelemetry()); + + expect(result.currentAttemptAssistant?.responseModel).toBe("gpt-5.4-codex-mini"); + expect(result.lastAssistant?.responseModel).toBe("gpt-5.4-codex-mini"); + expect(result).toMatchObject({ + terminalTurnId: "turn-1", + }); + }); + it("keeps reopened final answers as Activity candidates until turn completion selects one", async () => { const onAgentEvent = vi.fn(); const projector = await createProjector({ diff --git a/extensions/codex/src/app-server/event-projector.dynamic-tools.test.ts b/extensions/codex/src/app-server/event-projector.dynamic-tools.test.ts index bf28e859495f..3b411583a73c 100644 --- a/extensions/codex/src/app-server/event-projector.dynamic-tools.test.ts +++ b/extensions/codex/src/app-server/event-projector.dynamic-tools.test.ts @@ -39,6 +39,7 @@ describe("CodexAppServerEventProjector dynamic tool projection", () => { const result = projector.buildResult(buildEmptyToolTelemetry()); + expect(result.toolMetas).toEqual([{ toolName: "browser", isError: false }]); expect(result.messagesSnapshot.map((message) => message.role)).toEqual([ "user", "assistant", @@ -219,6 +220,7 @@ describe("CodexAppServerEventProjector dynamic tool projection", () => { toolName: "image_generate", meta: "lighthouse", asyncStarted: true, + isError: false, }, ]); expect(result.replayMetadata).toEqual({ diff --git a/extensions/codex/src/app-server/event-projector.native-finalization.test.ts b/extensions/codex/src/app-server/event-projector.native-finalization.test.ts index 48517e2333b2..1676a06ec0a4 100644 --- a/extensions/codex/src/app-server/event-projector.native-finalization.test.ts +++ b/extensions/codex/src/app-server/event-projector.native-finalization.test.ts @@ -23,6 +23,35 @@ import { registerCodexEventProjectorTestLifecycle(); describe("CodexAppServerEventProjector native tool finalization", () => { + it("marks only explicitly completed native tool metadata with false", async () => { + const projector = await createProjector(); + const command = { + type: "commandExecution", + command: "pnpm test extensions/codex", + cwd: "/workspace", + processId: null, + source: "agent", + commandActions: [], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }; + + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { ...command, id: "cmd-started-only", status: "inProgress" }, + }), + ); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { ...command, id: "cmd-completed", status: "completed" }, + }), + ); + + const result = projector.buildResult(buildEmptyToolTelemetry()); + expect(result.toolMetas.map((meta) => meta.isError)).toEqual([undefined, false]); + }); + it("keeps raw open-page status unknown until explicit completion", async () => { const diagnosticEvents: DiagnosticEventPayload[] = []; const unsubscribe = onInternalDiagnosticEvent((event) => diagnosticEvents.push(event)); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index 7090958aced2..a09ef5269c6a 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -222,7 +222,7 @@ export class CodexAppServerEventProjector { if (!params) { return; } - if (isHookNotificationMethod(notification.method)) { + if (notification.method === "hook/started" || notification.method === "hook/completed") { if (!this.isHookNotificationForCurrentThread(params)) { return; } @@ -235,6 +235,7 @@ export class CodexAppServerEventProjector { return; } this.nativeToolLifecycleProjector.handleNotification(notification); + this.assistantProjection.handleNotification(notification.method, params); switch (notification.method) { case "item/agentMessage/delta": @@ -317,7 +318,7 @@ export class CodexAppServerEventProjector { buildResult( toolTelemetry: CodexAppServerToolTelemetry, options?: { yieldDetected?: boolean }, - ): EmbeddedRunAttemptResult { + ): EmbeddedRunAttemptResult & { terminalTurnId: string } { // Result construction runs after the notification queue drains. Close any // tool lacking a terminal item so audit consumers never retain an open action. this.nativeToolLifecycleProjector.finalizeActive(); @@ -413,6 +414,7 @@ export class CodexAppServerEventProjector { promptErrorSource: promptError ? this.promptErrorSource || "prompt" : null, }), sessionIdUsed: this.params.sessionId, + terminalTurnId: this.turnId, ...(agentHarnessResultClassification ? { agentHarnessResultClassification } : {}), bootstrapPromptWarningSignaturesSeen: this.params.bootstrapPromptWarningSignaturesSeen, bootstrapPromptWarningSignature: this.params.bootstrapPromptWarningSignature, @@ -759,7 +761,3 @@ export class CodexAppServerEventProjector { return threadId === this.threadId && (turnId === this.turnId || turnId === null); } } - -function isHookNotificationMethod(method: string): method is "hook/started" | "hook/completed" { - return method === "hook/started" || method === "hook/completed"; -} diff --git a/extensions/copilot/src/attempt.test.ts b/extensions/copilot/src/attempt.test.ts index 1bce0940aba0..a59f407a0e3a 100644 --- a/extensions/copilot/src/attempt.test.ts +++ b/extensions/copilot/src/attempt.test.ts @@ -1506,7 +1506,7 @@ describe("runCopilotAttempt", () => { const result = await runCopilotAttempt(makeParams(), { pool }); - expect(result.toolMetas).toEqual([{ meta: "wrote file", toolName: "write" }]); + expect(result.toolMetas).toEqual([{ meta: "wrote file", toolName: "write", isError: false }]); expect(result.replayMetadata).toEqual({ hadPotentialSideEffects: true, replaySafe: false, diff --git a/extensions/copilot/src/event-bridge.test.ts b/extensions/copilot/src/event-bridge.test.ts index 8093d3a3f30f..4c1ab5243bac 100644 --- a/extensions/copilot/src/event-bridge.test.ts +++ b/extensions/copilot/src/event-bridge.test.ts @@ -195,7 +195,9 @@ describe("attachEventBridge", () => { expect(bridge.snapshot().assistantTexts).toEqual(["root"]); expect(bridge.snapshot().startedCount).toBe(0); - expect(bridge.snapshot().toolMetas).toEqual([{ meta: "child write", toolName: "write" }]); + expect(bridge.snapshot().toolMetas).toEqual([ + { meta: "child write", toolName: "write", isError: false }, + ]); expect( bridge.recordSendResult({ ...makeAssistantMessageEvent("child final"), @@ -797,7 +799,7 @@ describe("attachEventBridge", () => { ); expect(bridge.snapshot().toolMetas).toEqual([ - { meta: "details", toolName: "bash" }, + { meta: "details", toolName: "bash", isError: false }, { meta: "failed", toolName: "read", isError: true }, ]); }); diff --git a/extensions/copilot/src/event-bridge.ts b/extensions/copilot/src/event-bridge.ts index 09076ca7a4f4..12fd007daf88 100644 --- a/extensions/copilot/src/event-bridge.ts +++ b/extensions/copilot/src/event-bridge.ts @@ -350,7 +350,7 @@ export function attachEventBridge( toolMetas[toolMetaIndex] = { ...(meta ? { meta } : {}), toolName, - ...(event.data.success ? {} : { isError: true }), + isError: !event.data.success, }; } const projection = options.transcriptProjection; diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index 9c2828fbbeec..f8257d0d61f3 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -1045,4 +1045,17 @@ describe("runtime parity", () => { "tool-result-missing", ]); }); + + it("copies model-switch evidence into the runtime parity cell", async () => { + const modelSwitchEvidence = { + primary: { runId: "run-1", responseModel: "primary-model" }, + alternate: { runId: "run-2", responseModel: "alternate-model" }, + }; + const cell = await captureRuntimeParityWithMockRequests({ + requests: [], + scenarioResult: { status: "pass", modelSwitchEvidence }, + }); + + expect(cell.modelSwitchEvidence).toEqual(modelSwitchEvidence); + }); }); diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 841015d0ba28..edc8665b8422 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -62,6 +62,7 @@ export type RuntimeParityCell = { runtimeErrorClass?: string; bootStateLines: string[]; sentinelFindings?: GatewayLogSentinelFinding[]; + modelSwitchEvidence?: Record; }; type RuntimeParityResultCell = RuntimeParityCell & { @@ -154,6 +155,7 @@ type QaSuiteScenarioLike = { details?: string; status: "pass" | "fail" | "skip"; steps?: Array<{ details?: string; status?: "pass" | "fail" | "skip" }>; + modelSwitchEvidence?: Record; }; type RuntimeParityCaptureParams = { @@ -1510,6 +1512,9 @@ export async function captureRuntimeParityCell( : {}), bootStateLines: extractBootStateLines(gatewayLogs), ...(sentinelFindings.length > 0 ? { sentinelFindings } : {}), + ...(params.scenarioResult.modelSwitchEvidence + ? { modelSwitchEvidence: params.scenarioResult.modelSwitchEvidence } + : {}), }; } diff --git a/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts b/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts new file mode 100644 index 000000000000..a57337e4f59d --- /dev/null +++ b/extensions/qa-lab/src/scenario-catalog-model-switch-follow-up-proof.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it, vi } from "vitest"; +import { createQaBusState } from "./bus-state.js"; +import { runLoadedScenarioFlow } from "./scenario-flow-runner.test-support.js"; + +function splitModelRef(raw: string) { + const [provider, ...model] = raw.split("/"); + return provider && model.length + ? { provider: provider.toLowerCase(), model: model.join("/") } + : null; +} + +function normalizeModelRef(raw: string) { + const split = splitModelRef(raw); + if (!split) { + return null; + } + return split.provider === "openai" && split.model.toLowerCase() === "alternate-alias" + ? { provider: "openai", model: "alternate-model" } + : split; +} + +function terminalReceipt(params: { + runId: string; + provider: string; + model: string; + responseModel?: string; +}) { + const responseModel = params.responseModel ?? params.model; + return { + runId: params.runId, + sessionId: "session-model-switch", + turnId: `turn-${params.runId}`, + requested: { provider: params.provider, model: params.model }, + effective: { provider: params.provider, model: responseModel, responseModel }, + successfulToolNames: [], + rerouted: responseModel !== params.model, + terminalDisposition: "visible", + }; +} + +async function runFollowUp(params?: { + alternateModel?: string; + alternateReceiptRunId?: string; + primaryReplyText?: string; + primaryOutboundText?: string; + primaryDelivery?: { status: string; resultCount: number } | null; + alternateReplyText?: string; + alternateOutboundText?: string; + alternateDelivery?: { status: string; resultCount: number } | null; + unrelatedPrimaryOutboundText?: string; + unrelatedLaterOutboundText?: string; + onRun?: () => void; +}) { + const state = createQaBusState(); + let call = 0; + const runAgentPrompt = vi.fn( + async (_env: unknown, prompt: { provider?: string; model?: string; message: string }) => { + params?.onRun?.(); + call += 1; + const runId = `run-${call}`; + const provider = prompt.provider ?? "openai"; + const model = prompt.model ?? "primary-model"; + const replyText = + call === 1 + ? (params?.primaryReplyText ?? "hello from the primary model") + : (params?.alternateReplyText ?? "the model switch handoff completed"); + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: + call === 1 + ? (params?.primaryOutboundText ?? replyText) + : (params?.alternateOutboundText ?? replyText), + }); + if (call === 1 && params?.unrelatedPrimaryOutboundText) { + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: params.unrelatedPrimaryOutboundText, + }); + } + if (call === 2 && params?.unrelatedLaterOutboundText) { + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: params.unrelatedLaterOutboundText, + }); + } + const terminalDelivery = call === 1 ? params?.primaryDelivery : params?.alternateDelivery; + return { + started: { runId }, + waited: { + status: "ok", + ...(terminalDelivery === null + ? {} + : { + terminalDelivery: terminalDelivery ?? { status: "sent", resultCount: 1 }, + }), + terminalReply: { disposition: "visible", text: replyText }, + terminalReceipt: terminalReceipt({ + runId: call === 2 ? (params?.alternateReceiptRunId ?? runId) : runId, + provider, + model, + }), + }, + }; + }, + ); + const result = await runLoadedScenarioFlow("model-switch-follow-up", { + state, + api: { + env: { + providerMode: "mock-openai", + primaryModel: "openai/primary-model", + alternateModel: params?.alternateModel ?? "OPENAI/alternate-alias", + gateway: {}, + }, + runAgentPrompt, + splitModelRef, + normalizeModelRef, + normalizeLowercaseStringOrEmpty: (value: unknown) => + typeof value === "string" ? value.trim().toLowerCase() : "", + resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs, + }, + }); + return { result, runAgentPrompt }; +} + +describe("model-switch follow-up terminal evidence", () => { + it("invokes the canonical alias target and records exact run-owned evidence", async () => { + const { result, runAgentPrompt } = await runFollowUp({ + primaryReplyText: "hello **from the primary model**", + primaryOutboundText: "hello from the primary model", + alternateReplyText: "the **model switch** handoff completed", + alternateOutboundText: "the model switch handoff completed", + }); + + expect(result.status).toBe("pass"); + expect(runAgentPrompt.mock.calls[1]?.[1]).toMatchObject({ + provider: "openai", + model: "alternate-model", + }); + expect(result.modelSwitchEvidence).toMatchObject({ + primary: { runId: "run-1", effective: { responseModel: "primary-model" } }, + alternate: { runId: "run-2", effective: { responseModel: "alternate-model" } }, + terminalReply: { + disposition: "visible", + text: "the **model switch** handoff completed", + }, + terminalDelivery: { status: "sent", resultCount: 1 }, + }); + expect(result.steps[0]?.details).toBe("hello **from the primary model**"); + expect(result.steps[1]?.details).toBe("the **model switch** handoff completed"); + }); + + it("rejects a delayed prior-run receipt", async () => { + await expect(runFollowUp({ alternateReceiptRunId: "run-1" })).rejects.toThrow( + "alternate-model run did not return distinct exact owned model evidence", + ); + }); + + it("rejects normalized-identical refs before starting an agent run", async () => { + const onRun = vi.fn(); + await expect(runFollowUp({ alternateModel: "OPENAI/primary-model", onRun })).rejects.toThrow( + "primary and alternate models must normalize to different refs", + ); + expect(onRun).not.toHaveBeenCalled(); + }); + + it("rejects unrelated later continuity text when the alternate reply lacks it", async () => { + await expect( + runFollowUp({ + alternateReplyText: "the alternate run completed", + unrelatedLaterOutboundText: "the model switch handoff completed", + }), + ).rejects.toThrow("alternate-model terminal reply missed switch continuity"); + }); + + it.each([ + ["missing", null], + ["suppressed", { status: "suppressed", resultCount: 0 }], + ["zero-count", { status: "sent", resultCount: 0 }], + ] as const)( + "rejects %s primary delivery evidence despite identical and unrelated bus messages", + async (_, evidence) => { + await expect( + runFollowUp({ + primaryDelivery: evidence, + primaryOutboundText: "hello from the primary model", + unrelatedPrimaryOutboundText: "an unrelated run also replied", + }), + ).rejects.toThrow("default-model run did not return owned sent delivery evidence"); + }, + ); + + it.each([ + ["missing", null], + ["suppressed", { status: "suppressed", resultCount: 0 }], + ["zero-count", { status: "sent", resultCount: 0 }], + ] as const)( + "rejects %s delivery evidence despite an identical bus message", + async (_, evidence) => { + await expect( + runFollowUp({ + alternateDelivery: evidence, + alternateOutboundText: "the model switch handoff completed", + }), + ).rejects.toThrow("alternate-model run did not return owned sent delivery evidence"); + }, + ); +}); diff --git a/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts new file mode 100644 index 000000000000..97592e540ddc --- /dev/null +++ b/extensions/qa-lab/src/scenario-catalog-model-switch-tool-continuity-proof.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it, vi } from "vitest"; +import { createQaBusState } from "./bus-state.js"; +import { hasModelSwitchContinuitySignal } from "./model-switch-eval.js"; +import { runLoadedScenarioFlow } from "./scenario-flow-runner.test-support.js"; + +function splitModelRef(raw: string) { + const [provider, ...model] = raw.split("/"); + return provider && model.length + ? { provider: provider.toLowerCase(), model: model.join("/") } + : null; +} + +function normalizeModelRef(raw: string) { + const split = splitModelRef(raw); + if (!split) { + return null; + } + return split.provider === "openai" && split.model.toLowerCase() === "alternate-alias" + ? { provider: "openai", model: "alternate-model" } + : split; +} + +async function runToolContinuity( + alternateTools: string[], + params?: { + primaryOutboundText?: string; + primaryDelivery?: { status: string; resultCount: number } | null; + alternateReplyText?: string; + alternateOutboundText?: string; + alternateDelivery?: { status: string; resultCount: number } | null; + unrelatedPrimaryOutboundText?: string; + unrelatedLaterOutboundText?: string; + }, +) { + const state = createQaBusState(); + let call = 0; + const runAgentPrompt = vi.fn( + async (_env: unknown, prompt: { provider?: string; model?: string }) => { + call += 1; + const runId = `run-${call}`; + const provider = prompt.provider ?? "openai"; + const model = prompt.model ?? "primary-model"; + const replyText = + call === 1 + ? "the QA scenario pack verifies source and docs" + : (params?.alternateReplyText ?? + "the model handoff preserved the QA mission after rereading the scenario pack"); + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: + call === 1 + ? (params?.primaryOutboundText ?? replyText) + : (params?.alternateOutboundText ?? replyText), + }); + if (call === 1 && params?.unrelatedPrimaryOutboundText) { + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: params.unrelatedPrimaryOutboundText, + }); + } + if (call === 2 && params?.unrelatedLaterOutboundText) { + state.addOutboundMessage({ + accountId: "qa-channel", + to: "dm:qa-operator", + text: params.unrelatedLaterOutboundText, + }); + } + const terminalDelivery = call === 1 ? params?.primaryDelivery : params?.alternateDelivery; + return { + started: { runId }, + waited: { + status: "ok", + ...(terminalDelivery === null + ? {} + : { + terminalDelivery: terminalDelivery ?? { status: "sent", resultCount: 1 }, + }), + terminalReply: { disposition: "visible", text: replyText }, + terminalReceipt: { + runId, + sessionId: "session-tools", + turnId: `turn-${call}`, + requested: { provider, model }, + effective: { provider, model, responseModel: model }, + successfulToolNames: call === 1 ? ["read"] : alternateTools, + rerouted: false, + terminalDisposition: "visible", + }, + }, + }; + }, + ); + const result = await runLoadedScenarioFlow("model-switch-tool-continuity", { + state, + api: { + env: { + providerMode: "mock-openai", + primaryModel: "openai/primary-model", + alternateModel: "OPENAI/alternate-alias", + gateway: {}, + }, + splitModelRef, + normalizeModelRef, + normalizeLowercaseStringOrEmpty: (value: unknown) => + typeof value === "string" ? value.trim().toLowerCase() : "", + resolveQaLiveTurnTimeoutMs: (_env: unknown, timeoutMs: number) => timeoutMs, + hasModelSwitchContinuitySignal, + runAgentPrompt, + }, + }); + return { result, runAgentPrompt }; +} + +describe("model-switch tool continuity terminal evidence", () => { + it("invokes the canonical alias target and accepts run-owned delivery", async () => { + const { result, runAgentPrompt } = await runToolContinuity(["read"], { + alternateReplyText: + "the **model handoff** preserved the QA mission after rereading the scenario pack", + alternateOutboundText: + "the model handoff preserved the QA mission after rereading the scenario pack", + }); + + expect(result.status).toBe("pass"); + expect(runAgentPrompt.mock.calls[1]?.[1]).toMatchObject({ + provider: "openai", + model: "alternate-model", + }); + expect(result.modelSwitchEvidence).toMatchObject({ + primary: { runId: "run-1", successfulToolNames: ["read"] }, + alternate: { runId: "run-2", successfulToolNames: ["read"] }, + terminalReply: { + disposition: "visible", + text: "the **model handoff** preserved the QA mission after rereading the scenario pack", + }, + terminalDelivery: { status: "sent", resultCount: 1 }, + }); + expect(result.steps[0]?.details).toBe( + "the **model handoff** preserved the QA mission after rereading the scenario pack", + ); + }); + + it("does not let a successful prior-run read satisfy the alternate run", async () => { + await expect(runToolContinuity([])).rejects.toThrow( + "alternate-model run did not return exact owned successful read evidence", + ); + }); + + it("rejects unrelated later continuity text when the alternate reply lacks it", async () => { + await expect( + runToolContinuity(["read"], { + alternateReplyText: "the alternate tool run completed", + unrelatedLaterOutboundText: + "the model handoff preserved the QA mission after rereading the scenario pack", + }), + ).rejects.toThrow("alternate-model terminal reply missed kickoff continuity"); + }); + + it.each([ + ["missing", null], + ["suppressed", { status: "suppressed", resultCount: 0 }], + ["zero-count", { status: "sent", resultCount: 0 }], + ] as const)( + "rejects %s primary delivery evidence despite identical and unrelated bus messages", + async (_, evidence) => { + await expect( + runToolContinuity(["read"], { + primaryDelivery: evidence, + primaryOutboundText: "the QA scenario pack verifies source and docs", + unrelatedPrimaryOutboundText: "an unrelated tool run also replied", + }), + ).rejects.toThrow("default-model run did not return owned sent delivery evidence"); + }, + ); + + it.each([ + ["missing", null], + ["suppressed", { status: "suppressed", resultCount: 0 }], + ["zero-count", { status: "sent", resultCount: 0 }], + ] as const)( + "rejects %s delivery evidence despite an identical bus message", + async (_, evidence) => { + await expect( + runToolContinuity(["read"], { + alternateDelivery: evidence, + alternateOutboundText: + "the model handoff preserved the QA mission after rereading the scenario pack", + }), + ).rejects.toThrow("alternate-model run did not return owned sent delivery evidence"); + }, + ); +}); diff --git a/extensions/qa-lab/src/scenario-flow-runner.ts b/extensions/qa-lab/src/scenario-flow-runner.ts index 40608173f0f1..3a05f0d94484 100644 --- a/extensions/qa-lab/src/scenario-flow-runner.ts +++ b/extensions/qa-lab/src/scenario-flow-runner.ts @@ -17,6 +17,7 @@ type QaSuiteScenarioResult = { details?: string; }>; details?: string; + modelSwitchEvidence?: Record; }; type QaFlowApi = Record & { @@ -369,5 +370,8 @@ export async function runScenarioFlow(params: { return formatFlowDetails(details); }, })); - return await params.api.runScenario(params.scenarioTitle, steps); + const result = await params.api.runScenario(params.scenarioTitle, steps); + return isPlainObject(vars.modelSwitchEvidence) + ? { ...result, modelSwitchEvidence: vars.modelSwitchEvidence } + : result; } diff --git a/extensions/qa-lab/src/scenario-runtime-api.test.ts b/extensions/qa-lab/src/scenario-runtime-api.test.ts index 6e031bf45cfd..4e45ec371214 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.test.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.test.ts @@ -81,6 +81,7 @@ describe("createQaScenarioRuntimeApi", () => { waitForTransportReady: vi.fn(), waitForAgentHistoryReply: vi.fn(), browserRequest: vi.fn(), + normalizeModelRef: vi.fn(), }; const api = createQaScenarioRuntimeApi({ diff --git a/extensions/qa-lab/src/scenario-runtime-api.ts b/extensions/qa-lab/src/scenario-runtime-api.ts index a42db3cc0b1e..efab719b2ae4 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.ts @@ -94,6 +94,7 @@ export type QaScenarioRuntimeDeps = { formatErrorMessage: QaScenarioRuntimeFunction; liveTurnTimeoutMs: QaScenarioRuntimeFunction; resolveQaLiveTurnTimeoutMs: QaScenarioRuntimeFunction; + normalizeModelRef: QaScenarioRuntimeFunction; splitModelRef: QaScenarioRuntimeFunction; hasDiscoveryLabels: QaScenarioRuntimeFunction; reportsDiscoveryScopeLeak: QaScenarioRuntimeFunction; diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts index 7b50e6421935..3c487ee1db8a 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.test.ts @@ -536,10 +536,12 @@ describe("qa suite runtime agent process helpers", () => { }); it("accepts completed agent wait status as a successful terminal run", async () => { + const terminalReply = { disposition: "visible" as const, text: "completed reply" }; + const terminalDelivery = { status: "sent" as const, resultCount: 1 }; const gatewayCall = vi .fn() .mockResolvedValueOnce({ runId: "run-completed" }) - .mockResolvedValueOnce({ status: "completed" }); + .mockResolvedValueOnce({ status: "completed", terminalDelivery, terminalReply }); const env = createAgentPromptEnv(gatewayCall); await expect( @@ -549,7 +551,7 @@ describe("qa suite runtime agent process helpers", () => { }), ).resolves.toEqual({ started: { runId: "run-completed" }, - waited: { status: "completed" }, + waited: { status: "completed", terminalDelivery, terminalReply }, }); }); diff --git a/extensions/qa-lab/src/suite-runtime-agent-process.ts b/extensions/qa-lab/src/suite-runtime-agent-process.ts index 709fde09c9a1..f3675be4824d 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-process.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-process.ts @@ -44,10 +44,21 @@ type QaChatHistoryResponse = { messages?: unknown[]; }; +type QaAgentTerminalReply = + | { disposition: "visible"; text: string } + | { disposition: "silent" } + | { disposition: "empty" }; + type QaAgentWaitResult = { status?: string; error?: string; stopReason?: string; + terminalDelivery?: { + status: "sent" | "suppressed" | "partial_failed" | "failed"; + resultCount: number; + }; + terminalReceipt?: Record; + terminalReply?: QaAgentTerminalReply; }; const ANSI_ESCAPE_PATTERN = new RegExp(String.raw`\x1B\[[0-?]*[ -/]*[@-~]`, "g"); diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index 6d1279398332..355c6408ab10 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -1,4 +1,5 @@ // Qa Lab tests cover suite runtime flow plugin behavior. +import { parseModelRef, resolveModelRefFromString } from "openclaw/plugin-sdk/agent-runtime"; import { describe, expect, it, vi } from "vitest"; const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn()); @@ -110,7 +111,15 @@ describe("qa suite runtime flow", () => { primaryModel: "openai/gpt-5.6-luna", alternateModel: "openai/gpt-5.6-luna-mini", mock: null, - cfg: {} as QaSuiteRuntimeEnv["cfg"], + cfg: { + agents: { + defaults: { + models: { + "anthropic/claude-opus-5": { alias: "opus" }, + }, + }, + }, + }, } satisfies Parameters[0]["env"]; const scenario = { id: "session-memory-ranking", @@ -126,7 +135,7 @@ describe("qa suite runtime flow", () => { }, }; const runScenario = vi.fn(); - const splitModelRef = vi.fn(); + const splitModelRef = vi.fn((raw: string) => parseModelRef(raw, "openai")); const formatErrorMessage = vi.fn(); const liveTurnTimeoutMs = vi.fn(); const resolveQaLiveTurnTimeoutMs = vi.fn(); @@ -202,6 +211,22 @@ describe("qa suite runtime flow", () => { for (const [name, helper] of Object.entries(aliasedDependencies)) { expect((call.deps as Record)[name]).toBe(helper); } + const canonicalOpus = resolveModelRefFromString({ + cfg: env.cfg, + raw: "anthropic/opus", + defaultProvider: "anthropic", + })?.ref; + const normalizeModelRef = call.deps.normalizeModelRef as ( + raw: string, + ) => { provider: string; model: string } | null; + expect(canonicalOpus).toEqual({ provider: "anthropic", model: "claude-opus-5" }); + expect(normalizeModelRef("anthropic/opus")).toEqual(canonicalOpus); + expect(normalizeModelRef("AnThRoPiC/OPUS")).toEqual(canonicalOpus); + expect(normalizeModelRef("OPENAI/gpt-5.6-luna")).toEqual({ + provider: "openai", + model: "gpt-5.6-luna", + }); + expect(normalizeModelRef("")).toBeNull(); expect(call.deps.waitForOutboundMessage).toBeTypeOf("function"); const outboundPredicate = vi.fn(); call.deps.waitForOutboundMessage(env.transport.state, outboundPredicate, 123); diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 98a68324fab4..9407ed2c0965 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -3,6 +3,7 @@ import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; +import { resolveModelRefFromString } from "openclaw/plugin-sdk/agent-runtime"; import { formatErrorMessage as formatQaErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status"; import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-host-core"; @@ -219,6 +220,16 @@ function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) { formatErrorMessage: params.formatErrorMessage, liveTurnTimeoutMs: params.liveTurnTimeoutMs, resolveQaLiveTurnTimeoutMs: params.resolveQaLiveTurnTimeoutMs, + normalizeModelRef: (raw: string) => { + const split = params.splitModelRef(raw); + return split + ? (resolveModelRefFromString({ + cfg: params.env.cfg, + raw, + defaultProvider: split.provider, + })?.ref ?? null) + : null; + }, splitModelRef: params.splitModelRef, } satisfies QaScenarioRuntimeDeps; } diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 4ecafa37ef5c..0c6b85ab612a 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -71,6 +71,7 @@ export type QaSuiteScenarioResult = { details?: string; timing?: QaEvidenceTiming; runtimeParity?: RuntimeParityResult; + modelSwitchEvidence?: Record; }; type QaSuiteEnvironment = { diff --git a/packages/ai/src/providers/openai-responses-shared.test.ts b/packages/ai/src/providers/openai-responses-shared.test.ts index 9a4088b1bec9..6e5e5f347eea 100644 --- a/packages/ai/src/providers/openai-responses-shared.test.ts +++ b/packages/ai/src/providers/openai-responses-shared.test.ts @@ -969,6 +969,28 @@ describe("processResponsesStream", () => { expect(output.stopReason).toBe("stop"); }); + it("records the effective model from the terminal response", async () => { + const output = createAssistantOutput(); + + await processResponsesStream( + responseEvents([ + { + type: "response.completed", + response: { + id: "resp_rerouted", + status: "completed", + model: "gpt-5.5-rerouted", + }, + }, + ]), + output, + new AssistantMessageEventStream(), + nativeOpenAIModel, + ); + + expect(output.responseModel).toBe("gpt-5.5-rerouted"); + }); + it("keeps interleaved reasoning items bound to their output indices", async () => { const output = createAssistantOutput(); const { stream, events } = createCapturedAssistantMessageEventStream(); diff --git a/packages/ai/src/transports/openai-responses-stream-terminal-internal.ts b/packages/ai/src/transports/openai-responses-stream-terminal-internal.ts index 18308a1e3ade..18ed0534126e 100644 --- a/packages/ai/src/transports/openai-responses-stream-terminal-internal.ts +++ b/packages/ai/src/transports/openai-responses-stream-terminal-internal.ts @@ -43,6 +43,7 @@ type TerminalOutput = { content: Array; usage: Usage & { reasoningTokens?: number }; stopReason: string; + responseModel?: string; responseId?: string; errorMessage?: string; }; @@ -279,6 +280,7 @@ export function createResponsesTerminalController(params: { params.markFinalized(); backfillReasoning(response.output ?? []); output.responseId = response.id || output.responseId; + output.responseModel = response.model?.trim() || undefined; const usage = mapResponsesTerminalUsage(response.usage); const reasoningTokens = readResponsesReasoningTokens(response.usage); if (usage) { diff --git a/qa/scenarios/models/model-switch-follow-up.yaml b/qa/scenarios/models/model-switch-follow-up.yaml index 93fac0549b6b..df61728131a7 100644 --- a/qa/scenarios/models/model-switch-follow-up.yaml +++ b/qa/scenarios/models/model-switch-follow-up.yaml @@ -29,8 +29,15 @@ flow: steps: - name: runs on the default configured model actions: + - set: expectedAlternate + value: + expr: normalizeModelRef(env.alternateModel) + - assert: + expr: "normalizeModelRef(env.primaryModel) && expectedAlternate && (normalizeModelRef(env.primaryModel).provider !== expectedAlternate.provider || normalizeModelRef(env.primaryModel).model !== expectedAlternate.model)" + message: primary and alternate models must normalize to different refs - call: reset - call: runAgentPrompt + saveAs: primaryRun args: - ref: env - sessionKey: agent:qa:model-switch @@ -38,51 +45,48 @@ flow: expr: config.initialPrompt timeoutMs: expr: liveTurnTimeoutMs(env, 30000) - - call: waitForOutboundMessage - saveAs: outbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator'" - detailsExpr: "env.mock ? String((await fetchJson(`${env.mock.baseUrl}/debug/last-request`))?.body?.model ?? '') : outbound.text" + - assert: + expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean'; })()" + message: default-model run did not return an exact owned terminal receipt + - assert: + expr: "primaryRun?.waited?.terminalReply?.disposition === 'visible'" + message: default-model run did not return an owned visible terminal reply + - assert: + expr: "primaryRun?.waited?.terminalDelivery?.status === 'sent' && typeof primaryRun.waited.terminalDelivery.resultCount === 'number' && primaryRun.waited.terminalDelivery.resultCount > 0" + message: default-model run did not return owned sent delivery evidence + detailsExpr: primaryRun.waited.terminalReply.text - name: switches to the alternate model and continues actions: - - set: alternate - value: - expr: splitModelRef(env.alternateModel) - - set: switchRequestCursor - value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt + saveAs: alternateRun args: - ref: env - sessionKey: agent:qa:model-switch message: expr: config.followupPrompt provider: - expr: alternate?.provider + expr: expectedAlternate.provider model: - expr: alternate?.model + expr: expectedAlternate.model timeoutMs: expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel) - - call: waitForCondition - saveAs: outbound - args: - - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && (() => { const lower = normalizeLowercaseStringOrEmpty(candidate.text); return lower.includes('switch') || lower.includes('handoff'); })()).at(-1)" - - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.alternateModel) - - if: - expr: "Boolean(env.mock)" - then: - - set: switchDebugRequests - value: - expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${switchRequestCursor}`)" - - set: switchRequest - value: - expr: "switchDebugRequests.find((request) => String(request.allInputText ?? '').includes(config.followupPrompt))" - - assert: - expr: "String(switchRequest?.model ?? '') === String(alternate?.model ?? '')" - message: - expr: "`expected alternate model ${String(alternate?.model ?? '')}, got ${String(switchRequest?.model ?? '')}`" - detailsExpr: outbound.text + - assert: + expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()" + message: alternate-model run did not return distinct exact owned model evidence + - assert: + expr: "(() => { const reply = alternateRun?.waited?.terminalReply; if (reply?.disposition !== 'visible') return false; const lower = normalizeLowercaseStringOrEmpty(reply.text); return lower.includes('switch') || lower.includes('handoff'); })()" + message: alternate-model terminal reply missed switch continuity + - assert: + expr: "alternateRun?.waited?.terminalDelivery?.status === 'sent' && typeof alternateRun.waited.terminalDelivery.resultCount === 'number' && alternateRun.waited.terminalDelivery.resultCount > 0" + message: alternate-model run did not return owned sent delivery evidence + - set: modelSwitchEvidence + value: + primary: + ref: primaryRun.waited.terminalReceipt + alternate: + ref: alternateRun.waited.terminalReceipt + terminalReply: + ref: alternateRun.waited.terminalReply + terminalDelivery: + ref: alternateRun.waited.terminalDelivery + detailsExpr: alternateRun.waited.terminalReply.text diff --git a/qa/scenarios/models/model-switch-tool-continuity.yaml b/qa/scenarios/models/model-switch-tool-continuity.yaml index 197dc2d94aad..b2b878264a7c 100644 --- a/qa/scenarios/models/model-switch-tool-continuity.yaml +++ b/qa/scenarios/models/model-switch-tool-continuity.yaml @@ -32,12 +32,19 @@ flow: steps: - name: keeps using tools after switching models actions: + - set: expectedAlternate + value: + expr: normalizeModelRef(env.alternateModel) + - assert: + expr: "normalizeModelRef(env.primaryModel) && expectedAlternate && (normalizeModelRef(env.primaryModel).provider !== expectedAlternate.provider || normalizeModelRef(env.primaryModel).model !== expectedAlternate.model)" + message: primary and alternate models must normalize to different refs - call: waitForGatewayHealthy args: - ref: env - 60000 - call: reset - call: runAgentPrompt + saveAs: primaryRun args: - ref: env - sessionKey: agent:qa:model-switch-tools @@ -45,52 +52,42 @@ flow: expr: config.initialPrompt timeoutMs: expr: liveTurnTimeoutMs(env, 30000) - - set: alternate - value: - expr: splitModelRef(env.alternateModel) - - set: beforeSwitchCursor - value: - expr: state.getSnapshot().messages.length - - set: beforeSwitchRequestCursor - value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" + - assert: + expr: "(() => { const expected = normalizeModelRef(env.primaryModel); const receipt = primaryRun?.waited?.terminalReceipt; return receipt?.runId === primaryRun?.started?.runId && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expected.provider && receipt.requested?.model === expected.model && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible'; })()" + message: default-model run did not return owned successful read evidence + - assert: + expr: "primaryRun?.waited?.terminalDelivery?.status === 'sent' && typeof primaryRun.waited.terminalDelivery.resultCount === 'number' && primaryRun.waited.terminalDelivery.resultCount > 0" + message: default-model run did not return owned sent delivery evidence - call: runAgentPrompt + saveAs: alternateRun args: - ref: env - sessionKey: agent:qa:model-switch-tools message: expr: config.followupPrompt provider: - expr: alternate?.provider + expr: expectedAlternate.provider model: - expr: alternate?.model + expr: expectedAlternate.model timeoutMs: expr: resolveQaLiveTurnTimeoutMs(env, 30000, env.alternateModel) - - call: waitForCondition - saveAs: outbound - args: - - lambda: - expr: "state.getSnapshot().messages.slice(beforeSwitchCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && hasModelSwitchContinuitySignal(candidate.text)).at(-1)" - - expr: resolveQaLiveTurnTimeoutMs(env, 20000, env.alternateModel) - assert: - expr: hasModelSwitchContinuitySignal(outbound.text) - message: - expr: "`switch reply missed kickoff continuity: ${outbound.text}`" - - if: - expr: "Boolean(env.mock)" - then: - - set: switchDebugRequests - value: - expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${beforeSwitchRequestCursor}`)" - - set: switchRequest - value: - expr: "switchDebugRequests.find((request) => String(request.allInputText ?? '').includes(config.promptSnippet))" - - assert: - expr: "switchRequest?.plannedToolName === 'read'" - message: - expr: "`expected read after switch, got ${String(switchRequest?.plannedToolName ?? '')}`" - - assert: - expr: "String(switchRequest?.model ?? '') === String(alternate?.model ?? '')" - message: - expr: "`expected alternate model, got ${String(switchRequest?.model ?? '')}`" - detailsExpr: outbound.text + expr: "(() => { const receipt = alternateRun?.waited?.terminalReceipt; return receipt?.runId === alternateRun?.started?.runId && Boolean(receipt.sessionId) && Boolean(receipt.turnId) && normalizeLowercaseStringOrEmpty(receipt.requested?.provider) === expectedAlternate.provider && receipt.requested?.model === expectedAlternate.model && receipt.effective?.model === receipt.effective?.responseModel && receipt.successfulToolNames?.includes('read') && receipt.terminalDisposition === 'visible' && typeof receipt.rerouted === 'boolean' && `${receipt.effective?.provider}/${receipt.effective?.responseModel}` !== `${primaryRun.waited.terminalReceipt.effective?.provider}/${primaryRun.waited.terminalReceipt.effective?.responseModel}`; })()" + message: alternate-model run did not return exact owned successful read evidence + - assert: + expr: "alternateRun?.waited?.terminalReply?.disposition === 'visible' && hasModelSwitchContinuitySignal(alternateRun.waited.terminalReply.text)" + message: alternate-model terminal reply missed kickoff continuity + - assert: + expr: "alternateRun?.waited?.terminalDelivery?.status === 'sent' && typeof alternateRun.waited.terminalDelivery.resultCount === 'number' && alternateRun.waited.terminalDelivery.resultCount > 0" + message: alternate-model run did not return owned sent delivery evidence + - set: modelSwitchEvidence + value: + primary: + ref: primaryRun.waited.terminalReceipt + alternate: + ref: alternateRun.waited.terminalReceipt + terminalReply: + ref: alternateRun.waited.terminalReply + terminalDelivery: + ref: alternateRun.waited.terminalDelivery + detailsExpr: alternateRun.waited.terminalReply.text diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index cfbe953e1e26..236d2ac8f871 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -1475,6 +1475,41 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { ); }); + it("preserves bounded delivery evidence when strict post-turn delivery throws", async () => { + setupSingleAttemptFallback(); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); + const secret = ["sk", "strict-delivery-secret-value"].join("-"); + state.deliverAgentCommandResultMock.mockImplementation(async (params: unknown) => { + ( + params as { + onDeliveryResult?: (result: { deliveryStatus: Record }) => void; + } + ).onDeliveryResult?.({ + deliveryStatus: { + status: "failed", + errorMessage: `Authorization: Bearer ${secret}`, + target: "discord:dm:private", + }, + }); + throw new Error("strict delivery failed"); + }); + + await expect(runDiscordDelivery()).rejects.toThrow("strict delivery failed"); + + const lifecycleError = state.emitAgentEventMock.mock.calls + .map((call) => call[0] as { stream?: string; data?: Record }) + .find((event) => event.stream === "lifecycle" && event.data?.phase === "error"); + expect(lifecycleError?.data?.terminalDelivery).toEqual({ + status: "failed", + resultCount: 0, + }); + for (const field of ["stopReason", "terminalReceipt", "terminalReply"]) { + expect(lifecycleError?.data).not.toHaveProperty(field); + } + expect(JSON.stringify(lifecycleError)).not.toContain(secret); + expect(JSON.stringify(lifecycleError)).not.toContain("discord:dm:private"); + }); + it("preserves restart ownership when an aborted attempt resolves normally", async () => { setupSingleAttemptFallback(); const controller = new AbortController(); diff --git a/src/agents/agent-run-terminal-delivery.ts b/src/agents/agent-run-terminal-delivery.ts new file mode 100644 index 000000000000..5cd8cbbd04c6 --- /dev/null +++ b/src/agents/agent-run-terminal-delivery.ts @@ -0,0 +1,30 @@ +export type AgentRunTerminalDeliverySnapshot = { + status: "sent" | "suppressed" | "partial_failed" | "failed"; + resultCount: number; +}; + +/** Rejects malformed lifecycle/RPC input and projects only the bounded delivery fact. */ +export function normalizeAgentRunTerminalDeliverySnapshot( + value: unknown, +): AgentRunTerminalDeliverySnapshot | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const delivery = value as { status?: unknown; resultCount?: unknown }; + if ( + typeof delivery.resultCount !== "number" || + !Number.isSafeInteger(delivery.resultCount) || + delivery.resultCount < 0 + ) { + return undefined; + } + switch (delivery.status) { + case "sent": + case "suppressed": + case "partial_failed": + case "failed": + return { status: delivery.status, resultCount: delivery.resultCount }; + default: + return undefined; + } +} diff --git a/src/agents/agent-run-terminal-receipt.ts b/src/agents/agent-run-terminal-receipt.ts new file mode 100644 index 000000000000..c6acaa50c132 --- /dev/null +++ b/src/agents/agent-run-terminal-receipt.ts @@ -0,0 +1,27 @@ +type AgentRunTerminalModelRef = { provider: string; model: string }; + +export type AgentRunTerminalReceipt = { + runId: string; + sessionId: string; + turnId: string; + requested: AgentRunTerminalModelRef; + effective: AgentRunTerminalModelRef & { responseModel: string }; + successfulToolNames: string[]; + rerouted: boolean; + terminalDisposition: "visible" | "not-visible"; +}; + +export function normalizeAgentRunTerminalReceipt( + value: unknown, +): AgentRunTerminalReceipt | undefined { + const receipt = value as AgentRunTerminalReceipt | undefined; + return receipt && + typeof receipt.runId === "string" && + typeof receipt.sessionId === "string" && + typeof receipt.turnId === "string" && + receipt.requested && + receipt.effective && + Array.isArray(receipt.successfulToolNames) + ? receipt + : undefined; +} diff --git a/src/agents/command/lifecycle.test.ts b/src/agents/command/lifecycle.test.ts index 3386c70b5d0d..ae15287a7dfc 100644 --- a/src/agents/command/lifecycle.test.ts +++ b/src/agents/command/lifecycle.test.ts @@ -24,6 +24,12 @@ describe("createAgentCommandLifecycle", () => { yielded: true, replayInvalid: true, error: { message: `Authorization: Bearer ${secret}`, nested: { secret } }, + terminalDelivery: { + status: "sent", + resultCount: 2, + errorMessage: secret, + target: "private-target", + }, unsafeMetadata: { credential: secret }, }; const lifecycle = createAgentCommandLifecycle({ @@ -76,8 +82,10 @@ describe("createAgentCommandLifecycle", () => { }), }), ); - expect(JSON.stringify(emitAgentEvent.mock.calls[0]?.[0])).not.toContain(secret); - expect(emitAgentEvent.mock.calls[0]?.[0]?.data).not.toHaveProperty("unsafeMetadata"); + const event = emitAgentEvent.mock.calls[0]?.[0]; + expect(event.data.terminalDelivery).toEqual({ status: "sent", resultCount: 2 }); + expect(JSON.stringify(event)).not.toContain(secret); + expect(event.data).not.toHaveProperty("unsafeMetadata"); }, ); @@ -105,7 +113,7 @@ describe("createAgentCommandLifecycle", () => { }; if (source === "post-turn error") { - lifecycle.emitPostTurnError(new Error(error)); + lifecycle.emitPostTurnError(new Error(error), terminal); } else { lifecycle.emitResultError( { @@ -124,6 +132,60 @@ describe("createAgentCommandLifecycle", () => { }, ); + it("keeps post-turn errors narrow while publishing bounded delivery evidence", () => { + emitAgentEvent.mockClear(); + const secret = ["sk", "abcdefghijklmnopqrstuv"].join("-"); + const lifecycle = createAgentCommandLifecycle({ + runId: "post-turn-delivery-owner", + lifecycleGeneration: () => "test-generation", + startedAt: 100, + state: { + currentTurnUserMessagePersisted: true, + lifecycleFinishing: false, + lifecycleEnded: false, + }, + }); + lifecycle.emitPostTurnError(new Error("delivery failed"), { + metadata: { + terminalDelivery: { + status: "failed", + resultCount: 0, + errorMessage: secret, + }, + terminalReceipt: { runId: "unrelated-receipt", secret }, + terminalReply: { disposition: "visible", text: secret }, + unsafeMetadata: { secret }, + }, + outcome: buildAgentRunTerminalOutcome({ + status: "timeout", + stopReason: "timeout", + livenessState: "blocked", + timeoutPhase: "provider", + providerStarted: true, + }), + }); + + const event = emitAgentEvent.mock.calls[0]?.[0]; + expect(event.data).toMatchObject({ + phase: "error", + error: "delivery failed", + terminalDelivery: { status: "failed", resultCount: 0 }, + }); + expect(JSON.stringify(event)).not.toContain(secret); + for (const field of [ + "aborted", + "stopReason", + "livenessState", + "timeoutPhase", + "providerStarted", + "terminalReceipt", + "terminalReply", + "unsafeMetadata", + ]) { + expect(event.data).not.toHaveProperty(field); + } + }); + it.each(["finishing", "end", "error"] as const)( "rejects malformed canonical metadata on %s events", (phase) => { @@ -149,6 +211,8 @@ describe("createAgentCommandLifecycle", () => { providerStarted: malicious, livenessState: malicious, replayInvalid: malicious, + terminalReceipt: malicious, + terminalDelivery: malicious, error: malicious, unknownMetadata: malicious, }, @@ -172,6 +236,8 @@ describe("createAgentCommandLifecycle", () => { "providerStarted", "livenessState", "replayInvalid", + "terminalDelivery", + "terminalReceipt", "unknownMetadata", ]) { expect(event.data).not.toHaveProperty(field); diff --git a/src/agents/command/lifecycle.ts b/src/agents/command/lifecycle.ts index 25f45c0f2e09..8a5289fed2a5 100644 --- a/src/agents/command/lifecycle.ts +++ b/src/agents/command/lifecycle.ts @@ -1,10 +1,12 @@ import { emitAgentEvent } from "../../infra/agent-events.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; +import { normalizeAgentRunTerminalDeliverySnapshot } from "../agent-run-terminal-delivery.js"; import { buildAgentRunTerminalOutcomeFromLifecycleEvent, type AgentRunTerminalOutcome, } from "../agent-run-terminal-outcome.js"; +import { normalizeAgentRunTerminalReceipt } from "../agent-run-terminal-receipt.js"; import type { EmbeddedAgentRunEntryTerminal } from "../embedded-agent-runner/run-entry.js"; import { resolveAgentRunAbortLifecycleFields, @@ -79,6 +81,10 @@ export function createAgentCommandLifecycle(params: { fallbackExhausted?: boolean, ) => { const { aborted, yielded, replayInvalid, terminalReply } = terminal.metadata; + const terminalDelivery = normalizeAgentRunTerminalDeliverySnapshot( + terminal.metadata.terminalDelivery, + ); + const terminalReceipt = normalizeAgentRunTerminalReceipt(terminal.metadata.terminalReceipt); const { stopReason, livenessState, timeoutPhase, providerStarted } = terminal.outcome; emitAgentEvent({ runId: params.runId, @@ -97,6 +103,8 @@ export function createAgentCommandLifecycle(params: { ...(providerStarted !== undefined ? { providerStarted } : {}), ...(error ? { error: formatErrorMessage(error) } : {}), ...(fallbackExhausted ? { fallbackExhaustedFailure: true } : {}), + ...(terminalDelivery ? { terminalDelivery } : {}), + ...(terminalReceipt ? { terminalReceipt } : {}), ...(terminalReply ? { terminalReply } : {}), ...resolveAgentRunAbortLifecycleFields(params.abortSignal), }, @@ -143,11 +151,14 @@ export function createAgentCommandLifecycle(params: { (fallbackExhausted ? "All model fallback candidates failed" : "Agent run failed"); emitTerminalPhase("error", terminal, error, fallbackExhausted); }, - emitPostTurnError(error: unknown) { + emitPostTurnError(error: unknown, terminal: EmbeddedAgentRunEntryTerminal) { if (params.state.lifecycleEnded) { return; } params.state.lifecycleEnded = true; + const terminalDelivery = normalizeAgentRunTerminalDeliverySnapshot( + terminal.metadata.terminalDelivery, + ); emitAgentEvent({ runId: params.runId, lifecycleGeneration: params.lifecycleGeneration(), @@ -157,6 +168,7 @@ export function createAgentCommandLifecycle(params: { startedAt: params.startedAt, endedAt: Date.now(), error: formatErrorMessage(error), + ...(terminalDelivery ? { terminalDelivery } : {}), ...resolveAgentRunErrorLifecycleFields(error, params.abortSignal), }, }); diff --git a/src/agents/command/post-run.ts b/src/agents/command/post-run.ts index bcc018863181..11cedf6a5fe3 100644 --- a/src/agents/command/post-run.ts +++ b/src/agents/command/post-run.ts @@ -13,6 +13,7 @@ import { constrainRestartRecoveryDeliveryPayloads, shouldPersistCurrentRunSessionCleanup, } from "../agent-command-restart-recovery.js"; +import { normalizeAgentRunTerminalDeliverySnapshot } from "../agent-run-terminal-delivery.js"; import { isHeartbeatLifecycleRunKind } from "../bootstrap-mode.js"; import { persistPendingFinalDeliveryMarker } from "../pending-final-delivery-marker.js"; import type { AgentRunSessionTarget } from "../run-session-target.js"; @@ -340,6 +341,16 @@ export async function finalizeEmbeddedAgentCommand(params: { NonNullable[0]["onDeliveryResult"]> >[0], ) => { + const deliveryStatus = deliveryResult.deliveryStatus; + const terminalDelivery = normalizeAgentRunTerminalDeliverySnapshot( + deliveryStatus && { + status: deliveryStatus.status, + resultCount: deliveryStatus.resultCount ?? 0, + }, + ); + if (terminalDelivery) { + terminal.metadata.terminalDelivery = terminalDelivery; + } params.onTerminalDeliveryEvidenceChanged( buildRestartRecoveryTerminalDeliveryEvidence(deliveryResult), ); @@ -396,7 +407,7 @@ export async function finalizeEmbeddedAgentCommand(params: { sessionReboundDuringRun, }; } catch (error) { - lifecycle.emitPostTurnError(error); + lifecycle.emitPostTurnError(error, terminal); throw error; } } diff --git a/src/agents/embedded-agent-runner/run-entry.test.ts b/src/agents/embedded-agent-runner/run-entry.test.ts index 5bb7c69bb8a5..1d71d4629c74 100644 --- a/src/agents/embedded-agent-runner/run-entry.test.ts +++ b/src/agents/embedded-agent-runner/run-entry.test.ts @@ -780,6 +780,7 @@ describe("runEmbeddedAgentEntry", () => { expected: { disposition: "empty" }, }, ])("records the producer-owned terminal snapshot for $name", async ({ name, meta, expected }) => { + const runId = `terminal-${name}`; state.runWithModelFallback.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ outcome: "completed" as const, result: await params.run(params.provider, params.model), @@ -790,7 +791,7 @@ describe("runEmbeddedAgentEntry", () => { const { runEmbeddedAgentEntry } = await import("./run-entry.js"); const result = await runEmbeddedAgentEntry({ selection: { cfg: {}, provider: "provider", model: "model" }, - identity: { runId: `terminal-${name}`, agentId: "main", sessionId: "session-1" }, + identity: { runId, agentId: "main", sessionId: "session-1" }, harness: { workspaceDir: "/tmp/workspace", preparation: { kind: "direct" }, @@ -800,10 +801,35 @@ describe("runEmbeddedAgentEntry", () => { sessionOverride: { kind: "preserve" }, runCandidate: async (provider, model) => ({ ...makeResult({ provider, model }), - meta: { ...makeResult({ provider, model }).meta, ...meta }, + meta: { + ...makeResult({ provider, model }).meta, + ...meta, + agentMeta: Object.assign( + { + sessionId: "session-1", + provider, + model, + }, + { + terminalReceipt: { + runId, + sessionId: "session-1", + turnId: "turn-1", + requested: { provider, model }, + effective: { provider, model, responseModel: model }, + successfulToolNames: ["read"], + rerouted: false, + }, + }, + ), + }, }), }); expect(result.terminal.metadata.terminalReply).toEqual(expected); + expect(result.terminal.metadata.terminalReceipt).toMatchObject({ + runId, + terminalDisposition: expected.disposition === "visible" ? "visible" : "not-visible", + }); }); }); diff --git a/src/agents/embedded-agent-runner/run-entry.ts b/src/agents/embedded-agent-runner/run-entry.ts index 82ef483f9136..73cb11085401 100644 --- a/src/agents/embedded-agent-runner/run-entry.ts +++ b/src/agents/embedded-agent-runner/run-entry.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { ContextEngineHostSupport } from "../../context-engine/host-compat.js"; import { requireActivePluginRegistry } from "../../plugins/runtime.js"; import { buildAgentRunTerminalOutcome } from "../agent-run-terminal-outcome.js"; +import { normalizeAgentRunTerminalReceipt } from "../agent-run-terminal-receipt.js"; import { buildAgentRunTerminalReplySnapshot, normalizeAgentRunTerminalReplySnapshot, @@ -189,6 +190,7 @@ function buildTerminal(params: { result: EmbeddedAgentRunResult; fallbackExhausted: boolean; behavior: RunEntryBehavior; + runId: string; }): EmbeddedAgentRunEntryTerminal { const meta = params.result.meta; const outcome = buildAgentRunTerminalOutcome({ @@ -199,14 +201,23 @@ function buildTerminal(params: { timeoutPhase: meta.timeoutPhase, providerStarted: meta.providerStarted, }); - const metadata: Record = {}; - metadata.terminalReply = + const terminalReply = normalizeAgentRunTerminalReplySnapshot(meta.terminalReply) ?? buildAgentRunTerminalReplySnapshot({ visibleText: meta.finalAssistantVisibleText, rawText: meta.finalAssistantRawText, terminalReplyKind: meta.terminalReplyKind, }); + const metadata: Record = { terminalReply }; + const terminalReceipt = normalizeAgentRunTerminalReceipt( + (meta.agentMeta as { terminalReceipt?: unknown } | undefined)?.terminalReceipt, + ); + if (terminalReceipt?.runId === params.runId) { + metadata.terminalReceipt = { + ...terminalReceipt, + terminalDisposition: terminalReply.disposition === "visible" ? "visible" : "not-visible", + }; + } if (params.behavior.kind === "channel-delivery" || params.behavior.kind === "followup-delivery") { for (const key of [ "stopReason", @@ -447,6 +458,7 @@ export async function runEmbeddedAgentEntry( result, fallbackExhausted: settledResult.outcome === "exhausted", behavior: params.behavior, + runId: params.identity.runId, }); if (fallbackResult.result.turnAttempt) { if ( diff --git a/src/agents/embedded-agent-runner/run/attempt-result.test.ts b/src/agents/embedded-agent-runner/run/attempt-result.test.ts index 63118489e90b..d1fc9e130c53 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.test.ts @@ -14,7 +14,7 @@ function completeResult(params?: { toolName: string; meta?: string; replaySafe?: boolean; - isError?: true; + isError?: boolean; asyncStarted?: boolean; asyncTaskRunId?: string; asyncTaskId?: string; @@ -97,6 +97,7 @@ describe("attempt result projection", () => { completeResult({ toolMetas: [ { toolName: "", replaySafe: true }, + { toolName: "read", isError: false }, { toolName: "exec", meta: "done", @@ -109,6 +110,12 @@ describe("attempt result projection", () => { ], }).toolMetas, ).toEqual([ + { + toolName: "read", + meta: undefined, + replaySafe: false, + isError: false, + }, { toolName: "exec", meta: "done", diff --git a/src/agents/embedded-agent-runner/run/attempt-result.ts b/src/agents/embedded-agent-runner/run/attempt-result.ts index c812ca39abcf..23313be1c129 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.ts @@ -94,7 +94,7 @@ function normalizeEmbeddedAttemptToolMetas( toolName: string; meta?: string; replaySafe?: boolean; - isError?: true; + isError?: boolean; asyncStarted?: boolean; asyncTaskRunId?: string; asyncTaskId?: string; @@ -106,8 +106,8 @@ function normalizeEmbeddedAttemptToolMetas( meta: entry.meta, replaySafe: entry.replaySafe === true, }; - if (entry.isError === true) { - normalized.isError = true; + if (typeof entry.isError === "boolean") { + normalized.isError = entry.isError; } if (entry.asyncStarted === true) { normalized.asyncStarted = true; diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts index 57764e33a41f..8cb3168b659c 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.test.ts @@ -112,12 +112,15 @@ describe("prepareEmbeddedRunTerminal", () => { describe("prepareEmbeddedRunTerminal run stats", () => { type StatsInput = { - attempt?: Partial; + attempt?: Partial & { + terminalTurnId?: string; + }; assistantTurns?: number; bridgeCalls?: { search: number; describe: number; call: number }; config?: unknown; provider?: string; model?: string; + responseModel?: string; usage?: Partial< Pick< ReturnType, @@ -134,6 +137,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => { ...assistantMessage("stop"), provider, model, + ...(statsInput.responseModel ? { responseModel: statsInput.responseModel } : {}), }; const usageAccumulator = createUsageAccumulator(); Object.assign(usageAccumulator, statsInput.usage); @@ -243,4 +247,39 @@ describe("prepareEmbeddedRunTerminal run stats", () => { const prepared = await prepareStats({ config: COST_CONFIG }); expect(prepared.agentMeta).not.toHaveProperty("costUsd"); }); + + it("builds exact terminal model and successful-tool evidence", async () => { + const prepared = await prepareStats({ + responseModel: "cost-model-rerouted", + attempt: { + terminalTurnId: "turn-7", + toolMetas: [ + { toolName: "started" }, + { toolName: "unknown" }, + { toolName: "write", isError: true }, + { toolName: "read", isError: false }, + { toolName: "read", isError: false }, + ], + }, + }); + + expect( + (prepared.agentMeta as { terminalReceipt?: Record }).terminalReceipt, + ).toMatchObject({ + runId: "run-1", + sessionId: "session-1", + turnId: "turn-7", + requested: { provider: "cost-test-provider", model: "cost-model" }, + effective: { + provider: "cost-test-provider", + model: "cost-model-rerouted", + responseModel: "cost-model-rerouted", + }, + successfulToolNames: ["read"], + rerouted: true, + }); + expect( + (prepared.agentMeta as { terminalReceipt?: Record }).terminalReceipt, + ).not.toHaveProperty("terminalDisposition"); + }); }); diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.ts index c13acad65f5b..2953b1113a1b 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.ts @@ -2,6 +2,7 @@ import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js"; import type { AssistantMessage } from "../../../llm/types.js"; import { estimateUsageCost, resolveModelCostConfig } from "../../../utils/usage-format.js"; import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; +import type { AgentRunTerminalReceipt } from "../../agent-run-terminal-receipt.js"; import type { AuthProfileStore } from "../../auth-profiles.js"; import type { NormalizedUsage, UsageLike } from "../../usage.js"; import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js"; @@ -72,11 +73,13 @@ export function prepareEmbeddedRunTerminal(input: { lastAssistantUsage: terminalAssistant?.usage as UsageLike | undefined, lastRunPromptUsage: input.lastRunPromptUsage, }); - const reportedModelRef = resolveReportedModelRef({ + const resolvedModelRef = resolveReportedModelRef({ provider: input.provider, model: input.model, assistant: terminalAssistant, }); + const responseModel = terminalAssistant?.responseModel?.trim() || resolvedModelRef.model; + const reportedModelRef = { ...resolvedModelRef, model: responseModel }; const finalAssistantStopReason = (terminalAssistant?.stopReason ?? "").trim().toLowerCase(); const terminalAssistantCanOwnFinalText = finalAssistantStopReason !== "error" && finalAssistantStopReason !== "aborted"; @@ -129,6 +132,30 @@ export function prepareEmbeddedRunTerminal(input: { const finalAssistantRawText = terminalAssistantCanOwnFinalText ? (resolveFinalAssistantRawText(terminalAssistant) ?? attemptFinalText) : undefined; + const terminalTurnId = (attempt as { terminalTurnId?: string }).terminalTurnId; + const successfulToolNames = [ + ...new Set( + attempt.toolMetas + .filter((entry) => entry.isError === false) + .map((entry) => entry.toolName.trim()) + .filter(Boolean), + ), + ]; + Object.assign(agentMeta, { + terminalReceipt: { + runId: runParams.runId, + sessionId: input.sessionIdUsed, + turnId: terminalTurnId?.trim() || runParams.runId, + requested: { provider: input.provider, model: input.model }, + effective: { + provider: reportedModelRef.provider, + model: reportedModelRef.model, + responseModel, + }, + successfulToolNames, + rerouted: responseModel !== input.model, + } satisfies Omit, + }); // A yielded attempt ends before message_end. Its aborted tool-call assistant, // not an earlier completed cycle, owns paused-turn classification. const payloadAssistant = attempt.yieldDetected diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts index 439499d8cbc6..eee1d7f05612 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.test.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.test.ts @@ -1977,6 +1977,48 @@ describe("handleToolExecutionEnd mutating failure recovery", () => { }); describe("handleToolExecutionEnd timeout metadata", () => { + it("marks every finalized built-in call with its explicit outcome", async () => { + const { ctx } = createTestContext(); + + await endTool(ctx, { + toolName: "read", + toolCallId: "tool-read-complete", + isError: false, + result: { content: "ok" }, + }); + await endTool(ctx, { + toolName: "process", + toolCallId: "tool-process-running", + isError: false, + result: { details: { status: "running" } }, + }); + await endTool(ctx, { + toolName: "image_generate", + toolCallId: "tool-image-async-started", + isError: false, + result: { details: { async: true, status: "started" } }, + }); + await endTool(ctx, { + toolName: "write", + toolCallId: "tool-write-failed", + isError: true, + result: { error: "failed" }, + }); + + expect( + ctx.state.toolMetas.map(({ toolName, isError }) => ({ + toolName, + isError, + })), + ).toEqual([ + { toolName: "read", isError: false }, + { toolName: "process", isError: false }, + { toolName: "image_generate", isError: false }, + { toolName: "write", isError: true }, + ]); + expect(ctx.state.toolMetas[2]?.asyncStarted).toBe(true); + }); + it("retains every failed call after later successes change the last-error slot", async () => { const { ctx } = createTestContext(); @@ -1995,7 +2037,7 @@ describe("handleToolExecutionEnd timeout metadata", () => { expect(ctx.state.toolMetas.map(({ toolName, isError }) => ({ toolName, isError }))).toEqual([ { toolName: "read", isError: true }, - { toolName: "read", isError: undefined }, + { toolName: "read", isError: false }, { toolName: "exec", isError: true }, ]); }); @@ -2359,6 +2401,64 @@ describe("handleToolExecutionEnd exec approval prompts", () => { }), isError: true, }); + expect(ctx.state.toolMetas).toEqual([ + expect.objectContaining({ toolName: "exec", isError: true }), + ]); + const [ + { normalizeAgentRunTerminalReceipt }, + { createUsageAccumulator }, + { createEmbeddedRunContextRecoveryState }, + { prepareEmbeddedRunTerminal }, + ] = await Promise.all([ + import("./agent-run-terminal-receipt.js"), + import("./embedded-agent-runner/usage-accumulator.js"), + import("./embedded-agent-runner/run/context-recovery-state.js"), + import("./embedded-agent-runner/run/terminal-preparation.js"), + ]); + const prepared = prepareEmbeddedRunTerminal({ + runParams: { + sessionId: "session-test-id", + runId: "run-test", + workspaceDir: "/tmp/openclaw-test", + prompt: "run", + trigger: "user", + timeoutMs: 60_000, + }, + attempt: { + terminal: { kind: "ok" }, + sessionIdUsed: "session-test-id", + messagesSnapshot: [], + assistantTexts: [], + toolMetas: ctx.state.toolMetas.flatMap(({ toolName, ...entry }) => + toolName ? [{ ...entry, toolName }] : [], + ), + lastAssistant: undefined, + didSendViaMessagingTool: false, + messagingToolSentTexts: [], + messagingToolSentMediaUrls: [], + messagingToolSentTargets: [], + cloudCodeAssistFormatError: false, + replayMetadata: { hadPotentialSideEffects: false, replaySafe: true }, + itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 }, + }, + provider: "openai", + model: "gpt-5.4", + activeErrorContext: { provider: "openai", model: "gpt-5.4" }, + authProfileStore: { version: 1, profiles: {} }, + sessionIdUsed: "session-test-id", + outerContextTokenMeta: {}, + usageAccumulator: createUsageAccumulator(), + contextRecoveryState: createEmbeddedRunContextRecoveryState(), + resolvedToolResultFormat: "markdown", + terminalState: { + outcome: { reason: "completed", status: "ok", stopReason: "stop" }, + signalOwnedInterruption: false, + }, + }); + expect( + normalizeAgentRunTerminalReceipt(Reflect.get(prepared.agentMeta, "terminalReceipt")) + ?.successfulToolNames, + ).toEqual([]); expect(ctx.state.deterministicApprovalPromptSent).toBe(true); }); diff --git a/src/agents/embedded-agent-subscribe.handlers.tools.ts b/src/agents/embedded-agent-subscribe.handlers.tools.ts index 7e79bc644d1b..543b8d9ea48c 100644 --- a/src/agents/embedded-agent-subscribe.handlers.tools.ts +++ b/src/agents/embedded-agent-subscribe.handlers.tools.ts @@ -1467,7 +1467,7 @@ export async function handleToolExecutionEnd( toolName, meta, replaySafe: callSummary.replaySafe, - ...(isToolError ? { isError: true } : {}), + isError: observerIsError, ...(asyncStarted ? { asyncStarted: true, ...asyncTaskIds } : {}), }); const acceptedSessionSpawn = diff --git a/src/agents/embedded-agent-subscribe.handlers.types.ts b/src/agents/embedded-agent-subscribe.handlers.types.ts index 91762416a650..fc90fc63b75c 100644 --- a/src/agents/embedded-agent-subscribe.handlers.types.ts +++ b/src/agents/embedded-agent-subscribe.handlers.types.ts @@ -75,7 +75,7 @@ export type EmbeddedAgentSubscribeState = { toolName?: string; meta?: string; replaySafe?: boolean; - isError?: true; + isError?: boolean; asyncStarted?: boolean; asyncTaskRunId?: string; asyncTaskId?: string; diff --git a/src/gateway/server-methods/agent-job.ts b/src/gateway/server-methods/agent-job.ts index b40168cf17bd..d75b7fc79f35 100644 --- a/src/gateway/server-methods/agent-job.ts +++ b/src/gateway/server-methods/agent-job.ts @@ -2,6 +2,10 @@ // Gateway dedupe retains response payloads only for idempotent RPC replay. import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce"; +import { + normalizeAgentRunTerminalDeliverySnapshot, + type AgentRunTerminalDeliverySnapshot, +} from "../../agents/agent-run-terminal-delivery.js"; import { AGENT_RUN_TERMINAL_RETRY_GRACE_MS, buildAgentRunTerminalOutcome, @@ -10,6 +14,10 @@ import { mergeAgentRunTerminalOutcome, type AgentRunTerminalOutcome, } from "../../agents/agent-run-terminal-outcome.js"; +import { + normalizeAgentRunTerminalReceipt, + type AgentRunTerminalReceipt, +} from "../../agents/agent-run-terminal-receipt.js"; import { mergeAgentRunTerminalReplySnapshot, normalizeAgentRunTerminalReplySnapshot, @@ -35,6 +43,8 @@ type AgentJobTerminalSnapshot = { pendingError?: boolean; timeoutPhase?: AgentRunTerminalOutcome["timeoutPhase"]; providerStarted?: boolean; + terminalDelivery?: AgentRunTerminalDeliverySnapshot; + terminalReceipt?: AgentRunTerminalReceipt; terminalReply?: AgentRunTerminalReplySnapshot; }; @@ -168,11 +178,15 @@ function mergeSnapshot( existing.terminalReply, incoming.terminalReply, ); + const terminalDelivery = incoming.terminalDelivery ?? existing.terminalDelivery; + const terminalReceipt = incoming.terminalReceipt ?? existing.terminalReceipt; const canonical = shouldPreserveTerminalSnapshot(existing, incoming) ? existing : incoming; // Terminal status precedence and producer reply evidence are independent; // a late sticky timeout must not erase the final reply (or vice versa). return { ...canonical, + ...(terminalDelivery ? { terminalDelivery } : {}), + ...(terminalReceipt ? { terminalReceipt } : {}), ...(terminalReply ? { terminalReply } : {}), cachedAt: incoming.cachedAt, recordedAt: incoming.recordedAt, @@ -277,6 +291,7 @@ function createPendingErrorTimeoutSnapshot( ...(snapshot.providerStarted !== undefined ? { providerStarted: snapshot.providerStarted } : {}), + ...(snapshot.terminalDelivery ? { terminalDelivery: snapshot.terminalDelivery } : {}), }; } @@ -299,7 +314,11 @@ function createSnapshotFromLifecycleEvent(params: { // Modern explicit stop reasons keep the canonical cancellation projection. const legacyBareAbort = terminalOutcome.reason === "aborted" && data?.stopReason == null && data?.status == null; + const terminalDelivery = normalizeAgentRunTerminalDeliverySnapshot(data?.terminalDelivery); const terminalReply = normalizeAgentRunTerminalReplySnapshot(data?.terminalReply); + const normalizedTerminalReceipt = normalizeAgentRunTerminalReceipt(data?.terminalReceipt); + const terminalReceipt = + normalizedTerminalReceipt?.runId === runId ? normalizedTerminalReceipt : undefined; return { runId, source: "lifecycle", @@ -315,7 +334,9 @@ function createSnapshotFromLifecycleEvent(params: { ...(terminalOutcome.providerStarted !== undefined ? { providerStarted: terminalOutcome.providerStarted } : {}), + ...(terminalDelivery ? { terminalDelivery } : {}), ...(terminalReply ? { terminalReply } : {}), + ...(terminalReceipt ? { terminalReceipt } : {}), version: nextAgentRunVersion(), }; } @@ -560,6 +581,8 @@ function publicSnapshot(snapshot: AgentRunObservation): AgentJobTerminalSnapshot pendingError: snapshot.pendingError, timeoutPhase: snapshot.timeoutPhase, providerStarted: snapshot.providerStarted, + ...(snapshot.terminalDelivery ? { terminalDelivery: snapshot.terminalDelivery } : {}), + terminalReceipt: snapshot.terminalReceipt, terminalReply: snapshot.terminalReply, }; } diff --git a/src/gateway/server-methods/agent-wait-dedupe.test.ts b/src/gateway/server-methods/agent-wait-dedupe.test.ts index ec153ba0436a..f416ef11771d 100644 --- a/src/gateway/server-methods/agent-wait-dedupe.test.ts +++ b/src/gateway/server-methods/agent-wait-dedupe.test.ts @@ -33,6 +33,19 @@ function completeRun(dedupe: Map, runId: string): void { }); } +function terminalReceipt(runId: string) { + return { + runId, + sessionId: "session-1", + turnId: "turn-1", + requested: { provider: "openai", model: "gpt-primary" }, + effective: { provider: "openai", model: "gpt-alternate", responseModel: "gpt-alternate" }, + successfulToolNames: ["read"], + rerouted: true, + terminalDisposition: "visible", + }; +} + afterEach(() => { vi.useRealTimers(); }); @@ -144,7 +157,7 @@ describe("agent.wait gateway dedupe observations", () => { }); it.each(["lifecycle-first", "dedupe-first"] as const)( - "keeps reply evidence when sticky status arrives $0", + "keeps terminal evidence when sticky status arrives $0", async (order) => { const runId = `run-reply-merge-${order}`; const dedupe = new Map(); @@ -161,6 +174,12 @@ describe("agent.wait gateway dedupe observations", () => { phase: "end", startedAt: 100, endedAt: 300, + terminalDelivery: { + status: "sent", + resultCount: 1, + target: "private-target", + }, + terminalReceipt: terminalReceipt(runId), terminalReply: { disposition: "visible", text: "canonical reply" }, }, }); @@ -194,9 +213,12 @@ describe("agent.wait gateway dedupe observations", () => { expect.objectContaining({ runId, status: "timeout", + terminalDelivery: { status: "sent", resultCount: 1 }, + terminalReceipt: terminalReceipt(runId), terminalReply: { disposition: "visible", text: "canonical reply" }, }), ); + expect(JSON.stringify(waiter.respond.mock.calls[0]?.[1])).not.toContain("private-target"); }, ); }); diff --git a/src/gateway/server-methods/agent-wait.ts b/src/gateway/server-methods/agent-wait.ts index 876f66440767..0a9dc185a272 100644 --- a/src/gateway/server-methods/agent-wait.ts +++ b/src/gateway/server-methods/agent-wait.ts @@ -47,6 +47,8 @@ export const agentWaitHandler: GatewayRequestHandlers["agent.wait"] = async ({ pendingError: snapshot.pendingError, timeoutPhase: snapshot.timeoutPhase, providerStarted: snapshot.providerStarted, + ...(snapshot.terminalDelivery ? { terminalDelivery: snapshot.terminalDelivery } : {}), + terminalReceipt: snapshot.terminalReceipt, terminalReply: snapshot.terminalReply, }); };