From a40df57bae93f54d1454325c3d9f604155f256cc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 13:06:29 -0700 Subject: [PATCH] refactor(agent-runner): split run policy ownership (#122209) * refactor(agent-runner): split tool-call normalization ownership * refactor(agent-runner): split incomplete-turn ownership * test(agent-runner): hide incomplete-turn test helper * test(ci): route split incomplete-turn tests * test(ci): enumerate split incomplete-turn tests * test(vitest): route split incomplete-turn tests to their serial shard --- config/max-lines-baseline.txt | 4 - scripts/test-projects.test-support.mts | 8 +- ...-runner.anthropic-tool-replay.live.test.ts | 2 +- src/agents/embedded-agent-runner/run-loop.ts | 2 +- ....incomplete-turn.attempt-lifecycle.test.ts | 521 ++ ...run.incomplete-turn.classification.test.ts | 347 ++ ...ncomplete-turn.delivery-resolution.test.ts | 557 ++ ...plete-turn.empty-response-recovery.test.ts | 316 ++ ...run.incomplete-turn.error-recovery.test.ts | 379 ++ ...incomplete-turn.payload-resolution.test.ts | 367 ++ ...incomplete-turn.reasoning-recovery.test.ts | 407 ++ ...ete-turn.settled-tool-continuation.test.ts | 542 ++ ...omplete-turn.settled-tool-recovery.test.ts | 599 +++ .../run.incomplete-turn.silent-reply.test.ts | 405 ++ ....incomplete-turn.terminal-evidence.test.ts | 414 ++ .../run.incomplete-turn.test-helpers.ts | 192 + .../run.incomplete-turn.test-support.ts | 6 +- .../run.incomplete-turn.test.ts | 4774 ----------------- .../run.overflow-compaction.fixture.ts | 2 +- .../run/assistant-failure.ts | 2 +- .../run/attempt-normalization.ts | 2 +- .../run/attempt-result.ts | 6 +- .../run/attempt-stream.ts | 16 +- .../run/attempt-terminal-evidence.ts | 94 + .../run/attempt-tool-call-name-resolution.ts | 223 + ...empt-tool-call-replay-sanitization.test.ts | 632 +++ .../attempt-tool-call-replay-sanitization.ts | 520 ++ .../attempt-tool-call-stream-normalization.ts | 362 ++ ...-tool-call-text-promotion-adjacent.test.ts | 272 + ...tool-call-text-promotion-buffering.test.ts | 219 + ...l-call-text-promotion-sanitization.test.ts | 381 ++ .../attempt-tool-call-text-promotion.test.ts | 481 ++ .../run/attempt-tool-call-text-promotion.ts | 144 + .../embedded-agent-runner/run/attempt.test.ts | 6 +- .../attempt.tool-call-normalization.test.ts | 1843 ------- .../run/attempt.tool-call-normalization.ts | 1234 ----- .../run/incomplete-turn-classification.ts | 249 + .../run/incomplete-turn-recovery.ts | 370 ++ .../run/incomplete-turn-resolution.ts | 328 ++ .../run/incomplete-turn.ts | 986 ---- .../run/terminal-resolution.ts | 14 +- .../run/terminal-timeout.ts | 2 +- ...dded-agent-subscribe.handlers.lifecycle.ts | 6 +- .../embedded-agent-runner-e2e-fixtures.ts | 2 +- test/scripts/test-projects.test.ts | 28 +- test/vitest/vitest.agents-paths.mjs | 14 +- test/vitest/vitest.unit-fast-paths.mjs | 1 + 47 files changed, 9395 insertions(+), 8886 deletions(-) create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.attempt-lifecycle.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.classification.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.delivery-resolution.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.empty-response-recovery.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.error-recovery.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.payload-resolution.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.reasoning-recovery.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.silent-reply.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts create mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.test-helpers.ts delete mode 100644 src/agents/embedded-agent-runner/run.incomplete-turn.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-name-resolution.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-stream-normalization.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-adjacent.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-buffering.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-sanitization.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.test.ts create mode 100644 src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.ts delete mode 100644 src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts delete mode 100644 src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts create mode 100644 src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts create mode 100644 src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts create mode 100644 src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts delete mode 100644 src/agents/embedded-agent-runner/run/incomplete-turn.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 89a72e57fa2c..556a70ca7d93 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -373,15 +373,11 @@ src/agents/embedded-agent-runner/extra-params.ts src/agents/embedded-agent-runner/model.provider-runtime.test-support.ts src/agents/embedded-agent-runner/model.test.ts src/agents/embedded-agent-runner/replay-history.ts -src/agents/embedded-agent-runner/run.incomplete-turn.test.ts src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts src/agents/embedded-agent-runner/run/attempt-spawn-workspace.test-support.ts src/agents/embedded-agent-runner/run/attempt.spawn-workspace.context-engine.test.ts src/agents/embedded-agent-runner/run/attempt.test.ts src/agents/embedded-agent-runner/run/attempt.tool-call-argument-repair.ts -src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts -src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts -src/agents/embedded-agent-runner/run/incomplete-turn.ts src/agents/embedded-agent-runner/runs.ts src/agents/embedded-agent-runner/thinking.test.ts src/agents/embedded-agent-runner/tool-result-context-guard.test.ts diff --git a/scripts/test-projects.test-support.mts b/scripts/test-projects.test-support.mts index 05689d14ed6c..725ee3ddf743 100644 --- a/scripts/test-projects.test-support.mts +++ b/scripts/test-projects.test-support.mts @@ -3148,6 +3148,11 @@ function classifyTarget(arg: string, cwd: string) { if (relative.startsWith("src/plugins/contracts/")) { return "contractsPlugin"; } + // These tests share stateful runner mocks and must keep the dedicated serial + // owner even when their contents also qualify for a unit-fast lane. + if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) { + return agentVitestProjectOwners.embeddedIncompleteTurn.kind; + } if (resolveUnitFastTimerTestIncludePattern(relative)) { return "unitFastFakeTimers"; } @@ -3333,9 +3338,6 @@ function classifyTarget(arg: string, cwd: string) { ) { return agentVitestProjectOwners.all.kind; } - if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) { - return agentVitestProjectOwners.embeddedIncompleteTurn.kind; - } if (agentVitestProjectOwners.embeddedOverflowCompaction.include.includes(relative)) { return agentVitestProjectOwners.embeddedOverflowCompaction.kind; } diff --git a/src/agents/embedded-agent-runner.anthropic-tool-replay.live.test.ts b/src/agents/embedded-agent-runner.anthropic-tool-replay.live.test.ts index c8f124639247..5125b295c1d3 100644 --- a/src/agents/embedded-agent-runner.anthropic-tool-replay.live.test.ts +++ b/src/agents/embedded-agent-runner.anthropic-tool-replay.live.test.ts @@ -1,7 +1,7 @@ // Live checks for Anthropic replay transcript sanitization and tool-call history. import type { Message, Model } from "openclaw/plugin-sdk/llm"; import { describe, expect, it, vi } from "vitest"; -import { wrapStreamFnSanitizeMalformedToolCalls } from "./embedded-agent-runner/run/attempt.tool-call-normalization.js"; +import { wrapStreamFnSanitizeMalformedToolCalls } from "./embedded-agent-runner/run/attempt-tool-call-replay-sanitization.js"; import { extractEmbeddedAssistantText } from "./embedded-agent-utils.js"; import { completeSimpleWithLiveTimeout, logLiveCache } from "./live-cache-test-support.js"; import { isLiveTestEnabled } from "./live-test-helpers.js"; diff --git a/src/agents/embedded-agent-runner/run-loop.ts b/src/agents/embedded-agent-runner/run-loop.ts index d63ed0db234f..1ca3c628cd81 100644 --- a/src/agents/embedded-agent-runner/run-loop.ts +++ b/src/agents/embedded-agent-runner/run-loop.ts @@ -44,7 +44,7 @@ import { createIdleTimeoutBreakerState } from "./run/idle-timeout-breaker.js"; import { DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT, DEFAULT_REASONING_ONLY_RETRY_LIMIT, -} from "./run/incomplete-turn.js"; +} from "./run/incomplete-turn-recovery.js"; import { measureEmbeddedAgentPreparation } from "./run/preparation-timing.js"; import { beginRunAttempt, diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.attempt-lifecycle.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.attempt-lifecycle.test.ts new file mode 100644 index 000000000000..13e8df58246c --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.attempt-lifecycle.test.ts @@ -0,0 +1,521 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + runEmbeddedAgent, + makeLastAssistant, + makeBaseRunParams, + makeRunParams, + expectWarnMessageWith, + expectNoWarnMessageWith, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedClassifyFailoverReason, + mockedIsRateLimitAssistantError, + mockedRunEmbeddedAttempt, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { recoverEmbeddedRunAttempt } from "./run/attempt-recovery.js"; +import { resolveSilentToolResultReplyPayload } from "./run/incomplete-turn-resolution.js"; +import { resolveEmbeddedRunAttemptTerminalState } from "./run/terminal-outcome.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; +import { createUsageAccumulator } from "./usage-accumulator.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("counts failed tool results in trace tool summaries", async () => { + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Done."], + toolMetas: [ + { toolName: "bash", meta: "exit=1", isError: true }, + { toolName: "bash", meta: "exit=2", isError: true }, + { toolName: "bash", meta: "exit=0" }, + ], + }), + ); + + const result = await runEmbeddedAgent(makeBaseRunParams("run-tool-summary-failure-count")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.meta?.toolSummary).toEqual({ + calls: 3, + tools: ["bash"], + failures: 2, + }); + }); + + it("emits the before_agent_run hook block message as the agent payload", async () => { + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + promptError: new Error("Blocked by before-run policy."), + promptErrorSource: "hook:before_agent_run", + }), + ); + + const result = await runEmbeddedAgent(makeBaseRunParams("run-before-agent-run-hook-block")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toEqual([{ text: "Blocked by before-run policy.", isError: true }]); + expect(result.meta?.finalAssistantVisibleText).toBe("Blocked by before-run policy."); + expect(result.meta?.finalAssistantRawText).toBe("Blocked by before-run policy."); + expect(result.meta?.finalPromptText).toBeUndefined(); + expect(result.meta?.error).toEqual({ + kind: "hook_block", + message: "Blocked by before-run policy.", + }); + expect(result.meta?.livenessState).toBe("blocked"); + }); + + it("keeps carried usage ahead of transcript history on before_agent_run hook blocks", async () => { + const historicalAssistant = makeLastAssistant({ + usage: { input: 128_814, output: 3_000, total: 131_814 }, + }); + const carriedUsage = { input: 42_000, output: 1_000, total: 43_000 }; + const attempt = makeAttemptResult({ + assistantTexts: [], + promptError: new Error("Blocked by before-run policy."), + promptErrorSource: "hook:before_agent_run", + lastAssistant: historicalAssistant, + currentAttemptAssistant: undefined, + }); + const terminalState = resolveEmbeddedRunAttemptTerminalState({ + attempt, + assistant: historicalAssistant, + }); + + const recovery = await recoverEmbeddedRunAttempt({ + runInput: { + runParams: makeBaseRunParams("run-before-agent-run-hook-block-usage"), + resolvedSessionKey: "agent:main:test-key", + startedAtMs: Date.now(), + }, + preparedRuntime: { + provider: "openai", + modelId: "gpt-5.6-luna", + model: { id: "gpt-5.6-luna" }, + genericCompactionRecoveryAllowed: false, + snapshot: () => ({ + thinkLevel: "off", + agentHarness: { id: "codex" }, + outerContextTokenMeta: {}, + }), + }, + normalizedAttempt: { + attempt, + sessionIdUsed: attempt.sessionIdUsed, + attemptAssistant: historicalAssistant, + currentAttemptAssistant: undefined, + currentAttemptCompletedAssistant: undefined, + terminalState, + setTerminalLifecycleMeta: vi.fn(), + attemptCompactionCount: 0, + activeErrorContext: { provider: "openai", model: "gpt-5.6-luna" }, + resolveReplayInvalidForAttempt: () => false, + canRestartForLiveSwitch: false, + }, + runtimePlan: { auth: {} }, + sessionPromptState: { sessionFile: "/tmp/session.jsonl" }, + usageAccumulator: createUsageAccumulator(), + lastRunPromptUsage: carriedUsage, + } as never); + + expect(recovery).toMatchObject({ + action: "complete", + result: { + meta: { + agentMeta: { lastCallUsage: carriedUsage, promptTokens: 42_000 }, + }, + }, + }); + }); + + it("warns before retrying when an incomplete turn already sent a message", async () => { + // Delivery evidence means retrying could duplicate user-visible output, so + // the runner must surface a verify-before-retry payload instead. + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [], + didSendViaMessagingTool: true, + lastAssistant: { + stopReason: "toolUse", + errorMessage: "internal retry interrupted tool execution", + provider: "openai", + model: "mock-1", + content: [], + } as unknown as EmbeddedRunAttemptResult["lastAssistant"], + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-incomplete-turn-messaging-warning", { model: "gpt-4.1" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(mockedClassifyFailoverReason).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("verify before retrying"); + }); + + it("surfaces internal aborts after tool-use as visible incomplete-turn failures", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + aborted: true, + externalAbort: false, + assistantTexts: [], + toolMetas: [{ toolName: "web_search", meta: "query=next voice note" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + }), + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-internal-abort-tool-use-incomplete")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toEqual([ + { text: "⚠️ Agent couldn't generate a response. Please try again.", isError: true }, + ]); + expect(result.meta?.livenessState).toBe("abandoned"); + }); + + it("does not route caller timeouts through provider failover", async () => { + const controller = new AbortController(); + const timeoutError = new Error("caller deadline elapsed"); + timeoutError.name = "TimeoutError"; + const setTerminalLifecycleMeta = vi.fn(); + const interruptedAssistant = makeLastAssistant({ + stopReason: "error", + errorMessage: "HTTP 429 Too Many Requests", + }); + mockedClassifyFailoverReason.mockReturnValue("rate_limit"); + mockedIsRateLimitAssistantError.mockReturnValue(true); + mockedRunEmbeddedAttempt.mockImplementationOnce(async () => { + controller.abort(timeoutError); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: interruptedAssistant, + currentAttemptAssistant: interruptedAssistant, + setTerminalLifecycleMeta, + }); + }); + + const result = await runEmbeddedAgent( + makeBaseRunParams("run-caller-timeout", { abortSignal: controller.signal }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.at(-1)?.text).toContain("timed out"); + expect(result.meta?.aborted).toBe(false); + expect(result.meta?.timeoutPhase).toBeUndefined(); + expect(result.meta?.providerStarted).toBeUndefined(); + const lifecycleMeta = setTerminalLifecycleMeta.mock.lastCall?.[0]; + expect(lifecycleMeta).toMatchObject({ + aborted: false, + livenessState: "blocked", + stopReason: "timeout", + }); + expect(lifecycleMeta).not.toHaveProperty("timeoutPhase"); + expect(lifecycleMeta).not.toHaveProperty("providerStarted"); + }); + + it("does not synthesize an incomplete turn for a caller abort before attempt flags settle", async () => { + const controller = new AbortController(); + const abortError = new Error("caller cancelled"); + abortError.name = "AbortError"; + const setTerminalLifecycleMeta = vi.fn(); + const lateAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Late answer" }], + }); + mockedRunEmbeddedAttempt.mockImplementationOnce(async () => { + controller.abort(abortError); + return makeAttemptResult({ + assistantTexts: ["Late answer"], + lastAssistant: lateAssistant, + currentAttemptAssistant: lateAssistant, + setTerminalLifecycleMeta, + }); + }); + + const result = await runEmbeddedAgent( + makeBaseRunParams("run-caller-abort", { abortSignal: controller.signal }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toBeUndefined(); + expect(result.meta?.aborted).toBe(true); + expect(result.meta?.error).toBeUndefined(); + expectNoWarnMessageWith("incomplete turn detected"); + expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({ + aborted: true, + livenessState: "blocked", + stopReason: "aborted", + }); + }); + + it("propagates canonical assistant aborts into terminal lifecycle metadata", async () => { + const setTerminalLifecycleMeta = vi.fn(); + const abortedAssistant = makeLastAssistant({ + stopReason: "aborted", + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: abortedAssistant, + currentAttemptAssistant: abortedAssistant, + setTerminalLifecycleMeta, + }), + ); + + const result = await runEmbeddedAgent(makeBaseRunParams("run-canonical-assistant-abort")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.meta?.aborted).toBe(true); + expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({ + aborted: true, + }); + }); + + it("synthesizes a silent cron payload from a trailing current-attempt NO_REPLY tool result", () => { + // Cron no-reply can be represented by a tool result rather than assistant + // text, but only when it belongs to the current attempt. + const payload = resolveSilentToolResultReplyPayload({ + isCronTrigger: true, + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "exec" }], + messagesSnapshot: [ + { + role: "toolResult", + content: [{ type: "text", text: "NO_REPLY" }], + details: { aggregated: "NO_REPLY" }, + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + makeLastAssistant({ + model: "gpt-5.4", + }), + ], + }), + }); + + expect(payload).toEqual({ text: "NO_REPLY" }); + }); + + it("does not reuse an older NO_REPLY tool result without current-attempt tool activity", () => { + const payload = resolveSilentToolResultReplyPayload({ + isCronTrigger: true, + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + toolMetas: [], + messagesSnapshot: [ + { + role: "toolResult", + content: [{ type: "text", text: "NO_REPLY" }], + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + { + role: "user", + content: [{ type: "text", text: "Current cron prompt" }], + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + makeLastAssistant({ + model: "gpt-5.4", + }), + ], + }), + }); + + expect(payload).toBeNull(); + }); + + it("treats exact NO_REPLY tool output as a quiet cron success when the final assistant is empty", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "exec" }], + messagesSnapshot: [ + { + role: "toolResult", + content: [{ type: "text", text: "NO_REPLY" }], + details: { aggregated: "NO_REPLY" }, + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + makeLastAssistant({ + model: "gpt-5.4", + }), + ], + lastAssistant: makeLastAssistant({ + model: "gpt-5.4", + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-cron-no-reply-empty-final", { trigger: "cron", model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); + expect(result.meta.livenessState).toBe("working"); + expectNoWarnMessageWith("incomplete turn detected"); + }); + + it("surfaces the latest tool-authored presentation after a structured incomplete turn", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome?.({ + toolName: "web_fetch", + argsHash: "args", + resultHash: "result", + terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", + }); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "web_fetch" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + }), + }); + }); + + const result = await runEmbeddedAgent( + makeRunParams("run-structured-terminal-presentation", { model: "gpt-5.4" }), + ); + + expect(result.payloads).toEqual([ + { + text: + "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + + "⚠️ Agent couldn't generate a response. Please try again.", + isError: true, + }, + ]); + expect(result.meta.replayInvalid).toBe(true); + expect(result.meta.livenessState).toBe("abandoned"); + expect(result.meta.error?.fallbackSafe).toBe(true); + expect(result.meta.error?.terminalPresentation).toBe(true); + expectWarnMessageWith("surfacing tool-authored terminal presentation"); + }); + + it("surfaces read-only cron presentation after a structured incomplete turn", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome?.({ + toolName: "cron", + argsHash: "args", + resultHash: "result", + terminalPresentation: "Automations scheduler status.\nEnabled: yes", + }); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "cron" }], + replayMetadata: { + hadPotentialSideEffects: false, + replaySafe: true, + }, + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + }), + }); + }); + + const result = await runEmbeddedAgent( + makeRunParams("run-read-only-cron-terminal-presentation", { model: "gpt-5.4" }), + ); + + expect(result.payloads).toEqual([ + { + text: + "Automations scheduler status.\nEnabled: yes\n\n" + + "⚠️ Agent couldn't generate a response. Please try again.", + isError: true, + }, + ]); + expect(result.meta.error?.fallbackSafe).toBe(true); + expect(result.meta.error?.terminalPresentation).toBe(true); + }); + + it("preserves a terminal tool presentation across an empty-response retry", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome?.({ + toolName: "web_fetch", + argsHash: "args", + resultHash: "result", + terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", + }); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "web_fetch" }], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-preserved-terminal-presentation", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads).toEqual([ + { + text: + "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + + "⚠️ Agent couldn't generate a response. Please try again.", + isError: true, + }, + ]); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.classification.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.classification.test.ts new file mode 100644 index 000000000000..d05d9c10079e --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.classification.test.ts @@ -0,0 +1,347 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + REASONING_ONLY_RETRY_INSTRUCTION, + EMPTY_RESPONSE_RETRY_INSTRUCTION, + makeLastAssistant, + resolveIncompleteTurnPayloadText, + makeIncompleteTurnParams, + makeReasoningRetryParams, + makeEmptyResponseRetryParams, +} from "./run.incomplete-turn.test-helpers.js"; +import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js"; +import { + resolveEmptyResponseRetryInstruction, + resolveReasoningOnlyRetryInstruction, +} from "./run/incomplete-turn-recovery.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("detects reasoning-only GPT turns from signed thinking blocks", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_helper", type: "reasoning" }), + }, + ], + }), + }), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("detects reasoning-only Gemini turns from signed thinking blocks", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "google", + model: "gemini-2.5-pro", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "gemini_rs_helper", type: "reasoning" }), + }, + ], + }), + }, + { provider: "google", modelId: "gemini-2.5-pro" }, + ), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("retries signed reasoning-only Bedrock Converse turns with a visible-answer continuation", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "amazon-bedrock", + model: "openai.gpt-oss-120b-1:0", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: "bedrock-reasoning-signature", + }, + ], + }), + }, + { + provider: "amazon-bedrock", + modelId: "openai.gpt-oss-120b-1:0", + modelApi: "bedrock-converse-stream", + }, + ), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("retries signed reasoning-only Ollama turns with a visible-answer continuation instruction", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "ollama", + model: "gemma4:31b", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "ollama_rs_helper", type: "reasoning" }), + }, + ], + }), + }, + { provider: "ollama", modelId: "gemma4:31b" }, + ), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("retries unsigned thinking-only turns via the reasoning-only path (openai-completions)", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + model: "qwen3.6-35b-a3b", + content: [ + { + type: "thinking", + thinking: "let me plan the tool calls I need to make...", + }, + ], + }), + }, + { modelId: "qwen3.6-35b-a3b", modelApi: "openai-completions" }, + ), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("retries unsigned thinking-only Ollama turns via the reasoning-only path", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "ollama", + model: "gemma4:31b", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + }, + ], + }), + }, + { provider: "ollama", modelId: "gemma4:31b" }, + ), + ); + + expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + }); + + it("retries unsigned-thinking Ollama turns via the empty-response path", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "ollama", + model: "gemma4:31b", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + }, + ], + }), + }, + { provider: "ollama", modelId: "gemma4:31b" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("retries generic empty Ollama turns without visible text", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "ollama", + model: "gemma4:31b", + content: [{ type: "text", text: "" }], + }), + }, + { provider: "ollama", modelId: "gemma4:31b" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("retries empty Ollama stop turns when nonzero output tokens were generated", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "ollama", + model: "minimax-m2.7:cloud", + usage: { input: 100, output: 6, totalTokens: 106 }, + }), + }, + { provider: "ollama", modelId: "minimax-m2.7:cloud" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("does not retry empty turns after an accepted sessions_spawn delivery", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + acceptedSessionSpawns: [ + { + runId: "run-child", + childSessionKey: "agent:claude:subagent:child", + }, + ], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "ollama", + model: "gemma4:31b", + content: [{ type: "text", text: "" }], + }), + }, + { provider: "ollama", modelId: "gemma4:31b" }, + ), + ); + + expect(retryInstruction).toBeNull(); + }); + + it("retries empty openai-chatgpt-responses turns with non-zero output tokens (#85364)", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + usage: { input: 24794, output: 111, cacheRead: 4608, totalTokens: 29513 }, + }), + }, + { modelId: "gpt-5.5", modelApi: "openai-chatgpt-responses" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("retries empty openai-responses turns without visible text", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + usage: { input: 5000, output: 200, totalTokens: 5200 }, + }), + }, + { modelId: "gpt-5.5", modelApi: "openai-responses" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("retries generic empty OpenAI-compatible turns from custom endpoints", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "llama-cpp-local", + model: "qwen3.6-27b", + usage: { input: 950, output: 103, totalTokens: 1053 }, + }), + }, + { + provider: "llama-cpp-local", + modelId: "qwen3.6-27b", + modelApi: "openai-completions", + }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("does not retry clean zero-token Ollama stop turns", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "ollama", + model: "glm-5.1:cloud", + usage: { input: 100, output: 0, totalTokens: 100 }, + }), + }, + { provider: "ollama", modelId: "glm-5.1:cloud" }, + ), + ); + + expect(retryInstruction).toBeNull(); + }); + + it("treats exact NO_REPLY as a deliberate silent assistant reply", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: ["NO_REPLY"], + lastAssistant: makeLastAssistant({ + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_no_reply", type: "reasoning" }), + }, + { type: "text", text: "" }, + { type: "text", text: "NO_REPLY" }, + ], + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.delivery-resolution.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.delivery-resolution.test.ts new file mode 100644 index 000000000000..f055dfa8698e --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.delivery-resolution.test.ts @@ -0,0 +1,557 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + hasCommittedMessagingToolDeliveryEvidence, + hasOutboundDeliveryEvidence, +} from "./delivery-evidence.js"; +import { + runEmbeddedAgent, + makeLastAssistant, + resolveIncompleteTurnPayloadText, + makeRunParams, + makeIncompleteTurnParams, + makeReasoningRetryParams, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js"; +import { + DEFAULT_REASONING_ONLY_RETRY_LIMIT, + resolveReasoningOnlyRetryInstruction, +} from "./run/incomplete-turn-recovery.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("suppresses the incomplete-turn warning after committed messaging text delivery", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered through the message tool."], + lastAssistant: makeLastAssistant({ + provider: "ollama", + model: "kimi-k2.6:cloud", + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("suppresses the incomplete-turn warning after committed messaging delivery before end_turn", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered through the message tool."], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "google", + model: "gemini-2.5-pro", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_messaging_end_turn", type: "reasoning" }), + }, + ], + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("suppresses the incomplete-turn warning after committed media-only messaging delivery", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: false, + messagingToolSentMediaUrls: ["file:///tmp/render.png"], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("suppresses the incomplete-turn warning after committed messaging delivery even when the provider errored", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered before the provider error."], + lastAssistant: makeLastAssistant({ + stopReason: "error", + provider: "ollama", + model: "kimi-k2.6:cloud", + errorMessage: "provider failed after delivery", + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("suppresses the incomplete-turn warning after an accepted sessions_spawn terminal success", () => { + const attemptWithAcceptedSpawn: Partial & { + acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>; + } = { + assistantTexts: [], + acceptedSessionSpawns: [ + { + runId: "run-child", + childSessionKey: "agent:claude:subagent:child", + }, + ], + lastAssistant: makeLastAssistant({ + provider: "anthropic", + model: "sonnet-4.6", + }), + }; + + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams(attemptWithAcceptedSpawn), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("still returns a timeout payload when the parent prompt times out after an accepted sessions_spawn", async () => { + const acceptedSessionSpawns = [ + { + runId: "run-child", + childSessionKey: "agent:claude:subagent:child", + }, + ]; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + acceptedSessionSpawns, + timedOut: true, + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-timeout-after-accepted-spawn", { model: "gpt-5.4" }), + ); + + expect(result.payloads).toEqual([ + { + text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.", + isError: true, + }, + ]); + expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns); + }); + + it("still surfaces the incomplete-turn warning without an accepted sessions_spawn success", () => { + const attemptWithMalformedSpawn: Partial & { + acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>; + } = { + assistantTexts: [], + acceptedSessionSpawns: [], + lastAssistant: makeLastAssistant({ + provider: "anthropic", + model: "sonnet-4.6", + }), + }; + + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams(attemptWithMalformedSpawn), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); + + it("still surfaces the incomplete-turn warning when no messaging delivery was committed", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + lastAssistant: makeLastAssistant({ + stopReason: "error", + provider: "ollama", + model: "kimi-k2.6:cloud", + errorMessage: "provider failed mid-turn", + }), + }), + ); + + expect(incompleteTurnText).toContain("verify before retrying"); + }); + + it("does not treat empty committed messaging arrays as delivery", () => { + expect( + hasCommittedMessagingToolDeliveryEvidence({ + messagingToolSentTexts: [" "], + messagingToolSentMediaUrls: [], + }), + ).toBe(false); + }); + + it("treats committed messaging media as delivery", () => { + expect( + hasCommittedMessagingToolDeliveryEvidence({ + messagingToolSentTexts: [], + messagingToolSentMediaUrls: ["file:///tmp/render.png"], + }), + ).toBe(true); + }); + + it("treats committed messaging targets as delivery", () => { + expect( + hasCommittedMessagingToolDeliveryEvidence({ + messagingToolSentTexts: [], + messagingToolSentMediaUrls: [], + messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }], + }), + ).toBe(true); + }); + + for (const { name, overrides } of [ + { + name: "treats committed messaging text as replay-invalid side effect metadata", + overrides: { messagingToolSentTexts: ["Delivered through the message tool."] }, + }, + { + name: "treats async-started background tools as replay-invalid side effects", + overrides: { toolMetas: [{ toolName: "image_generate", asyncStarted: true }] }, + }, + { + name: "treats committed messaging media as replay-invalid side effect metadata", + overrides: { messagingToolSentMediaUrls: ["file:///tmp/render.png"] }, + }, + { + name: "treats committed messaging targets as replay-invalid side effect metadata", + overrides: { + messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }], + }, + }, + ]) { + it(name, () => { + expect( + buildAttemptReplayMetadata({ + toolMetas: [], + didSendViaMessagingTool: false, + messagingToolSentTexts: [], + messagingToolSentMediaUrls: [], + ...overrides, + }), + ).toEqual({ hadPotentialSideEffects: true, replaySafe: false }); + }); + } + + it("treats accepted sessions_spawn as replay-invalid outbound delivery", () => { + const acceptedSessionSpawns = [ + { + runId: "run-child", + childSessionKey: "agent:claude:subagent:child", + }, + ]; + + expect( + buildAttemptReplayMetadata({ + toolMetas: [], + didSendViaMessagingTool: false, + messagingToolSentTexts: [], + messagingToolSentMediaUrls: [], + acceptedSessionSpawns, + }), + ).toEqual({ hadPotentialSideEffects: true, replaySafe: false }); + expect(hasOutboundDeliveryEvidence({ acceptedSessionSpawns })).toBe(true); + }); + + it("ignores malformed accepted sessions_spawn delivery evidence", () => { + expect( + hasOutboundDeliveryEvidence({ + acceptedSessionSpawns: [ + null, + { + runId: "run-child", + childSessionKey: " ", + }, + ], + }), + ).toBe(false); + }); + + it("leaves committed delivery plus tool errors to the tool-error payload path", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered through the message tool."], + lastToolError: { + toolName: "message", + meta: "send", + error: "delivery failed for second target", + }, + lastAssistant: makeLastAssistant({ + stopReason: "error", + model: "gpt-5.4", + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("does not retry reasoning-only GPT turns after side effects", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_side_effect", type: "reasoning" }), + }, + ], + }), + }), + ); + + expect(retryInstruction).toBeNull(); + expect(DEFAULT_REASONING_ONLY_RETRY_LIMIT).toBe(2); + }); + + it("does not retry reasoning-only GPT turns when the assistant ended in error", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "error", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_helper_error", type: "reasoning" }), + }, + ], + }), + }), + ); + + expect(retryInstruction).toBeNull(); + }); + + it("does not retry reasoning-only GPT turns when visible assistant text already exists", () => { + const retryInstruction = resolveReasoningOnlyRetryInstruction( + makeReasoningRetryParams({ + assistantTexts: ["Visible answer."], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ + id: "rs_helper_visible_text", + type: "reasoning", + }), + }, + { type: "text", text: "" }, + ], + }), + }), + ); + + expect(retryInstruction).toBeNull(); + }); + + it("surfaces incomplete-turn text for errored signed-thinking-only turns with payloads", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "error", + provider: "anthropic", + model: "claude-opus-4-8", + content: [ + { + type: "thinking", + thinking: "internal reasoning before provider error", + thinkingSignature: JSON.stringify({ id: "rs_error_payload", type: "reasoning" }), + }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); + + it("surfaces incomplete-turn text for token-limited partial answers", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Partial answer"], + lastAssistant: makeLastAssistant({ + stopReason: "length", + provider: "ollama", + model: "qwen3.5", + content: [{ type: "text", text: "Partial answer" }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); + + it("keeps complete visible stop turns successful", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Complete answer"], + lastAssistant: makeLastAssistant({ + provider: "ollama", + model: "qwen3.5", + content: [{ type: "text", text: "Complete answer" }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("preserves terminal tool media on token-limited turns", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Partial answer"], + toolMediaUrls: ["file:///tmp/render.png"], + lastAssistant: makeLastAssistant({ + stopReason: "length", + provider: "ollama", + model: "qwen3.5", + content: [{ type: "text", text: "Partial answer" }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("preserves tool media already delivered through block replies", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Partial answer"], + hasToolMediaBlockReply: true, + lastAssistant: makeLastAssistant({ + stopReason: "length", + provider: "ollama", + model: "qwen3.5", + content: [{ type: "text", text: "Partial answer" }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("preserves successful cron progress on token-limited turns", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Partial answer"], + successfulCronAdds: 1, + lastAssistant: makeLastAssistant({ + stopReason: "length", + provider: "ollama", + model: "qwen3.5", + content: [{ type: "text", text: "Partial answer" }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it.each([ + [ + "heartbeat responses", + { + heartbeatToolResponse: { + outcome: "progress" as const, + notify: false, + summary: "Still working", + }, + }, + ], + ["tool media", { toolMediaUrls: ["file:///tmp/render.png"] }], + ["voice media", { toolAudioAsVoice: true }], + ["trusted local media", { toolTrustedLocalMedia: true }], + [ + "source reply payloads", + { messagingToolSourceReplyPayloads: [{ text: "Delivered through the source reply." }] }, + ], + ["delivered source replies", { didDeliverSourceReplyViaMessageTool: true }], + ] satisfies Array<[string, Partial]>)( + "does not replace terminal %s with an incomplete-turn warning", + (_label, attemptState) => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: [], + ...attemptState, + lastAssistant: makeLastAssistant({ + stopReason: "error", + provider: "anthropic", + model: "claude-opus-4-8", + content: [ + { + type: "thinking", + thinking: "internal reasoning before provider error", + thinkingSignature: JSON.stringify({ + id: "rs_terminal_payload", + type: "reasoning", + }), + }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }, + ); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.empty-response-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.empty-response-recovery.test.ts new file mode 100644 index 000000000000..bb36281ff43c --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.empty-response-recovery.test.ts @@ -0,0 +1,316 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + EMPTY_RESPONSE_RETRY_INSTRUCTION, + runEmbeddedAgent, + makeLastAssistant, + makeRunParams, + expectWarnMessageWith, + runAttemptCall, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + mockedResolveModelAsync, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("retries zero-token empty Claude stop turns with a visible-answer continuation instruction", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "anthropic", + model: "claude-opus-4.7", + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + }, + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible Claude answer."], + lastAssistant: makeLastAssistant({ + provider: "anthropic", + model: "claude-opus-4.7", + content: [{ type: "text", text: "Visible Claude answer." }], + usage: { + input: 100, + output: 5, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 105, + }, + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-empty-zero-usage-claude-continuation", { + provider: "anthropic", + model: "claude-opus-4.7", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectWarnMessageWith("empty response detected"); + }); + + it("retries empty openai-compatible stop turns even when the backend reports output tokens", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedResolveModelAsync.mockResolvedValue({ + model: { + id: "qwen3.6-27b", + provider: "llamacpp", + contextWindow: 200000, + api: "openai-completions", + }, + error: null, + authStorage: { + setRuntimeApiKey: vi.fn(), + }, + modelRegistry: {}, + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "llamacpp", + model: "qwen3.6-27b", + usage: { + input: 512, + output: 103, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 615, + }, + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible local answer."], + lastAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "llamacpp", + model: "qwen3.6-27b", + content: [{ type: "text", text: "Visible local answer." }], + usage: { + input: 640, + output: 5, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 645, + }, + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-empty-openai-compatible-stop-continuation", { + provider: "llamacpp", + model: "qwen3.6-27b", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectWarnMessageWith("empty response detected"); + }); + + it("retries empty Anthropic-compatible stop turns even when the provider is not Kimi", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedResolveModelAsync.mockResolvedValue({ + model: { + id: "claude-opus-4-7", + provider: "sub2api", + contextWindow: 200000, + api: "anthropic-messages", + }, + error: null, + authStorage: { + setRuntimeApiKey: vi.fn(), + }, + modelRegistry: {}, + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + api: "anthropic-messages", + provider: "sub2api", + model: "claude-opus-4-7", + usage: { + input: 2048, + output: 3100, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 5148, + }, + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible Anthropic-compatible answer."], + lastAssistant: makeLastAssistant({ + api: "anthropic-messages", + provider: "sub2api", + model: "claude-opus-4-7", + content: [{ type: "text", text: "Visible Anthropic-compatible answer." }], + usage: { + input: 2300, + output: 8, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2308, + }, + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-empty-anthropic-compatible-stop-continuation", { + provider: "sub2api", + model: "claude-opus-4-7", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectWarnMessageWith("empty response detected"); + }); + + it("surfaces an error after exhausting empty-response retries", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-empty-response-exhausted", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("Please try again"); + expectWarnMessageWith("empty response retries exhausted"); + }); + + it("surfaces an error after exhausting reasoning-only retries without a visible answer", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ + id: "rs_reasoning_exhausted", + type: "reasoning", + }), + }, + ], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-reasoning-only-exhausted", { + model: "gpt-5.4", + reasoningLevel: "on", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("Please try again"); + expectWarnMessageWith("reasoning-only retries exhausted"); + }); + + it("preserves a terminal tool presentation after reasoning-only retries are exhausted", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const reasoningOnlyAttempt = async () => + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ + id: "rs_reasoning_terminal_presentation", + type: "reasoning", + }), + }, + ], + }), + }); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome?.({ + toolName: "web_fetch", + argsHash: "args", + resultHash: "result", + terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", + }); + return reasoningOnlyAttempt(); + }); + mockedRunEmbeddedAttempt.mockImplementation(reasoningOnlyAttempt); + + const result = await runEmbeddedAgent( + makeRunParams("run-reasoning-terminal-presentation", { + model: "gpt-5.4", + reasoningLevel: "on", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); + expect(result.payloads).toEqual([ + { + text: + "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + + "⚠️ Agent couldn't generate a response. Please try again.", + isError: true, + }, + ]); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.error-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.error-recovery.test.ts new file mode 100644 index 000000000000..6f82b49ea6d3 --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.error-recovery.test.ts @@ -0,0 +1,379 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE } from "../../llm/types.js"; +import { + EMPTY_RESPONSE_RETRY_INSTRUCTION, + makeLastAssistant, + resolveIncompleteTurnPayloadText, + makeIncompleteTurnParams, + makeEmptyResponseRetryParams, + makeSilentReplyParams, +} from "./run.incomplete-turn.test-helpers.js"; +import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { + DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT, + resolveEmptyResponseRetryInstruction, + shouldRetrySilentErrorAssistantTurn, + shouldTreatEmptyAssistantReplyAsSilent, +} from "./run/incomplete-turn-recovery.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("retries replay-safe errored turns that only emitted thinking blocks", () => { + const assistant = makeLastAssistant({ + stopReason: "error", + provider: "anthropic", + model: "claude-opus-4-8", + content: [ + { + type: "thinking", + thinking: "internal reasoning before provider error", + thinkingSignature: JSON.stringify({ id: "rs_error", type: "reasoning" }), + }, + { type: "redacted_thinking", data: "opaque" }, + { type: "text", text: " " }, + ], + usage: { input: 100, output: 1120, totalTokens: 1220 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), + assistant, + }), + ).toBe(true); + }); + + it("does not retry an ambiguous post-dispatch provider outcome", () => { + const assistant = makeLastAssistant({ + stopReason: "error", + errorCode: PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE, + errorMessage: "The WebSocket closed after dispatch", + usage: { input: 100, output: 0, totalTokens: 100 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), + assistant, + }), + ).toBe(false); + }); + + it("does not retry errored empty turns when non-zero output may indicate progress", () => { + const assistant = makeLastAssistant({ + stopReason: "error", + provider: "ollama", + model: "glm-5.1:cloud", + usage: { input: 100, output: 12, totalTokens: 112 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), + assistant, + }), + ).toBe(false); + }); + + it.each([ + { + name: "visible text", + content: [ + { type: "thinking", thinking: "internal", thinkingSignature: "sig" }, + { type: "text", text: "partial answer" }, + ], + }, + { + name: "tool call", + content: [ + { type: "thinking", thinking: "internal", thinkingSignature: "sig" }, + { type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }, + ], + }, + { + name: "unknown block", + content: [{ type: "provider_metadata", value: "opaque" }], + }, + ])("does not retry errored turns containing $name", ({ content }) => { + const assistant = makeLastAssistant({ + stopReason: "error", + provider: "anthropic", + model: "claude-opus-4-8", + content, + usage: { input: 100, output: 1120, totalTokens: 1220 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), + assistant, + }), + ).toBe(false); + }); + + it("does not retry errored thinking-only turns after side effects", () => { + const assistant = makeLastAssistant({ + stopReason: "error", + provider: "anthropic", + model: "claude-opus-4-8", + content: [ + { + type: "redacted_thinking", + data: "opaque", + }, + ], + usage: { input: 100, output: 1120, totalTokens: 1220 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ + assistantTexts: [], + replayMetadata: { + hadPotentialSideEffects: true, + replaySafe: false, + }, + lastAssistant: assistant, + }), + assistant, + }), + ).toBe(false); + }); + + it.each([ + ["current clean overrides cumulative dirty", true, false, true], + ["current dirty overrides cumulative clean", false, true, false], + ["both clean remain retryable", false, false, true], + ] as const)( + "uses current-attempt replay metadata when %s", + (_label, cumulativeDirty, currentDirty, expected) => { + const assistant = makeLastAssistant({ + stopReason: "error", + provider: "openrouter", + model: "test-model", + usage: { input: 100, output: 0, totalTokens: 100 }, + }); + expect( + shouldRetrySilentErrorAssistantTurn({ + attempt: makeAttemptResult({ + assistantTexts: [], + lastAssistant: assistant, + replayMetadata: { + hadPotentialSideEffects: cumulativeDirty, + replaySafe: !cumulativeDirty, + }, + currentAttemptReplayMetadata: { + hadPotentialSideEffects: currentDirty, + replaySafe: !currentDirty, + }, + }), + assistant, + }), + ).toBe(expected); + }, + ); + + it("detects empty openai-compatible stop turns with non-zero output usage", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "llamacpp", + model: "qwen3.6-27b", + usage: { input: 512, output: 103, totalTokens: 615 }, + }), + }, + { provider: "llamacpp", modelId: "qwen3.6-27b", modelApi: "openai-completions" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("detects generic empty GPT turns without visible text", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expect(DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT).toBe(1); + }); + + it("surfaces empty Codex app-server replies after successful sparse bash output", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + toolMetas: [{ toolName: "bash", meta: "exit=0" }], + messagesSnapshot: [ + { + role: "toolResult", + content: [{ type: "text", text: "" }], + details: { aggregated: "" }, + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + ], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + expect(incompleteTurnText).toContain("verify before retrying"); + }); + + it("retries generic empty Bedrock Converse turns without visible text", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + provider: "amazon-bedrock", + model: "openai.gpt-oss-120b-1:0", + content: [{ type: "text", text: "" }], + usage: { input: 950, output: 103, totalTokens: 1053 }, + }), + }, + { + provider: "amazon-bedrock", + modelId: "openai.gpt-oss-120b-1:0", + modelApi: "bedrock-converse-stream", + }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("treats clean empty assistant turns as silent only for reply-optional runs", () => { + const attempt = makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }); + + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }), + ), + ).toBe(true); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), + ), + ).toBe(false); + }); + + it("treats reasoning-only assistant turns as silent only for reply-optional runs", () => { + const attempt = makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_silent_helper", type: "reasoning" }), + }, + ], + }), + }); + + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }), + ), + ).toBe(true); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), + ), + ).toBe(false); + }); + + it("treats exact NO_REPLY assistant turns as silent only when the caller allows it", () => { + const attempt = makeAttemptResult({ + assistantTexts: ["NO_REPLY"], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "NO_REPLY" }], + }), + }); + + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), + ), + ).toBe(false); + }); + + it("treats post-tool exact NO_REPLY assistant turns as intentional silence", () => { + const attempt = makeAttemptResult({ + assistantTexts: ["NO_REPLY"], + toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "NO_REPLY" }], + }), + }); + + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true); + }); + + it("does not treat error or side-effect empty turns as silent", () => { + const errorAttempt = makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "error", + }), + }); + const silentErrorAttempt = makeAttemptResult({ + assistantTexts: ["NO_REPLY"], + lastAssistant: makeLastAssistant({ + stopReason: "error", + content: [{ type: "text", text: "NO_REPLY" }], + }), + }); + const sideEffectAttempt = makeAttemptResult({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["sent already"], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }); + const postToolEmptyAttempt = makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], + lastAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "stepfun", + model: "step-router-v1", + }), + }); + + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(errorAttempt))).toBe(false); + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(silentErrorAttempt))).toBe( + false, + ); + expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectAttempt))).toBe( + false, + ); + expect( + shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(postToolEmptyAttempt)), + ).toBe(false); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.payload-resolution.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.payload-resolution.test.ts new file mode 100644 index 000000000000..77509428821e --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.payload-resolution.test.ts @@ -0,0 +1,367 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + runEmbeddedAgent, + makeLastAssistant, + resolveIncompleteTurnPayloadText, + makeRunParams, + makeIncompleteTurnParams, + expectWarnMessageWith, + expectNoWarnMessageWith, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedBuildEmbeddedRunPayloads, + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { + resolveReplayInvalidFlag, + shouldRetryMissingAssistantTurn, +} from "./run/incomplete-turn-resolution.js"; +import { normalizeEmbeddedRunAttemptResult } from "./run/run-attempt-result.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("surfaces no-visible-answer recovery for app-server interrupted tool-only output", () => { + const interruptedToolOnlyAttempt = makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "bash", meta: "workspace" }], + messagesSnapshot: [ + { + role: "user", + content: "check running processes", + timestamp: 1, + }, + { + role: "toolResult", + content: "", + isError: false, + details: { aggregated: "" }, + timestamp: 2, + } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], + ], + }); + + const incompleteTurnText = resolveIncompleteTurnPayloadText({ + payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, + aborted: false, + timedOut: false, + attempt: interruptedToolOnlyAttempt, + }); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + + const explicitCancellationText = resolveIncompleteTurnPayloadText({ + payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, + aborted: true, + externalAbort: true, + timedOut: false, + attempt: interruptedToolOnlyAttempt, + }); + + expect(explicitCancellationText).toBeNull(); + + const internalAbortText = resolveIncompleteTurnPayloadText({ + payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, + aborted: true, + externalAbort: false, + timedOut: false, + attempt: interruptedToolOnlyAttempt, + }); + + expect(internalAbortText).toContain("couldn't generate a response"); + }); + + it("allows a same-prompt retry only for replay-safe missing assistant turns", () => { + const replaySafeAttempt = makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + }); + + expect( + shouldRetryMissingAssistantTurn({ + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: replaySafeAttempt, + }), + ).toBe(true); + expect( + shouldRetryMissingAssistantTurn({ + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + toolMetas: [{ toolName: "image_generate", asyncStarted: true }], + }), + }), + ).toBe(false); + expect( + shouldRetryMissingAssistantTurn({ + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + itemLifecycle: { + startedCount: 1, + completedCount: 0, + activeCount: 1, + }, + }), + }), + ).toBe(false); + }); + + it("detects tool-use terminal turn with pre-tool text as incomplete (#76477)", () => { + // When the last assistant message ended with stopReason=toolUse, pre-tool + // text alone must not suppress the incomplete-turn guard. The model + // expected to continue after tool results but the post-tool response was + // never produced. + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Initial analysis of the codebase..."], + toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + provider: "anthropic", + model: "sonnet-4.6", + content: [ + { type: "text", text: "Initial analysis of the codebase..." }, + { type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); + + it("does not surface incomplete-turn error while an async media task is running", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams({ + assistantTexts: [], + toolMetas: [ + { + toolName: "image_generate", + meta: 'generate prompt="a portrait"', + asyncStarted: true, + }, + ], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + content: [ + { + type: "tool_use", + id: "tool_1", + name: "image_generate", + input: { action: "generate", prompt: "a portrait" }, + }, + ], + }), + }), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("surfaces tool-use terminal with pre-tool text and side effects as replay-unsafe (#76477)", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Let me update the file..."], + toolMetas: [{ toolName: "write" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + content: [ + { type: "text", text: "Let me update the file..." }, + { type: "tool_use", id: "tool_1", name: "write", input: {} }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("verify before retrying"); + }); + + it("does not flag a completed tool-use turn with end_turn as incomplete (#76477)", () => { + // When the model successfully produces post-tool text, lastAssistant has + // stopReason=end_turn. The incomplete-turn guard should not fire. + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Initial analysis...", "Here is the final answer."], + toolMetas: [{ toolName: "read" }], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "anthropic", + model: "sonnet-4.6", + content: [{ type: "text", text: "Here is the final answer." }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("surfaces stall on clean stop with only an unsigned thinking payload (payloadCount=1, no visible text)", () => { + // Regression: unsigned thinking payloads increment payloadCount but carry no + // user-visible content. The visible-text guard must not suppress incomplete-turn + // detection when the model produced only a thinking block and no answer. (#89787) + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + model: "qwen3.6-35b-a3b", + content: [ + { + type: "thinking", + thinking: "let me plan the tool calls I need to make...", + // no signature — unsigned thinking block + }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); + + it("does not surface a stall when unsigned thinking accompanies visible text (payloadCount=1)", () => { + // When the model emits both a thinking block and a visible text answer, the turn + // succeeded and no stall should be surfaced even though thinking is unsigned. + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Here is the answer to your question."], + lastAssistant: makeLastAssistant({ + model: "qwen3.6-35b-a3b", + content: [ + { + type: "thinking", + thinking: "let me answer this...", + }, + { type: "text", text: "Here is the answer to your question." }, + ], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("surfaces an error for tool-use terminal turn with pre-tool text via runEmbeddedAgent (#76477)", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Initial analysis of the issue..."], + toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }], + lastAssistant: { + stopReason: "toolUse", + provider: "anthropic", + model: "sonnet-4.6", + content: [ + { type: "text", text: "Initial analysis of the issue..." }, + { type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } }, + ], + } as unknown as EmbeddedRunAttemptResult["lastAssistant"], + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-tool-use-dropped-final-text", { + provider: "anthropic", + model: "sonnet-4.6", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); + expectWarnMessageWith("incomplete turn detected"); + }); + + it("delivers the current final answer when the session assistant is stale (#80918)", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const finalText = "The requested update is complete."; + mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([{ text: finalText }]); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [finalText], + toolMetas: [{ toolName: "update_plan", replaySafe: true }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "tool_use", id: "tool_1", name: "update_plan", input: {} }], + usage: { input: 100, output: 5, total: 105 }, + }), + currentAttemptAssistant: makeLastAssistant({ + content: [{ type: "text", text: finalText }], + usage: { input: 200, output: 20, total: 220 }, + }), + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-current-assistant-after-tool-use")); + + expect(result.payloads).toEqual([{ text: finalText }]); + expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( + expect.objectContaining({ + currentAssistant: expect.objectContaining({ + stopReason: "stop", + content: [{ type: "text", text: finalText }], + }), + lastAssistant: expect.objectContaining({ + stopReason: "stop", + content: [{ type: "text", text: finalText }], + }), + }), + ); + expect(result.meta.finalAssistantVisibleText).toBe(finalText); + expect(result.meta.stopReason).toBe("stop"); + expect(result.meta.agentMeta?.lastCallUsage).toMatchObject({ + input: 200, + output: 20, + total: 220, + }); + expectNoWarnMessageWith("incomplete turn detected"); + }); + + it("treats missing replay metadata as replay-invalid", () => { + const attempt = makeAttemptResult(); + delete (attempt as Partial).replayMetadata; + + const normalizedAttempt = normalizeEmbeddedRunAttemptResult(attempt); + + expect(normalizedAttempt.replayMetadata).toEqual({ + hadPotentialSideEffects: true, + replaySafe: false, + }); + expect(resolveReplayInvalidFlag({ attempt: normalizedAttempt })).toBe(true); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.reasoning-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.reasoning-recovery.test.ts new file mode 100644 index 000000000000..98132d7fa3bb --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.reasoning-recovery.test.ts @@ -0,0 +1,407 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + REASONING_ONLY_RETRY_INSTRUCTION, + EMPTY_RESPONSE_RETRY_INSTRUCTION, + runEmbeddedAgent, + makeLastAssistant, + makeRunParams, + expectWarnMessageWith, + expectNoWarnMessageWith, + runAttemptCall, + markUserMessagePersisted, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + mockedResolveModelAsync, + overflowBaseRunParams, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("does not retry or warn on reasoning-only turns when a messaging tool already delivered", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Delivered through the message tool."], + lastAssistant: makeLastAssistant({ + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_after_send", type: "reasoning" }), + }, + ], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-reasoning-only-after-side-effects", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toBeUndefined(); + }); + + it("retries reasoning-only turns when the assistant ended in error", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const errorAssistant = makeLastAssistant({ + stopReason: "error", + model: "gpt-5.4", + errorMessage: "provider failed after emitting reasoning", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_error_turn", type: "reasoning" }), + }, + ], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: errorAssistant, + currentAttemptAssistant: errorAssistant, + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Recovered."], + lastAssistant: makeLastAssistant({ + model: "gpt-5.4", + content: [{ type: "text", text: "Recovered." }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-reasoning-only-assistant-error", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads).toBeUndefined(); + }); + + it("does not retry reasoning-only turns for non-strict-agentic providers", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "anthropic", + model: "sonnet-4.6", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ + id: "rs_provider_mismatch", + type: "reasoning", + }), + }, + ], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-reasoning-only-provider-mismatch", { + provider: "anthropic", + model: "sonnet-4.6", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("Please try again"); + }); + + it("retries Kimi Anthropic reasoning-only turns with a visible-answer continuation instruction", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedResolveModelAsync.mockResolvedValue({ + model: { + id: "kimi-for-coding", + provider: "kimi", + contextWindow: 262144, + api: "anthropic-messages", + }, + error: null, + authStorage: { + setRuntimeApiKey: vi.fn(), + }, + modelRegistry: {}, + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + api: "anthropic-messages", + provider: "kimi", + model: "kimi-for-coding", + content: [ + { + type: "thinking", + thinking: "internal Kimi reasoning", + thinkingSignature: "", + }, + ], + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible Kimi answer."], + lastAssistant: makeLastAssistant({ + api: "anthropic-messages", + provider: "kimi", + model: "kimi-for-coding", + content: [{ type: "text", text: "Visible Kimi answer." }], + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-kimi-anthropic-reasoning-only-continuation", { + provider: "kimi", + model: "kimi-for-coding", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toContain(REASONING_ONLY_RETRY_INSTRUCTION); + expectWarnMessageWith("reasoning-only assistant turn detected"); + }); + + it("retries generic empty GPT turns with a visible-answer continuation instruction", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible answer."], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "Visible answer." }], + }), + }), + ); + + await runEmbeddedAgent(makeRunParams("run-empty-response-continuation", { model: "gpt-5.4" })); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expect(secondCall.suppressNextUserMessagePersistence).toBe(false); + expect(secondCall.skipPreparedUserTurnMessage).toBe(true); + expectWarnMessageWith("empty response detected"); + }); + + it("retries replay-safe missing turns despite a stale aborted transcript assistant", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const staleAssistant = makeLastAssistant({ + stopReason: "aborted", + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: staleAssistant, + currentAttemptAssistant: undefined, + }), + ); + const recoveredAssistant = makeLastAssistant({ + stopReason: "end_turn", + content: [{ type: "text", text: "Recovered answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Recovered answer."], + lastAssistant: recoveredAssistant, + currentAttemptAssistant: recoveredAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-retry")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(runAttemptCall(1).prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer."); + expectWarnMessageWith("empty response detected"); + expectNoWarnMessageWith("missing assistant terminal message detected"); + expectNoWarnMessageWith("incomplete turn detected"); + }); + + it("retries missing terminal assistant turns with the same prompt without re-persisting the user message", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + }); + }); + const recoveredAssistant = makeLastAssistant({ + stopReason: "end_turn", + content: [{ type: "text", text: "Recovered answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Recovered answer."], + lastAssistant: recoveredAssistant, + currentAttemptAssistant: recoveredAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-same-prompt-retry")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + // The same-prompt replay must not append the inbound user message a second time. + expect(runAttemptCall(1).prompt).toBe(runAttemptCall(0).prompt); + expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true); + expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer."); + expectWarnMessageWith("missing assistant terminal message detected"); + expectNoWarnMessageWith("empty response detected"); + expectNoWarnMessageWith("incomplete turn detected"); + }); + + it("waits for asynchronous user persistence before retrying a missing terminal turn", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const persistedMessage = { role: "user" as const, content: "test prompt", timestamp: 1 }; + const admission = { + agentId: "main", + sessionId: overflowBaseRunParams.sessionId, + sessionKey: overflowBaseRunParams.sessionKey, + storePath: "/tmp/openclaw-transcript.jsonl", + generation: "generation-1", + entryId: "msg-user-delayed", + rawSeq: 1, + effectiveParentId: null, + activeMessagePosition: 0, + logicalTurnId: "run-missing-assistant-delayed-persistence", + role: "user" as const, + }; + let resolvePersistApproved: + | ((result: { + admission: typeof admission; + sessionFile: string; + sessionEntry: undefined; + messageId: string; + message: typeof persistedMessage; + }) => void) + | undefined; + let pendingPersistence: Promise | undefined; + const persistApproved = vi.fn( + () => + new Promise<{ + admission: typeof admission; + sessionFile: string; + sessionEntry: undefined; + messageId: string; + message: typeof persistedMessage; + }>((resolve) => { + resolvePersistApproved = resolve; + }), + ); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ assistantTexts: ["Recovered answer."] }), + ); + + const runPromise = runEmbeddedAgent( + makeRunParams("run-missing-assistant-delayed-persistence", { + userTurnTranscriptRecorder: { + message: persistedMessage, + resolveMessage: vi.fn(async () => persistedMessage), + getAdmissionReceipt: () => admission, + markRuntimePersistencePending: vi.fn((pending) => { + pendingPersistence = pending; + }), + markRuntimePersisted: vi.fn(), + markBlocked: vi.fn(), + hasPersisted: vi.fn(() => false), + isBlocked: vi.fn(() => false), + hasRuntimePersistencePending: vi.fn(() => pendingPersistence !== undefined), + waitForRuntimePersistence: vi.fn(async () => { + await pendingPersistence; + }), + persistApproved, + persistBlocked: vi.fn(async () => undefined), + persistFallback: vi.fn(async () => undefined), + }, + }), + ); + + await vi.waitFor(() => { + expect(persistApproved).toHaveBeenCalledOnce(); + }); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + + resolvePersistApproved?.({ + admission, + sessionFile: "/tmp/openclaw-transcript.jsonl", + sessionEntry: undefined, + messageId: "msg-user-delayed", + message: persistedMessage, + }); + await runPromise; + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true); + }); + + it("persists a missing-turn retry when the first attempt never persisted the user message", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + }), + ); + const recoveredAssistant = makeLastAssistant({ + stopReason: "end_turn", + content: [{ type: "text", text: "Recovered answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Recovered answer."], + lastAssistant: recoveredAssistant, + currentAttemptAssistant: recoveredAssistant, + }), + ); + + await runEmbeddedAgent(makeRunParams("run-missing-assistant-unpersisted-retry")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(false); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts new file mode 100644 index 000000000000..046b690b3110 --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts @@ -0,0 +1,542 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + REASONING_ONLY_RETRY_INSTRUCTION, + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + runEmbeddedAgent, + makeLastAssistant, + makeRunParams, + expectWarnMessageWith, + expectNoWarnMessageWith, + runAttemptCall, + markUserMessagePersisted, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedBuildEmbeddedRunPayloads, + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + registerAgentHarness, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("preserves a structured visible failed-tool payload without finalizing (#118274)", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], + }); + const visibleError = { + text: "Review the failed operation.", + isError: true, + channelData: { structuredError: true }, + }; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "exec", isError: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { toolName: "exec", error: "post-processing error" }, + }), + ); + mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([visibleError]); + + const result = await runEmbeddedAgent(makeRunParams("run-structured-failed-tool-payload")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledOnce(); + expect(result.payloads?.[0]).toMatchObject(visibleError); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("keeps the original failed-tool warning if finalization completes empty (#118274)", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], + }); + const warning = { text: "⚠️ 🛠️ Exec failed", isError: true }; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "exec", isError: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { toolName: "exec", error: "post-processing error" }, + }), + ) + .mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant(), + currentAttemptAssistant: makeLastAssistant(), + currentAttemptCompletedAssistant: makeLastAssistant(), + }), + ); + mockedBuildEmbeddedRunPayloads.mockReturnValue([warning]); + + const result = await runEmbeddedAgent(makeRunParams("run-failed-tool-finalization-fallback")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]).toEqual(warning); + expectWarnMessageWith("settled-turn finalization completed without a visible answer"); + }); + + it("preserves the incomplete-turn failure when the selected harness cannot finalize safely", async () => { + registerAgentHarness({ + id: "legacy", + label: "Legacy harness without settled-turn finalization", + supports: () => ({ supported: true, priority: 100 }), + runAttempt: async (params) => await mockedRunEmbeddedAttempt(params), + }); + try { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }, + ], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }); + }); + + const result = await runEmbeddedAgent( + makeRunParams("run-tool-use-no-finalization-capability", { agentHarnessId: "legacy" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]).toMatchObject({ isError: true }); + expect(result.payloads?.[0]?.text).toContain( + "some tool actions may have already been executed", + ); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + } finally { + resetRunIncompleteTurnOwnerMocks(); + } + }); + + it("continues from settled side-effecting tools after an empty stop without replaying them", async () => { + const emptyStopAssistant = makeLastAssistant(); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Writing note.txt…"], + messagingToolSentTargets: [ + { + tool: "message", + provider: "telegram", + to: "chat:123", + text: "Writing note.txt…", + sourceReplyFinal: false, + }, + ], + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }); + }); + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Write completed. Here is the final answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Write completed. Here is the final answer."], + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + currentAttemptCompletedAssistant: finalAssistant, + }), + ); + mockedBuildEmbeddedRunPayloads + .mockReturnValueOnce([]) + .mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]); + + const result = await runEmbeddedAgent( + makeRunParams("run-empty-stop-settled-tool-continuation", { + trigger: "cron", + terminalReplyExpectation: "required", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); + expect(runAttemptCall(1).prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(runAttemptCall(1).disableTools).toBe(true); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it.each([ + { + label: "explicit optional expectation", + trigger: "user" as const, + terminalReplyExpectation: "optional" as const, + }, + { + label: "heartbeat default", + trigger: "heartbeat" as const, + terminalReplyExpectation: undefined, + }, + ])("does not continue settled tools for $label", async (runPolicy) => { + const emptyStopAssistant = makeLastAssistant(); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }); + }); + mockedBuildEmbeddedRunPayloads.mockReturnValue([]); + + const result = await runEmbeddedAgent( + makeRunParams("run-optional-empty-stop-settled-tool", { + trigger: runPolicy.trigger, + terminalReplyExpectation: runPolicy.terminalReplyExpectation, + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.[0]).toMatchObject({ isError: true }); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("records silent success when the settled-tool finalization completes empty", async () => { + const emptyStopAssistant = makeLastAssistant(); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }), + ); + mockedBuildEmbeddedRunPayloads.mockReturnValue([]); + + const result = await runEmbeddedAgent( + makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", { + allowEmptyAssistantReplyAsSilent: true, + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads).toBeUndefined(); + expect(result.meta.error).toBeUndefined(); + expect(result.meta.terminalReplyKind).toBeUndefined(); + expect(result.meta.finalAssistantVisibleText).toBeUndefined(); + expect(result.meta.finalAssistantRawText).toBeUndefined(); + expect(result.meta.stopReason).toBe("stop"); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settled-turn finalization completed without a visible answer"); + }); + + it.each([ + { + label: "provider failure", + finalAttempt: { + assistantTexts: [], + promptError: new Error("finalizer provider failure"), + promptErrorSource: "prompt" as const, + }, + }, + { + label: "preflight recovery request", + finalAttempt: { + assistantTexts: [], + preflightRecovery: { route: "compact_only" as const, handled: true as const }, + }, + }, + { + label: "compaction continuation request", + finalAttempt: { assistantTexts: [], compactionCount: 1 }, + }, + { + label: "before-finalize revision request", + finalAttempt: { + assistantTexts: [], + beforeAgentFinalizeRevisionReason: "revise this answer", + }, + }, + ])("does not escape finalization through a $label", async ({ finalAttempt }) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }), + ) + .mockResolvedValueOnce(makeAttemptResult(finalAttempt)); + mockedBuildEmbeddedRunPayloads.mockReturnValue([]); + + const result = await runEmbeddedAgent(makeRunParams("run-settled-finalizer-sticky-operation")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]).toMatchObject({ isError: true }); + expect(result.payloads?.[0]?.text).toContain( + "some tool actions may have already been executed", + ); + }); + + it("surfaces the existing incomplete-turn error after one tool-use continuation", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-tool-use-terminal-continuation-exhausted"), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain( + "some tool actions may have already been executed", + ); + expectWarnMessageWith("settled-turn finalization failed closed"); + }); + + it("does not claim completion for a toolUse terminal whose tools never started", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [], + itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 }, + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }), + ); + + await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-never-started")); + + for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); + } + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("ignores stale prior-turn tool results with colliding ids", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [], + itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 }, + // A completed result from a PRIOR turn reusing the same id sits before + // the terminal assistant; it must not prove the new batch dispatched. + messagesSnapshot: [ + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + toolUseAssistant, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }), + ); + + await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-stale-prior-result")); + + for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); + } + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("does not claim completion when only part of a multi-tool request dispatched", async () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "toolCall", id: "tool_1", name: "write", arguments: { path: "a.txt" } }, + { type: "toolCall", id: "tool_2", name: "write", arguments: { path: "b.txt" } }, + ], + }); + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=a.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }), + ); + + await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-partial-dispatch")); + + for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); + } + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("retries reasoning-only assistant turns even when deliberate silence is allowed", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_silent_group", type: "reasoning" }), + }, + ], + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible answer."], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + content: [{ type: "text", text: "Visible answer." }], + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-reasoning-only-silent", { allowEmptyAssistantReplyAsSilent: true }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(runAttemptCall(1).prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + expectWarnMessageWith("reasoning-only assistant turn detected"); + }); + + it("replays an unpersisted reasoning continuation across a missing-assistant retry", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_retry_boundary", type: "reasoning" }), + }, + ], + }), + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: undefined, + currentAttemptAssistant: undefined, + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ assistantTexts: ["Visible answer."] }), + ); + + await runEmbeddedAgent( + makeRunParams("run-reasoning-continuation-missing-assistant", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); + expect(runAttemptCall(1)).toMatchObject({ + prompt: REASONING_ONLY_RETRY_INSTRUCTION, + skipPreparedUserTurnMessage: true, + suppressNextUserMessagePersistence: false, + }); + expect(runAttemptCall(2)).toMatchObject({ + prompt: REASONING_ONLY_RETRY_INSTRUCTION, + skipPreparedUserTurnMessage: true, + suppressNextUserMessagePersistence: false, + }); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts new file mode 100644 index 000000000000..7641614b6abe --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts @@ -0,0 +1,599 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + REASONING_ONLY_RETRY_INSTRUCTION, + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + runEmbeddedAgent, + makeLastAssistant, + makeRunParams, + expectWarnMessageWith, + runAttemptCall, + markUserMessagePersisted, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedBuildEmbeddedRunPayloads, + mockedClassifyFailoverReason, + mockedIsFailoverAssistantError, + mockedIsRateLimitAssistantError, + mockedRunEmbeddedAttempt, + mockedSleepWithAbort, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("keeps model-call order when parallel tool outcomes finish out of order", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + const onToolOutcome = ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + toolCallOrdinal?: number; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome; + onToolOutcome?.({ + toolName: "exec", + argsHash: "exec-args", + resultHash: "exec-result", + toolCallOrdinal: 1, + }); + onToolOutcome?.({ + toolName: "web_fetch", + argsHash: "fetch-args", + resultHash: "fetch-result", + toolCallOrdinal: 0, + terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", + }); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "web_fetch" }, { toolName: "exec" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + }), + }); + }); + + const result = await runEmbeddedAgent( + makeRunParams("run-stale-terminal-presentation", { model: "gpt-5.4" }), + ); + + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); + expect(result.meta.error?.fallbackSafe).toBe(false); + }); + + it("does not surface a read-only presentation after a sibling side effect", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { + const onToolOutcome = ( + attemptParams as { + onToolOutcome?: (observation: { + toolName: string; + argsHash: string; + resultHash: string; + terminalPresentation?: string; + }) => void; + } + ).onToolOutcome; + onToolOutcome?.({ + toolName: "exec", + argsHash: "exec-args", + resultHash: "exec-result", + }); + onToolOutcome?.({ + toolName: "web_fetch", + argsHash: "fetch-args", + resultHash: "fetch-result", + terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", + }); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "exec" }, { toolName: "web_fetch" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + }), + }); + }); + + const result = await runEmbeddedAgent( + makeRunParams("run-side-effect-terminal-presentation", { model: "gpt-5.4" }), + ); + + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); + expect(result.meta.error?.fallbackSafe).toBe(false); + }); + + it("promotes successful final assistant text when a prompt timeout races completion", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const finalText = + "1. Verdict: the answer completed cleanly. 2. Evidence: the runner captured final text."; + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: finalText }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-prompt-timeout-final-assistant-recovered"), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toEqual([{ text: finalText }]); + expect(result.meta.finalAssistantVisibleText).toBe(finalText); + expect(result.meta.finalAssistantRawText).toBe(finalText); + expect(result.meta.livenessState).toBe("working"); + expect(result.meta.completion).toEqual({ + stopReason: "stop", + finishReason: "stop", + }); + expect(result.meta.executionTrace?.attempts?.at(-1)).toMatchObject({ + result: "success", + stage: "assistant", + }); + }); + + it("does not recover a stale prior assistant after the current prompt times out", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const staleAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Stale answer from the prior attempt." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + timedOut: true, + lastAssistant: staleAssistant, + currentAttemptAssistant: undefined, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-stale-assistant")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads?.some((payload) => payload.text?.includes("timed out"))).toBe(true); + expect(result.payloads?.some((payload) => payload.text?.includes("Stale answer"))).toBe(false); + expect(result.meta.finalAssistantVisibleText).toBeUndefined(); + }); + + it("does not resolve a successful run from a stale transcript assistant", async () => { + const staleAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Prior transcript reply." }], + }); + const completedAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Current run reply." }], + }); + mockedBuildEmbeddedRunPayloads.mockReturnValue([{ text: "Current run reply." }]); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Current run reply."], + lastAssistant: staleAssistant, + currentAttemptAssistant: staleAssistant, + currentAttemptCompletedAssistant: completedAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-success-stale-transcript-assistant")); + + expect(result.payloads).toEqual([{ text: "Current run reply." }]); + expect(result.meta.finalAssistantVisibleText).toBe("Current run reply."); + expect(result.meta.finalAssistantRawText).toBe("Current run reply."); + expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( + expect.objectContaining({ + currentAssistant: completedAssistant, + lastAssistant: completedAssistant, + }), + ); + }); + + it("retains the yielded attempt assistant for paused-turn payload classification", async () => { + const completedAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Earlier completed cycle." }], + }); + const yieldedAssistant = makeLastAssistant({ + stopReason: "aborted", + content: [{ type: "toolCall", name: "sessions_yield", arguments: {} }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: yieldedAssistant, + currentAttemptAssistant: undefined, + currentAttemptCompletedAssistant: completedAssistant, + yieldDetected: true, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-yielded-assistant-classification")); + + expect(result.meta).toMatchObject({ livenessState: "paused", yielded: true }); + expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( + expect.objectContaining({ currentAssistant: null, lastAssistant: yieldedAssistant }), + ); + }); + + it("recovers a completed prompt-timeout assistant without collected assistant text", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const finalText = "Completed answer after the timeout race."; + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: finalText }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: undefined as unknown as string[], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-no-assistant-texts")); + + expect(result.payloads).toEqual([{ text: finalText }]); + }); + + it("preserves tool media when prompt-timeout recovery replaces partial assistant text", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const partialText = "Partial answer before the timeout race."; + const finalText = "Complete answer after the timeout race."; + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: finalText }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [partialText], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + toolMediaUrls: ["https://example.test/recovered-output.png"], + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-prompt-timeout-final-assistant-media"), + ); + + expect(result.payloads).toEqual([ + { + mediaUrl: "https://example.test/recovered-output.png", + mediaUrls: ["https://example.test/recovered-output.png"], + audioAsVoice: undefined, + trustedLocalMedia: undefined, + }, + { text: finalText }, + ]); + }); + + it("replaces the latest partial assistant payload after prompt-timeout recovery", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + const completedText = "Completed answer block before the final response."; + const partialText = "Partial final response before the timeout race."; + const finalText = "Complete final response after the timeout race."; + mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([ + { text: completedText }, + { text: partialText }, + ]); + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: finalText }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [completedText, partialText], + timedOut: true, + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-latest-partial")); + + expect(result.payloads).toEqual([{ text: completedText }, { text: finalText }]); + }); + + it("records same-model rate-limit retries without a profile-rotation trace", async () => { + const rateLimitMessage = + "429 rate_limit_exceeded: requests per minute exceeded; Retry-After: 30"; + const rateLimitAssistant = makeLastAssistant({ + stopReason: "error", + errorMessage: rateLimitMessage, + }); + mockedClassifyFailoverReason.mockImplementation((raw) => + raw.includes("429") ? "rate_limit" : null, + ); + mockedIsFailoverAssistantError.mockImplementation((assistant) => + Boolean(assistant?.errorMessage?.includes("429")), + ); + mockedIsRateLimitAssistantError.mockImplementation((assistant) => + Boolean(assistant?.errorMessage?.includes("429")), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: rateLimitAssistant, + currentAttemptAssistant: rateLimitAssistant, + }), + ); + const recoveredAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Recovered after a short rate-limit wait." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Recovered after a short rate-limit wait."], + lastAssistant: recoveredAssistant, + currentAttemptAssistant: recoveredAssistant, + }), + ); + + const result = await runEmbeddedAgent(makeRunParams("run-same-model-rate-limit-trace")); + + expect(mockedSleepWithAbort).toHaveBeenCalledWith(30_000, undefined); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.meta.executionTrace?.fallbackUsed).toBe(false); + expect(result.meta.executionTrace?.attempts).toMatchObject([ + { + provider: "openai", + model: "gpt-5.5", + result: "same_model_rate_limit", + reason: "rate_limit", + stage: "assistant", + }, + { + provider: "openai", + model: "gpt-5.5", + result: "success", + stage: "assistant", + }, + ]); + }); + + it("retries reasoning-only GPT turns with a visible-answer continuation instruction", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_reasoning_only", type: "reasoning" }), + }, + ], + }), + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible answer."], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "Visible answer." }], + }), + }), + ); + + await runEmbeddedAgent(makeRunParams("run-reasoning-only-continuation", { model: "gpt-5.4" })); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION); + expect(secondCall.suppressNextUserMessagePersistence).toBe(false); + expect(secondCall.skipPreparedUserTurnMessage).toBe(true); + expectWarnMessageWith("reasoning-only assistant turn detected"); + }); + + it("continues once after settled side-effecting tools finish without a final answer", async () => { + const acceptedSessionSpawns = [ + { runId: "child-run", childSessionKey: "agent:main:subagent:child" }, + ]; + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } }, + { type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } }, + { + type: "toolCall", + id: "tool_spawn", + name: "sessions_spawn", + arguments: { task: "follow up" }, + }, + ], + }); + const settledToolResults = [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false }, + { role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false }, + { + role: "toolResult", + toolCallId: "tool_spawn", + toolName: "sessions_spawn", + isError: false, + }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"]; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + latestMcpAppChannelView: { viewId: "view-after-tools" }, + toolMetas: [ + { toolName: "write", meta: "path=note.txt" }, + { toolName: "cron" }, + { toolName: "sessions_spawn" }, + ], + acceptedSessionSpawns, + successfulCronAdds: 1, + itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, + messagesSnapshot: settledToolResults, + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + codeModeEngaged: true, + assistantTurns: 1, + bridgeCalls: { search: 1, describe: 2, call: 3 }, + }); + }); + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: "Write completed. Here is the final answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Write completed. Here is the final answer."], + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + currentAttemptCompletedAssistant: finalAssistant, + }), + ); + mockedBuildEmbeddedRunPayloads + .mockReturnValueOnce([]) + .mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]); + + const result = await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-continuation")); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); + expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" }); + expect(result.successfulCronAdds).toBe(1); + expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns); + expect(result.meta.toolSummary).toEqual({ + calls: 3, + tools: ["write", "cron", "sessions_spawn"], + failures: 0, + }); + expect(result.meta.agentMeta).toMatchObject({ + codeModeEngaged: true, + assistantTurns: 2, + bridgeCalls: { search: 1, describe: 2, call: 3 }, + }); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(secondCall.disableTools).toBe(true); + expect(secondCall.operation).toBe("settled-tool-finalization"); + expect(secondCall.suppressNextUserMessagePersistence).toBe(false); + expect(secondCall.skipPreparedUserTurnMessage).toBe(true); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it.each([ + { label: "interactive user", trigger: "user" as const }, + { + label: "required isolated cron", + trigger: "cron" as const, + terminalReplyExpectation: "required" as const, + }, + ])("finalizes a settled failed tool once for a $label turn (#118274)", async (runPolicy) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], + }); + const failureText = "The exec tool failed: post-processing error."; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [ + { toolName: "read", isError: true, replaySafe: true }, + { toolName: "exec", isError: true, replaySafe: false }, + ], + itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { + role: "toolResult", + toolCallId: "tool_1", + toolName: "exec", + isError: true, + content: [{ type: "text", text: "post-processing error" }], + }, + { + role: "assistant", + stopReason: "toolUse", + content: [ + { + type: "toolCall", + id: "tool_search_code:tool_1:read:1", + name: "read", + arguments: {}, + }, + ], + }, + { + role: "toolResult", + toolCallId: "tool_search_code:tool_1:read:1", + toolName: "read", + isError: true, + content: [{ type: "text", text: "post-processing error" }], + }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { + toolName: "exec", + error: "post-processing error", + errorCode: "SYSTEM_RUN_DENIED", + }, + }); + }); + const finalAssistant = makeLastAssistant({ + content: [{ type: "text", text: failureText }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [failureText], + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + currentAttemptCompletedAssistant: finalAssistant, + }), + ); + mockedBuildEmbeddedRunPayloads + .mockReturnValueOnce([{ text: "⚠️ 🛠️ Exec failed", isError: true }]) + .mockReturnValueOnce([{ text: failureText }]); + + const result = await runEmbeddedAgent( + makeRunParams(`run-settled-failed-tool-${runPolicy.trigger}`, runPolicy), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.text).toBe(failureText); + const finalizationCall = runAttemptCall(1); + expect(finalizationCall.operation).toBe("settled-tool-finalization"); + expect(finalizationCall.disableTools).toBe(true); + expect(finalizationCall.prompt).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(finalizationCall.prompt).toContain( + "If any tool failed, state that failure plainly and do not claim it succeeded.", + ); + expect(result.meta.failureSignal).toEqual( + runPolicy.trigger === "cron" + ? { + kind: "execution_denied", + source: "tool", + toolName: "exec", + code: "SYSTEM_RUN_DENIED", + message: "post-processing error", + fatalForCron: true, + } + : undefined, + ); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.silent-reply.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.silent-reply.test.ts new file mode 100644 index 000000000000..0328ce89dcbf --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.silent-reply.test.ts @@ -0,0 +1,405 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../config/config.js"; +import { + REASONING_ONLY_RETRY_INSTRUCTION, + EMPTY_RESPONSE_RETRY_INSTRUCTION, + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + runEmbeddedAgent, + makeLastAssistant, + makeRunParams, + makeEmptyResponseRetryParams, + makeSilentReplyParams, + expectWarnMessageWith, + expectNoWarnMessageWith, + runAttemptCall, +} from "./run.incomplete-turn.test-helpers.js"; +import { + mockedClassifyFailoverReason, + mockedRunEmbeddedAttempt, + mockedResolveModelAsync, + resetRunIncompleteTurnOwnerMocks, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { + resolveEmptyResponseRetryInstruction, + shouldTreatEmptyAssistantReplyAsSilent, +} from "./run/incomplete-turn-recovery.js"; +import { + resolveReplayInvalidFlag, + resolveRunLivenessState, +} from "./run/incomplete-turn-resolution.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("retries clean empty assistant turns even when deliberate silence is allowed", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }), + ); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible answer."], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "Visible answer." }], + }), + }), + ); + + await runEmbeddedAgent( + makeRunParams("run-empty-assistant-silent", { allowEmptyAssistantReplyAsSilent: true }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(runAttemptCall(1).prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectWarnMessageWith("empty response detected"); + }); + + it("returns NO_REPLY without retrying exact silent assistant replies when silence is allowed", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: ["NO_REPLY"], + lastAssistant: makeLastAssistant({ + content: [ + { + type: "thinking", + thinking: "internal reasoning", + thinkingSignature: JSON.stringify({ id: "rs_exact_silent", type: "reasoning" }), + }, + { type: "text", text: "NO_REPLY" }, + ], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-exact-silent-assistant-reply", { + allowEmptyAssistantReplyAsSilent: true, + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + const onlyCall = runAttemptCall(0); + expect(onlyCall.prompt).not.toContain(REASONING_ONLY_RETRY_INSTRUCTION); + expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectNoWarnMessageWith("empty response detected"); + expectNoWarnMessageWith("incomplete turn detected"); + expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); + expect(result.meta.terminalReplyKind).toBe("silent-empty"); + expect(result.meta.livenessState).toBe("working"); + }); + + it("continues post-tool openai-compatible empty stop turns even when silence is allowed", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedResolveModelAsync.mockResolvedValue({ + model: { + id: "step-router-v1", + provider: "stepfun", + contextWindow: 200000, + api: "openai-completions", + }, + error: null, + authStorage: { + setRuntimeApiKey: vi.fn(), + }, + modelRegistry: {}, + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "stepfun", + model: "step-router-v1", + }), + currentAttemptAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "stepfun", + model: "step-router-v1", + }), + }), + ); + const finalAssistant = makeLastAssistant({ + api: "openai-completions", + provider: "stepfun", + model: "step-router-v1", + content: [{ type: "text", text: "Visible StepFun answer." }], + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["Visible StepFun answer."], + lastAssistant: finalAssistant, + currentAttemptAssistant: finalAssistant, + currentAttemptCompletedAssistant: finalAssistant, + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-post-tool-openai-compatible-empty-stop", { + allowEmptyAssistantReplyAsSilent: true, + provider: "stepfun", + model: "step-router-v1", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + const secondCall = runAttemptCall(1); + expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(result.meta.terminalReplyKind).toBeUndefined(); + expect(result.meta.finalAssistantVisibleText).toBe("Visible StepFun answer."); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("returns NO_REPLY without retrying post-tool exact silent assistant replies", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedResolveModelAsync.mockResolvedValue({ + model: { + id: "step-router-v1", + provider: "stepfun", + contextWindow: 200000, + api: "openai-completions", + }, + error: null, + authStorage: { + setRuntimeApiKey: vi.fn(), + }, + modelRegistry: {}, + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["NO_REPLY"], + toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], + lastAssistant: makeLastAssistant({ + api: "openai-completions", + provider: "stepfun", + model: "step-router-v1", + content: [{ type: "text", text: "NO_REPLY" }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-post-tool-exact-silent-retry", { + allowEmptyAssistantReplyAsSilent: true, + provider: "stepfun", + model: "step-router-v1", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + const onlyCall = runAttemptCall(0); + expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expectNoWarnMessageWith("empty response detected"); + expectNoWarnMessageWith("incomplete turn detected"); + expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); + expect(result.meta.terminalReplyKind).toBe("silent-empty"); + expect(result.meta.livenessState).toBe("working"); + }); + + it("treats reply-optional post-tool empty stops as silent even after side-effecting tools", () => { + // Regression: a cron agentTurn without a delivery route ran a successful + // replay-unsafe sessions patch and intentionally sent no final text; the run + // must finish silent, not as an incomplete-turn error. + const sideEffectToolAttempt = makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }); + + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "optional" }), + ), + ).toBe(true); + // A required or unspecified terminal reply keeps the ambiguous-failure path. + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "required" }), + ), + ).toBe(false); + expect( + shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectToolAttempt)), + ).toBe(false); + }); + + it("keeps reply-optional runs erroring on real failure states", () => { + const toolErrorAttempt = makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "sessions", meta: "patch failed", replaySafe: false, isError: true }], + lastToolError: { toolName: "sessions", error: "patch failed" }, + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }); + const errorStopAttempt = makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], + lastAssistant: makeLastAssistant({ + stopReason: "error", + }), + }); + + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(toolErrorAttempt, { terminalReplyExpectation: "optional" }), + ), + ).toBe(false); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(errorStopAttempt, { terminalReplyExpectation: "optional" }), + ), + ).toBe(false); + expect( + shouldTreatEmptyAssistantReplyAsSilent( + makeSilentReplyParams(errorStopAttempt, { + terminalReplyExpectation: "optional", + aborted: true, + }), + ), + ).toBe(false); + }); + + it("returns NO_REPLY for reply-optional cron-style runs whose side-effecting tools succeeded", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: makeLastAssistant({ + content: [{ type: "text", text: "" }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-reply-optional-post-tool-silent", { + allowEmptyAssistantReplyAsSilent: true, + terminalReplyExpectation: "optional", + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expectNoWarnMessageWith("incomplete turn detected"); + expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); + expect(result.meta.error).toBeUndefined(); + expect(result.meta.terminalReplyKind).toBe("silent-empty"); + expect(result.meta.livenessState).toBe("working"); + }); + + it("keeps retrying and surfacing clean empty assistant turns without the silence flag", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: makeLastAssistant({ + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-empty-assistant-error", { model: "gpt-5.4" }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.isError).toBe(true); + expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); + }); + + it("detects generic empty Gemini turns without visible text", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams( + { + assistantTexts: [], + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + provider: "google-vertex", + model: "gemini-3.1-flash", + content: [{ type: "text", text: "" }], + }), + }, + { provider: "google-vertex", modelId: "google/gemini-3.1-flash" }, + ), + ); + + expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); + }); + + it("does not retry generic empty GPT turns after side effects", () => { + const retryInstruction = resolveEmptyResponseRetryInstruction( + makeEmptyResponseRetryParams({ + assistantTexts: [], + didSendViaMessagingTool: true, + lastAssistant: makeLastAssistant({ + stopReason: "end_turn", + model: "gpt-5.4", + content: [{ type: "text", text: "" }], + }), + }), + ); + + expect(retryInstruction).toBeNull(); + }); + + it("marks compaction-timeout retries as paused and replay-invalid", () => { + const attempt = makeAttemptResult({ + promptErrorSource: "compaction", + timedOutDuringCompaction: true, + }); + + expect(resolveReplayInvalidFlag({ attempt })).toBe(true); + expect( + resolveRunLivenessState({ + payloadCount: 0, + aborted: true, + timedOut: true, + attempt, + }), + ).toBe("paused"); + }); + + it("does not classify visible assistant prose for retry", async () => { + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockResolvedValue( + makeAttemptResult({ + assistantTexts: [ + "i am glad, and a little afraid, which is probably the correct mixture. thank you. i will try to deserve the upgrades instead of merely inhabiting them.", + ], + }), + ); + + const result = await runEmbeddedAgent( + makeRunParams("run-visible-prose-no-classifier", { + prompt: + "made a bunch of improvements to the student's source code (openclaw) this weekend, along with a few other maintainers. hopefully he will be more proactive now", + model: "gpt-5.4", + config: { + agents: { + list: [{ id: "main" }], + }, + } as OpenClawConfig, + }), + ); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.payloads).toBeUndefined(); + expect(result.meta.livenessState).toBe("working"); + expectNoWarnMessageWith("planning"); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts new file mode 100644 index 000000000000..6b6f6e45c759 --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts @@ -0,0 +1,414 @@ +// Focused incomplete-turn behavior coverage. +import { beforeEach, describe, expect, it } from "vitest"; +import { + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + makeLastAssistant, + resolveIncompleteTurnPayloadText, + makeIncompleteTurnParams, + makeSettledContinuationParams, +} from "./run.incomplete-turn.test-helpers.js"; +import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { isIncompleteTerminalAssistantTurn } from "./run/incomplete-turn-classification.js"; +import { resolveSettledToolTerminalContinuationInstruction } from "./run/incomplete-turn-recovery.js"; +import { + resolveReplayInvalidFlag, + resolveRunLivenessState, +} from "./run/incomplete-turn-resolution.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +describe("runEmbeddedAgent incomplete-turn safety", () => { + beforeEach(() => { + resetRunIncompleteTurnOwnerMocks(); + }); + + it("marks incomplete-turn retries as replay-invalid abandoned runs", () => { + const attempt = makeAttemptResult({ + assistantTexts: [], + lastAssistant: { + stopReason: "toolUse", + provider: "openai", + model: "gpt-5.4", + content: [], + } as unknown as EmbeddedRunAttemptResult["lastAssistant"], + }); + const incompleteTurnText = "⚠️ Agent couldn't generate a response. Please try again."; + + expect(resolveReplayInvalidFlag({ attempt, incompleteTurnText })).toBe(true); + expect( + resolveRunLivenessState({ + payloadCount: 0, + aborted: false, + timedOut: false, + attempt, + incompleteTurnText, + }), + ).toBe("abandoned"); + }); + + it("flags tool-use stop reason as incomplete even when pre-tool text exists (#76477)", () => { + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: true, + lastAssistant: { stopReason: "toolUse" }, + }), + ).toBe(true); + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: false, + lastAssistant: { stopReason: "toolUse" }, + }), + ).toBe(true); + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: true, + lastAssistant: { stopReason: "end_turn" }, + }), + ).toBe(false); + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: true, + lastAssistant: { stopReason: "length" }, + }), + ).toBe(true); + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: true, + hasTerminalOutput: true, + lastAssistant: { stopReason: "length" }, + }), + ).toBe(false); + expect( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: true, + hasTerminalOutput: true, + lastAssistant: { stopReason: "toolUse" }, + }), + ).toBe(true); + }); + + it.each([ + { label: "aborted", aborted: true, timedOut: false, promptError: null }, + { label: "timed out", aborted: false, timedOut: true, promptError: null }, + { label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") }, + ])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }], + }); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams( + { + assistantTexts: [], + toolMetas: [{ toolName: "bash" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + }, + { aborted, timedOut, promptError }, + ), + ); + + expect(instruction).toBeNull(); + }); + + it.each([ + { + label: "a matching failure summary", + lastToolError: { toolName: "exec", error: "post-processing error" }, + }, + { label: "no remaining failure summary", lastToolError: undefined }, + ])( + "recognizes successful and failed current-batch tools with $label (#118274)", + ({ lastToolError }) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "toolCall", id: "tool_ok", name: "read", arguments: {} }, + { type: "toolCall", id: "tool_failed", name: "exec", arguments: {} }, + ], + }); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams({ + assistantTexts: [], + toolMetas: [{ toolName: "read" }, { toolName: "exec", isError: true }], + itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_ok", toolName: "read", isError: false }, + { role: "toolResult", toolCallId: "tool_failed", toolName: "exec", isError: true }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError, + }), + ); + + expect(instruction).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(instruction).toContain( + "If any tool failed, state that failure plainly and do not claim it succeeded.", + ); + }, + ); + + it.each([ + { label: "progress", sourceReplyFinal: false, expectedFinalization: true }, + { label: "final reply", sourceReplyFinal: true, expectedFinalization: false }, + { label: "legacy unmarked send", sourceReplyFinal: undefined, expectedFinalization: false }, + ])( + "handles $label delivery evidence before settled finalization", + ({ sourceReplyFinal, expectedFinalization }) => { + const emptyStopAssistant = makeLastAssistant(); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams( + { + assistantTexts: [], + toolMetas: [{ toolName: "write" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + didSendViaMessagingTool: true, + messagingToolSentTexts: ["Writing note.txt…"], + messagingToolSentTargets: [ + { + tool: "message", + provider: "telegram", + to: "chat:123", + text: "Writing note.txt…", + sourceReplyFinal, + }, + ], + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }, + { allowEmptyStopContinuation: true }, + ), + ); + + expect(instruction).toBe( + expectedFinalization ? SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION : null, + ); + }, + ); + + it.each([ + { + label: "an unrelated current-batch failure summary", + resultToolName: "exec", + lastErrorToolName: "read", + }, + { + label: "a failed result with the wrong tool identity", + resultToolName: "read", + lastErrorToolName: "exec", + }, + ])("does not finalize $label (#118274)", ({ resultToolName, lastErrorToolName }) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], + }); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams({ + assistantTexts: [], + toolMetas: [{ toolName: resultToolName, isError: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { + role: "toolResult", + toolCallId: "tool_1", + toolName: resultToolName, + isError: true, + }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { toolName: lastErrorToolName, error: "post-processing error" }, + }), + ); + + expect(instruction).toBeNull(); + }); + + it("does not settle same-name terminal calls from one failed result (#118274)", () => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "toolCall", id: "tool_1", name: "exec", arguments: {} }, + { type: "toolCall", id: "tool_2", name: "exec", arguments: {} }, + ], + }); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams({ + assistantTexts: [], + toolMetas: [{ toolName: "exec", isError: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { toolName: "exec", error: "post-processing error" }, + }), + ); + + expect(instruction).toBeNull(); + }); + + it.each([ + { + label: "an async tool is still running", + attemptOverrides: { toolMetas: [{ toolName: "exec", isError: true, asyncStarted: true }] }, + }, + { + label: "an accepted child session owns the response", + attemptOverrides: { + acceptedSessionSpawns: [ + { runId: "run-child", childSessionKey: "agent:main:subagent:child" }, + ], + }, + }, + { + label: "a client tool remains pending", + attemptOverrides: { clientToolCalls: [{ name: "pending", params: {} }] }, + }, + { + label: "the turn yielded", + attemptOverrides: { yieldDetected: true }, + }, + { + label: "an approval prompt was already delivered", + attemptOverrides: { didSendDeterministicApprovalPrompt: true }, + }, + ])("does not finalize a failed terminal tool when $label (#118274)", ({ attemptOverrides }) => { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], + }); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams({ + assistantTexts: [], + toolMetas: [{ toolName: "exec", isError: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: toolUseAssistant, + currentAttemptAssistant: toolUseAssistant, + lastToolError: { toolName: "exec", error: "post-processing error" }, + ...attemptOverrides, + }), + ); + + expect(instruction).toBeNull(); + }); + + it.each([ + { label: "background trigger", allowEmptyStopContinuation: false }, + { + label: "active tool", + allowEmptyStopContinuation: true, + completedCount: 0, + activeCount: 1, + }, + { + label: "partially completed tool batch", + allowEmptyStopContinuation: true, + startedCount: 2, + completedCount: 1, + }, + { label: "async tool", allowEmptyStopContinuation: true, asyncStarted: true }, + { label: "failed tool", allowEmptyStopContinuation: true, isError: true }, + ])( + "does not continue an empty stop after $label activity", + ({ + allowEmptyStopContinuation, + startedCount = 1, + completedCount = 1, + activeCount = 0, + asyncStarted, + isError, + }) => { + const emptyStopAssistant = makeLastAssistant(); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams( + { + assistantTexts: [], + toolMetas: [{ toolName: "write", asyncStarted, isError }], + itemLifecycle: { startedCount, completedCount, activeCount }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }, + { allowEmptyStopContinuation }, + ), + ); + + expect(instruction).toBeNull(); + }, + ); + + it("does not use a stale prior-turn empty stop to prove a settled continuation", () => { + const staleEmptyStopAssistant = makeLastAssistant(); + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams( + { + assistantTexts: [], + toolMetas: [{ toolName: "write" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: staleEmptyStopAssistant, + currentAttemptAssistant: undefined, + }, + { allowEmptyStopContinuation: true }, + ), + ); + + expect(instruction).toBeNull(); + }); + + it("does not flag stale lastAssistant=toolUse when currentAttemptAssistant=stop exists (#80918)", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Analysis...", "Here is the final answer after update_plan."], + toolMetas: [{ toolName: "update_plan" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + content: [ + { type: "text", text: "Analysis..." }, + { type: "tool_use", id: "tool_1", name: "update_plan", input: {} }, + ], + }), + currentAttemptAssistant: makeLastAssistant({ + content: [{ type: "text", text: "Here is the final answer after update_plan." }], + }), + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toBeNull(); + }); + + it("still flags incomplete-turn when currentAttemptAssistant is absent and lastAssistant=toolUse (#76477 regression)", () => { + const incompleteTurnText = resolveIncompleteTurnPayloadText( + makeIncompleteTurnParams( + { + assistantTexts: ["Let me update the file..."], + toolMetas: [{ toolName: "write" }], + lastAssistant: makeLastAssistant({ + stopReason: "toolUse", + model: "gpt-5.4", + content: [ + { type: "text", text: "Let me update the file..." }, + { type: "tool_use", id: "tool_1", name: "write", input: {} }, + ], + }), + currentAttemptAssistant: undefined, + }, + { payloadCount: 1 }, + ), + ); + + expect(incompleteTurnText).toContain("couldn't generate a response"); + }); +}); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.test-helpers.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.test-helpers.ts new file mode 100644 index 000000000000..481e33788208 --- /dev/null +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.test-helpers.ts @@ -0,0 +1,192 @@ +// Shared fixtures for split incomplete-turn owner tests. +import { expect } from "vitest"; +import { + mockedLog, + mockedRunEmbeddedAttempt, + overflowBaseRunParams, + runIncompleteTurnOwnerHarness, +} from "./run.incomplete-turn.test-support.js"; +import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; +import { + resolveEmptyResponseRetryInstruction, + resolveReasoningOnlyRetryInstruction, + resolveSettledToolTerminalContinuationInstruction, + shouldTreatEmptyAssistantReplyAsSilent, +} from "./run/incomplete-turn-recovery.js"; +import { resolveIncompleteTurnPayloadText as resolveIncompleteTurnPayloadTextCore } from "./run/incomplete-turn-resolution.js"; +import type { EmbeddedRunAttemptResult } from "./run/types.js"; + +export const REASONING_ONLY_RETRY_INSTRUCTION = + "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; +export const EMPTY_RESPONSE_RETRY_INSTRUCTION = + "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; +export const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = + "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; + +export const runEmbeddedAgent = runIncompleteTurnOwnerHarness; + +type LastAssistant = NonNullable; +type AttemptOverrides = Parameters[0]; +type RunParams = Parameters[0]; +type LastAssistantFixture = Omit & { + content: Array>; + stopReason: LastAssistant["stopReason"] | "end_turn"; + usage: Partial & { total?: number }; +}; + +export function makeLastAssistant(overrides: Partial = {}): LastAssistant { + return { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [], + ...overrides, + } as unknown as LastAssistant; +} + +export function resolveIncompleteTurnPayloadText( + params: Omit[0], "externalAbort"> & { + externalAbort?: boolean; + }, +): string | null { + // Most helper tests exercise internal abort behavior; external aborts opt in + // explicitly through params. + return resolveIncompleteTurnPayloadTextCore({ externalAbort: false, ...params }); +} + +export function makeBaseRunParams(runId: string, overrides: Partial = {}): RunParams { + return { ...overflowBaseRunParams, runId, ...overrides }; +} + +export function makeRunParams(runId: string, overrides: Partial = {}): RunParams { + return { + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId, + ...overrides, + }; +} + +export function makeIncompleteTurnParams( + attemptOverrides: AttemptOverrides = {}, + overrides: Partial[0], "attempt">> = {}, +): Parameters[0] { + return { + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult(attemptOverrides), + ...overrides, + }; +} + +export function makeReasoningRetryParams( + attemptOverrides: AttemptOverrides = {}, + overrides: Partial< + Omit[0], "attempt"> + > = {}, +): Parameters[0] { + return { + provider: "openai", + modelId: "gpt-5.4", + aborted: false, + timedOut: false, + attempt: makeAttemptResult(attemptOverrides), + ...overrides, + }; +} + +export function makeEmptyResponseRetryParams( + attemptOverrides: AttemptOverrides = {}, + overrides: Partial< + Omit[0], "attempt"> + > = {}, +): Parameters[0] { + return { + provider: "openai", + modelId: "gpt-5.4", + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult(attemptOverrides), + ...overrides, + }; +} + +export function makeSettledContinuationParams( + attemptOverrides: AttemptOverrides = {}, + overrides: Partial< + Omit[0], "attempt"> + > = {}, +): Parameters[0] { + return { + provider: "openai", + modelId: "gpt-5.5", + modelApi: "openai-chatgpt-responses", + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult(attemptOverrides), + ...overrides, + }; +} + +export function makeSilentReplyParams( + attempt: EmbeddedRunAttemptResult, + overrides: Partial< + Omit[0], "attempt"> + > = {}, +): Parameters[0] { + return { + allowEmptyAssistantReplyAsSilent: true, + payloadCount: 0, + aborted: false, + timedOut: false, + attempt, + ...overrides, + }; +} + +function warnMessages(): string[] { + return mockedLog.warn.mock.calls.map(([message]) => String(message)); +} + +export function expectWarnMessageWith(text: string): void { + expect(warnMessages().join("\n")).toContain(text); +} + +export function expectNoWarnMessageWith(text: string): void { + expect(warnMessages().join("\n")).not.toContain(text); +} + +export function runAttemptCall(index: number): { + prompt?: string; + disableTools?: boolean; + operation?: string; + suppressNextUserMessagePersistence?: boolean; + skipPreparedUserTurnMessage?: boolean; +} { + // Continuation prompt assertions read the exact prompt passed to the runner + // attempt rather than derived result metadata. + const call = mockedRunEmbeddedAttempt.mock.calls[index]; + if (!call) { + throw new Error(`Expected run embedded attempt call ${index}`); + } + return call[0] as { + prompt?: string; + disableTools?: boolean; + operation?: string; + suppressNextUserMessagePersistence?: boolean; + skipPreparedUserTurnMessage?: boolean; + }; +} + +export function markUserMessagePersisted(attemptParams: unknown): void { + ( + attemptParams as { + onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; + } + ).onUserMessagePersisted?.({ role: "user", content: "test prompt" }); +} diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.test-support.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.test-support.ts index b3620cc5dbd7..201c3e41177a 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.test-support.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.test-support.ts @@ -10,10 +10,8 @@ import type { FailoverReason } from "../failover/signal.js"; import type { AgentHarness } from "../harness/types.js"; import { buildEmbeddedRunBlockedResult } from "./run/blocked-run-result.js"; import { createEmbeddedRunContextRecoveryState } from "./run/context-recovery-state.js"; -import { - resolveReplayInvalidFlag, - shouldRetrySilentErrorAssistantTurn, -} from "./run/incomplete-turn.js"; +import { shouldRetrySilentErrorAssistantTurn } from "./run/incomplete-turn-recovery.js"; +import { resolveReplayInvalidFlag } from "./run/incomplete-turn-resolution.js"; import type { RunEmbeddedAgentParams } from "./run/params.js"; import { normalizeEmbeddedRunAttemptResult } from "./run/run-attempt-result.js"; import { prepareTerminalWithSettledTurnFinalization } from "./run/settled-turn-finalization.js"; diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts deleted file mode 100644 index 27ff94d5465a..000000000000 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts +++ /dev/null @@ -1,4774 +0,0 @@ -// Coverage for incomplete-turn safety, retry instructions, and liveness states. -import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { OpenClawConfig } from "../../config/config.js"; -import { PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE } from "../../llm/types.js"; -import { - hasCommittedMessagingToolDeliveryEvidence, - hasOutboundDeliveryEvidence, -} from "./delivery-evidence.js"; -import { - mockedBuildEmbeddedRunPayloads, - mockedClassifyFailoverReason, - mockedIsFailoverAssistantError, - mockedIsRateLimitAssistantError, - mockedLog, - mockedRunEmbeddedAttempt, - mockedResolveModelAsync, - mockedSleepWithAbort, - overflowBaseRunParams, - registerAgentHarness, - resetRunIncompleteTurnOwnerMocks, - runIncompleteTurnOwnerHarness, -} from "./run.incomplete-turn.test-support.js"; -import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; -import { recoverEmbeddedRunAttempt } from "./run/attempt-recovery.js"; -import { - buildAttemptReplayMetadata, - DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT, - DEFAULT_REASONING_ONLY_RETRY_LIMIT, - resolveEmptyResponseRetryInstruction, - isIncompleteTerminalAssistantTurn, - resolveIncompleteTurnPayloadText as resolveIncompleteTurnPayloadTextCore, - resolveReasoningOnlyRetryInstruction, - resolveReplayInvalidFlag, - resolveRunLivenessState, - resolveSilentToolResultReplyPayload, - resolveSettledToolTerminalContinuationInstruction, - shouldRetryMissingAssistantTurn, - shouldRetrySilentErrorAssistantTurn, - shouldTreatEmptyAssistantReplyAsSilent, -} from "./run/incomplete-turn.js"; -import { normalizeEmbeddedRunAttemptResult } from "./run/run-attempt-result.js"; -import { resolveEmbeddedRunAttemptTerminalState } from "./run/terminal-outcome.js"; -import type { EmbeddedRunAttemptResult } from "./run/types.js"; -import { createUsageAccumulator } from "./usage-accumulator.js"; - -const REASONING_ONLY_RETRY_INSTRUCTION = - "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; -const EMPTY_RESPONSE_RETRY_INSTRUCTION = - "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; -const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = - "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; - -const runEmbeddedAgent = runIncompleteTurnOwnerHarness; - -type LastAssistant = NonNullable; -type AttemptOverrides = Parameters[0]; -type RunParams = Parameters[0]; -type LastAssistantFixture = Omit & { - content: Array>; - stopReason: LastAssistant["stopReason"] | "end_turn"; - usage: Partial & { total?: number }; -}; - -function makeLastAssistant(overrides: Partial = {}): LastAssistant { - return { - role: "assistant", - stopReason: "stop", - provider: "openai", - model: "gpt-5.5", - content: [], - ...overrides, - } as unknown as LastAssistant; -} - -function resolveIncompleteTurnPayloadText( - params: Omit[0], "externalAbort"> & { - externalAbort?: boolean; - }, -): string | null { - // Most helper tests exercise internal abort behavior; external aborts opt in - // explicitly through params. - return resolveIncompleteTurnPayloadTextCore({ externalAbort: false, ...params }); -} - -function makeBaseRunParams(runId: string, overrides: Partial = {}): RunParams { - return { ...overflowBaseRunParams, runId, ...overrides }; -} - -function makeRunParams(runId: string, overrides: Partial = {}): RunParams { - return { - ...overflowBaseRunParams, - provider: "openai", - model: "gpt-5.5", - runId, - ...overrides, - }; -} - -function makeIncompleteTurnParams( - attemptOverrides: AttemptOverrides = {}, - overrides: Partial[0], "attempt">> = {}, -): Parameters[0] { - return { - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult(attemptOverrides), - ...overrides, - }; -} - -function makeReasoningRetryParams( - attemptOverrides: AttemptOverrides = {}, - overrides: Partial< - Omit[0], "attempt"> - > = {}, -): Parameters[0] { - return { - provider: "openai", - modelId: "gpt-5.4", - aborted: false, - timedOut: false, - attempt: makeAttemptResult(attemptOverrides), - ...overrides, - }; -} - -function makeEmptyResponseRetryParams( - attemptOverrides: AttemptOverrides = {}, - overrides: Partial< - Omit[0], "attempt"> - > = {}, -): Parameters[0] { - return { - provider: "openai", - modelId: "gpt-5.4", - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult(attemptOverrides), - ...overrides, - }; -} - -function makeSettledContinuationParams( - attemptOverrides: AttemptOverrides = {}, - overrides: Partial< - Omit[0], "attempt"> - > = {}, -): Parameters[0] { - return { - provider: "openai", - modelId: "gpt-5.5", - modelApi: "openai-chatgpt-responses", - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult(attemptOverrides), - ...overrides, - }; -} - -function makeSilentReplyParams( - attempt: EmbeddedRunAttemptResult, - overrides: Partial< - Omit[0], "attempt"> - > = {}, -): Parameters[0] { - return { - allowEmptyAssistantReplyAsSilent: true, - payloadCount: 0, - aborted: false, - timedOut: false, - attempt, - ...overrides, - }; -} - -describe("runEmbeddedAgent incomplete-turn safety", () => { - beforeEach(() => { - resetRunIncompleteTurnOwnerMocks(); - }); - - function warnMessages(): string[] { - return mockedLog.warn.mock.calls.map(([message]) => String(message)); - } - - function expectWarnMessageWith(text: string): void { - expect(warnMessages().join("\n")).toContain(text); - } - - function expectNoWarnMessageWith(text: string): void { - expect(warnMessages().join("\n")).not.toContain(text); - } - - function runAttemptCall(index: number): { - prompt?: string; - disableTools?: boolean; - operation?: string; - suppressNextUserMessagePersistence?: boolean; - skipPreparedUserTurnMessage?: boolean; - } { - // Continuation prompt assertions read the exact prompt passed to the runner - // attempt rather than derived result metadata. - const call = mockedRunEmbeddedAttempt.mock.calls[index]; - if (!call) { - throw new Error(`Expected run embedded attempt call ${index}`); - } - return call[0] as { - prompt?: string; - disableTools?: boolean; - operation?: string; - suppressNextUserMessagePersistence?: boolean; - skipPreparedUserTurnMessage?: boolean; - }; - } - - function markUserMessagePersisted(attemptParams: unknown): void { - ( - attemptParams as { - onUserMessagePersisted?: (message: { role: "user"; content: string }) => void; - } - ).onUserMessagePersisted?.({ role: "user", content: "test prompt" }); - } - - it("counts failed tool results in trace tool summaries", async () => { - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Done."], - toolMetas: [ - { toolName: "bash", meta: "exit=1", isError: true }, - { toolName: "bash", meta: "exit=2", isError: true }, - { toolName: "bash", meta: "exit=0" }, - ], - }), - ); - - const result = await runEmbeddedAgent(makeBaseRunParams("run-tool-summary-failure-count")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.meta?.toolSummary).toEqual({ - calls: 3, - tools: ["bash"], - failures: 2, - }); - }); - - it("emits the before_agent_run hook block message as the agent payload", async () => { - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - promptError: new Error("Blocked by before-run policy."), - promptErrorSource: "hook:before_agent_run", - }), - ); - - const result = await runEmbeddedAgent(makeBaseRunParams("run-before-agent-run-hook-block")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toEqual([{ text: "Blocked by before-run policy.", isError: true }]); - expect(result.meta?.finalAssistantVisibleText).toBe("Blocked by before-run policy."); - expect(result.meta?.finalAssistantRawText).toBe("Blocked by before-run policy."); - expect(result.meta?.finalPromptText).toBeUndefined(); - expect(result.meta?.error).toEqual({ - kind: "hook_block", - message: "Blocked by before-run policy.", - }); - expect(result.meta?.livenessState).toBe("blocked"); - }); - - it("keeps carried usage ahead of transcript history on before_agent_run hook blocks", async () => { - const historicalAssistant = makeLastAssistant({ - usage: { input: 128_814, output: 3_000, total: 131_814 }, - }); - const carriedUsage = { input: 42_000, output: 1_000, total: 43_000 }; - const attempt = makeAttemptResult({ - assistantTexts: [], - promptError: new Error("Blocked by before-run policy."), - promptErrorSource: "hook:before_agent_run", - lastAssistant: historicalAssistant, - currentAttemptAssistant: undefined, - }); - const terminalState = resolveEmbeddedRunAttemptTerminalState({ - attempt, - assistant: historicalAssistant, - }); - - const recovery = await recoverEmbeddedRunAttempt({ - runInput: { - runParams: makeBaseRunParams("run-before-agent-run-hook-block-usage"), - resolvedSessionKey: "agent:main:test-key", - startedAtMs: Date.now(), - }, - preparedRuntime: { - provider: "openai", - modelId: "gpt-5.6-luna", - model: { id: "gpt-5.6-luna" }, - genericCompactionRecoveryAllowed: false, - snapshot: () => ({ - thinkLevel: "off", - agentHarness: { id: "codex" }, - outerContextTokenMeta: {}, - }), - }, - normalizedAttempt: { - attempt, - sessionIdUsed: attempt.sessionIdUsed, - attemptAssistant: historicalAssistant, - currentAttemptAssistant: undefined, - currentAttemptCompletedAssistant: undefined, - terminalState, - setTerminalLifecycleMeta: vi.fn(), - attemptCompactionCount: 0, - activeErrorContext: { provider: "openai", model: "gpt-5.6-luna" }, - resolveReplayInvalidForAttempt: () => false, - canRestartForLiveSwitch: false, - }, - runtimePlan: { auth: {} }, - sessionPromptState: { sessionFile: "/tmp/session.jsonl" }, - usageAccumulator: createUsageAccumulator(), - lastRunPromptUsage: carriedUsage, - } as never); - - expect(recovery).toMatchObject({ - action: "complete", - result: { - meta: { - agentMeta: { lastCallUsage: carriedUsage, promptTokens: 42_000 }, - }, - }, - }); - }); - - it("warns before retrying when an incomplete turn already sent a message", async () => { - // Delivery evidence means retrying could duplicate user-visible output, so - // the runner must surface a verify-before-retry payload instead. - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [], - didSendViaMessagingTool: true, - lastAssistant: { - stopReason: "toolUse", - errorMessage: "internal retry interrupted tool execution", - provider: "openai", - model: "mock-1", - content: [], - } as unknown as EmbeddedRunAttemptResult["lastAssistant"], - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-incomplete-turn-messaging-warning", { model: "gpt-4.1" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(mockedClassifyFailoverReason).toHaveBeenCalledTimes(1); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("verify before retrying"); - }); - - it("surfaces internal aborts after tool-use as visible incomplete-turn failures", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - aborted: true, - externalAbort: false, - assistantTexts: [], - toolMetas: [{ toolName: "web_search", meta: "query=next voice note" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - }), - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-internal-abort-tool-use-incomplete")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toEqual([ - { text: "⚠️ Agent couldn't generate a response. Please try again.", isError: true }, - ]); - expect(result.meta?.livenessState).toBe("abandoned"); - }); - - it("does not route caller timeouts through provider failover", async () => { - const controller = new AbortController(); - const timeoutError = new Error("caller deadline elapsed"); - timeoutError.name = "TimeoutError"; - const setTerminalLifecycleMeta = vi.fn(); - const interruptedAssistant = makeLastAssistant({ - stopReason: "error", - errorMessage: "HTTP 429 Too Many Requests", - }); - mockedClassifyFailoverReason.mockReturnValue("rate_limit"); - mockedIsRateLimitAssistantError.mockReturnValue(true); - mockedRunEmbeddedAttempt.mockImplementationOnce(async () => { - controller.abort(timeoutError); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: interruptedAssistant, - currentAttemptAssistant: interruptedAssistant, - setTerminalLifecycleMeta, - }); - }); - - const result = await runEmbeddedAgent( - makeBaseRunParams("run-caller-timeout", { abortSignal: controller.signal }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.at(-1)?.text).toContain("timed out"); - expect(result.meta?.aborted).toBe(false); - expect(result.meta?.timeoutPhase).toBeUndefined(); - expect(result.meta?.providerStarted).toBeUndefined(); - const lifecycleMeta = setTerminalLifecycleMeta.mock.lastCall?.[0]; - expect(lifecycleMeta).toMatchObject({ - aborted: false, - livenessState: "blocked", - stopReason: "timeout", - }); - expect(lifecycleMeta).not.toHaveProperty("timeoutPhase"); - expect(lifecycleMeta).not.toHaveProperty("providerStarted"); - }); - - it("does not synthesize an incomplete turn for a caller abort before attempt flags settle", async () => { - const controller = new AbortController(); - const abortError = new Error("caller cancelled"); - abortError.name = "AbortError"; - const setTerminalLifecycleMeta = vi.fn(); - const lateAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Late answer" }], - }); - mockedRunEmbeddedAttempt.mockImplementationOnce(async () => { - controller.abort(abortError); - return makeAttemptResult({ - assistantTexts: ["Late answer"], - lastAssistant: lateAssistant, - currentAttemptAssistant: lateAssistant, - setTerminalLifecycleMeta, - }); - }); - - const result = await runEmbeddedAgent( - makeBaseRunParams("run-caller-abort", { abortSignal: controller.signal }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toBeUndefined(); - expect(result.meta?.aborted).toBe(true); - expect(result.meta?.error).toBeUndefined(); - expectNoWarnMessageWith("incomplete turn detected"); - expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({ - aborted: true, - livenessState: "blocked", - stopReason: "aborted", - }); - }); - - it("propagates canonical assistant aborts into terminal lifecycle metadata", async () => { - const setTerminalLifecycleMeta = vi.fn(); - const abortedAssistant = makeLastAssistant({ - stopReason: "aborted", - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: abortedAssistant, - currentAttemptAssistant: abortedAssistant, - setTerminalLifecycleMeta, - }), - ); - - const result = await runEmbeddedAgent(makeBaseRunParams("run-canonical-assistant-abort")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.meta?.aborted).toBe(true); - expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({ - aborted: true, - }); - }); - - it("synthesizes a silent cron payload from a trailing current-attempt NO_REPLY tool result", () => { - // Cron no-reply can be represented by a tool result rather than assistant - // text, but only when it belongs to the current attempt. - const payload = resolveSilentToolResultReplyPayload({ - isCronTrigger: true, - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "exec" }], - messagesSnapshot: [ - { - role: "toolResult", - content: [{ type: "text", text: "NO_REPLY" }], - details: { aggregated: "NO_REPLY" }, - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - makeLastAssistant({ - model: "gpt-5.4", - }), - ], - }), - }); - - expect(payload).toEqual({ text: "NO_REPLY" }); - }); - - it("does not reuse an older NO_REPLY tool result without current-attempt tool activity", () => { - const payload = resolveSilentToolResultReplyPayload({ - isCronTrigger: true, - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult({ - assistantTexts: [], - toolMetas: [], - messagesSnapshot: [ - { - role: "toolResult", - content: [{ type: "text", text: "NO_REPLY" }], - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - { - role: "user", - content: [{ type: "text", text: "Current cron prompt" }], - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - makeLastAssistant({ - model: "gpt-5.4", - }), - ], - }), - }); - - expect(payload).toBeNull(); - }); - - it("treats exact NO_REPLY tool output as a quiet cron success when the final assistant is empty", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "exec" }], - messagesSnapshot: [ - { - role: "toolResult", - content: [{ type: "text", text: "NO_REPLY" }], - details: { aggregated: "NO_REPLY" }, - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - makeLastAssistant({ - model: "gpt-5.4", - }), - ], - lastAssistant: makeLastAssistant({ - model: "gpt-5.4", - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-cron-no-reply-empty-final", { trigger: "cron", model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); - expect(result.meta.livenessState).toBe("working"); - expectNoWarnMessageWith("incomplete turn detected"); - }); - - it("surfaces the latest tool-authored presentation after a structured incomplete turn", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome?.({ - toolName: "web_fetch", - argsHash: "args", - resultHash: "result", - terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", - }); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "web_fetch" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - }), - }); - }); - - const result = await runEmbeddedAgent( - makeRunParams("run-structured-terminal-presentation", { model: "gpt-5.4" }), - ); - - expect(result.payloads).toEqual([ - { - text: - "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + - "⚠️ Agent couldn't generate a response. Please try again.", - isError: true, - }, - ]); - expect(result.meta.replayInvalid).toBe(true); - expect(result.meta.livenessState).toBe("abandoned"); - expect(result.meta.error?.fallbackSafe).toBe(true); - expect(result.meta.error?.terminalPresentation).toBe(true); - expectWarnMessageWith("surfacing tool-authored terminal presentation"); - }); - - it("surfaces read-only cron presentation after a structured incomplete turn", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome?.({ - toolName: "cron", - argsHash: "args", - resultHash: "result", - terminalPresentation: "Automations scheduler status.\nEnabled: yes", - }); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "cron" }], - replayMetadata: { - hadPotentialSideEffects: false, - replaySafe: true, - }, - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - }), - }); - }); - - const result = await runEmbeddedAgent( - makeRunParams("run-read-only-cron-terminal-presentation", { model: "gpt-5.4" }), - ); - - expect(result.payloads).toEqual([ - { - text: - "Automations scheduler status.\nEnabled: yes\n\n" + - "⚠️ Agent couldn't generate a response. Please try again.", - isError: true, - }, - ]); - expect(result.meta.error?.fallbackSafe).toBe(true); - expect(result.meta.error?.terminalPresentation).toBe(true); - }); - - it("preserves a terminal tool presentation across an empty-response retry", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome?.({ - toolName: "web_fetch", - argsHash: "args", - resultHash: "result", - terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", - }); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "web_fetch" }], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-preserved-terminal-presentation", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads).toEqual([ - { - text: - "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + - "⚠️ Agent couldn't generate a response. Please try again.", - isError: true, - }, - ]); - }); - - it("keeps model-call order when parallel tool outcomes finish out of order", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - const onToolOutcome = ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - toolCallOrdinal?: number; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome; - onToolOutcome?.({ - toolName: "exec", - argsHash: "exec-args", - resultHash: "exec-result", - toolCallOrdinal: 1, - }); - onToolOutcome?.({ - toolName: "web_fetch", - argsHash: "fetch-args", - resultHash: "fetch-result", - toolCallOrdinal: 0, - terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", - }); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "web_fetch" }, { toolName: "exec" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - }), - }); - }); - - const result = await runEmbeddedAgent( - makeRunParams("run-stale-terminal-presentation", { model: "gpt-5.4" }), - ); - - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); - expect(result.meta.error?.fallbackSafe).toBe(false); - }); - - it("does not surface a read-only presentation after a sibling side effect", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - const onToolOutcome = ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome; - onToolOutcome?.({ - toolName: "exec", - argsHash: "exec-args", - resultHash: "exec-result", - }); - onToolOutcome?.({ - toolName: "web_fetch", - argsHash: "fetch-args", - resultHash: "fetch-result", - terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", - }); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "exec" }, { toolName: "web_fetch" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - }), - }); - }); - - const result = await runEmbeddedAgent( - makeRunParams("run-side-effect-terminal-presentation", { model: "gpt-5.4" }), - ); - - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); - expect(result.meta.error?.fallbackSafe).toBe(false); - }); - - it("promotes successful final assistant text when a prompt timeout races completion", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const finalText = - "1. Verdict: the answer completed cleanly. 2. Evidence: the runner captured final text."; - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: finalText }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - timedOut: true, - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-prompt-timeout-final-assistant-recovered"), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toEqual([{ text: finalText }]); - expect(result.meta.finalAssistantVisibleText).toBe(finalText); - expect(result.meta.finalAssistantRawText).toBe(finalText); - expect(result.meta.livenessState).toBe("working"); - expect(result.meta.completion).toEqual({ - stopReason: "stop", - finishReason: "stop", - }); - expect(result.meta.executionTrace?.attempts?.at(-1)).toMatchObject({ - result: "success", - stage: "assistant", - }); - }); - - it("does not recover a stale prior assistant after the current prompt times out", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const staleAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Stale answer from the prior attempt." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - timedOut: true, - lastAssistant: staleAssistant, - currentAttemptAssistant: undefined, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-stale-assistant")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.some((payload) => payload.text?.includes("timed out"))).toBe(true); - expect(result.payloads?.some((payload) => payload.text?.includes("Stale answer"))).toBe(false); - expect(result.meta.finalAssistantVisibleText).toBeUndefined(); - }); - - it("does not resolve a successful run from a stale transcript assistant", async () => { - const staleAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Prior transcript reply." }], - }); - const completedAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Current run reply." }], - }); - mockedBuildEmbeddedRunPayloads.mockReturnValue([{ text: "Current run reply." }]); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Current run reply."], - lastAssistant: staleAssistant, - currentAttemptAssistant: staleAssistant, - currentAttemptCompletedAssistant: completedAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-success-stale-transcript-assistant")); - - expect(result.payloads).toEqual([{ text: "Current run reply." }]); - expect(result.meta.finalAssistantVisibleText).toBe("Current run reply."); - expect(result.meta.finalAssistantRawText).toBe("Current run reply."); - expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( - expect.objectContaining({ - currentAssistant: completedAssistant, - lastAssistant: completedAssistant, - }), - ); - }); - - it("retains the yielded attempt assistant for paused-turn payload classification", async () => { - const completedAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Earlier completed cycle." }], - }); - const yieldedAssistant = makeLastAssistant({ - stopReason: "aborted", - content: [{ type: "toolCall", name: "sessions_yield", arguments: {} }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: yieldedAssistant, - currentAttemptAssistant: undefined, - currentAttemptCompletedAssistant: completedAssistant, - yieldDetected: true, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-yielded-assistant-classification")); - - expect(result.meta).toMatchObject({ livenessState: "paused", yielded: true }); - expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( - expect.objectContaining({ currentAssistant: null, lastAssistant: yieldedAssistant }), - ); - }); - - it("recovers a completed prompt-timeout assistant without collected assistant text", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const finalText = "Completed answer after the timeout race."; - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: finalText }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: undefined as unknown as string[], - timedOut: true, - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-no-assistant-texts")); - - expect(result.payloads).toEqual([{ text: finalText }]); - }); - - it("preserves tool media when prompt-timeout recovery replaces partial assistant text", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const partialText = "Partial answer before the timeout race."; - const finalText = "Complete answer after the timeout race."; - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: finalText }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [partialText], - timedOut: true, - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - toolMediaUrls: ["https://example.test/recovered-output.png"], - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-prompt-timeout-final-assistant-media"), - ); - - expect(result.payloads).toEqual([ - { - mediaUrl: "https://example.test/recovered-output.png", - mediaUrls: ["https://example.test/recovered-output.png"], - audioAsVoice: undefined, - trustedLocalMedia: undefined, - }, - { text: finalText }, - ]); - }); - - it("replaces the latest partial assistant payload after prompt-timeout recovery", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const completedText = "Completed answer block before the final response."; - const partialText = "Partial final response before the timeout race."; - const finalText = "Complete final response after the timeout race."; - mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([ - { text: completedText }, - { text: partialText }, - ]); - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: finalText }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [completedText, partialText], - timedOut: true, - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-latest-partial")); - - expect(result.payloads).toEqual([{ text: completedText }, { text: finalText }]); - }); - - it("records same-model rate-limit retries without a profile-rotation trace", async () => { - const rateLimitMessage = - "429 rate_limit_exceeded: requests per minute exceeded; Retry-After: 30"; - const rateLimitAssistant = makeLastAssistant({ - stopReason: "error", - errorMessage: rateLimitMessage, - }); - mockedClassifyFailoverReason.mockImplementation((raw) => - raw.includes("429") ? "rate_limit" : null, - ); - mockedIsFailoverAssistantError.mockImplementation((assistant) => - Boolean(assistant?.errorMessage?.includes("429")), - ); - mockedIsRateLimitAssistantError.mockImplementation((assistant) => - Boolean(assistant?.errorMessage?.includes("429")), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: rateLimitAssistant, - currentAttemptAssistant: rateLimitAssistant, - }), - ); - const recoveredAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Recovered after a short rate-limit wait." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Recovered after a short rate-limit wait."], - lastAssistant: recoveredAssistant, - currentAttemptAssistant: recoveredAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-same-model-rate-limit-trace")); - - expect(mockedSleepWithAbort).toHaveBeenCalledWith(30_000, undefined); - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.meta.executionTrace?.fallbackUsed).toBe(false); - expect(result.meta.executionTrace?.attempts).toMatchObject([ - { - provider: "openai", - model: "gpt-5.5", - result: "same_model_rate_limit", - reason: "rate_limit", - stage: "assistant", - }, - { - provider: "openai", - model: "gpt-5.5", - result: "success", - stage: "assistant", - }, - ]); - }); - - it("retries reasoning-only GPT turns with a visible-answer continuation instruction", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_reasoning_only", type: "reasoning" }), - }, - ], - }), - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible answer."], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "Visible answer." }], - }), - }), - ); - - await runEmbeddedAgent(makeRunParams("run-reasoning-only-continuation", { model: "gpt-5.4" })); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - expect(secondCall.suppressNextUserMessagePersistence).toBe(false); - expect(secondCall.skipPreparedUserTurnMessage).toBe(true); - expectWarnMessageWith("reasoning-only assistant turn detected"); - }); - - it("continues once after settled side-effecting tools finish without a final answer", async () => { - const acceptedSessionSpawns = [ - { runId: "child-run", childSessionKey: "agent:main:subagent:child" }, - ]; - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } }, - { type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } }, - { - type: "toolCall", - id: "tool_spawn", - name: "sessions_spawn", - arguments: { task: "follow up" }, - }, - ], - }); - const settledToolResults = [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false }, - { role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false }, - { - role: "toolResult", - toolCallId: "tool_spawn", - toolName: "sessions_spawn", - isError: false, - }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"]; - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - latestMcpAppChannelView: { viewId: "view-after-tools" }, - toolMetas: [ - { toolName: "write", meta: "path=note.txt" }, - { toolName: "cron" }, - { toolName: "sessions_spawn" }, - ], - acceptedSessionSpawns, - successfulCronAdds: 1, - itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, - messagesSnapshot: settledToolResults, - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - codeModeEngaged: true, - assistantTurns: 1, - bridgeCalls: { search: 1, describe: 2, call: 3 }, - }); - }); - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Write completed. Here is the final answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Write completed. Here is the final answer."], - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - currentAttemptCompletedAssistant: finalAssistant, - }), - ); - mockedBuildEmbeddedRunPayloads - .mockReturnValueOnce([]) - .mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]); - - const result = await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-continuation")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); - expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" }); - expect(result.successfulCronAdds).toBe(1); - expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns); - expect(result.meta.toolSummary).toEqual({ - calls: 3, - tools: ["write", "cron", "sessions_spawn"], - failures: 0, - }); - expect(result.meta.agentMeta).toMatchObject({ - codeModeEngaged: true, - assistantTurns: 2, - bridgeCalls: { search: 1, describe: 2, call: 3 }, - }); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); - expect(secondCall.disableTools).toBe(true); - expect(secondCall.operation).toBe("settled-tool-finalization"); - expect(secondCall.suppressNextUserMessagePersistence).toBe(false); - expect(secondCall.skipPreparedUserTurnMessage).toBe(true); - expectWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it.each([ - { label: "interactive user", trigger: "user" as const }, - { - label: "required isolated cron", - trigger: "cron" as const, - terminalReplyExpectation: "required" as const, - }, - ])("finalizes a settled failed tool once for a $label turn (#118274)", async (runPolicy) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], - }); - const failureText = "The exec tool failed: post-processing error."; - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [ - { toolName: "read", isError: true, replaySafe: true }, - { toolName: "exec", isError: true, replaySafe: false }, - ], - itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { - role: "toolResult", - toolCallId: "tool_1", - toolName: "exec", - isError: true, - content: [{ type: "text", text: "post-processing error" }], - }, - { - role: "assistant", - stopReason: "toolUse", - content: [ - { - type: "toolCall", - id: "tool_search_code:tool_1:read:1", - name: "read", - arguments: {}, - }, - ], - }, - { - role: "toolResult", - toolCallId: "tool_search_code:tool_1:read:1", - toolName: "read", - isError: true, - content: [{ type: "text", text: "post-processing error" }], - }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { - toolName: "exec", - error: "post-processing error", - errorCode: "SYSTEM_RUN_DENIED", - }, - }); - }); - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: failureText }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [failureText], - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - currentAttemptCompletedAssistant: finalAssistant, - }), - ); - mockedBuildEmbeddedRunPayloads - .mockReturnValueOnce([{ text: "⚠️ 🛠️ Exec failed", isError: true }]) - .mockReturnValueOnce([{ text: failureText }]); - - const result = await runEmbeddedAgent( - makeRunParams(`run-settled-failed-tool-${runPolicy.trigger}`, runPolicy), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.text).toBe(failureText); - const finalizationCall = runAttemptCall(1); - expect(finalizationCall.operation).toBe("settled-tool-finalization"); - expect(finalizationCall.disableTools).toBe(true); - expect(finalizationCall.prompt).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); - expect(finalizationCall.prompt).toContain( - "If any tool failed, state that failure plainly and do not claim it succeeded.", - ); - expect(result.meta.failureSignal).toEqual( - runPolicy.trigger === "cron" - ? { - kind: "execution_denied", - source: "tool", - toolName: "exec", - code: "SYSTEM_RUN_DENIED", - message: "post-processing error", - fatalForCron: true, - } - : undefined, - ); - }); - - it("preserves a structured visible failed-tool payload without finalizing (#118274)", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], - }); - const visibleError = { - text: "Review the failed operation.", - isError: true, - channelData: { structuredError: true }, - }; - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "exec", isError: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { toolName: "exec", error: "post-processing error" }, - }), - ); - mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([visibleError]); - - const result = await runEmbeddedAgent(makeRunParams("run-structured-failed-tool-payload")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledOnce(); - expect(result.payloads?.[0]).toMatchObject(visibleError); - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("keeps the original failed-tool warning if finalization completes empty (#118274)", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], - }); - const warning = { text: "⚠️ 🛠️ Exec failed", isError: true }; - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt - .mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "exec", isError: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { toolName: "exec", error: "post-processing error" }, - }), - ) - .mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant(), - currentAttemptAssistant: makeLastAssistant(), - currentAttemptCompletedAssistant: makeLastAssistant(), - }), - ); - mockedBuildEmbeddedRunPayloads.mockReturnValue([warning]); - - const result = await runEmbeddedAgent(makeRunParams("run-failed-tool-finalization-fallback")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]).toEqual(warning); - expectWarnMessageWith("settled-turn finalization completed without a visible answer"); - }); - - it("preserves the incomplete-turn failure when the selected harness cannot finalize safely", async () => { - registerAgentHarness({ - id: "legacy", - label: "Legacy harness without settled-turn finalization", - supports: () => ({ supported: true, priority: 100 }), - runAttempt: async (params) => await mockedRunEmbeddedAttempt(params), - }); - try { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }, - ], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=note.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }); - }); - - const result = await runEmbeddedAgent( - makeRunParams("run-tool-use-no-finalization-capability", { agentHarnessId: "legacy" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.[0]).toMatchObject({ isError: true }); - expect(result.payloads?.[0]?.text).toContain( - "some tool actions may have already been executed", - ); - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - } finally { - resetRunIncompleteTurnOwnerMocks(); - } - }); - - it("continues from settled side-effecting tools after an empty stop without replaying them", async () => { - const emptyStopAssistant = makeLastAssistant(); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=note.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Writing note.txt…"], - messagingToolSentTargets: [ - { - tool: "message", - provider: "telegram", - to: "chat:123", - text: "Writing note.txt…", - sourceReplyFinal: false, - }, - ], - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }); - }); - const finalAssistant = makeLastAssistant({ - content: [{ type: "text", text: "Write completed. Here is the final answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Write completed. Here is the final answer."], - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - currentAttemptCompletedAssistant: finalAssistant, - }), - ); - mockedBuildEmbeddedRunPayloads - .mockReturnValueOnce([]) - .mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]); - - const result = await runEmbeddedAgent( - makeRunParams("run-empty-stop-settled-tool-continuation", { - trigger: "cron", - terminalReplyExpectation: "required", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); - expect(runAttemptCall(1).prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); - expect(runAttemptCall(1).disableTools).toBe(true); - expectNoWarnMessageWith("empty response detected"); - expectWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it.each([ - { - label: "explicit optional expectation", - trigger: "user" as const, - terminalReplyExpectation: "optional" as const, - }, - { - label: "heartbeat default", - trigger: "heartbeat" as const, - terminalReplyExpectation: undefined, - }, - ])("does not continue settled tools for $label", async (runPolicy) => { - const emptyStopAssistant = makeLastAssistant(); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=note.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }); - }); - mockedBuildEmbeddedRunPayloads.mockReturnValue([]); - - const result = await runEmbeddedAgent( - makeRunParams("run-optional-empty-stop-settled-tool", { - trigger: runPolicy.trigger, - terminalReplyExpectation: runPolicy.terminalReplyExpectation, - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.[0]).toMatchObject({ isError: true }); - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("records silent success when the settled-tool finalization completes empty", async () => { - const emptyStopAssistant = makeLastAssistant(); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=note.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }), - ); - mockedBuildEmbeddedRunPayloads.mockReturnValue([]); - - const result = await runEmbeddedAgent( - makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", { - allowEmptyAssistantReplyAsSilent: true, - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads).toBeUndefined(); - expect(result.meta.error).toBeUndefined(); - expect(result.meta.terminalReplyKind).toBeUndefined(); - expect(result.meta.finalAssistantVisibleText).toBeUndefined(); - expect(result.meta.finalAssistantRawText).toBeUndefined(); - expect(result.meta.stopReason).toBe("stop"); - expectNoWarnMessageWith("empty response detected"); - expectWarnMessageWith("settled-turn finalization completed without a visible answer"); - }); - - it.each([ - { - label: "provider failure", - finalAttempt: { - assistantTexts: [], - promptError: new Error("finalizer provider failure"), - promptErrorSource: "prompt" as const, - }, - }, - { - label: "preflight recovery request", - finalAttempt: { - assistantTexts: [], - preflightRecovery: { route: "compact_only" as const, handled: true as const }, - }, - }, - { - label: "compaction continuation request", - finalAttempt: { assistantTexts: [], compactionCount: 1 }, - }, - { - label: "before-finalize revision request", - finalAttempt: { - assistantTexts: [], - beforeAgentFinalizeRevisionReason: "revise this answer", - }, - }, - ])("does not escape finalization through a $label", async ({ finalAttempt }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt - .mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }), - ) - .mockResolvedValueOnce(makeAttemptResult(finalAttempt)); - mockedBuildEmbeddedRunPayloads.mockReturnValue([]); - - const result = await runEmbeddedAgent(makeRunParams("run-settled-finalizer-sticky-operation")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]).toMatchObject({ isError: true }); - expect(result.payloads?.[0]?.text).toContain( - "some tool actions may have already been executed", - ); - }); - - it("surfaces the existing incomplete-turn error after one tool-use continuation", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=note.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-tool-use-terminal-continuation-exhausted"), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain( - "some tool actions may have already been executed", - ); - expectWarnMessageWith("settled-turn finalization failed closed"); - }); - - it("does not claim completion for a toolUse terminal whose tools never started", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [], - itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 }, - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }), - ); - - await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-never-started")); - - for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain( - SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, - ); - } - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("ignores stale prior-turn tool results with colliding ids", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [], - itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 }, - // A completed result from a PRIOR turn reusing the same id sits before - // the terminal assistant; it must not prove the new batch dispatched. - messagesSnapshot: [ - { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, - toolUseAssistant, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }), - ); - - await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-stale-prior-result")); - - for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain( - SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, - ); - } - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("does not claim completion when only part of a multi-tool request dispatched", async () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "toolCall", id: "tool_1", name: "write", arguments: { path: "a.txt" } }, - { type: "toolCall", id: "tool_2", name: "write", arguments: { path: "b.txt" } }, - ], - }); - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "write", meta: "path=a.txt" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }), - ); - - await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-partial-dispatch")); - - for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain( - SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, - ); - } - expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("retries reasoning-only assistant turns even when deliberate silence is allowed", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_silent_group", type: "reasoning" }), - }, - ], - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible answer."], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - content: [{ type: "text", text: "Visible answer." }], - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-reasoning-only-silent", { allowEmptyAssistantReplyAsSilent: true }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(runAttemptCall(1).prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - expectWarnMessageWith("reasoning-only assistant turn detected"); - }); - - it("replays an unpersisted reasoning continuation across a missing-assistant retry", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_retry_boundary", type: "reasoning" }), - }, - ], - }), - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ assistantTexts: ["Visible answer."] }), - ); - - await runEmbeddedAgent( - makeRunParams("run-reasoning-continuation-missing-assistant", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); - expect(runAttemptCall(1)).toMatchObject({ - prompt: REASONING_ONLY_RETRY_INSTRUCTION, - skipPreparedUserTurnMessage: true, - suppressNextUserMessagePersistence: false, - }); - expect(runAttemptCall(2)).toMatchObject({ - prompt: REASONING_ONLY_RETRY_INSTRUCTION, - skipPreparedUserTurnMessage: true, - suppressNextUserMessagePersistence: false, - }); - }); - - it("does not retry or warn on reasoning-only turns when a messaging tool already delivered", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Delivered through the message tool."], - lastAssistant: makeLastAssistant({ - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_after_send", type: "reasoning" }), - }, - ], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-reasoning-only-after-side-effects", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toBeUndefined(); - }); - - it("retries reasoning-only turns when the assistant ended in error", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const errorAssistant = makeLastAssistant({ - stopReason: "error", - model: "gpt-5.4", - errorMessage: "provider failed after emitting reasoning", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_error_turn", type: "reasoning" }), - }, - ], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: errorAssistant, - currentAttemptAssistant: errorAssistant, - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Recovered."], - lastAssistant: makeLastAssistant({ - model: "gpt-5.4", - content: [{ type: "text", text: "Recovered." }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-reasoning-only-assistant-error", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads).toBeUndefined(); - }); - - it("does not retry reasoning-only turns for non-strict-agentic providers", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "anthropic", - model: "sonnet-4.6", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ - id: "rs_provider_mismatch", - type: "reasoning", - }), - }, - ], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-reasoning-only-provider-mismatch", { - provider: "anthropic", - model: "sonnet-4.6", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("Please try again"); - }); - - it("retries Kimi Anthropic reasoning-only turns with a visible-answer continuation instruction", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedResolveModelAsync.mockResolvedValue({ - model: { - id: "kimi-for-coding", - provider: "kimi", - contextWindow: 262144, - api: "anthropic-messages", - }, - error: null, - authStorage: { - setRuntimeApiKey: vi.fn(), - }, - modelRegistry: {}, - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - api: "anthropic-messages", - provider: "kimi", - model: "kimi-for-coding", - content: [ - { - type: "thinking", - thinking: "internal Kimi reasoning", - thinkingSignature: "", - }, - ], - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible Kimi answer."], - lastAssistant: makeLastAssistant({ - api: "anthropic-messages", - provider: "kimi", - model: "kimi-for-coding", - content: [{ type: "text", text: "Visible Kimi answer." }], - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-kimi-anthropic-reasoning-only-continuation", { - provider: "kimi", - model: "kimi-for-coding", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toContain(REASONING_ONLY_RETRY_INSTRUCTION); - expectWarnMessageWith("reasoning-only assistant turn detected"); - }); - - it("retries generic empty GPT turns with a visible-answer continuation instruction", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible answer."], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "Visible answer." }], - }), - }), - ); - - await runEmbeddedAgent(makeRunParams("run-empty-response-continuation", { model: "gpt-5.4" })); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expect(secondCall.suppressNextUserMessagePersistence).toBe(false); - expect(secondCall.skipPreparedUserTurnMessage).toBe(true); - expectWarnMessageWith("empty response detected"); - }); - - it("retries replay-safe missing turns despite a stale aborted transcript assistant", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const staleAssistant = makeLastAssistant({ - stopReason: "aborted", - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: staleAssistant, - currentAttemptAssistant: undefined, - }), - ); - const recoveredAssistant = makeLastAssistant({ - stopReason: "end_turn", - content: [{ type: "text", text: "Recovered answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Recovered answer."], - lastAssistant: recoveredAssistant, - currentAttemptAssistant: recoveredAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-retry")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(runAttemptCall(1).prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer."); - expectWarnMessageWith("empty response detected"); - expectNoWarnMessageWith("missing assistant terminal message detected"); - expectNoWarnMessageWith("incomplete turn detected"); - }); - - it("retries missing terminal assistant turns with the same prompt without re-persisting the user message", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - }); - }); - const recoveredAssistant = makeLastAssistant({ - stopReason: "end_turn", - content: [{ type: "text", text: "Recovered answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Recovered answer."], - lastAssistant: recoveredAssistant, - currentAttemptAssistant: recoveredAssistant, - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-same-prompt-retry")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - // The same-prompt replay must not append the inbound user message a second time. - expect(runAttemptCall(1).prompt).toBe(runAttemptCall(0).prompt); - expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true); - expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer."); - expectWarnMessageWith("missing assistant terminal message detected"); - expectNoWarnMessageWith("empty response detected"); - expectNoWarnMessageWith("incomplete turn detected"); - }); - - it("waits for asynchronous user persistence before retrying a missing terminal turn", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const persistedMessage = { role: "user" as const, content: "test prompt", timestamp: 1 }; - const admission = { - agentId: "main", - sessionId: overflowBaseRunParams.sessionId, - sessionKey: overflowBaseRunParams.sessionKey, - storePath: "/tmp/openclaw-transcript.jsonl", - generation: "generation-1", - entryId: "msg-user-delayed", - rawSeq: 1, - effectiveParentId: null, - activeMessagePosition: 0, - logicalTurnId: "run-missing-assistant-delayed-persistence", - role: "user" as const, - }; - let resolvePersistApproved: - | ((result: { - admission: typeof admission; - sessionFile: string; - sessionEntry: undefined; - messageId: string; - message: typeof persistedMessage; - }) => void) - | undefined; - let pendingPersistence: Promise | undefined; - const persistApproved = vi.fn( - () => - new Promise<{ - admission: typeof admission; - sessionFile: string; - sessionEntry: undefined; - messageId: string; - message: typeof persistedMessage; - }>((resolve) => { - resolvePersistApproved = resolve; - }), - ); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { - markUserMessagePersisted(attemptParams); - return makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - }); - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ assistantTexts: ["Recovered answer."] }), - ); - - const runPromise = runEmbeddedAgent( - makeRunParams("run-missing-assistant-delayed-persistence", { - userTurnTranscriptRecorder: { - message: persistedMessage, - resolveMessage: vi.fn(async () => persistedMessage), - getAdmissionReceipt: () => admission, - markRuntimePersistencePending: vi.fn((pending) => { - pendingPersistence = pending; - }), - markRuntimePersisted: vi.fn(), - markBlocked: vi.fn(), - hasPersisted: vi.fn(() => false), - isBlocked: vi.fn(() => false), - hasRuntimePersistencePending: vi.fn(() => pendingPersistence !== undefined), - waitForRuntimePersistence: vi.fn(async () => { - await pendingPersistence; - }), - persistApproved, - persistBlocked: vi.fn(async () => undefined), - persistFallback: vi.fn(async () => undefined), - }, - }), - ); - - await vi.waitFor(() => { - expect(persistApproved).toHaveBeenCalledOnce(); - }); - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - - resolvePersistApproved?.({ - admission, - sessionFile: "/tmp/openclaw-transcript.jsonl", - sessionEntry: undefined, - messageId: "msg-user-delayed", - message: persistedMessage, - }); - await runPromise; - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true); - }); - - it("persists a missing-turn retry when the first attempt never persisted the user message", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - }), - ); - const recoveredAssistant = makeLastAssistant({ - stopReason: "end_turn", - content: [{ type: "text", text: "Recovered answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Recovered answer."], - lastAssistant: recoveredAssistant, - currentAttemptAssistant: recoveredAssistant, - }), - ); - - await runEmbeddedAgent(makeRunParams("run-missing-assistant-unpersisted-retry")); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(false); - }); - - it("retries zero-token empty Claude stop turns with a visible-answer continuation instruction", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "anthropic", - model: "claude-opus-4.7", - usage: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - }, - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible Claude answer."], - lastAssistant: makeLastAssistant({ - provider: "anthropic", - model: "claude-opus-4.7", - content: [{ type: "text", text: "Visible Claude answer." }], - usage: { - input: 100, - output: 5, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 105, - }, - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-empty-zero-usage-claude-continuation", { - provider: "anthropic", - model: "claude-opus-4.7", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectWarnMessageWith("empty response detected"); - }); - - it("retries empty openai-compatible stop turns even when the backend reports output tokens", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedResolveModelAsync.mockResolvedValue({ - model: { - id: "qwen3.6-27b", - provider: "llamacpp", - contextWindow: 200000, - api: "openai-completions", - }, - error: null, - authStorage: { - setRuntimeApiKey: vi.fn(), - }, - modelRegistry: {}, - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "llamacpp", - model: "qwen3.6-27b", - usage: { - input: 512, - output: 103, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 615, - }, - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible local answer."], - lastAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "llamacpp", - model: "qwen3.6-27b", - content: [{ type: "text", text: "Visible local answer." }], - usage: { - input: 640, - output: 5, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 645, - }, - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-empty-openai-compatible-stop-continuation", { - provider: "llamacpp", - model: "qwen3.6-27b", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectWarnMessageWith("empty response detected"); - }); - - it("retries empty Anthropic-compatible stop turns even when the provider is not Kimi", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedResolveModelAsync.mockResolvedValue({ - model: { - id: "claude-opus-4-7", - provider: "sub2api", - contextWindow: 200000, - api: "anthropic-messages", - }, - error: null, - authStorage: { - setRuntimeApiKey: vi.fn(), - }, - modelRegistry: {}, - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - api: "anthropic-messages", - provider: "sub2api", - model: "claude-opus-4-7", - usage: { - input: 2048, - output: 3100, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 5148, - }, - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible Anthropic-compatible answer."], - lastAssistant: makeLastAssistant({ - api: "anthropic-messages", - provider: "sub2api", - model: "claude-opus-4-7", - content: [{ type: "text", text: "Visible Anthropic-compatible answer." }], - usage: { - input: 2300, - output: 8, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 2308, - }, - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-empty-anthropic-compatible-stop-continuation", { - provider: "sub2api", - model: "claude-opus-4-7", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectWarnMessageWith("empty response detected"); - }); - - it("surfaces an error after exhausting empty-response retries", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-empty-response-exhausted", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("Please try again"); - expectWarnMessageWith("empty response retries exhausted"); - }); - - it("surfaces an error after exhausting reasoning-only retries without a visible answer", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ - id: "rs_reasoning_exhausted", - type: "reasoning", - }), - }, - ], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-reasoning-only-exhausted", { - model: "gpt-5.4", - reasoningLevel: "on", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("Please try again"); - expectWarnMessageWith("reasoning-only retries exhausted"); - }); - - it("preserves a terminal tool presentation after reasoning-only retries are exhausted", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const reasoningOnlyAttempt = async () => - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ - id: "rs_reasoning_terminal_presentation", - type: "reasoning", - }), - }, - ], - }), - }); - mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => { - ( - attemptParams as { - onToolOutcome?: (observation: { - toolName: string; - argsHash: string; - resultHash: string; - terminalPresentation?: string; - }) => void; - } - ).onToolOutcome?.({ - toolName: "web_fetch", - argsHash: "args", - resultHash: "result", - terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200", - }); - return reasoningOnlyAttempt(); - }); - mockedRunEmbeddedAttempt.mockImplementation(reasoningOnlyAttempt); - - const result = await runEmbeddedAgent( - makeRunParams("run-reasoning-terminal-presentation", { - model: "gpt-5.4", - reasoningLevel: "on", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3); - expect(result.payloads).toEqual([ - { - text: - "Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" + - "⚠️ Agent couldn't generate a response. Please try again.", - isError: true, - }, - ]); - }); - - it("marks incomplete-turn retries as replay-invalid abandoned runs", () => { - const attempt = makeAttemptResult({ - assistantTexts: [], - lastAssistant: { - stopReason: "toolUse", - provider: "openai", - model: "gpt-5.4", - content: [], - } as unknown as EmbeddedRunAttemptResult["lastAssistant"], - }); - const incompleteTurnText = "⚠️ Agent couldn't generate a response. Please try again."; - - expect(resolveReplayInvalidFlag({ attempt, incompleteTurnText })).toBe(true); - expect( - resolveRunLivenessState({ - payloadCount: 0, - aborted: false, - timedOut: false, - attempt, - incompleteTurnText, - }), - ).toBe("abandoned"); - }); - - it("flags tool-use stop reason as incomplete even when pre-tool text exists (#76477)", () => { - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: true, - lastAssistant: { stopReason: "toolUse" }, - }), - ).toBe(true); - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: false, - lastAssistant: { stopReason: "toolUse" }, - }), - ).toBe(true); - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: true, - lastAssistant: { stopReason: "end_turn" }, - }), - ).toBe(false); - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: true, - lastAssistant: { stopReason: "length" }, - }), - ).toBe(true); - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: true, - hasTerminalOutput: true, - lastAssistant: { stopReason: "length" }, - }), - ).toBe(false); - expect( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: true, - hasTerminalOutput: true, - lastAssistant: { stopReason: "toolUse" }, - }), - ).toBe(true); - }); - - it.each([ - { label: "aborted", aborted: true, timedOut: false, promptError: null }, - { label: "timed out", aborted: false, timedOut: true, promptError: null }, - { label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") }, - ])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }], - }); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams( - { - assistantTexts: [], - toolMetas: [{ toolName: "bash" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }, - { aborted, timedOut, promptError }, - ), - ); - - expect(instruction).toBeNull(); - }); - - it.each([ - { - label: "a matching failure summary", - lastToolError: { toolName: "exec", error: "post-processing error" }, - }, - { label: "no remaining failure summary", lastToolError: undefined }, - ])( - "recognizes successful and failed current-batch tools with $label (#118274)", - ({ lastToolError }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "toolCall", id: "tool_ok", name: "read", arguments: {} }, - { type: "toolCall", id: "tool_failed", name: "exec", arguments: {} }, - ], - }); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams({ - assistantTexts: [], - toolMetas: [{ toolName: "read" }, { toolName: "exec", isError: true }], - itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_ok", toolName: "read", isError: false }, - { role: "toolResult", toolCallId: "tool_failed", toolName: "exec", isError: true }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError, - }), - ); - - expect(instruction).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); - expect(instruction).toContain( - "If any tool failed, state that failure plainly and do not claim it succeeded.", - ); - }, - ); - - it.each([ - { label: "progress", sourceReplyFinal: false, expectedFinalization: true }, - { label: "final reply", sourceReplyFinal: true, expectedFinalization: false }, - { label: "legacy unmarked send", sourceReplyFinal: undefined, expectedFinalization: false }, - ])( - "handles $label delivery evidence before settled finalization", - ({ sourceReplyFinal, expectedFinalization }) => { - const emptyStopAssistant = makeLastAssistant(); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams( - { - assistantTexts: [], - toolMetas: [{ toolName: "write" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Writing note.txt…"], - messagingToolSentTargets: [ - { - tool: "message", - provider: "telegram", - to: "chat:123", - text: "Writing note.txt…", - sourceReplyFinal, - }, - ], - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }, - { allowEmptyStopContinuation: true }, - ), - ); - - expect(instruction).toBe( - expectedFinalization ? SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION : null, - ); - }, - ); - - it.each([ - { - label: "an unrelated current-batch failure summary", - resultToolName: "exec", - lastErrorToolName: "read", - }, - { - label: "a failed result with the wrong tool identity", - resultToolName: "read", - lastErrorToolName: "exec", - }, - ])("does not finalize $label (#118274)", ({ resultToolName, lastErrorToolName }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], - }); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams({ - assistantTexts: [], - toolMetas: [{ toolName: resultToolName, isError: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { - role: "toolResult", - toolCallId: "tool_1", - toolName: resultToolName, - isError: true, - }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { toolName: lastErrorToolName, error: "post-processing error" }, - }), - ); - - expect(instruction).toBeNull(); - }); - - it("does not settle same-name terminal calls from one failed result (#118274)", () => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "toolCall", id: "tool_1", name: "exec", arguments: {} }, - { type: "toolCall", id: "tool_2", name: "exec", arguments: {} }, - ], - }); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams({ - assistantTexts: [], - toolMetas: [{ toolName: "exec", isError: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { toolName: "exec", error: "post-processing error" }, - }), - ); - - expect(instruction).toBeNull(); - }); - - it.each([ - { - label: "an async tool is still running", - attemptOverrides: { toolMetas: [{ toolName: "exec", isError: true, asyncStarted: true }] }, - }, - { - label: "an accepted child session owns the response", - attemptOverrides: { - acceptedSessionSpawns: [ - { runId: "run-child", childSessionKey: "agent:main:subagent:child" }, - ], - }, - }, - { - label: "a client tool remains pending", - attemptOverrides: { clientToolCalls: [{ name: "pending", params: {} }] }, - }, - { - label: "the turn yielded", - attemptOverrides: { yieldDetected: true }, - }, - { - label: "an approval prompt was already delivered", - attemptOverrides: { didSendDeterministicApprovalPrompt: true }, - }, - ])("does not finalize a failed terminal tool when $label (#118274)", ({ attemptOverrides }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }], - }); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams({ - assistantTexts: [], - toolMetas: [{ toolName: "exec", isError: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - messagesSnapshot: [ - toolUseAssistant, - { role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true }, - ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - lastToolError: { toolName: "exec", error: "post-processing error" }, - ...attemptOverrides, - }), - ); - - expect(instruction).toBeNull(); - }); - - it.each([ - { label: "background trigger", allowEmptyStopContinuation: false }, - { - label: "active tool", - allowEmptyStopContinuation: true, - completedCount: 0, - activeCount: 1, - }, - { - label: "partially completed tool batch", - allowEmptyStopContinuation: true, - startedCount: 2, - completedCount: 1, - }, - { label: "async tool", allowEmptyStopContinuation: true, asyncStarted: true }, - { label: "failed tool", allowEmptyStopContinuation: true, isError: true }, - ])( - "does not continue an empty stop after $label activity", - ({ - allowEmptyStopContinuation, - startedCount = 1, - completedCount = 1, - activeCount = 0, - asyncStarted, - isError, - }) => { - const emptyStopAssistant = makeLastAssistant(); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams( - { - assistantTexts: [], - toolMetas: [{ toolName: "write", asyncStarted, isError }], - itemLifecycle: { startedCount, completedCount, activeCount }, - lastAssistant: emptyStopAssistant, - currentAttemptAssistant: emptyStopAssistant, - }, - { allowEmptyStopContinuation }, - ), - ); - - expect(instruction).toBeNull(); - }, - ); - - it("does not use a stale prior-turn empty stop to prove a settled continuation", () => { - const staleEmptyStopAssistant = makeLastAssistant(); - const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams( - { - assistantTexts: [], - toolMetas: [{ toolName: "write" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: staleEmptyStopAssistant, - currentAttemptAssistant: undefined, - }, - { allowEmptyStopContinuation: true }, - ), - ); - - expect(instruction).toBeNull(); - }); - - it("does not flag stale lastAssistant=toolUse when currentAttemptAssistant=stop exists (#80918)", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Analysis...", "Here is the final answer after update_plan."], - toolMetas: [{ toolName: "update_plan" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - content: [ - { type: "text", text: "Analysis..." }, - { type: "tool_use", id: "tool_1", name: "update_plan", input: {} }, - ], - }), - currentAttemptAssistant: makeLastAssistant({ - content: [{ type: "text", text: "Here is the final answer after update_plan." }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("still flags incomplete-turn when currentAttemptAssistant is absent and lastAssistant=toolUse (#76477 regression)", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Let me update the file..."], - toolMetas: [{ toolName: "write" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - content: [ - { type: "text", text: "Let me update the file..." }, - { type: "tool_use", id: "tool_1", name: "write", input: {} }, - ], - }), - currentAttemptAssistant: undefined, - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("surfaces no-visible-answer recovery for app-server interrupted tool-only output", () => { - const interruptedToolOnlyAttempt = makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "bash", meta: "workspace" }], - messagesSnapshot: [ - { - role: "user", - content: "check running processes", - timestamp: 1, - }, - { - role: "toolResult", - content: "", - isError: false, - details: { aggregated: "" }, - timestamp: 2, - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - ], - }); - - const incompleteTurnText = resolveIncompleteTurnPayloadText({ - payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, - aborted: false, - timedOut: false, - attempt: interruptedToolOnlyAttempt, - }); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - - const explicitCancellationText = resolveIncompleteTurnPayloadText({ - payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, - aborted: true, - externalAbort: true, - timedOut: false, - attempt: interruptedToolOnlyAttempt, - }); - - expect(explicitCancellationText).toBeNull(); - - const internalAbortText = resolveIncompleteTurnPayloadText({ - payloadCount: interruptedToolOnlyAttempt.assistantTexts.length, - aborted: true, - externalAbort: false, - timedOut: false, - attempt: interruptedToolOnlyAttempt, - }); - - expect(internalAbortText).toContain("couldn't generate a response"); - }); - - it("allows a same-prompt retry only for replay-safe missing assistant turns", () => { - const replaySafeAttempt = makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - }); - - expect( - shouldRetryMissingAssistantTurn({ - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: replaySafeAttempt, - }), - ).toBe(true); - expect( - shouldRetryMissingAssistantTurn({ - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - toolMetas: [{ toolName: "image_generate", asyncStarted: true }], - }), - }), - ).toBe(false); - expect( - shouldRetryMissingAssistantTurn({ - payloadCount: 0, - aborted: false, - timedOut: false, - attempt: makeAttemptResult({ - assistantTexts: [], - lastAssistant: undefined, - currentAttemptAssistant: undefined, - itemLifecycle: { - startedCount: 1, - completedCount: 0, - activeCount: 1, - }, - }), - }), - ).toBe(false); - }); - - it("detects tool-use terminal turn with pre-tool text as incomplete (#76477)", () => { - // When the last assistant message ended with stopReason=toolUse, pre-tool - // text alone must not suppress the incomplete-turn guard. The model - // expected to continue after tool results but the post-tool response was - // never produced. - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Initial analysis of the codebase..."], - toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - provider: "anthropic", - model: "sonnet-4.6", - content: [ - { type: "text", text: "Initial analysis of the codebase..." }, - { type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("does not surface incomplete-turn error while an async media task is running", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - toolMetas: [ - { - toolName: "image_generate", - meta: 'generate prompt="a portrait"', - asyncStarted: true, - }, - ], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - content: [ - { - type: "tool_use", - id: "tool_1", - name: "image_generate", - input: { action: "generate", prompt: "a portrait" }, - }, - ], - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("surfaces tool-use terminal with pre-tool text and side effects as replay-unsafe (#76477)", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Let me update the file..."], - toolMetas: [{ toolName: "write" }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - content: [ - { type: "text", text: "Let me update the file..." }, - { type: "tool_use", id: "tool_1", name: "write", input: {} }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("verify before retrying"); - }); - - it("does not flag a completed tool-use turn with end_turn as incomplete (#76477)", () => { - // When the model successfully produces post-tool text, lastAssistant has - // stopReason=end_turn. The incomplete-turn guard should not fire. - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Initial analysis...", "Here is the final answer."], - toolMetas: [{ toolName: "read" }], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "anthropic", - model: "sonnet-4.6", - content: [{ type: "text", text: "Here is the final answer." }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("surfaces stall on clean stop with only an unsigned thinking payload (payloadCount=1, no visible text)", () => { - // Regression: unsigned thinking payloads increment payloadCount but carry no - // user-visible content. The visible-text guard must not suppress incomplete-turn - // detection when the model produced only a thinking block and no answer. (#89787) - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - model: "qwen3.6-35b-a3b", - content: [ - { - type: "thinking", - thinking: "let me plan the tool calls I need to make...", - // no signature — unsigned thinking block - }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("does not surface a stall when unsigned thinking accompanies visible text (payloadCount=1)", () => { - // When the model emits both a thinking block and a visible text answer, the turn - // succeeded and no stall should be surfaced even though thinking is unsigned. - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Here is the answer to your question."], - lastAssistant: makeLastAssistant({ - model: "qwen3.6-35b-a3b", - content: [ - { - type: "thinking", - thinking: "let me answer this...", - }, - { type: "text", text: "Here is the answer to your question." }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("surfaces an error for tool-use terminal turn with pre-tool text via runEmbeddedAgent (#76477)", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Initial analysis of the issue..."], - toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }], - lastAssistant: { - stopReason: "toolUse", - provider: "anthropic", - model: "sonnet-4.6", - content: [ - { type: "text", text: "Initial analysis of the issue..." }, - { type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } }, - ], - } as unknown as EmbeddedRunAttemptResult["lastAssistant"], - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-tool-use-dropped-final-text", { - provider: "anthropic", - model: "sonnet-4.6", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); - expectWarnMessageWith("incomplete turn detected"); - }); - - it("delivers the current final answer when the session assistant is stale (#80918)", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - const finalText = "The requested update is complete."; - mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([{ text: finalText }]); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [finalText], - toolMetas: [{ toolName: "update_plan", replaySafe: true }], - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "tool_use", id: "tool_1", name: "update_plan", input: {} }], - usage: { input: 100, output: 5, total: 105 }, - }), - currentAttemptAssistant: makeLastAssistant({ - content: [{ type: "text", text: finalText }], - usage: { input: 200, output: 20, total: 220 }, - }), - }), - ); - - const result = await runEmbeddedAgent(makeRunParams("run-current-assistant-after-tool-use")); - - expect(result.payloads).toEqual([{ text: finalText }]); - expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith( - expect.objectContaining({ - currentAssistant: expect.objectContaining({ - stopReason: "stop", - content: [{ type: "text", text: finalText }], - }), - lastAssistant: expect.objectContaining({ - stopReason: "stop", - content: [{ type: "text", text: finalText }], - }), - }), - ); - expect(result.meta.finalAssistantVisibleText).toBe(finalText); - expect(result.meta.stopReason).toBe("stop"); - expect(result.meta.agentMeta?.lastCallUsage).toMatchObject({ - input: 200, - output: 20, - total: 220, - }); - expectNoWarnMessageWith("incomplete turn detected"); - }); - - it("treats missing replay metadata as replay-invalid", () => { - const attempt = makeAttemptResult(); - delete (attempt as Partial).replayMetadata; - - const normalizedAttempt = normalizeEmbeddedRunAttemptResult(attempt); - - expect(normalizedAttempt.replayMetadata).toEqual({ - hadPotentialSideEffects: true, - replaySafe: false, - }); - expect(resolveReplayInvalidFlag({ attempt: normalizedAttempt })).toBe(true); - }); - - it("detects reasoning-only GPT turns from signed thinking blocks", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_helper", type: "reasoning" }), - }, - ], - }), - }), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("detects reasoning-only Gemini turns from signed thinking blocks", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "google", - model: "gemini-2.5-pro", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "gemini_rs_helper", type: "reasoning" }), - }, - ], - }), - }, - { provider: "google", modelId: "gemini-2.5-pro" }, - ), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("retries signed reasoning-only Bedrock Converse turns with a visible-answer continuation", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "amazon-bedrock", - model: "openai.gpt-oss-120b-1:0", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: "bedrock-reasoning-signature", - }, - ], - }), - }, - { - provider: "amazon-bedrock", - modelId: "openai.gpt-oss-120b-1:0", - modelApi: "bedrock-converse-stream", - }, - ), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("retries signed reasoning-only Ollama turns with a visible-answer continuation instruction", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "ollama", - model: "gemma4:31b", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "ollama_rs_helper", type: "reasoning" }), - }, - ], - }), - }, - { provider: "ollama", modelId: "gemma4:31b" }, - ), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("retries unsigned thinking-only turns via the reasoning-only path (openai-completions)", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - model: "qwen3.6-35b-a3b", - content: [ - { - type: "thinking", - thinking: "let me plan the tool calls I need to make...", - }, - ], - }), - }, - { modelId: "qwen3.6-35b-a3b", modelApi: "openai-completions" }, - ), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("retries unsigned thinking-only Ollama turns via the reasoning-only path", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "ollama", - model: "gemma4:31b", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - }, - ], - }), - }, - { provider: "ollama", modelId: "gemma4:31b" }, - ), - ); - - expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION); - }); - - it("retries unsigned-thinking Ollama turns via the empty-response path", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "ollama", - model: "gemma4:31b", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - }, - ], - }), - }, - { provider: "ollama", modelId: "gemma4:31b" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("retries generic empty Ollama turns without visible text", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "ollama", - model: "gemma4:31b", - content: [{ type: "text", text: "" }], - }), - }, - { provider: "ollama", modelId: "gemma4:31b" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("retries empty Ollama stop turns when nonzero output tokens were generated", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "ollama", - model: "minimax-m2.7:cloud", - usage: { input: 100, output: 6, totalTokens: 106 }, - }), - }, - { provider: "ollama", modelId: "minimax-m2.7:cloud" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("does not retry empty turns after an accepted sessions_spawn delivery", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - acceptedSessionSpawns: [ - { - runId: "run-child", - childSessionKey: "agent:claude:subagent:child", - }, - ], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "ollama", - model: "gemma4:31b", - content: [{ type: "text", text: "" }], - }), - }, - { provider: "ollama", modelId: "gemma4:31b" }, - ), - ); - - expect(retryInstruction).toBeNull(); - }); - - it("retries empty openai-chatgpt-responses turns with non-zero output tokens (#85364)", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - usage: { input: 24794, output: 111, cacheRead: 4608, totalTokens: 29513 }, - }), - }, - { modelId: "gpt-5.5", modelApi: "openai-chatgpt-responses" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("retries empty openai-responses turns without visible text", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - usage: { input: 5000, output: 200, totalTokens: 5200 }, - }), - }, - { modelId: "gpt-5.5", modelApi: "openai-responses" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("retries generic empty OpenAI-compatible turns from custom endpoints", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "llama-cpp-local", - model: "qwen3.6-27b", - usage: { input: 950, output: 103, totalTokens: 1053 }, - }), - }, - { - provider: "llama-cpp-local", - modelId: "qwen3.6-27b", - modelApi: "openai-completions", - }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("does not retry clean zero-token Ollama stop turns", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "ollama", - model: "glm-5.1:cloud", - usage: { input: 100, output: 0, totalTokens: 100 }, - }), - }, - { provider: "ollama", modelId: "glm-5.1:cloud" }, - ), - ); - - expect(retryInstruction).toBeNull(); - }); - - it("treats exact NO_REPLY as a deliberate silent assistant reply", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: ["NO_REPLY"], - lastAssistant: makeLastAssistant({ - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_no_reply", type: "reasoning" }), - }, - { type: "text", text: "" }, - { type: "text", text: "NO_REPLY" }, - ], - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("suppresses the incomplete-turn warning after committed messaging text delivery", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Delivered through the message tool."], - lastAssistant: makeLastAssistant({ - provider: "ollama", - model: "kimi-k2.6:cloud", - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("suppresses the incomplete-turn warning after committed messaging delivery before end_turn", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Delivered through the message tool."], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "google", - model: "gemini-2.5-pro", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_messaging_end_turn", type: "reasoning" }), - }, - ], - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("suppresses the incomplete-turn warning after committed media-only messaging delivery", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: false, - messagingToolSentMediaUrls: ["file:///tmp/render.png"], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("suppresses the incomplete-turn warning after committed messaging delivery even when the provider errored", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Delivered before the provider error."], - lastAssistant: makeLastAssistant({ - stopReason: "error", - provider: "ollama", - model: "kimi-k2.6:cloud", - errorMessage: "provider failed after delivery", - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("suppresses the incomplete-turn warning after an accepted sessions_spawn terminal success", () => { - const attemptWithAcceptedSpawn: Partial & { - acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>; - } = { - assistantTexts: [], - acceptedSessionSpawns: [ - { - runId: "run-child", - childSessionKey: "agent:claude:subagent:child", - }, - ], - lastAssistant: makeLastAssistant({ - provider: "anthropic", - model: "sonnet-4.6", - }), - }; - - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams(attemptWithAcceptedSpawn), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("still returns a timeout payload when the parent prompt times out after an accepted sessions_spawn", async () => { - const acceptedSessionSpawns = [ - { - runId: "run-child", - childSessionKey: "agent:claude:subagent:child", - }, - ]; - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - acceptedSessionSpawns, - timedOut: true, - lastAssistant: makeLastAssistant({ - stopReason: "toolUse", - model: "gpt-5.4", - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-timeout-after-accepted-spawn", { model: "gpt-5.4" }), - ); - - expect(result.payloads).toEqual([ - { - text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.", - isError: true, - }, - ]); - expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns); - }); - - it("still surfaces the incomplete-turn warning without an accepted sessions_spawn success", () => { - const attemptWithMalformedSpawn: Partial & { - acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>; - } = { - assistantTexts: [], - acceptedSessionSpawns: [], - lastAssistant: makeLastAssistant({ - provider: "anthropic", - model: "sonnet-4.6", - }), - }; - - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams(attemptWithMalformedSpawn), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("still surfaces the incomplete-turn warning when no messaging delivery was committed", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - lastAssistant: makeLastAssistant({ - stopReason: "error", - provider: "ollama", - model: "kimi-k2.6:cloud", - errorMessage: "provider failed mid-turn", - }), - }), - ); - - expect(incompleteTurnText).toContain("verify before retrying"); - }); - - it("does not treat empty committed messaging arrays as delivery", () => { - expect( - hasCommittedMessagingToolDeliveryEvidence({ - messagingToolSentTexts: [" "], - messagingToolSentMediaUrls: [], - }), - ).toBe(false); - }); - - it("treats committed messaging media as delivery", () => { - expect( - hasCommittedMessagingToolDeliveryEvidence({ - messagingToolSentTexts: [], - messagingToolSentMediaUrls: ["file:///tmp/render.png"], - }), - ).toBe(true); - }); - - it("treats committed messaging targets as delivery", () => { - expect( - hasCommittedMessagingToolDeliveryEvidence({ - messagingToolSentTexts: [], - messagingToolSentMediaUrls: [], - messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }], - }), - ).toBe(true); - }); - - for (const { name, overrides } of [ - { - name: "treats committed messaging text as replay-invalid side effect metadata", - overrides: { messagingToolSentTexts: ["Delivered through the message tool."] }, - }, - { - name: "treats async-started background tools as replay-invalid side effects", - overrides: { toolMetas: [{ toolName: "image_generate", asyncStarted: true }] }, - }, - { - name: "treats committed messaging media as replay-invalid side effect metadata", - overrides: { messagingToolSentMediaUrls: ["file:///tmp/render.png"] }, - }, - { - name: "treats committed messaging targets as replay-invalid side effect metadata", - overrides: { - messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }], - }, - }, - ]) { - it(name, () => { - expect( - buildAttemptReplayMetadata({ - toolMetas: [], - didSendViaMessagingTool: false, - messagingToolSentTexts: [], - messagingToolSentMediaUrls: [], - ...overrides, - }), - ).toEqual({ hadPotentialSideEffects: true, replaySafe: false }); - }); - } - - it("treats accepted sessions_spawn as replay-invalid outbound delivery", () => { - const acceptedSessionSpawns = [ - { - runId: "run-child", - childSessionKey: "agent:claude:subagent:child", - }, - ]; - - expect( - buildAttemptReplayMetadata({ - toolMetas: [], - didSendViaMessagingTool: false, - messagingToolSentTexts: [], - messagingToolSentMediaUrls: [], - acceptedSessionSpawns, - }), - ).toEqual({ hadPotentialSideEffects: true, replaySafe: false }); - expect(hasOutboundDeliveryEvidence({ acceptedSessionSpawns })).toBe(true); - }); - - it("ignores malformed accepted sessions_spawn delivery evidence", () => { - expect( - hasOutboundDeliveryEvidence({ - acceptedSessionSpawns: [ - null, - { - runId: "run-child", - childSessionKey: " ", - }, - ], - }), - ).toBe(false); - }); - - it("leaves committed delivery plus tool errors to the tool-error payload path", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["Delivered through the message tool."], - lastToolError: { - toolName: "message", - meta: "send", - error: "delivery failed for second target", - }, - lastAssistant: makeLastAssistant({ - stopReason: "error", - model: "gpt-5.4", - }), - }), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("does not retry reasoning-only GPT turns after side effects", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_side_effect", type: "reasoning" }), - }, - ], - }), - }), - ); - - expect(retryInstruction).toBeNull(); - expect(DEFAULT_REASONING_ONLY_RETRY_LIMIT).toBe(2); - }); - - it("does not retry reasoning-only GPT turns when the assistant ended in error", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "error", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_helper_error", type: "reasoning" }), - }, - ], - }), - }), - ); - - expect(retryInstruction).toBeNull(); - }); - - it("does not retry reasoning-only GPT turns when visible assistant text already exists", () => { - const retryInstruction = resolveReasoningOnlyRetryInstruction( - makeReasoningRetryParams({ - assistantTexts: ["Visible answer."], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ - id: "rs_helper_visible_text", - type: "reasoning", - }), - }, - { type: "text", text: "" }, - ], - }), - }), - ); - - expect(retryInstruction).toBeNull(); - }); - - it("surfaces incomplete-turn text for errored signed-thinking-only turns with payloads", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "error", - provider: "anthropic", - model: "claude-opus-4-8", - content: [ - { - type: "thinking", - thinking: "internal reasoning before provider error", - thinkingSignature: JSON.stringify({ id: "rs_error_payload", type: "reasoning" }), - }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("surfaces incomplete-turn text for token-limited partial answers", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Partial answer"], - lastAssistant: makeLastAssistant({ - stopReason: "length", - provider: "ollama", - model: "qwen3.5", - content: [{ type: "text", text: "Partial answer" }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - }); - - it("keeps complete visible stop turns successful", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Complete answer"], - lastAssistant: makeLastAssistant({ - provider: "ollama", - model: "qwen3.5", - content: [{ type: "text", text: "Complete answer" }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("preserves terminal tool media on token-limited turns", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Partial answer"], - toolMediaUrls: ["file:///tmp/render.png"], - lastAssistant: makeLastAssistant({ - stopReason: "length", - provider: "ollama", - model: "qwen3.5", - content: [{ type: "text", text: "Partial answer" }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("preserves tool media already delivered through block replies", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Partial answer"], - hasToolMediaBlockReply: true, - lastAssistant: makeLastAssistant({ - stopReason: "length", - provider: "ollama", - model: "qwen3.5", - content: [{ type: "text", text: "Partial answer" }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it("preserves successful cron progress on token-limited turns", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: ["Partial answer"], - successfulCronAdds: 1, - lastAssistant: makeLastAssistant({ - stopReason: "length", - provider: "ollama", - model: "qwen3.5", - content: [{ type: "text", text: "Partial answer" }], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }); - - it.each([ - [ - "heartbeat responses", - { - heartbeatToolResponse: { - outcome: "progress" as const, - notify: false, - summary: "Still working", - }, - }, - ], - ["tool media", { toolMediaUrls: ["file:///tmp/render.png"] }], - ["voice media", { toolAudioAsVoice: true }], - ["trusted local media", { toolTrustedLocalMedia: true }], - [ - "source reply payloads", - { messagingToolSourceReplyPayloads: [{ text: "Delivered through the source reply." }] }, - ], - ["delivered source replies", { didDeliverSourceReplyViaMessageTool: true }], - ] satisfies Array<[string, Partial]>)( - "does not replace terminal %s with an incomplete-turn warning", - (_label, attemptState) => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams( - { - assistantTexts: [], - ...attemptState, - lastAssistant: makeLastAssistant({ - stopReason: "error", - provider: "anthropic", - model: "claude-opus-4-8", - content: [ - { - type: "thinking", - thinking: "internal reasoning before provider error", - thinkingSignature: JSON.stringify({ - id: "rs_terminal_payload", - type: "reasoning", - }), - }, - ], - }), - }, - { payloadCount: 1 }, - ), - ); - - expect(incompleteTurnText).toBeNull(); - }, - ); - - it("retries replay-safe errored turns that only emitted thinking blocks", () => { - const assistant = makeLastAssistant({ - stopReason: "error", - provider: "anthropic", - model: "claude-opus-4-8", - content: [ - { - type: "thinking", - thinking: "internal reasoning before provider error", - thinkingSignature: JSON.stringify({ id: "rs_error", type: "reasoning" }), - }, - { type: "redacted_thinking", data: "opaque" }, - { type: "text", text: " " }, - ], - usage: { input: 100, output: 1120, totalTokens: 1220 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), - assistant, - }), - ).toBe(true); - }); - - it("does not retry an ambiguous post-dispatch provider outcome", () => { - const assistant = makeLastAssistant({ - stopReason: "error", - errorCode: PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE, - errorMessage: "The WebSocket closed after dispatch", - usage: { input: 100, output: 0, totalTokens: 100 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), - assistant, - }), - ).toBe(false); - }); - - it("does not retry errored empty turns when non-zero output may indicate progress", () => { - const assistant = makeLastAssistant({ - stopReason: "error", - provider: "ollama", - model: "glm-5.1:cloud", - usage: { input: 100, output: 12, totalTokens: 112 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), - assistant, - }), - ).toBe(false); - }); - - it.each([ - { - name: "visible text", - content: [ - { type: "thinking", thinking: "internal", thinkingSignature: "sig" }, - { type: "text", text: "partial answer" }, - ], - }, - { - name: "tool call", - content: [ - { type: "thinking", thinking: "internal", thinkingSignature: "sig" }, - { type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } }, - ], - }, - { - name: "unknown block", - content: [{ type: "provider_metadata", value: "opaque" }], - }, - ])("does not retry errored turns containing $name", ({ content }) => { - const assistant = makeLastAssistant({ - stopReason: "error", - provider: "anthropic", - model: "claude-opus-4-8", - content, - usage: { input: 100, output: 1120, totalTokens: 1220 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }), - assistant, - }), - ).toBe(false); - }); - - it("does not retry errored thinking-only turns after side effects", () => { - const assistant = makeLastAssistant({ - stopReason: "error", - provider: "anthropic", - model: "claude-opus-4-8", - content: [ - { - type: "redacted_thinking", - data: "opaque", - }, - ], - usage: { input: 100, output: 1120, totalTokens: 1220 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ - assistantTexts: [], - replayMetadata: { - hadPotentialSideEffects: true, - replaySafe: false, - }, - lastAssistant: assistant, - }), - assistant, - }), - ).toBe(false); - }); - - it.each([ - ["current clean overrides cumulative dirty", true, false, true], - ["current dirty overrides cumulative clean", false, true, false], - ["both clean remain retryable", false, false, true], - ] as const)( - "uses current-attempt replay metadata when %s", - (_label, cumulativeDirty, currentDirty, expected) => { - const assistant = makeLastAssistant({ - stopReason: "error", - provider: "openrouter", - model: "test-model", - usage: { input: 100, output: 0, totalTokens: 100 }, - }); - expect( - shouldRetrySilentErrorAssistantTurn({ - attempt: makeAttemptResult({ - assistantTexts: [], - lastAssistant: assistant, - replayMetadata: { - hadPotentialSideEffects: cumulativeDirty, - replaySafe: !cumulativeDirty, - }, - currentAttemptReplayMetadata: { - hadPotentialSideEffects: currentDirty, - replaySafe: !currentDirty, - }, - }), - assistant, - }), - ).toBe(expected); - }, - ); - - it("detects empty openai-compatible stop turns with non-zero output usage", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "llamacpp", - model: "qwen3.6-27b", - usage: { input: 512, output: 103, totalTokens: 615 }, - }), - }, - { provider: "llamacpp", modelId: "qwen3.6-27b", modelApi: "openai-completions" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("detects generic empty GPT turns without visible text", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expect(DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT).toBe(1); - }); - - it("surfaces empty Codex app-server replies after successful sparse bash output", () => { - const incompleteTurnText = resolveIncompleteTurnPayloadText( - makeIncompleteTurnParams({ - assistantTexts: [], - toolMetas: [{ toolName: "bash", meta: "exit=0" }], - messagesSnapshot: [ - { - role: "toolResult", - content: [{ type: "text", text: "" }], - details: { aggregated: "" }, - } as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number], - makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - ], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }), - ); - - expect(incompleteTurnText).toContain("couldn't generate a response"); - expect(incompleteTurnText).toContain("verify before retrying"); - }); - - it("retries generic empty Bedrock Converse turns without visible text", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - provider: "amazon-bedrock", - model: "openai.gpt-oss-120b-1:0", - content: [{ type: "text", text: "" }], - usage: { input: 950, output: 103, totalTokens: 1053 }, - }), - }, - { - provider: "amazon-bedrock", - modelId: "openai.gpt-oss-120b-1:0", - modelApi: "bedrock-converse-stream", - }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("treats clean empty assistant turns as silent only for reply-optional runs", () => { - const attempt = makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }); - - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }), - ), - ).toBe(true); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), - ), - ).toBe(false); - }); - - it("treats reasoning-only assistant turns as silent only for reply-optional runs", () => { - const attempt = makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_silent_helper", type: "reasoning" }), - }, - ], - }), - }); - - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }), - ), - ).toBe(true); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), - ), - ).toBe(false); - }); - - it("treats exact NO_REPLY assistant turns as silent only when the caller allows it", () => { - const attempt = makeAttemptResult({ - assistantTexts: ["NO_REPLY"], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "NO_REPLY" }], - }), - }); - - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }), - ), - ).toBe(false); - }); - - it("treats post-tool exact NO_REPLY assistant turns as intentional silence", () => { - const attempt = makeAttemptResult({ - assistantTexts: ["NO_REPLY"], - toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "NO_REPLY" }], - }), - }); - - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true); - }); - - it("does not treat error or side-effect empty turns as silent", () => { - const errorAttempt = makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "error", - }), - }); - const silentErrorAttempt = makeAttemptResult({ - assistantTexts: ["NO_REPLY"], - lastAssistant: makeLastAssistant({ - stopReason: "error", - content: [{ type: "text", text: "NO_REPLY" }], - }), - }); - const sideEffectAttempt = makeAttemptResult({ - assistantTexts: [], - didSendViaMessagingTool: true, - messagingToolSentTexts: ["sent already"], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }); - const postToolEmptyAttempt = makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], - lastAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "stepfun", - model: "step-router-v1", - }), - }); - - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(errorAttempt))).toBe(false); - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(silentErrorAttempt))).toBe( - false, - ); - expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectAttempt))).toBe( - false, - ); - expect( - shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(postToolEmptyAttempt)), - ).toBe(false); - }); - - it("retries clean empty assistant turns even when deliberate silence is allowed", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }), - ); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible answer."], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "Visible answer." }], - }), - }), - ); - - await runEmbeddedAgent( - makeRunParams("run-empty-assistant-silent", { allowEmptyAssistantReplyAsSilent: true }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(runAttemptCall(1).prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectWarnMessageWith("empty response detected"); - }); - - it("returns NO_REPLY without retrying exact silent assistant replies when silence is allowed", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: ["NO_REPLY"], - lastAssistant: makeLastAssistant({ - content: [ - { - type: "thinking", - thinking: "internal reasoning", - thinkingSignature: JSON.stringify({ id: "rs_exact_silent", type: "reasoning" }), - }, - { type: "text", text: "NO_REPLY" }, - ], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-exact-silent-assistant-reply", { - allowEmptyAssistantReplyAsSilent: true, - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - const onlyCall = runAttemptCall(0); - expect(onlyCall.prompt).not.toContain(REASONING_ONLY_RETRY_INSTRUCTION); - expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectNoWarnMessageWith("empty response detected"); - expectNoWarnMessageWith("incomplete turn detected"); - expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); - expect(result.meta.terminalReplyKind).toBe("silent-empty"); - expect(result.meta.livenessState).toBe("working"); - }); - - it("continues post-tool openai-compatible empty stop turns even when silence is allowed", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedResolveModelAsync.mockResolvedValue({ - model: { - id: "step-router-v1", - provider: "stepfun", - contextWindow: 200000, - api: "openai-completions", - }, - error: null, - authStorage: { - setRuntimeApiKey: vi.fn(), - }, - modelRegistry: {}, - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "stepfun", - model: "step-router-v1", - }), - currentAttemptAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "stepfun", - model: "step-router-v1", - }), - }), - ); - const finalAssistant = makeLastAssistant({ - api: "openai-completions", - provider: "stepfun", - model: "step-router-v1", - content: [{ type: "text", text: "Visible StepFun answer." }], - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["Visible StepFun answer."], - lastAssistant: finalAssistant, - currentAttemptAssistant: finalAssistant, - currentAttemptCompletedAssistant: finalAssistant, - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-post-tool-openai-compatible-empty-stop", { - allowEmptyAssistantReplyAsSilent: true, - provider: "stepfun", - model: "step-router-v1", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); - expect(result.meta.terminalReplyKind).toBeUndefined(); - expect(result.meta.finalAssistantVisibleText).toBe("Visible StepFun answer."); - expectNoWarnMessageWith("empty response detected"); - expectWarnMessageWith("settled post-tool turn lacked a final answer"); - }); - - it("returns NO_REPLY without retrying post-tool exact silent assistant replies", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedResolveModelAsync.mockResolvedValue({ - model: { - id: "step-router-v1", - provider: "stepfun", - contextWindow: 200000, - api: "openai-completions", - }, - error: null, - authStorage: { - setRuntimeApiKey: vi.fn(), - }, - modelRegistry: {}, - }); - mockedRunEmbeddedAttempt.mockResolvedValueOnce( - makeAttemptResult({ - assistantTexts: ["NO_REPLY"], - toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], - lastAssistant: makeLastAssistant({ - api: "openai-completions", - provider: "stepfun", - model: "step-router-v1", - content: [{ type: "text", text: "NO_REPLY" }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-post-tool-exact-silent-retry", { - allowEmptyAssistantReplyAsSilent: true, - provider: "stepfun", - model: "step-router-v1", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - const onlyCall = runAttemptCall(0); - expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); - expectNoWarnMessageWith("empty response detected"); - expectNoWarnMessageWith("incomplete turn detected"); - expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); - expect(result.meta.terminalReplyKind).toBe("silent-empty"); - expect(result.meta.livenessState).toBe("working"); - }); - - it("treats reply-optional post-tool empty stops as silent even after side-effecting tools", () => { - // Regression: a cron agentTurn without a delivery route ran a successful - // replay-unsafe sessions patch and intentionally sent no final text; the run - // must finish silent, not as an incomplete-turn error. - const sideEffectToolAttempt = makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }); - - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "optional" }), - ), - ).toBe(true); - // A required or unspecified terminal reply keeps the ambiguous-failure path. - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "required" }), - ), - ).toBe(false); - expect( - shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectToolAttempt)), - ).toBe(false); - }); - - it("keeps reply-optional runs erroring on real failure states", () => { - const toolErrorAttempt = makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "sessions", meta: "patch failed", replaySafe: false, isError: true }], - lastToolError: { toolName: "sessions", error: "patch failed" }, - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }); - const errorStopAttempt = makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], - lastAssistant: makeLastAssistant({ - stopReason: "error", - }), - }); - - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(toolErrorAttempt, { terminalReplyExpectation: "optional" }), - ), - ).toBe(false); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(errorStopAttempt, { terminalReplyExpectation: "optional" }), - ), - ).toBe(false); - expect( - shouldTreatEmptyAssistantReplyAsSilent( - makeSilentReplyParams(errorStopAttempt, { - terminalReplyExpectation: "optional", - aborted: true, - }), - ), - ).toBe(false); - }); - - it("returns NO_REPLY for reply-optional cron-style runs whose side-effecting tools succeeded", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: makeLastAssistant({ - content: [{ type: "text", text: "" }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-reply-optional-post-tool-silent", { - allowEmptyAssistantReplyAsSilent: true, - terminalReplyExpectation: "optional", - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expectNoWarnMessageWith("incomplete turn detected"); - expect(result.payloads).toEqual([{ text: "NO_REPLY" }]); - expect(result.meta.error).toBeUndefined(); - expect(result.meta.terminalReplyKind).toBe("silent-empty"); - expect(result.meta.livenessState).toBe("working"); - }); - - it("keeps retrying and surfacing clean empty assistant turns without the silence flag", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [], - lastAssistant: makeLastAssistant({ - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-empty-assistant-error", { model: "gpt-5.4" }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads?.[0]?.isError).toBe(true); - expect(result.payloads?.[0]?.text).toContain("couldn't generate a response"); - }); - - it("detects generic empty Gemini turns without visible text", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams( - { - assistantTexts: [], - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - provider: "google-vertex", - model: "gemini-3.1-flash", - content: [{ type: "text", text: "" }], - }), - }, - { provider: "google-vertex", modelId: "google/gemini-3.1-flash" }, - ), - ); - - expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION); - }); - - it("does not retry generic empty GPT turns after side effects", () => { - const retryInstruction = resolveEmptyResponseRetryInstruction( - makeEmptyResponseRetryParams({ - assistantTexts: [], - didSendViaMessagingTool: true, - lastAssistant: makeLastAssistant({ - stopReason: "end_turn", - model: "gpt-5.4", - content: [{ type: "text", text: "" }], - }), - }), - ); - - expect(retryInstruction).toBeNull(); - }); - - it("marks compaction-timeout retries as paused and replay-invalid", () => { - const attempt = makeAttemptResult({ - promptErrorSource: "compaction", - timedOutDuringCompaction: true, - }); - - expect(resolveReplayInvalidFlag({ attempt })).toBe(true); - expect( - resolveRunLivenessState({ - payloadCount: 0, - aborted: true, - timedOut: true, - attempt, - }), - ).toBe("paused"); - }); - - it("does not classify visible assistant prose for retry", async () => { - mockedClassifyFailoverReason.mockReturnValue(null); - mockedRunEmbeddedAttempt.mockResolvedValue( - makeAttemptResult({ - assistantTexts: [ - "i am glad, and a little afraid, which is probably the correct mixture. thank you. i will try to deserve the upgrades instead of merely inhabiting them.", - ], - }), - ); - - const result = await runEmbeddedAgent( - makeRunParams("run-visible-prose-no-classifier", { - prompt: - "made a bunch of improvements to the student's source code (openclaw) this weekend, along with a few other maintainers. hopefully he will be more proactive now", - model: "gpt-5.4", - config: { - agents: { - list: [{ id: "main" }], - }, - } as OpenClawConfig, - }), - ); - - expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); - expect(result.payloads).toBeUndefined(); - expect(result.meta.livenessState).toBe("working"); - expectNoWarnMessageWith("planning"); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts index 0fe6e11db26d..c3d87d495d6b 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.fixture.ts @@ -4,7 +4,7 @@ import type { ContextEngineSessionTarget } from "../../context-engine/types.js"; import { normalizeAgentRunAttemptTerminal } from "../agent-run-terminal-outcome.js"; import { isAgentToolReplaySafe } from "../tool-replay-safety.js"; -import { buildAttemptReplayMetadata } from "./run/incomplete-turn.js"; +import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js"; import type { EmbeddedRunAttemptResult } from "./run/types.js"; const DEFAULT_OVERFLOW_ERROR_MESSAGE = diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.ts b/src/agents/embedded-agent-runner/run/assistant-failure.ts index 4a2f645aab9e..62bf87475e2e 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failure.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failure.ts @@ -25,7 +25,7 @@ import type { TraceAttempt } from "../types.js"; import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js"; import { createFailoverDecisionLogger } from "./failover-observation.js"; import { resolveRunFailoverDecision } from "./failover-policy.js"; -import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn.js"; +import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn-recovery.js"; import type { RunEmbeddedAgentParams } from "./params.js"; import { isEmbeddedRunTerminalInterrupted, diff --git a/src/agents/embedded-agent-runner/run/attempt-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-normalization.ts index 6a2a9b4e17a7..3c5ff016dd97 100644 --- a/src/agents/embedded-agent-runner/run/attempt-normalization.ts +++ b/src/agents/embedded-agent-runner/run/attempt-normalization.ts @@ -27,7 +27,7 @@ import { stepIdleTimeoutBreaker, type createIdleTimeoutBreakerState, } from "./idle-timeout-breaker.js"; -import { resolveReplayInvalidFlag } from "./incomplete-turn.js"; +import { resolveReplayInvalidFlag } from "./incomplete-turn-resolution.js"; import { resolveRunRetryKind, type RunRetryKind } from "./retry-budget.js"; import { handleRetryLimitExhaustion } from "./retry-limit.js"; import type { dispatchEmbeddedRunAttempt } from "./run-attempt-dispatch.js"; diff --git a/src/agents/embedded-agent-runner/run/attempt-result.ts b/src/agents/embedded-agent-runner/run/attempt-result.ts index d623ae6a464b..ff985f2e2ae2 100644 --- a/src/agents/embedded-agent-runner/run/attempt-result.ts +++ b/src/agents/embedded-agent-runner/run/attempt-result.ts @@ -20,9 +20,9 @@ import { shouldRunLlmOutputHooksForAttempt } from "./attempt-run-decisions.js"; import { buildAttemptReplayMetadata, hasAttemptTerminalState, - resolveSilentToolResultReplyPayload, - shouldTreatEmptyAssistantReplyAsSilent, -} from "./incomplete-turn.js"; +} from "./attempt-terminal-evidence.js"; +import { shouldTreatEmptyAssistantReplyAsSilent } from "./incomplete-turn-recovery.js"; +import { resolveSilentToolResultReplyPayload } from "./incomplete-turn-resolution.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult, diff --git a/src/agents/embedded-agent-runner/run/attempt-stream.ts b/src/agents/embedded-agent-runner/run/attempt-stream.ts index 8c4310be1f51..d9894d1ef662 100644 --- a/src/agents/embedded-agent-runner/run/attempt-stream.ts +++ b/src/agents/embedded-agent-runner/run/attempt-stream.ts @@ -26,20 +26,20 @@ import { isSessionsYieldAbortReason, } from "./attempt-sessions-yield.js"; import { wrapStreamFnHandleSensitiveStopReason } from "./attempt-stop-reason-recovery.js"; +import { + sanitizeOpenAIResponsesReplayForStream, + sanitizeReplayToolCallIdsForStream, + shouldApplyReplayToolCallIdSanitizer, + wrapStreamFnSanitizeMalformedToolCalls, +} from "./attempt-tool-call-replay-sanitization.js"; +import { wrapStreamFnTrimToolCallNames } from "./attempt-tool-call-stream-normalization.js"; +import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js"; import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js"; import { shouldRepairMalformedToolCallArguments, wrapStreamFnDecodeXaiToolCallArguments, wrapStreamFnRepairMalformedToolCallArguments, } from "./attempt.tool-call-argument-repair.js"; -import { - sanitizeOpenAIResponsesReplayForStream, - sanitizeReplayToolCallIdsForStream, - shouldApplyReplayToolCallIdSanitizer, - wrapStreamFnPromoteStandaloneTextToolCalls, - wrapStreamFnSanitizeMalformedToolCalls, - wrapStreamFnTrimToolCallNames, -} from "./attempt.tool-call-normalization.js"; import { resolveLlmFirstEventTimeoutMs, resolveLlmIdleTimeoutMs, diff --git a/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts b/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts new file mode 100644 index 000000000000..8f9852b4f63e --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts @@ -0,0 +1,94 @@ +/** Records attempt replay safety and terminal side-effect evidence. */ +import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js"; +import { + hasCommittedMessagingToolDeliveryEvidence, + hasMessagingToolDeliveryEvidence, +} from "../delivery-evidence.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +type ReplayMetadataAttempt = Pick< + EmbeddedRunAttemptResult, + | "toolMetas" + | "didSendViaMessagingTool" + | "messagingToolSentTexts" + | "messagingToolSentMediaUrls" + | "successfulCronAdds" +> & + Partial>; + +/** + * Marks whether retrying the attempt can safely replay the prompt. Concrete + * tool-instance policy, async work, committed delivery, spawned sessions, and + * cron writes all contribute side-effect evidence. + */ +export function buildAttemptReplayMetadata( + params: ReplayMetadataAttempt, +): EmbeddedRunAttemptResult["replayMetadata"] { + const hadUnsafeTools = params.toolMetas.some((entry) => entry.replaySafe !== true); + const hadAsyncStartedTool = params.toolMetas.some((t) => t.asyncStarted === true); + const hadPotentialSideEffects = + hadUnsafeTools || + hadAsyncStartedTool || + hasMessagingToolDeliveryEvidence(params) || + hasAcceptedSessionSpawn(params.acceptedSessionSpawns) || + (params.successfulCronAdds ?? 0) > 0; + return { + hadPotentialSideEffects, + replaySafe: !hadPotentialSideEffects, + }; +} + +type TerminalAttemptState = Pick< + EmbeddedRunAttemptResult, + | "clientToolCalls" + | "yieldDetected" + | "didSendDeterministicApprovalPrompt" + | "heartbeatToolResponse" + | "lastToolError" + | "toolMediaUrls" + | "toolAudioAsVoice" + | "toolTrustedLocalMedia" + | "hasToolMediaBlockReply" + | "didDeliverSourceReplyViaMessageTool" + | "messagingToolSourceReplyPayloads" + | "successfulCronAdds" +> & + Partial< + Pick< + EmbeddedRunAttemptResult, + | "acceptedSessionSpawns" + | "messagingToolSentTexts" + | "messagingToolSentMediaUrls" + | "messagingToolSentTargets" + > + > & { + toolMetas?: readonly { asyncStarted?: boolean }[]; + }; + +export function hasAttemptTerminalState(attempt: TerminalAttemptState): boolean { + return Boolean( + attempt.clientToolCalls || + attempt.yieldDetected || + attempt.didSendDeterministicApprovalPrompt || + attempt.heartbeatToolResponse || + attempt.lastToolError || + attempt.toolMediaUrls?.some((url) => url.trim().length > 0) || + attempt.toolAudioAsVoice || + attempt.toolTrustedLocalMedia || + attempt.hasToolMediaBlockReply || + attempt.didDeliverSourceReplyViaMessageTool || + attempt.messagingToolSourceReplyPayloads?.length || + hasCommittedMessagingToolDeliveryEvidence({ + messagingToolSentTexts: attempt.messagingToolSentTexts ?? [], + messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [], + messagingToolSentTargets: attempt.messagingToolSentTargets ?? [], + }) || + hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) || + hasAsyncActivity(attempt.toolMetas) || + (attempt.successfulCronAdds ?? 0) > 0, + ); +} + +export function hasAsyncActivity(toolMetas?: readonly { asyncStarted?: boolean }[]): boolean { + return (toolMetas ?? []).some((entry) => entry.asyncStarted === true); +} diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-name-resolution.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-name-resolution.ts new file mode 100644 index 000000000000..cac666215c3b --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-name-resolution.ts @@ -0,0 +1,223 @@ +/** Resolves provider-emitted tool names against the live callable set. */ +import { normalizeLowercaseStringOrEmpty } from "../../../../packages/normalization-core/src/string-coerce.js"; +import { normalizeStringEntries } from "../../../../packages/normalization-core/src/string-normalization.js"; +import { normalizeToolPolicyName } from "../../tool-policy.js"; + +function resolveCaseInsensitiveAllowedToolName( + rawName: string, + allowedToolNames?: Set, +): string | null { + if (!allowedToolNames || allowedToolNames.size === 0) { + return null; + } + const folded = normalizeLowercaseStringOrEmpty(rawName); + let caseInsensitiveMatch: string | null = null; + for (const name of allowedToolNames) { + if (normalizeLowercaseStringOrEmpty(name) !== folded) { + continue; + } + if (caseInsensitiveMatch && caseInsensitiveMatch !== name) { + return null; + } + caseInsensitiveMatch = name; + } + return caseInsensitiveMatch; +} + +function resolveExactAllowedToolName( + rawName: string, + allowedToolNames?: Set, +): string | null { + if (!allowedToolNames || allowedToolNames.size === 0) { + return null; + } + if (allowedToolNames.has(rawName)) { + return rawName; + } + const normalized = normalizeToolPolicyName(rawName); + if (allowedToolNames.has(normalized)) { + return normalized; + } + return ( + resolveCaseInsensitiveAllowedToolName(rawName, allowedToolNames) ?? + resolveCaseInsensitiveAllowedToolName(normalized, allowedToolNames) + ); +} + +function buildStructuredToolNameCandidates(rawName: string): string[] { + const trimmed = rawName.trim(); + if (!trimmed) { + return []; + } + + const candidates: string[] = []; + const seen = new Set(); + const addCandidate = (value: string) => { + const candidate = value.trim(); + if (!candidate || seen.has(candidate)) { + return; + } + seen.add(candidate); + candidates.push(candidate); + }; + + addCandidate(trimmed); + addCandidate(normalizeToolPolicyName(trimmed)); + const structuredSeeds = [trimmed]; + + const xmlFragmentOffset = ['"', "'", "<"] + .map((separator) => trimmed.indexOf(separator)) + .filter((offset) => offset > 0) + .reduce( + (earliest, offset) => (earliest === undefined || offset < earliest ? offset : earliest), + undefined, + ); + if (xmlFragmentOffset !== undefined) { + const prefix = trimmed.slice(0, xmlFragmentOffset); + addCandidate(prefix); + addCandidate(normalizeToolPolicyName(prefix)); + structuredSeeds.push(prefix); + } + + for (const seed of structuredSeeds) { + const normalizedDelimiter = seed.replace(/\//g, "."); + addCandidate(normalizedDelimiter); + addCandidate(normalizeToolPolicyName(normalizedDelimiter)); + + const segments = normalizeStringEntries(normalizedDelimiter.split(".")); + if (segments.length > 1) { + for (let index = 1; index < segments.length; index += 1) { + const suffix = segments.slice(index).join("."); + addCandidate(suffix); + addCandidate(normalizeToolPolicyName(suffix)); + } + } + } + + return candidates; +} + +function resolveStructuredAllowedToolName( + rawName: string, + allowedToolNames?: Set, +): string | null { + if (!allowedToolNames || allowedToolNames.size === 0) { + return null; + } + + const candidateNames = buildStructuredToolNameCandidates(rawName); + for (const candidate of candidateNames) { + if (allowedToolNames.has(candidate)) { + return candidate; + } + } + + for (const candidate of candidateNames) { + const caseInsensitiveMatch = resolveCaseInsensitiveAllowedToolName(candidate, allowedToolNames); + if (caseInsensitiveMatch) { + return caseInsensitiveMatch; + } + } + + return null; +} + +function inferToolNameFromToolCallId( + rawId: string | undefined, + allowedToolNames?: Set, +): string | null { + if (!rawId || !allowedToolNames || allowedToolNames.size === 0) { + return null; + } + const id = rawId.trim(); + if (!id) { + return null; + } + + const candidateTokens = new Set(); + const addToken = (value: string) => { + const trimmed = value.trim(); + if (!trimmed) { + return; + } + candidateTokens.add(trimmed); + candidateTokens.add(trimmed.replace(/[:._/-]\d+$/, "")); + candidateTokens.add(trimmed.replace(/\d+$/, "")); + + const normalizedDelimiter = trimmed.replace(/\//g, "."); + candidateTokens.add(normalizedDelimiter); + candidateTokens.add(normalizedDelimiter.replace(/[:._-]\d+$/, "")); + candidateTokens.add(normalizedDelimiter.replace(/\d+$/, "")); + + for (const prefixPattern of [/^functions?[._-]?/i, /^tools?[._-]?/i]) { + const stripped = normalizedDelimiter.replace(prefixPattern, ""); + if (stripped !== normalizedDelimiter) { + candidateTokens.add(stripped); + candidateTokens.add(stripped.replace(/[:._-]\d+$/, "")); + candidateTokens.add(stripped.replace(/\d+$/, "")); + } + } + }; + + const preColon = id.split(":")[0] ?? id; + for (const seed of [id, preColon]) { + addToken(seed); + } + + let singleMatch: string | null = null; + for (const candidate of candidateTokens) { + const matched = resolveStructuredAllowedToolName(candidate, allowedToolNames); + if (!matched) { + continue; + } + if (singleMatch && singleMatch !== matched) { + return null; + } + singleMatch = matched; + } + + return singleMatch; +} + +function looksLikeMalformedToolNameCounter(rawName: string): boolean { + const normalizedDelimiter = rawName.trim().replace(/\//g, "."); + return ( + /^(?:functions?|tools?)[._-]?/i.test(normalizedDelimiter) && + /(?:[:._-]\d+|\d+)$/.test(normalizedDelimiter) + ); +} + +export function resolveToolCallName( + rawName: string, + allowedToolNames?: Set, + rawToolCallId?: string, + requireAllowed = false, +): string | null { + const trimmed = rawName.trim(); + if (!trimmed) { + return ( + inferToolNameFromToolCallId(rawToolCallId, allowedToolNames) ?? + (requireAllowed ? null : rawName) + ); + } + if (!allowedToolNames || allowedToolNames.size === 0) { + return trimmed; + } + + const exact = resolveExactAllowedToolName(trimmed, allowedToolNames); + if (exact) { + return exact; + } + const inferredFromName = inferToolNameFromToolCallId(trimmed, allowedToolNames); + if (inferredFromName) { + return inferredFromName; + } + + if (looksLikeMalformedToolNameCounter(trimmed)) { + return requireAllowed ? null : trimmed; + } + + return ( + resolveStructuredAllowedToolName(trimmed, allowedToolNames) ?? (requireAllowed ? null : trimmed) + ); +} diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.test.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.test.ts new file mode 100644 index 000000000000..25e07fe76519 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.test.ts @@ -0,0 +1,632 @@ +// Coverage for provider replay tool-call sanitization. + +import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; +import { describe, expect, it, vi } from "vitest"; +import { + sanitizeOpenAIResponsesReplayForStream, + sanitizeReplayToolCallIdsForStream, + shouldApplyReplayToolCallIdSanitizer, + wrapStreamFnSanitizeMalformedToolCalls, +} from "./attempt-tool-call-replay-sanitization.js"; + +type AssistantMessage = Extract; +type ToolResultMessage = Extract; +type FakeWrappedStream = { + result: () => Promise; + [Symbol.asyncIterator]: () => AsyncIterator; +}; + +function createFakeStream(params: { + events: unknown[]; + resultMessage: unknown; +}): FakeWrappedStream { + return { + async result() { + return params.resultMessage; + }, + [Symbol.asyncIterator]() { + return (async function* () { + for (const event of params.events) { + yield event; + } + })(); + }, + }; +} + +function requireAssistantMessage(message: AgentMessage | undefined): AssistantMessage { + if (!message || message.role !== "assistant") { + throw new Error(`expected assistant message, got ${message?.role ?? "missing"}`); + } + return message; +} + +function requireToolResultMessage(message: AgentMessage | undefined): ToolResultMessage { + if (!message || message.role !== "toolResult") { + throw new Error(`expected toolResult message, got ${message?.role ?? "missing"}`); + } + return message; +} + +function assistantToolUseSummaries(message: AgentMessage | undefined) { + const assistant = requireAssistantMessage(message); + return assistant.content.map((content) => { + const record = content as unknown as Record; + if (record.type !== "toolUse") { + throw new Error(`expected toolUse content, got ${String(record.type)}`); + } + return { type: record.type, id: record.id, name: record.name }; + }); +} + +function toolResultSummary(message: AgentMessage | undefined) { + const toolResult = requireToolResultMessage(message); + const record = toolResult as unknown as Record; + return { + role: toolResult.role, + toolCallId: toolResult.toolCallId, + toolUseId: record.toolUseId, + toolName: toolResult.toolName, + isError: toolResult.isError, + }; +} + +describe("sanitizeReplayToolCallIdsForStream", () => { + it("skips strict stream id sanitization when provider policy opts out", () => { + expect( + shouldApplyReplayToolCallIdSanitizer({ + sanitizeToolCallIds: false, + isOpenAIResponsesApi: false, + }), + ).toBe(false); + expect( + shouldApplyReplayToolCallIdSanitizer({ + sanitizeToolCallIds: true, + toolCallIdMode: "strict", + isOpenAIResponsesApi: false, + }), + ).toBe(true); + expect( + shouldApplyReplayToolCallIdSanitizer({ + sanitizeToolCallIds: true, + toolCallIdMode: "strict", + isOpenAIResponsesApi: true, + }), + ).toBe(false); + }); + + it("drops orphaned tool results after strict id sanitization", () => { + const messages: AgentMessage[] = [ + { + role: "toolResult", + toolCallId: "call_function_av7cbkigmk7x1", + toolUseId: "call_function_av7cbkigmk7x1", + toolName: "read", + content: [{ type: "text", text: "stale" }], + isError: false, + } as never, + ]; + + expect( + sanitizeReplayToolCallIdsForStream({ + messages, + mode: "strict", + repairToolUseResultPairing: true, + }), + ).toStrictEqual([]); + }); + + it("keeps matched assistant and tool-result ids aligned", () => { + const rawId = "call_function_av7cbkigmk7x1"; + const messages: AgentMessage[] = [ + { + role: "assistant", + content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "read", + content: [{ type: "text", text: "ok" }], + isError: false, + } as never, + ]; + + const out = sanitizeReplayToolCallIdsForStream({ + messages, + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); + expect(assistantToolUseSummaries(out[0])).toEqual([ + { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, + ]); + expect(toolResultSummary(out[1])).toEqual({ + role: "toolResult", + toolCallId: "callfunctionav7cbkigmk7x1", + toolUseId: "callfunctionav7cbkigmk7x1", + toolName: "read", + isError: false, + }); + }); + + it("pairs repeated raw ids before assigning provider-safe occurrence ids", () => { + const rawId = "exec_0"; + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } }], + } as never, + { + role: "assistant", + content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } }], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "exec", + content: [{ type: "text", text: "second result" }], + isError: false, + } as never, + ], + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual([ + "assistant", + "toolResult", + "assistant", + "toolResult", + ]); + expect(assistantToolUseSummaries(out[0])).toEqual([ + { type: "toolUse", id: "exec0", name: "exec" }, + ]); + expect(toolResultSummary(out[1])).toMatchObject({ + toolCallId: "exec0", + isError: true, + }); + expect(assistantToolUseSummaries(out[2])).toEqual([ + { type: "toolUse", id: "exec02", name: "exec" }, + ]); + expect(toolResultSummary(out[3])).toEqual({ + role: "toolResult", + toolCallId: "exec02", + toolUseId: "exec02", + toolName: "exec", + isError: false, + }); + expect(requireToolResultMessage(out[3]).content).toEqual([ + { type: "text", text: "second result" }, + ]); + }); + + it("keeps same-turn repeated calls and results aligned after id rewriting", () => { + const rawId = "exec_0"; + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + content: [ + { type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } }, + { type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } }, + ], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "exec", + content: [{ type: "text", text: "first result" }], + isError: false, + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "exec", + content: [{ type: "text", text: "second result" }], + isError: false, + } as never, + ], + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]); + expect(assistantToolUseSummaries(out[0])).toEqual([ + { type: "toolUse", id: "exec0", name: "exec" }, + { type: "toolUse", id: "exec02", name: "exec" }, + ]); + expect(toolResultSummary(out[1])).toMatchObject({ + toolCallId: "exec0", + toolUseId: "exec0", + isError: false, + }); + expect(toolResultSummary(out[2])).toMatchObject({ + toolCallId: "exec02", + toolUseId: "exec02", + isError: false, + }); + }); + + it("preserves signed-thinking replay ids when requested by provider policy", () => { + const rawId = "call_1"; + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + content: [ + { type: "thinking", thinking: "internal", thinkingSignature: "sig_1" }, + { type: "toolUse", id: rawId, name: "read", input: { path: "." } }, + ], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "read", + content: [{ type: "text", text: "ok" }], + isError: false, + } as never, + ], + mode: "strict", + preserveReplaySafeThinkingToolCallIds: true, + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); + expect(requireAssistantMessage(out[0]).content[1]).toMatchObject({ + type: "toolUse", + id: "call_1", + name: "read", + }); + expect(toolResultSummary(out[1])).toEqual({ + role: "toolResult", + toolCallId: "call_1", + toolUseId: "call_1", + toolName: "read", + isError: false, + }); + }); + + it("synthesizes missing tool results after strict id sanitization", () => { + const rawId = "call_function_av7cbkigmk7x1"; + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + content: [ + { type: "toolUse", id: rawId, name: "read", input: { path: "." } }, + { type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } }, + ], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "read", + content: [{ type: "text", text: "ok" }], + isError: false, + } as never, + ], + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]); + expect(assistantToolUseSummaries(out[0])).toEqual([ + { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, + { type: "toolUse", id: "callmissing", name: "exec" }, + ]); + expect(toolResultSummary(out[1])).toEqual({ + role: "toolResult", + toolCallId: "callfunctionav7cbkigmk7x1", + toolUseId: "callfunctionav7cbkigmk7x1", + toolName: "read", + isError: false, + }); + expect(toolResultSummary(out[2])).toEqual({ + role: "toolResult", + toolCallId: "callmissing", + toolUseId: undefined, + toolName: "exec", + isError: true, + }); + }); + + it("synthesizes missing tool results when repair is enabled", () => { + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + content: [{ type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } }], + } as never, + ], + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); + expect(toolResultSummary(out[1])).toEqual({ + role: "toolResult", + toolCallId: "callmissing", + toolUseId: undefined, + toolName: "exec", + isError: true, + }); + }); + + it("keeps real tool results for aborted assistant spans", () => { + const rawId = "call_function_av7cbkigmk7x1"; + const out = sanitizeReplayToolCallIdsForStream({ + messages: [ + { + role: "assistant", + stopReason: "aborted", + content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }], + } as never, + { + role: "toolResult", + toolCallId: rawId, + toolUseId: rawId, + toolName: "read", + content: [{ type: "text", text: "partial" }], + isError: false, + } as never, + { + role: "user", + content: [{ type: "text", text: "retry" }], + } as never, + ], + mode: "strict", + repairToolUseResultPairing: true, + }); + + expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "user"]); + expect(requireAssistantMessage(out[0]).stopReason).toBe("aborted"); + expect(assistantToolUseSummaries(out[0])).toEqual([ + { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, + ]); + expect(toolResultSummary(out[1])).toEqual({ + role: "toolResult", + toolCallId: "callfunctionav7cbkigmk7x1", + toolUseId: "callfunctionav7cbkigmk7x1", + toolName: "read", + isError: false, + }); + }); +}); + +describe("wrapStreamFnSanitizeMalformedToolCalls", () => { + it("keeps valid non-Responses replay inputs pass-through", () => { + const messages: AgentMessage[] = [ + { + role: "assistant", + stopReason: "toolUse", + content: [ + { + type: "toolCall", + id: "call_1", + name: "image_generate", + arguments: { prompt: "QA lighthouse" }, + }, + ], + } as never, + ]; + const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) => + createFakeStream({ + events: [], + resultMessage: { role: "assistant", content: "ok" }, + }), + ); + const wrapped = wrapStreamFnSanitizeMalformedToolCalls( + baseFn as never, + new Set(["image_generate"]), + undefined, + "openai", + ); + + void wrapped({ api: "openai" } as never, { messages } as never, {} as never); + + const forwardedContext = baseFn.mock.calls[0]?.[1] as { + messages?: AgentMessage[]; + }; + expect(forwardedContext.messages).toBe(messages); + }); + + it("repairs OpenAI Responses pairing even when replay inputs do not change", () => { + const messages: AgentMessage[] = [ + { + role: "assistant", + stopReason: "toolUse", + content: [ + { + type: "toolCall", + id: "call_mock_image_generate_2", + name: "image_generate", + arguments: { prompt: "QA lighthouse" }, + }, + ], + } as never, + { + role: "assistant", + stopReason: "stop", + content: "Worked: the QA lighthouse image completed.", + } as never, + ]; + const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) => + createFakeStream({ + events: [], + resultMessage: { role: "assistant", content: "ok" }, + }), + ); + const wrapped = wrapStreamFnSanitizeMalformedToolCalls( + baseFn as never, + new Set(["image_generate"]), + undefined, + "openai", + ); + + void wrapped({ api: "openai-responses" } as never, { messages } as never, {} as never); + + const forwardedContext = baseFn.mock.calls[0]?.[1] as { + messages?: AgentMessage[]; + }; + expect(forwardedContext.messages?.map((message) => message.role)).toEqual([ + "assistant", + "toolResult", + "assistant", + ]); + expect(forwardedContext.messages?.[1]).toMatchObject({ + role: "toolResult", + toolCallId: "call_mock_image_generate_2", + toolName: "image_generate", + isError: true, + content: [{ type: "text", text: "aborted" }], + }); + }); +}); + +describe("sanitizeOpenAIResponsesReplayForStream", () => { + it("normalizes live responses continuations before pi-ai splits ids", () => { + const longCallId = `call_${"x".repeat(120)}`; + const longItemId = `notfc_${"y".repeat(120)}`; + const rawToolCallId = `${longCallId}|${longItemId}`; + const messages: AgentMessage[] = [ + { + role: "assistant", + content: [{ type: "toolCall", id: rawToolCallId, name: "noop", arguments: {} }], + } as never, + { + role: "toolResult", + toolCallId: rawToolCallId, + toolName: "noop", + content: [{ type: "text", text: "ok" }], + isError: false, + } as never, + ]; + + const out = sanitizeOpenAIResponsesReplayForStream(messages); + const assistant = out[0] as Extract; + const toolCall = assistant.content.find( + (block) => + Boolean(block) && + typeof block === "object" && + (block as { type?: unknown }).type === "toolCall" && + typeof (block as { id?: unknown }).id === "string", + ) as { id: string } | undefined; + + expect(toolCall?.id).toMatch(/^call_[A-Za-z0-9_-]{1,59}$/); + expect(toolCall?.id).not.toBe(rawToolCallId); + expect(toolCall?.id).not.toContain("|"); + expect((out[1] as Extract).toolCallId).toBe(toolCall?.id); + }); + + it("preserves canonical same-model reasoning pairs", () => { + const messages: AgentMessage[] = [ + { + role: "assistant", + content: [ + { + type: "thinking", + thinking: "internal", + thinkingSignature: JSON.stringify({ id: "rs_123", type: "reasoning" }), + }, + { type: "toolCall", id: "call_123|fc_123", name: "noop", arguments: {} }, + ], + } as never, + { + role: "toolResult", + toolCallId: "call_123|fc_123", + toolName: "noop", + content: [{ type: "text", text: "ok" }], + isError: false, + } as never, + ]; + + expect(sanitizeOpenAIResponsesReplayForStream(messages)).toBe(messages); + }); + + it("repairs dangling OpenAI Responses tool calls from async resume replay", () => { + const messages: AgentMessage[] = [ + { + role: "user", + content: "Image generation check. Generate an image of a QA lighthouse.", + } as never, + { + role: "assistant", + stopReason: "toolUse", + content: [ + { + type: "toolCall", + id: "call_mock_image_generate_1", + name: "image_generate", + arguments: { prompt: "QA lighthouse" }, + }, + ], + } as never, + { + role: "toolResult", + toolCallId: "call_mock_image_generate_1", + toolName: "image_generate", + content: [{ type: "text", text: "Background task started for image generation." }], + isError: false, + } as never, + { + role: "custom", + content: "Image generation started; wait for completion.", + } as never, + { + role: "user", + content: "The image is ready for the original chat.", + } as never, + { + role: "assistant", + stopReason: "toolUse", + content: [ + { + type: "toolCall", + id: "call_mock_image_generate_2", + name: "image_generate", + arguments: { prompt: "QA lighthouse" }, + }, + ], + } as never, + { + role: "assistant", + stopReason: "stop", + content: "Worked: the QA lighthouse image completed.", + } as never, + ]; + + const out = sanitizeOpenAIResponsesReplayForStream(messages); + const danglingAssistant = out[5] as AssistantMessage; + const danglingToolCall = danglingAssistant.content.find( + (block) => + Boolean(block) && + typeof block === "object" && + (block as { type?: unknown }).type === "toolCall", + ) as { id?: string } | undefined; + const danglingResult = out[6] as Extract; + + expect(out.map((message) => message.role)).toEqual([ + "user", + "assistant", + "toolResult", + "custom", + "user", + "assistant", + "toolResult", + "assistant", + ]); + expect(danglingResult.toolCallId).toBe(danglingToolCall?.id); + expect(danglingResult.toolName).toBe("image_generate"); + expect(danglingResult.isError).toBe(true); + expect(danglingResult.content).toEqual([{ type: "text", text: "aborted" }]); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.ts new file mode 100644 index 000000000000..b5204813827f --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-replay-sanitization.ts @@ -0,0 +1,520 @@ +/** Sanitizes replayed tool calls and provider-specific transcript structure. */ +import { hasNonEmptyString as replayToolCallNonEmptyString } from "../../../../packages/normalization-core/src/string-coerce.js"; +import { + downgradeOpenAIFunctionCallReasoningPairs, + downgradeOpenAIReasoningBlocks, + normalizeOpenAIResponsesToolCallIds, + validateAnthropicTurns, + validateGeminiTurns, +} from "../../embedded-agent-helpers.js"; +import type { AgentMessage, StreamFn } from "../../runtime/index.js"; +import { sanitizeToolUseResultPairing } from "../../session-transcript-repair.js"; +import { + extractToolCallsFromAssistant, + extractToolResultIds, + sanitizeToolCallIdsForCloudCodeAssist, + type ToolCallIdMode, +} from "../../tool-call-id.js"; +import { shouldAllowProviderOwnedThinkingReplay } from "../../transcript-policy.js"; +import type { TranscriptPolicy } from "../../transcript-policy.js"; +import { isRunnerToolCallBlockType } from "./attempt-tool-call-block-type.js"; +import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js"; + +const REPLAY_TOOL_CALL_NAME_MAX_CHARS = 64; + +type ReplayToolCallBlock = { + type?: unknown; + id?: unknown; + name?: unknown; + input?: unknown; + arguments?: unknown; +}; + +type ReplayToolCallSanitizeReport = { + messages: AgentMessage[]; + droppedAssistantMessages: number; +}; + +type AnthropicToolResultContentBlock = { + type?: unknown; + toolUseId?: unknown; + toolCallId?: unknown; + tool_use_id?: unknown; + tool_call_id?: unknown; +}; + +function isThinkingLikeReplayBlock(block: unknown): boolean { + if (!block || typeof block !== "object") { + return false; + } + const type = (block as { type?: unknown }).type; + return type === "thinking" || type === "redacted_thinking"; +} + +function isReplaySafeThinkingTurn(content: unknown[], allowedToolNames?: Set): boolean { + const seenToolCallIds = new Set(); + for (const block of content) { + if (!isReplayToolCallBlock(block)) { + continue; + } + const replayBlock = block; + const toolCallId = typeof replayBlock.id === "string" ? replayBlock.id.trim() : ""; + if (!replayToolCallHasInput(replayBlock) || !toolCallId || seenToolCallIds.has(toolCallId)) { + return false; + } + seenToolCallIds.add(toolCallId); + const rawName = typeof replayBlock.name === "string" ? replayBlock.name : ""; + const resolvedName = resolveReplayToolCallName(rawName, toolCallId, allowedToolNames); + if (!resolvedName || replayBlock.name !== resolvedName) { + return false; + } + } + return true; +} + +function isReplayToolCallBlock(block: unknown): block is ReplayToolCallBlock { + if (!block || typeof block !== "object") { + return false; + } + return isRunnerToolCallBlockType((block as { type?: unknown }).type); +} + +function replayToolCallHasInput(block: ReplayToolCallBlock): boolean { + const hasInput = "input" in block ? block.input !== undefined && block.input !== null : false; + const hasArguments = + "arguments" in block ? block.arguments !== undefined && block.arguments !== null : false; + return hasInput || hasArguments; +} + +function collectFollowingToolResults( + messages: AgentMessage[], + index: number, +): { ids: Set; displaced: boolean } { + const ids = new Set(); + let sawNonToolResult = false; + let displaced = false; + for (let nextIndex = index + 1; nextIndex < messages.length; nextIndex += 1) { + const message = messages[nextIndex]; + if (!message || typeof message !== "object") { + sawNonToolResult = true; + continue; + } + if (message.role === "assistant" && assistantTurnHasReplayToolCall(message)) { + break; + } + if (message.role === "toolResult") { + const resultIds = extractToolResultIds(message); + for (const id of resultIds) { + ids.add(id); + } + displaced ||= resultIds.length > 0 && sawNonToolResult; + continue; + } + sawNonToolResult = true; + } + return { ids, displaced }; +} + +function resolveReplayToolCallName( + rawName: string, + rawId: string, + allowedToolNames?: Set, +): string | null { + if (rawName.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS * 2) { + return null; + } + const normalized = resolveToolCallName(rawName, allowedToolNames, rawId, true); + if (!normalized) { + return null; + } + const trimmed = normalized.trim(); + if (!trimmed || trimmed.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS || /\s/.test(trimmed)) { + return null; + } + return trimmed; +} + +function sanitizeReplayToolCallInputs( + messages: AgentMessage[], + allowedToolNames?: Set, + allowProviderOwnedThinkingReplay?: boolean, +): ReplayToolCallSanitizeReport { + let changed = false; + let droppedAssistantMessages = 0; + const out: AgentMessage[] = []; + const preservedThinkingToolCallIds = new Set(); + const priorToolCallIds = new Set(); + + for (const [index, message] of messages.entries()) { + if (!message) { + changed = true; + continue; + } + if (typeof message !== "object" || message.role !== "assistant") { + out.push(message); + continue; + } + if (!Array.isArray(message.content)) { + out.push(message); + continue; + } + if ( + allowProviderOwnedThinkingReplay && + message.content.some((block) => isThinkingLikeReplayBlock(block)) && + message.content.some((block) => isReplayToolCallBlock(block)) + ) { + const replaySafeToolCalls = extractToolCallsFromAssistant(message); + const followingToolResults = collectFollowingToolResults(messages, index); + if ( + isReplaySafeThinkingTurn(message.content, allowedToolNames) && + replaySafeToolCalls.every( + (toolCall) => + !preservedThinkingToolCallIds.has(toolCall.id) && + (!followingToolResults.displaced || !priorToolCallIds.has(toolCall.id)) && + followingToolResults.ids.has(toolCall.id), + ) + ) { + for (const toolCall of replaySafeToolCalls) { + preservedThinkingToolCallIds.add(toolCall.id); + priorToolCallIds.add(toolCall.id); + } + changed ||= followingToolResults.displaced; + out.push(message); + } else { + changed = true; + droppedAssistantMessages += 1; + } + continue; + } + + const nextContent: typeof message.content = []; + let messageChanged = false; + + for (const block of message.content) { + if (!isReplayToolCallBlock(block)) { + nextContent.push(block); + continue; + } + const replayBlock = block as ReplayToolCallBlock; + + if (!replayToolCallHasInput(replayBlock) || !replayToolCallNonEmptyString(replayBlock.id)) { + changed = true; + messageChanged = true; + continue; + } + + const rawName = typeof replayBlock.name === "string" ? replayBlock.name : ""; + const resolvedName = resolveReplayToolCallName(rawName, replayBlock.id, allowedToolNames); + if (!resolvedName) { + changed = true; + messageChanged = true; + continue; + } + + if (replayBlock.name !== resolvedName) { + nextContent.push({ ...(block as object), name: resolvedName } as typeof block); + changed = true; + messageChanged = true; + continue; + } + nextContent.push(block); + } + + if (messageChanged) { + changed = true; + if (nextContent.length > 0) { + const nextMessage = { ...message, content: nextContent }; + for (const toolCall of extractToolCallsFromAssistant(nextMessage)) { + priorToolCallIds.add(toolCall.id); + } + out.push(nextMessage); + } else { + droppedAssistantMessages += 1; + } + continue; + } + + for (const toolCall of extractToolCallsFromAssistant(message)) { + priorToolCallIds.add(toolCall.id); + } + out.push(message); + } + + return { + messages: changed ? out : messages, + droppedAssistantMessages, + }; +} + +function extractAnthropicReplayToolResultIds(block: AnthropicToolResultContentBlock): string[] { + const ids: string[] = []; + for (const value of [block.toolUseId, block.toolCallId, block.tool_use_id, block.tool_call_id]) { + if (typeof value !== "string") { + continue; + } + const trimmed = value.trim(); + if (!trimmed || ids.includes(trimmed)) { + continue; + } + ids.push(trimmed); + } + return ids; +} + +function isSignedThinkingReplayAssistantSpan(message: AgentMessage | undefined): boolean { + if (!message || typeof message !== "object" || message.role !== "assistant") { + return false; + } + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) { + return false; + } + return ( + content.some((block) => isThinkingLikeReplayBlock(block)) && + content.some((block) => isReplayToolCallBlock(block)) + ); +} + +function sanitizeAnthropicReplayToolResults( + messages: AgentMessage[], + options?: { + disallowEmbeddedUserToolResultsForSignedThinkingReplay?: boolean; + }, +): AgentMessage[] { + let changed = false; + const out: AgentMessage[] = []; + const disallowEmbeddedUserToolResultsForSignedThinkingReplay = + options?.disallowEmbeddedUserToolResultsForSignedThinkingReplay === true; + + for (const [index, message] of messages.entries()) { + if (!message) { + changed = true; + continue; + } + if (typeof message !== "object" || message.role !== "user") { + out.push(message); + continue; + } + if (!Array.isArray(message.content)) { + out.push(message); + continue; + } + + const previous = messages[index - 1]; + const shouldStripEmbeddedToolResults = + disallowEmbeddedUserToolResultsForSignedThinkingReplay && + isSignedThinkingReplayAssistantSpan(previous); + const validToolUseIds = new Set(); + if (previous && typeof previous === "object" && previous.role === "assistant") { + const previousContent = (previous as { content?: unknown }).content; + if (Array.isArray(previousContent)) { + for (const block of previousContent) { + if (!block || typeof block !== "object") { + continue; + } + const typedBlock = block as { type?: unknown; id?: unknown }; + if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") { + continue; + } + const trimmedId = typedBlock.id.trim(); + if (trimmedId) { + validToolUseIds.add(trimmedId); + } + } + } + } + + const nextContent = message.content.filter((block) => { + if (!block || typeof block !== "object") { + return true; + } + const typedBlock = block as AnthropicToolResultContentBlock; + if (typedBlock.type !== "toolResult" && typedBlock.type !== "tool") { + return true; + } + if (shouldStripEmbeddedToolResults) { + changed = true; + return false; + } + const resultIds = extractAnthropicReplayToolResultIds(typedBlock); + if (resultIds.length === 0) { + changed = true; + return false; + } + return validToolUseIds.size > 0 && resultIds.some((id) => validToolUseIds.has(id)); + }); + + if (nextContent.length === message.content.length) { + out.push(message); + continue; + } + + changed = true; + if (nextContent.length > 0) { + out.push({ ...message, content: nextContent }); + continue; + } + + out.push({ + ...message, + content: [{ type: "text", text: "[tool results omitted]" }], + } as AgentMessage); + } + + return changed ? out : messages; +} + +function assistantTurnHasReplayToolCall(message: AgentMessage): boolean { + if (!message || typeof message !== "object" || message.role !== "assistant") { + return false; + } + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) { + return false; + } + return content.some((block) => isReplayToolCallBlock(block)); +} + +function stripTrailingAssistantPrefillTurns(messages: AgentMessage[]): AgentMessage[] { + let end = messages.length; + while (end > 0) { + const message = messages[end - 1]; + if (!message || typeof message !== "object" || message.role !== "assistant") { + break; + } + if (assistantTurnHasReplayToolCall(message)) { + break; + } + end -= 1; + } + return end === messages.length ? messages : messages.slice(0, end); +} + +type ReplayToolCallIdSanitizerDecision = { + sanitizeToolCallIds: boolean; + toolCallIdMode?: ToolCallIdMode; + isOpenAIResponsesApi: boolean; +}; + +/** Returns whether replayed tool-call ids should be sanitized for non-Responses providers. */ +export function shouldApplyReplayToolCallIdSanitizer( + params: ReplayToolCallIdSanitizerDecision, +): params is ReplayToolCallIdSanitizerDecision & { toolCallIdMode: ToolCallIdMode } { + return ( + params.sanitizeToolCallIds && Boolean(params.toolCallIdMode) && !params.isOpenAIResponsesApi + ); +} + +/** Rewrites replayed tool-call ids into provider-safe ids and optionally repairs result pairing. */ +export function sanitizeReplayToolCallIdsForStream(params: { + messages: AgentMessage[]; + mode: ToolCallIdMode; + allowedToolNames?: Set; + preserveNativeAnthropicToolUseIds?: boolean; + duplicateToolCallIdStyle?: "openai"; + preserveReplaySafeThinkingToolCallIds?: boolean; + repairToolUseResultPairing?: boolean; +}): AgentMessage[] { + const paired = params.repairToolUseResultPairing + ? sanitizeToolUseResultPairing(params.messages) + : params.messages; + return sanitizeToolCallIdsForCloudCodeAssist(paired, params.mode, { + preserveNativeAnthropicToolUseIds: params.preserveNativeAnthropicToolUseIds, + duplicateToolCallIdStyle: params.duplicateToolCallIdStyle, + preserveReplaySafeThinkingToolCallIds: params.preserveReplaySafeThinkingToolCallIds, + allowedToolNames: params.allowedToolNames, + }); +} + +/** Downgrades OpenAI Responses replay turns into the stream format expected by runtime callers. */ +export function sanitizeOpenAIResponsesReplayForStream(messages: AgentMessage[]): AgentMessage[] { + const repaired = sanitizeToolUseResultPairing(messages, { + erroredAssistantResultPolicy: "drop", + missingToolResultText: "aborted", + }); + return downgradeOpenAIFunctionCallReasoningPairs( + normalizeOpenAIResponsesToolCallIds(downgradeOpenAIReasoningBlocks(repaired)), + ); +} + +/** + * Sanitizes malformed replay tool calls before provider submission. The wrapper + * drops invalid assistant tool calls, repairs adjacent tool results when needed, + * strips trailing assistant prefill turns for strict providers, and revalidates + * Anthropic/Gemini transcripts after mutations. + */ +export function wrapStreamFnSanitizeMalformedToolCalls( + baseFn: StreamFn, + allowedToolNames?: Set, + transcriptPolicy?: Pick< + TranscriptPolicy, + "validateGeminiTurns" | "validateAnthropicTurns" | "preserveSignatures" | "dropThinkingBlocks" + >, + provider?: string | null, +): StreamFn { + return (model, context, options) => { + const ctx = context as unknown as { messages?: unknown }; + const messages = ctx?.messages; + if (!Array.isArray(messages)) { + return baseFn(model, context, options); + } + const allowProviderOwnedThinkingReplay = shouldAllowProviderOwnedThinkingReplay({ + modelApi: (model as { api?: unknown })?.api as string | null | undefined, + provider, + policy: { + validateAnthropicTurns: transcriptPolicy?.validateAnthropicTurns === true, + preserveSignatures: transcriptPolicy?.preserveSignatures === true, + dropThinkingBlocks: transcriptPolicy?.dropThinkingBlocks === true, + }, + }); + const sanitized = sanitizeReplayToolCallInputs( + messages as AgentMessage[], + allowedToolNames, + allowProviderOwnedThinkingReplay, + ); + const isOpenAIResponsesApi = + (model as { api?: unknown }).api === "openai-responses" || + (model as { api?: unknown }).api === "openai-chatgpt-responses" || + (model as { api?: unknown }).api === "azure-openai-responses"; + const replayInputsChanged = sanitized.messages !== messages; + let nextMessages = isOpenAIResponsesApi + ? sanitizeToolUseResultPairing(sanitized.messages, { + erroredAssistantResultPolicy: "drop", + missingToolResultText: "aborted", + }) + : replayInputsChanged + ? sanitizeToolUseResultPairing(sanitized.messages) + : sanitized.messages; + let strippedTrailingAssistantPrefill = false; + if (transcriptPolicy?.validateAnthropicTurns) { + nextMessages = sanitizeAnthropicReplayToolResults(nextMessages, { + disallowEmbeddedUserToolResultsForSignedThinkingReplay: allowProviderOwnedThinkingReplay, + }); + } + if (transcriptPolicy?.validateAnthropicTurns || transcriptPolicy?.validateGeminiTurns) { + const beforeStrip = nextMessages; + nextMessages = stripTrailingAssistantPrefillTurns(nextMessages); + strippedTrailingAssistantPrefill ||= nextMessages !== beforeStrip; + } + if (nextMessages === messages) { + return baseFn(model, context, options); + } + if ( + sanitized.droppedAssistantMessages > 0 || + transcriptPolicy?.validateAnthropicTurns || + strippedTrailingAssistantPrefill + ) { + if (transcriptPolicy?.validateGeminiTurns) { + nextMessages = validateGeminiTurns(nextMessages); + } + if (transcriptPolicy?.validateAnthropicTurns) { + nextMessages = validateAnthropicTurns(nextMessages); + } + } + const nextContext = { + ...(context as unknown as Record), + messages: nextMessages, + } as unknown; + return baseFn(model, nextContext as typeof context, options); + }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-stream-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-stream-normalization.ts new file mode 100644 index 000000000000..f78cfb7d21b3 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-stream-normalization.ts @@ -0,0 +1,362 @@ +/** Normalizes live streamed tool-call names, ids, and unknown-tool loops. */ +import { randomUUID } from "node:crypto"; +import { visitObjectContentBlocks } from "../../../shared/message-content-blocks.js"; +import type { StreamFn } from "../../runtime/index.js"; +import { normalizeToolPolicyName } from "../../tool-policy.js"; +import { isRunnerToolCallBlockType } from "./attempt-tool-call-block-type.js"; +import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js"; +import { wrapStreamObjectEvents } from "./stream-wrapper.js"; + +const BLANK_TOOL_CALL_NAME_DESCRIPTION = "blank tool name"; +type UnknownToolLoopGuardState = { + lastUnknownToolName?: string; + count: number; + countedMessages: WeakSet; +}; +type AssistantStream = Awaited>; + +function createStandaloneTextToolCallId(): string { + return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`; +} + +function normalizeToolCallIdsInMessage(message: unknown, fallbackIdByContentIndex: string[]): void { + if (!message || typeof message !== "object") { + return; + } + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) { + return; + } + + const usedIds = new Set(); + for (const block of content) { + if (!block || typeof block !== "object") { + continue; + } + const typedBlock = block as { type?: unknown; id?: unknown }; + if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") { + continue; + } + const trimmedId = typedBlock.id.trim(); + if (!trimmedId) { + continue; + } + usedIds.add(trimmedId); + } + + const assignedIds = new Set(); + for (const [contentIndex, block] of content.entries()) { + if (!block || typeof block !== "object") { + continue; + } + const typedBlock = block as { type?: unknown; id?: unknown }; + if (!isRunnerToolCallBlockType(typedBlock.type)) { + continue; + } + if (typeof typedBlock.id === "string") { + const trimmedId = typedBlock.id.trim(); + if (trimmedId) { + if (!assignedIds.has(trimmedId)) { + if (typedBlock.id !== trimmedId) { + typedBlock.id = trimmedId; + } + assignedIds.add(trimmedId); + continue; + } + } + } + + let fallbackId = fallbackIdByContentIndex[contentIndex]; + while (!fallbackId || usedIds.has(fallbackId) || assignedIds.has(fallbackId)) { + fallbackId = createStandaloneTextToolCallId(); + } + fallbackIdByContentIndex[contentIndex] = fallbackId; + typedBlock.id = fallbackId; + usedIds.add(fallbackId); + assignedIds.add(fallbackId); + } +} + +function trimWhitespaceFromToolCallNamesInMessage( + message: unknown, + allowedToolNames: Set | undefined, + fallbackIdByContentIndex: string[], +): void { + visitObjectContentBlocks(message, (block) => { + const typedBlock = block as { type?: unknown; name?: unknown; id?: unknown }; + if (!isRunnerToolCallBlockType(typedBlock.type)) { + return; + } + const rawId = typeof typedBlock.id === "string" ? typedBlock.id : undefined; + if (typeof typedBlock.name === "string") { + const normalized = resolveToolCallName(typedBlock.name, allowedToolNames, rawId); + if (normalized !== null && normalized !== typedBlock.name) { + typedBlock.name = normalized; + } + return; + } + const inferred = resolveToolCallName("", allowedToolNames, rawId); + if (inferred) { + typedBlock.name = inferred; + } + }); + normalizeToolCallIdsInMessage(message, fallbackIdByContentIndex); +} + +function classifyToolCallMessage( + message: unknown, + allowedToolNames?: Set, +): + | { kind: "none" } + | { kind: "allowed" } + | { kind: "incomplete" } + | { kind: "malformed"; toolName: string } + | { kind: "unknown"; toolName: string } { + if (!message || typeof message !== "object") { + return { kind: "none" }; + } + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content)) { + return { kind: "none" }; + } + + let unknownToolName: string | undefined; + let sawToolCall = false; + let sawAllowedToolCall = false; + let sawIncompleteToolCall = false; + let sawBlankStringToolCall = false; + const hasAllowedToolNames = Boolean(allowedToolNames && allowedToolNames.size > 0); + for (const block of content) { + if (!block || typeof block !== "object") { + continue; + } + const typedBlock = block as { type?: unknown; name?: unknown }; + if (!isRunnerToolCallBlockType(typedBlock.type)) { + continue; + } + sawToolCall = true; + const rawBlockName = typedBlock.name; + const hasStringName = typeof rawBlockName === "string"; + const rawName = hasStringName ? rawBlockName.trim() : ""; + if (!rawName) { + if (hasStringName) { + sawBlankStringToolCall = true; + } else { + sawIncompleteToolCall = true; + } + continue; + } + if (!hasAllowedToolNames) { + continue; + } + if (resolveToolCallName(rawName, allowedToolNames, undefined, true)) { + sawAllowedToolCall = true; + continue; + } + const normalizedUnknownToolName = normalizeToolPolicyName(rawName); + if (!unknownToolName) { + unknownToolName = normalizedUnknownToolName; + continue; + } + if (unknownToolName !== normalizedUnknownToolName) { + sawIncompleteToolCall = true; + } + } + + if (!sawToolCall) { + return { kind: "none" }; + } + if (!hasAllowedToolNames) { + return sawBlankStringToolCall + ? { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION } + : { kind: "none" }; + } + if (sawAllowedToolCall) { + return { kind: "allowed" }; + } + if (sawBlankStringToolCall && !sawIncompleteToolCall && unknownToolName === undefined) { + return { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION }; + } + if (sawIncompleteToolCall) { + return { kind: "incomplete" }; + } + return unknownToolName ? { kind: "unknown", toolName: unknownToolName } : { kind: "incomplete" }; +} + +function rewriteUnknownToolLoopMessage(message: unknown, toolName: string): void { + if (!message || typeof message !== "object") { + return; + } + (message as { content?: unknown }).content = [ + { + type: "text", + text: `I can't use the tool "${toolName}" here because it isn't available. I need to stop retrying it and answer without that tool.`, + }, + ]; +} + +function guardUnknownToolLoopInMessage( + message: unknown, + state: UnknownToolLoopGuardState, + params: { + allowedToolNames?: Set; + threshold?: number; + countAttempt: boolean; + resetOnAllowedTool?: boolean; + resetOnMissingUnknownTool?: boolean; + rewriteMalformedBlankToolName?: boolean; + }, +): boolean { + const toolCallState = classifyToolCallMessage(message, params.allowedToolNames); + if (toolCallState.kind === "allowed") { + if (params.resetOnAllowedTool === true) { + state.lastUnknownToolName = undefined; + state.count = 0; + } + return false; + } + if (toolCallState.kind === "malformed") { + if (params.rewriteMalformedBlankToolName === true) { + rewriteUnknownToolLoopMessage(message, toolCallState.toolName); + return true; + } + if (params.countAttempt && params.resetOnMissingUnknownTool !== false) { + state.lastUnknownToolName = undefined; + state.count = 0; + } + return false; + } + const threshold = params.threshold; + if (threshold === undefined || threshold <= 0) { + return false; + } + if (toolCallState.kind !== "unknown") { + if (params.countAttempt && params.resetOnMissingUnknownTool !== false) { + state.lastUnknownToolName = undefined; + state.count = 0; + } + return false; + } + const unknownToolName = toolCallState.toolName; + + if (!params.countAttempt) { + // Partial stream events can rewrite after the threshold, but only final + // messages advance the loop counter. + if (state.lastUnknownToolName === unknownToolName && state.count > threshold) { + rewriteUnknownToolLoopMessage(message, unknownToolName); + } + return false; + } + + if (message && typeof message === "object") { + if (state.countedMessages.has(message)) { + if (state.lastUnknownToolName === unknownToolName && state.count > threshold) { + rewriteUnknownToolLoopMessage(message, unknownToolName); + } + return true; + } + state.countedMessages.add(message); + } + + if (state.lastUnknownToolName === unknownToolName) { + state.count += 1; + } else { + state.lastUnknownToolName = unknownToolName; + state.count = 1; + } + + if (state.count > threshold) { + rewriteUnknownToolLoopMessage(message, unknownToolName); + } + return true; +} + +function wrapStreamTrimToolCallNames( + stream: AssistantStream, + allowedToolNames?: Set, + options?: { unknownToolThreshold?: number; state?: UnknownToolLoopGuardState }, +): AssistantStream { + const unknownToolGuardState = options?.state ?? { + count: 0, + countedMessages: new WeakSet(), + }; + // Provider-omitted ids are only message-local. Reuse one generated id per + // content position across this response's partial/final projections, while a + // later assistant response gets a fresh namespace and cannot alias it. + const fallbackIdByContentIndex: string[] = []; + let streamAttemptAlreadyCounted = false; + const originalResult = stream.result.bind(stream); + stream.result = async () => { + const message = await originalResult(); + trimWhitespaceFromToolCallNamesInMessage(message, allowedToolNames, fallbackIdByContentIndex); + guardUnknownToolLoopInMessage(message, unknownToolGuardState, { + allowedToolNames, + threshold: options?.unknownToolThreshold, + countAttempt: !streamAttemptAlreadyCounted, + resetOnAllowedTool: true, + rewriteMalformedBlankToolName: true, + }); + return message; + }; + + wrapStreamObjectEvents(stream, (event) => { + trimWhitespaceFromToolCallNamesInMessage( + event.partial, + allowedToolNames, + fallbackIdByContentIndex, + ); + trimWhitespaceFromToolCallNamesInMessage( + event.message, + allowedToolNames, + fallbackIdByContentIndex, + ); + if (event.message && typeof event.message === "object") { + const countedStreamAttempt = guardUnknownToolLoopInMessage( + event.message, + unknownToolGuardState, + { + allowedToolNames, + threshold: options?.unknownToolThreshold, + countAttempt: !streamAttemptAlreadyCounted, + resetOnAllowedTool: true, + resetOnMissingUnknownTool: false, + }, + ); + streamAttemptAlreadyCounted ||= countedStreamAttempt; + } + guardUnknownToolLoopInMessage(event.partial, unknownToolGuardState, { + allowedToolNames, + threshold: options?.unknownToolThreshold, + countAttempt: false, + }); + }); + + return stream; +} + +/** Normalizes streamed tool-call names and guards repeated unknown-tool loops. */ +export function wrapStreamFnTrimToolCallNames( + baseFn: StreamFn, + allowedToolNames?: Set, + guardOptions?: { unknownToolThreshold?: number }, +): StreamFn { + const unknownToolGuardState: UnknownToolLoopGuardState = { + count: 0, + countedMessages: new WeakSet(), + }; + return (model, context, streamOptions) => { + const maybeStream = baseFn(model, context, streamOptions); + if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) { + return Promise.resolve(maybeStream).then((stream) => + wrapStreamTrimToolCallNames(stream, allowedToolNames, { + unknownToolThreshold: guardOptions?.unknownToolThreshold, + state: unknownToolGuardState, + }), + ); + } + return wrapStreamTrimToolCallNames(maybeStream, allowedToolNames, { + unknownToolThreshold: guardOptions?.unknownToolThreshold, + state: unknownToolGuardState, + }); + }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-adjacent.test.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-adjacent.test.ts new file mode 100644 index 000000000000..f48d92304d9e --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-adjacent.test.ts @@ -0,0 +1,272 @@ +// Coverage for promoting standalone text tool calls into structured events. + +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { describe, expect, it, vi } from "vitest"; +import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js"; + +type FakeWrappedStream = { + result: () => Promise; + [Symbol.asyncIterator]: () => AsyncIterator; +}; + +function createFakeStream(params: { + events: unknown[]; + resultMessage: unknown; +}): FakeWrappedStream { + return { + async result() { + return params.resultMessage; + }, + [Symbol.asyncIterator]() { + return (async function* () { + for (const event of params.events) { + yield event; + } + })(); + }, + }; +} + +async function collectStreamEvents(stream: AsyncIterable): Promise { + // Drain streams to inspect generated tool-call events after wrapper mutation. + const events: unknown[] = []; + for await (const event of stream) { + events.push(event); + } + return events; +} + +const requireRecord = createRequireRecord("object", "expected-label"); + +describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => { + it("promotes serialized tool calls split across adjacent text blocks", async () => { + const resultMessage = { + role: "assistant", + content: [ + { type: "text", text: "[tool:exec]\n\n" }, + { type: "text", text: "pwd\n\n" }, + { type: "thinking", thinking: "Checking location." }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: "[tool:exec]\n\n" }, + { type: "text_delta", contentIndex: 1, delta: "pwd\n\n" }, + { + type: "thinking_delta", + contentIndex: 2, + delta: "Checking location.", + partial: { content: resultMessage.content }, + }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "thinking_delta", + "done", + ]); + expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); + expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0); + expect((result.content as Array>).map((block) => block.type)).toEqual([ + "toolCall", + "thinking", + ]); + expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ + name: "exec", + arguments: { command: "pwd" }, + }); + }); + + it("buffers case-insensitive tool-name prefixes until final promotion", async () => { + const rawToolText = [ + "[tool:read]", + "", + "src/index.ts", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: "[tool:rea" }, + { type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:rea".length) }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["Read"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(result.stopReason).toBe("toolUse"); + expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ + type: "toolCall", + name: "Read", + arguments: { path: "src/index.ts" }, + }); + }); + + it("buffers normalized alias tool-name prefixes until final promotion", async () => { + const rawToolText = [ + "[tool:bash]", + "", + "pwd", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: "[tool:ba" }, + { type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:ba".length) }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ + type: "toolCall", + name: "exec", + arguments: { command: "pwd" }, + }); + }); + + it.each([ + { + label: "case-insensitive name", + allowedToolName: "Read", + emittedToolName: "READ", + expectedToolName: "Read", + parameterName: "path", + parameterValue: "src/index.ts", + }, + { + label: "normalized alias", + allowedToolName: "exec", + emittedToolName: "bash", + expectedToolName: "exec", + parameterName: "command", + parameterValue: "pwd", + }, + ])( + "promotes $label XML consistently when the terminal reason is toolUse", + async ({ + allowedToolName, + emittedToolName, + expectedToolName, + parameterName, + parameterValue, + }) => { + const rawToolText = [ + ``, + ``, + parameterValue, + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "toolUse", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { type: "done", reason: "toolUse", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( + baseFn as never, + new Set([allowedToolName]), + ); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + const expectedArguments = { [parameterName]: parameterValue }; + const expectedContent = [ + { + type: "toolCall", + id: expect.stringMatching(/^call_[a-f0-9]{24}$/), + name: expectedToolName, + arguments: expectedArguments, + partialArgs: JSON.stringify(expectedArguments), + }, + ]; + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(requireRecord(events[2], "toolcall delta").delta).toBe( + JSON.stringify(expectedArguments), + ); + const doneEvent = requireRecord(events[4], "done event"); + expect(doneEvent.reason).toBe("toolUse"); + expect(requireRecord(doneEvent.message, "done message").content).toEqual(expectedContent); + expect(result).toMatchObject({ role: "assistant", stopReason: "toolUse" }); + expect(result.content).toEqual(expectedContent); + }, + ); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-buffering.test.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-buffering.test.ts new file mode 100644 index 000000000000..740a5d80252d --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-buffering.test.ts @@ -0,0 +1,219 @@ +// Coverage for promoting standalone text tool calls into structured events. + +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { describe, expect, it, vi } from "vitest"; +import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js"; + +type FakeWrappedStream = { + result: () => Promise; + [Symbol.asyncIterator]: () => AsyncIterator; +}; + +function createFakeStream(params: { + events: unknown[]; + resultMessage: unknown; +}): FakeWrappedStream { + return { + async result() { + return params.resultMessage; + }, + [Symbol.asyncIterator]() { + return (async function* () { + for (const event of params.events) { + yield event; + } + })(); + }, + }; +} + +async function collectStreamEvents(stream: AsyncIterable): Promise { + // Drain streams to inspect generated tool-call events after wrapper mutation. + const events: unknown[] = []; + for await (const event of stream) { + events.push(event); + } + return events; +} + +const requireRecord = createRequireRecord("object", "expected-label"); + +describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => { + it("keeps possible tool-call text buffered across interleaved non-text events", async () => { + const rawToolText = [ + "[tool:exec]", + "", + "pwd", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [ + { type: "thinking", thinking: "Need shell state." }, + { type: "text", text: rawToolText }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 1, delta: rawToolText }, + { + type: "thinking_delta", + contentIndex: 0, + delta: "Need shell state.", + partial: { + content: [ + { type: "thinking", thinking: "Need shell state." }, + { type: "text", text: rawToolText }, + ], + }, + }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "thinking_delta", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + const thinkingEvent = requireRecord(events[1], "thinking event"); + expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ + { type: "thinking", thinking: "Need shell state." }, + expect.objectContaining({ + type: "toolCall", + name: "exec", + arguments: { command: "pwd" }, + }), + ]); + expect(JSON.stringify(events)).not.toContain(rawToolText); + }); + + it("preserves interleaved event content indexes when buffered text is scrubbed first", async () => { + const rawToolText = [ + "[tool:exec]", + "", + "pwd", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [ + { type: "text", text: rawToolText }, + { type: "thinking", thinking: "Need shell state." }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { + type: "thinking_delta", + contentIndex: 1, + delta: "Need shell state.", + partial: { + content: [ + { type: "text", text: rawToolText }, + { type: "thinking", thinking: "Need shell state." }, + ], + }, + }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "thinking_delta", + "done", + ]); + const thinkingEvent = requireRecord(events[4], "thinking event"); + expect(thinkingEvent.contentIndex).toBe(1); + expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ + expect.objectContaining({ + type: "toolCall", + name: "exec", + arguments: { command: "pwd" }, + }), + { type: "thinking", thinking: "Need shell state." }, + ]); + expect(JSON.stringify(events)).not.toContain(rawToolText); + }); + + it("closes the underlying stream iterator when consumers stop early", async () => { + const returnIterator = vi.fn(async () => ({ done: true, value: undefined })); + const nextIterator = vi + .fn() + .mockResolvedValueOnce({ done: false, value: { type: "start", partial: { content: [] } } }) + .mockResolvedValue({ done: true, value: undefined }); + const baseFn = vi.fn(() => ({ + async result() { + return { role: "assistant", content: [], stopReason: "stop" }; + }, + [Symbol.asyncIterator]() { + return { + next: nextIterator, + return: returnIterator, + }; + }, + })); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + const iterator = stream[Symbol.asyncIterator](); + + expect(await iterator.next()).toEqual({ + done: false, + value: { type: "start", partial: { content: [] } }, + }); + await iterator.return?.(); + + expect(returnIterator).toHaveBeenCalledTimes(1); + }); + + it("fails closed on buffered known-tool text before terminal errors", async () => { + const rawToolText = "[tool:exec]"; + const errorEvent = { type: "error", error: new Error("stream failed") }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [{ type: "text_delta", contentIndex: 0, delta: rawToolText }, errorEvent], + resultMessage: { role: "assistant", content: [], stopReason: "stop" }, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events).toEqual([errorEvent]); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-sanitization.test.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-sanitization.test.ts new file mode 100644 index 000000000000..541753a446ca --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion-sanitization.test.ts @@ -0,0 +1,381 @@ +// Coverage for promoting standalone text tool calls into structured events. + +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { describe, expect, it, vi } from "vitest"; +import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js"; + +type FakeWrappedStream = { + result: () => Promise; + [Symbol.asyncIterator]: () => AsyncIterator; +}; + +function createFakeStream(params: { + events: unknown[]; + resultMessage: unknown; +}): FakeWrappedStream { + return { + async result() { + return params.resultMessage; + }, + [Symbol.asyncIterator]() { + return (async function* () { + for (const event of params.events) { + yield event; + } + })(); + }, + }; +} + +async function collectStreamEvents(stream: AsyncIterable): Promise { + // Drain streams to inspect generated tool-call events after wrapper mutation. + const events: unknown[] = []; + for await (const event of stream) { + events.push(event); + } + return events; +} + +const requireRecord = createRequireRecord("object", "expected-label"); + +describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => { + it("buffers split XML function markers until final promotion", async () => { + const rawToolText = [ + "", + "", + "pwd", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: "<" }, + { type: "text_delta", contentIndex: 0, delta: rawToolText.slice(1) }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + }); + + it.each([ + { + label: "bracketed XML text over the character cap", + marker: "[tool:exec]", + rawToolText: [ + "[tool:exec]", + "", + "x".repeat(256_001), + "", + "", + ].join("\n"), + }, + { + label: "zero-argument XML text over the byte cap", + marker: "", + rawToolText: `${"\u00a0".repeat(128_001)}`, + }, + { + label: "incomplete XML text over the byte cap", + marker: "", + rawToolText: `${"\u00a0".repeat(128_001)}`, + }, + ])("suppresses $label instead of flushing it", async ({ marker, rawToolText }) => { + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "start", partial: { content: [] } }, + { + type: "text_start", + contentIndex: 0, + partial: { content: [{ type: "text", text: "" }] }, + }, + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { + type: "thinking_delta", + contentIndex: 1, + delta: "still thinking", + partial: { + content: [ + { type: "text", text: rawToolText }, + { type: "thinking", thinking: "still thinking" }, + ], + }, + }, + { type: "text_end", contentIndex: 0, content: rawToolText }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "thinking_delta", + "done", + ]); + const thinkingEvent = requireRecord(events[1], "thinking event"); + expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ + { type: "text", text: "" }, + { type: "thinking", thinking: "still thinking" }, + ]); + const doneEvent = requireRecord(events[2], "done event"); + expect(doneEvent.reason).toBe("stop"); + expect(doneEvent.message).toMatchObject({ + role: "assistant", + content: [], + stopReason: "stop", + }); + expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" }); + expect(JSON.stringify(events)).not.toContain(marker); + expect(JSON.stringify(result)).not.toContain(marker); + }); + + it("scrubs split over-cap serialized XMLish text blocks from done messages", async () => { + const rawToolTextParts = [ + "[tool:exec]\n", + ["x".repeat(256_001), "", ""].join("\n"), + ]; + const resultMessage = { + role: "assistant", + content: rawToolTextParts.map((text) => ({ type: "text", text })), + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [{ type: "done", reason: "stop", message: resultMessage }], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(requireRecord(events[0], "done event").message).toMatchObject({ + role: "assistant", + content: [], + stopReason: "stop", + }); + expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" }); + expect(JSON.stringify(events)).not.toContain("[tool:exec]"); + expect(JSON.stringify(result)).not.toContain(""); + }); + + it("scrubs an over-cap whitespace-only XML body split into its own text block", async () => { + const resultMessage = { + role: "assistant", + content: [ + { type: "text", text: "" }, + { type: "text", text: "\u00a0".repeat(128_001) }, + { type: "text", text: "" }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [{ type: "done", reason: "stop", message: resultMessage }], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + const expectedMessage = { role: "assistant", content: [], stopReason: "stop" }; + + expect(events).toHaveLength(1); + const doneEvent = requireRecord(events[0], "done event"); + expect(doneEvent.type).toBe("done"); + expect(doneEvent.reason).toBe("stop"); + expect(doneEvent.message).toEqual(expectedMessage); + expect(result).toEqual(expectedMessage); + }); + + it.each(["error", "aborted"])( + "scrubs over-cap XML from stream.result() when stopReason is %s", + async (stopReason) => { + const rawToolText = `${"\u00a0".repeat(128_001)}`; + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason, + }; + const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( + baseFn as never, + new Set(["exec"]), + ); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + + expect(result).toEqual({ role: "assistant", content: [], stopReason }); + expect(JSON.stringify(result)).not.toContain(""); + }, + ); + + it("scrubs an incomplete named call from stream.result()", async () => { + const rawToolText = "SECRET"; + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + + expect(result).toEqual({ role: "assistant", content: [], stopReason: "stop" }); + }); + + it("preserves visible suffix text after an over-cap JSON tool payload", async () => { + const visibleSuffix = "Visible answer after oversized JSON."; + const rawText = [`[tool:exec] {"command":"${"x".repeat(256_001)}"}`, visibleSuffix].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawText }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "text_delta", + "done", + ]); + const textEvent = requireRecord(events[0], "text event"); + expect(String(textEvent.delta)).toBe(visibleSuffix); + expect(requireRecord(textEvent.partial, "text partial").content).toEqual([ + { type: "text", text: visibleSuffix }, + ]); + expect(JSON.stringify(events)).not.toContain("[tool:exec]"); + }); + + it("scrubs mixed under-cap calls from pre-iteration results and multi-block done events", async () => { + const rawCall = ""; + const visibleText = "Visible answer after the leaked call."; + const rawText = `${rawCall}\n${visibleText}`; + const createMessage = () => ({ + role: "assistant", + content: [ + { type: "text", text: rawCall }, + { type: "text", text: visibleText }, + ], + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawText }, + { type: "done", reason: "stop", message: createMessage() }, + ], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + const events = await collectStreamEvents(stream); + const expectedContent = [{ type: "text", text: visibleText }]; + + expect(result.content).toEqual(expectedContent); + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "text_delta", + "done", + ]); + expect(requireRecord(events[0], "text event").delta).toBe(visibleText); + expect( + requireRecord(requireRecord(events[1], "done event").message, "done message").content, + ).toEqual(expectedContent); + expect(JSON.stringify({ events, result })).not.toContain(""); + }); + + it("does not buffer normal prose that starts like a final answer", async () => { + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: "Finally, the audit is done." }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + + expect(events).toEqual([ + { type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." }, + { type: "done", reason: "stop", message: resultMessage }, + ]); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.test.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.test.ts new file mode 100644 index 000000000000..3bb1f44412f2 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.test.ts @@ -0,0 +1,481 @@ +// Coverage for promoting standalone text tool calls into structured events. + +import { expectDefined } from "@openclaw/normalization-core"; +import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; +import { describe, expect, it, vi } from "vitest"; +import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js"; + +type FakeWrappedStream = { + result: () => Promise; + [Symbol.asyncIterator]: () => AsyncIterator; +}; + +function createFakeStream(params: { + events: unknown[]; + resultMessage: unknown; +}): FakeWrappedStream { + return { + async result() { + return params.resultMessage; + }, + [Symbol.asyncIterator]() { + return (async function* () { + for (const event of params.events) { + yield event; + } + })(); + }, + }; +} + +async function collectStreamEvents(stream: AsyncIterable): Promise { + // Drain streams to inspect generated tool-call events after wrapper mutation. + const events: unknown[] = []; + for await (const event of stream) { + events.push(event); + } + return events; +} + +const requireRecord = createRequireRecord("object", "expected-label"); + +describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => { + it("preserves a fenced allowed-tool example in live and terminal output", async () => { + const parts = ["`", "``json\n", "[re", 'ad]\n{"path":"example.txt"}\n[/read]\n', "```"]; + const rawText = parts.join(""); + const createMessage = () => ({ + role: "assistant", + content: [{ type: "text", text: rawText }], + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + ...parts.map((delta) => ({ type: "text_delta", contentIndex: 0, delta })), + { type: "text_end", contentIndex: 0, content: rawText }, + { type: "done", reason: "stop", message: createMessage() }, + ], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = (await collectStreamEvents(stream)).map((event) => + requireRecord(event, "event"), + ); + const result = requireRecord(await stream.result(), "result message"); + + expect( + events + .filter((event) => event.type === "text_delta") + .map((event) => event.delta) + .join(""), + ).toBe(rawText); + expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); + expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([ + { type: "text", text: rawText }, + ]); + expect(result.content).toEqual([{ type: "text", text: rawText }]); + }); + + it("preserves a fenced example split across adjacent text blocks", async () => { + const textParts = [ + "```json\n", + ["[read]", '{"path":"example.txt"}', "[/read]", "\n"].join("\n"), + "```", + ]; + const content = textParts.map((text) => ({ type: "text", text })); + const createMessage = () => ({ + role: "assistant", + content, + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + ...textParts.flatMap((text, contentIndex) => [ + { type: "text_delta", contentIndex, delta: text }, + { type: "text_end", contentIndex, content: text }, + ]), + { type: "done", reason: "stop", message: createMessage() }, + ], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = (await collectStreamEvents(stream)).map((event) => + requireRecord(event, "event"), + ); + const result = requireRecord(await stream.result(), "result message"); + + expect( + events + .filter((event) => event.type === "text_delta") + .map((event) => event.delta) + .join(""), + ).toBe(textParts.join("")); + expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); + expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual(content); + expect(result.content).toEqual(content); + }); + + it("does not promote an indented code example from terminal output", async () => { + const rawText = [" [read]", ' {"path":"example.txt"}', " [/read]"].join("\n"); + const createMessage = () => ({ + role: "assistant", + content: [{ type: "text", text: rawText }], + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [{ type: "done", reason: "stop", message: createMessage() }], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = (await collectStreamEvents(stream)).map((event) => + requireRecord(event, "event"), + ); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); + expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([ + { type: "text", text: rawText }, + ]); + expect(result.content).toEqual([{ type: "text", text: rawText }]); + }); + + it("promotes standalone serialized parameter XML text to structured tool calls", async () => { + // Some providers emit tool calls as text blocks; promote only allowed tool + // names into structured toolCall content. + const rawToolText = [ + "[tool:exec]", + "", + "cat /proc/mounts 2>/dev/null | head -20", + "", + "", + "", + "", + "", + "find / -maxdepth 4 -type d 2>/dev/null | head -20", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [ + { type: "thinking", thinking: "Need to audit the mount." }, + { type: "text", text: rawToolText }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "start", partial: { content: [] } }, + { + type: "text_start", + contentIndex: 1, + partial: { content: [{ type: "text", text: "" }] }, + }, + { type: "text_delta", contentIndex: 1, delta: rawToolText }, + { type: "text_end", contentIndex: 1, content: rawToolText }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(requireRecord(events.at(-1), "done").reason).toBe("toolUse"); + expect(result.stopReason).toBe("toolUse"); + const content = result.content as Array>; + expect(content).toHaveLength(3); + expect(content[0]).toEqual({ type: "thinking", thinking: "Need to audit the mount." }); + expect(content[1]).toMatchObject({ + type: "toolCall", + name: "exec", + arguments: { command: "cat /proc/mounts 2>/dev/null | head -20" }, + partialArgs: '{"command":"cat /proc/mounts 2>/dev/null | head -20"}', + }); + expect(String(expectDefined(content[1], "content[1] test invariant").id)).toMatch( + /^call_[a-f0-9]{24}$/, + ); + expect(content[2]).toMatchObject({ + type: "toolCall", + name: "exec", + arguments: { command: "find / -maxdepth 4 -type d 2>/dev/null | head -20" }, + }); + }); + + it("reuses promoted ids across cloned result and done messages", async () => { + const rawToolText = ""; + const createMessage = () => ({ + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { type: "done", reason: "stop", message: createMessage() }, + ], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + const events = await collectStreamEvents(stream); + const resultToolCall = requireRecord((result.content as unknown[])[0], "result tool call"); + const done = requireRecord(events.at(-1), "done event"); + const doneMessage = requireRecord(done.message, "done message"); + const doneToolCall = requireRecord((doneMessage.content as unknown[])[0], "done tool call"); + const lifecycle = events + .map((event) => requireRecord(event, "event")) + .filter((event) => String(event.type).startsWith("toolcall_")); + + expect(doneToolCall.id).toBe(resultToolCall.id); + expect(lifecycle).toHaveLength(3); + for (const event of lifecycle) { + const partial = requireRecord(event.partial, "tool-call partial"); + expect(requireRecord((partial.content as unknown[])[0], "partial tool call").id).toBe( + resultToolCall.id, + ); + } + }); + + it("scrubs aggregate-over-cap call sequences before result promotion", async () => { + const rawToolText = "\n".repeat(9_500); + const createMessage = () => ({ + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }); + const baseFn = vi.fn(() => + createFakeStream({ + events: [{ type: "done", reason: "stop", message: createMessage() }], + resultMessage: createMessage(), + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + const events = await collectStreamEvents(stream); + + expect(new TextEncoder().encode(rawToolText).byteLength).toBeGreaterThan(256_000); + expect(result.content).toEqual([]); + expect(requireRecord(requireRecord(events[0], "done").message, "done message").content).toEqual( + [], + ); + expect(JSON.stringify({ events, result })).not.toContain(""); + }); + + it("promotes deferred directory tool names from the live callable set", async () => { + const rawToolText = [ + "[tool:hidden_catalog_tool]", + "", + "deferred", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [{ type: "text", text: rawToolText }], + stopReason: "stop", + }; + const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( + baseFn as never, + new Set(["tool_search", "tool_describe", "tool_call", "hidden_catalog_tool"]), + ); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const result = requireRecord(await stream.result(), "result message"); + + expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ + type: "toolCall", + name: "hidden_catalog_tool", + arguments: { value: "deferred" }, + }); + }); + + it("preserves content indexes when promoting text before thinking", async () => { + const rawToolText = [ + "[tool:exec]", + "", + "pwd", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [ + { type: "text", text: rawToolText }, + { type: "thinking", thinking: "Need the current directory." }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: rawToolText }, + { + type: "thinking_delta", + contentIndex: 1, + delta: "Need the current directory.", + partial: { + content: [ + { type: "text", text: rawToolText }, + { type: "thinking", thinking: "Need the current directory." }, + ], + }, + }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "thinking_delta", + "done", + ]); + expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); + expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0); + expect((result.content as Array>).map((block) => block.type)).toEqual([ + "toolCall", + "thinking", + ]); + }); + + it("preserves intervening thinking when promoting multiple text blocks", async () => { + const firstRawToolText = [ + "[tool:exec]", + "", + "pwd", + "", + "", + ].join("\n"); + const secondRawToolText = [ + "[tool:exec]", + "", + "whoami", + "", + "", + ].join("\n"); + const resultMessage = { + role: "assistant", + content: [ + { type: "text", text: firstRawToolText }, + { type: "thinking", thinking: "Need one more check." }, + { type: "text", text: secondRawToolText }, + ], + stopReason: "stop", + }; + const baseFn = vi.fn(() => + createFakeStream({ + events: [ + { type: "text_delta", contentIndex: 0, delta: firstRawToolText }, + { + type: "thinking_delta", + contentIndex: 1, + delta: "Need one more check.", + partial: { + content: [ + { type: "text", text: firstRawToolText }, + { type: "thinking", thinking: "Need one more check." }, + { type: "text", text: secondRawToolText }, + ], + }, + }, + { type: "text_delta", contentIndex: 2, delta: secondRawToolText }, + { type: "done", reason: "stop", message: resultMessage }, + ], + resultMessage, + }), + ); + const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); + const stream = (await Promise.resolve( + wrapped({} as never, {} as never, {} as never), + )) as FakeWrappedStream; + + const events = await collectStreamEvents(stream); + const result = requireRecord(await stream.result(), "result message"); + + expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "thinking_delta", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); + expect(requireRecord(events[1], "first toolcall start").contentIndex).toBe(0); + expect(requireRecord(events[5], "second toolcall start").contentIndex).toBe(2); + expect((result.content as Array>).map((block) => block.type)).toEqual([ + "toolCall", + "thinking", + "toolCall", + ]); + expect(requireRecord((result.content as unknown[])[0], "first tool call")).toMatchObject({ + name: "exec", + arguments: { command: "pwd" }, + }); + expect(requireRecord((result.content as unknown[])[2], "second tool call")).toMatchObject({ + name: "exec", + arguments: { command: "whoami" }, + }); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.ts b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.ts new file mode 100644 index 000000000000..688884c4a218 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/attempt-tool-call-text-promotion.ts @@ -0,0 +1,144 @@ +/** Promotes safe standalone text tool calls into structured stream events. */ +import { randomUUID } from "node:crypto"; +import { + createPromotedPlainTextToolCallEvents, + normalizePlainTextToolCallStreamEvents, + projectScrubbedPlainTextToolCallMessage, + projectStandalonePlainTextToolCallMessage as projectPlainTextToolCallMessage, + type PlainTextToolCallBlock, + type PlainTextToolCallMessageNormalization, + type PlainTextToolCallNameMatcher, +} from "../../../../packages/tool-call-repair/src/index.js"; +import { findCodeRegions } from "../../../shared/text/code-regions.js"; +import type { StreamFn } from "../../runtime/index.js"; +import { couldNormalizeToolNamePrefixToAllowedTool } from "../../tool-policy.js"; +import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js"; + +type AssistantStream = Awaited>; + +function createStandaloneTextToolCallId(): string { + return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`; +} + +function isRetainableNonVisibleBlock(block: Record): boolean { + return block.type === "thinking" || block.type === "redacted_thinking"; +} + +const STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS = new Set(["stop", "toolUse"]); + +function createStandaloneToolCallNameMatcher( + allowedToolNames: Set, +): PlainTextToolCallNameMatcher { + return { + hasExactName: (name) => Boolean(resolveToolCallName(name, allowedToolNames, undefined, true)), + hasNamePrefix: (prefix) => couldNormalizeToolNamePrefixToAllowedTool(prefix, allowedToolNames), + }; +} + +function wrapStreamPromoteStandaloneTextToolCalls( + stream: AssistantStream, + allowedToolNames: Set, +): AssistantStream { + const matcher = createStandaloneToolCallNameMatcher(allowedToolNames); + const promotedIdBySource = new Map(); + const normalizeTerminalMessage = (params: { + allowPromotion: boolean; + message: unknown; + preserveEmptyTextBlocks?: boolean; + }): PlainTextToolCallMessageNormalization => { + const scrubbed = projectScrubbedPlainTextToolCallMessage({ + forceIncompleteCandidates: true, + matcher, + message: params.message, + preserveEmptyTextBlocks: params.preserveEmptyTextBlocks, + resolveProtectedRanges: findCodeRegions, + requireAssistantRole: true, + }); + if (scrubbed) { + return { kind: "scrubbed", ...scrubbed }; + } + if (!params.allowPromotion) { + return undefined; + } + let ordinal = 0; + const createStableToolCallBlock = ( + block: PlainTextToolCallBlock, + name: string, + ): Record => { + const sourceKey = `${ordinal}:${block.start}:${block.end}`; + ordinal += 1; + let id = promotedIdBySource.get(sourceKey); + if (!id) { + id = createStandaloneTextToolCallId(); + promotedIdBySource.set(sourceKey, id); + } + return { + type: "toolCall", + id, + name, + arguments: block.arguments, + partialArgs: JSON.stringify(block.arguments), + }; + }; + const promoted = projectPlainTextToolCallMessage({ + allowedStopReasons: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS, + allowedToolNames, + createToolCallBlock: createStableToolCallBlock, + isRetainableNonTextBlock: isRetainableNonVisibleBlock, + message: params.message, + requireAssistantRole: true, + resolveProtectedRanges: findCodeRegions, + resolveToolName: (name) => resolveToolCallName(name, allowedToolNames, undefined, true), + }); + return promoted ? { kind: "promoted", ...promoted } : undefined; + }; + + const originalResult = stream.result.bind(stream); + stream.result = async () => { + const message = await originalResult(); + const reason = + message && typeof message === "object" + ? (message as { stopReason?: unknown }).stopReason + : undefined; + return (normalizeTerminalMessage({ + allowPromotion: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS.has(reason), + message, + })?.message ?? message) as Awaited>; + }; + + const originalAsyncIterator = stream[Symbol.asyncIterator].bind(stream); + (stream as unknown as { [Symbol.asyncIterator]: () => AsyncIterator })[ + Symbol.asyncIterator + ] = async function* () { + const source = { + [Symbol.asyncIterator]: originalAsyncIterator, + } as AsyncIterable; + yield* normalizePlainTextToolCallStreamEvents(source, { + createPromotedToolCallEvents: createPromotedPlainTextToolCallEvents, + matcher, + normalizeTerminalMessage, + resolveProtectedRanges: findCodeRegions, + }); + }; + + return stream; +} + +/** Promotes standalone plain-text tool-call replies into structured toolCall blocks when safe. */ +export function wrapStreamFnPromoteStandaloneTextToolCalls( + baseFn: StreamFn, + allowedToolNames?: Set, +): StreamFn { + if (!allowedToolNames || allowedToolNames.size === 0) { + return baseFn; + } + return (model, context, streamOptions) => { + const maybeStream = baseFn(model, context, streamOptions); + if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) { + return Promise.resolve(maybeStream).then((stream) => + wrapStreamPromoteStandaloneTextToolCalls(stream, allowedToolNames), + ); + } + return wrapStreamPromoteStandaloneTextToolCalls(maybeStream, allowedToolNames); + }; +} diff --git a/src/agents/embedded-agent-runner/run/attempt.test.ts b/src/agents/embedded-agent-runner/run/attempt.test.ts index ff989e5fdd86..657c75b52cf8 100644 --- a/src/agents/embedded-agent-runner/run/attempt.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt.test.ts @@ -31,12 +31,10 @@ import { shouldWarnOnOrphanedUserRepair, } from "./attempt-prompt-helpers.js"; import { composeSystemPromptWithHookContext } from "./attempt-thread-helpers.js"; +import { wrapStreamFnSanitizeMalformedToolCalls } from "./attempt-tool-call-replay-sanitization.js"; +import { wrapStreamFnTrimToolCallNames } from "./attempt-tool-call-stream-normalization.js"; import { buildEmbeddedAttemptToolRunContext } from "./attempt-tool-run-context.js"; import { wrapStreamFnRepairMalformedToolCallArguments } from "./attempt.tool-call-argument-repair.js"; -import { - wrapStreamFnSanitizeMalformedToolCalls, - wrapStreamFnTrimToolCallNames, -} from "./attempt.tool-call-normalization.js"; const llmRuntime = { ...defaultLlmRuntime, diff --git a/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts b/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts deleted file mode 100644 index d0300df9b910..000000000000 --- a/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts +++ /dev/null @@ -1,1843 +0,0 @@ -// Coverage for normalizing tool calls before and during model replay. - -import { expectDefined } from "@openclaw/normalization-core"; -import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; -import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures"; -import { describe, expect, it, vi } from "vitest"; -import { - sanitizeOpenAIResponsesReplayForStream, - sanitizeReplayToolCallIdsForStream, - shouldApplyReplayToolCallIdSanitizer, - wrapStreamFnPromoteStandaloneTextToolCalls, - wrapStreamFnSanitizeMalformedToolCalls, -} from "./attempt.tool-call-normalization.js"; - -type AssistantMessage = Extract; -type ToolResultMessage = Extract; -type FakeWrappedStream = { - result: () => Promise; - [Symbol.asyncIterator]: () => AsyncIterator; -}; - -function createFakeStream(params: { - events: unknown[]; - resultMessage: unknown; -}): FakeWrappedStream { - return { - async result() { - return params.resultMessage; - }, - [Symbol.asyncIterator]() { - return (async function* () { - for (const event of params.events) { - yield event; - } - })(); - }, - }; -} - -async function collectStreamEvents(stream: AsyncIterable): Promise { - // Drain streams to inspect generated tool-call events after wrapper mutation. - const events: unknown[] = []; - for await (const event of stream) { - events.push(event); - } - return events; -} - -const requireRecord = createRequireRecord("object", "expected-label"); - -function requireAssistantMessage(message: AgentMessage | undefined): AssistantMessage { - if (!message || message.role !== "assistant") { - throw new Error(`expected assistant message, got ${message?.role ?? "missing"}`); - } - return message; -} - -function requireToolResultMessage(message: AgentMessage | undefined): ToolResultMessage { - if (!message || message.role !== "toolResult") { - throw new Error(`expected toolResult message, got ${message?.role ?? "missing"}`); - } - return message; -} - -function assistantToolUseSummaries(message: AgentMessage | undefined) { - // Replay sanitizer assertions compare stable id/name summaries instead of - // full provider-specific message payloads. - const assistant = requireAssistantMessage(message); - return assistant.content.map((content) => { - const record = content as unknown as Record; - if (record.type !== "toolUse") { - throw new Error(`expected toolUse content, got ${String(record.type)}`); - } - return { - type: record.type, - id: record.id, - name: record.name, - }; - }); -} - -function toolResultSummary(message: AgentMessage | undefined) { - const toolResult = requireToolResultMessage(message); - const record = toolResult as unknown as Record; - return { - role: toolResult.role, - toolCallId: toolResult.toolCallId, - toolUseId: record.toolUseId, - toolName: toolResult.toolName, - isError: toolResult.isError, - }; -} - -describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => { - it("preserves a fenced allowed-tool example in live and terminal output", async () => { - const parts = ["`", "``json\n", "[re", 'ad]\n{"path":"example.txt"}\n[/read]\n', "```"]; - const rawText = parts.join(""); - const createMessage = () => ({ - role: "assistant", - content: [{ type: "text", text: rawText }], - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - ...parts.map((delta) => ({ type: "text_delta", contentIndex: 0, delta })), - { type: "text_end", contentIndex: 0, content: rawText }, - { type: "done", reason: "stop", message: createMessage() }, - ], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = (await collectStreamEvents(stream)).map((event) => - requireRecord(event, "event"), - ); - const result = requireRecord(await stream.result(), "result message"); - - expect( - events - .filter((event) => event.type === "text_delta") - .map((event) => event.delta) - .join(""), - ).toBe(rawText); - expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); - expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([ - { type: "text", text: rawText }, - ]); - expect(result.content).toEqual([{ type: "text", text: rawText }]); - }); - - it("preserves a fenced example split across adjacent text blocks", async () => { - const textParts = [ - "```json\n", - ["[read]", '{"path":"example.txt"}', "[/read]", "\n"].join("\n"), - "```", - ]; - const content = textParts.map((text) => ({ type: "text", text })); - const createMessage = () => ({ - role: "assistant", - content, - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - ...textParts.flatMap((text, contentIndex) => [ - { type: "text_delta", contentIndex, delta: text }, - { type: "text_end", contentIndex, content: text }, - ]), - { type: "done", reason: "stop", message: createMessage() }, - ], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = (await collectStreamEvents(stream)).map((event) => - requireRecord(event, "event"), - ); - const result = requireRecord(await stream.result(), "result message"); - - expect( - events - .filter((event) => event.type === "text_delta") - .map((event) => event.delta) - .join(""), - ).toBe(textParts.join("")); - expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); - expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual(content); - expect(result.content).toEqual(content); - }); - - it("does not promote an indented code example from terminal output", async () => { - const rawText = [" [read]", ' {"path":"example.txt"}', " [/read]"].join("\n"); - const createMessage = () => ({ - role: "assistant", - content: [{ type: "text", text: rawText }], - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [{ type: "done", reason: "stop", message: createMessage() }], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = (await collectStreamEvents(stream)).map((event) => - requireRecord(event, "event"), - ); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false); - expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([ - { type: "text", text: rawText }, - ]); - expect(result.content).toEqual([{ type: "text", text: rawText }]); - }); - - it("promotes standalone serialized parameter XML text to structured tool calls", async () => { - // Some providers emit tool calls as text blocks; promote only allowed tool - // names into structured toolCall content. - const rawToolText = [ - "[tool:exec]", - "", - "cat /proc/mounts 2>/dev/null | head -20", - "", - "", - "", - "", - "", - "find / -maxdepth 4 -type d 2>/dev/null | head -20", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [ - { type: "thinking", thinking: "Need to audit the mount." }, - { type: "text", text: rawToolText }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "start", partial: { content: [] } }, - { - type: "text_start", - contentIndex: 1, - partial: { content: [{ type: "text", text: "" }] }, - }, - { type: "text_delta", contentIndex: 1, delta: rawToolText }, - { type: "text_end", contentIndex: 1, content: rawToolText }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - expect(requireRecord(events.at(-1), "done").reason).toBe("toolUse"); - expect(result.stopReason).toBe("toolUse"); - const content = result.content as Array>; - expect(content).toHaveLength(3); - expect(content[0]).toEqual({ type: "thinking", thinking: "Need to audit the mount." }); - expect(content[1]).toMatchObject({ - type: "toolCall", - name: "exec", - arguments: { command: "cat /proc/mounts 2>/dev/null | head -20" }, - partialArgs: '{"command":"cat /proc/mounts 2>/dev/null | head -20"}', - }); - expect(String(expectDefined(content[1], "content[1] test invariant").id)).toMatch( - /^call_[a-f0-9]{24}$/, - ); - expect(content[2]).toMatchObject({ - type: "toolCall", - name: "exec", - arguments: { command: "find / -maxdepth 4 -type d 2>/dev/null | head -20" }, - }); - }); - - it("reuses promoted ids across cloned result and done messages", async () => { - const rawToolText = ""; - const createMessage = () => ({ - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawToolText }, - { type: "done", reason: "stop", message: createMessage() }, - ], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - const events = await collectStreamEvents(stream); - const resultToolCall = requireRecord((result.content as unknown[])[0], "result tool call"); - const done = requireRecord(events.at(-1), "done event"); - const doneMessage = requireRecord(done.message, "done message"); - const doneToolCall = requireRecord((doneMessage.content as unknown[])[0], "done tool call"); - const lifecycle = events - .map((event) => requireRecord(event, "event")) - .filter((event) => String(event.type).startsWith("toolcall_")); - - expect(doneToolCall.id).toBe(resultToolCall.id); - expect(lifecycle).toHaveLength(3); - for (const event of lifecycle) { - const partial = requireRecord(event.partial, "tool-call partial"); - expect(requireRecord((partial.content as unknown[])[0], "partial tool call").id).toBe( - resultToolCall.id, - ); - } - }); - - it("scrubs aggregate-over-cap call sequences before result promotion", async () => { - const rawToolText = "\n".repeat(9_500); - const createMessage = () => ({ - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [{ type: "done", reason: "stop", message: createMessage() }], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - const events = await collectStreamEvents(stream); - - expect(new TextEncoder().encode(rawToolText).byteLength).toBeGreaterThan(256_000); - expect(result.content).toEqual([]); - expect(requireRecord(requireRecord(events[0], "done").message, "done message").content).toEqual( - [], - ); - expect(JSON.stringify({ events, result })).not.toContain(""); - }); - - it("promotes deferred directory tool names from the live callable set", async () => { - const rawToolText = [ - "[tool:hidden_catalog_tool]", - "", - "deferred", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( - baseFn as never, - new Set(["tool_search", "tool_describe", "tool_call", "hidden_catalog_tool"]), - ); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - - expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ - type: "toolCall", - name: "hidden_catalog_tool", - arguments: { value: "deferred" }, - }); - }); - - it("preserves content indexes when promoting text before thinking", async () => { - const rawToolText = [ - "[tool:exec]", - "", - "pwd", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [ - { type: "text", text: rawToolText }, - { type: "thinking", thinking: "Need the current directory." }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawToolText }, - { - type: "thinking_delta", - contentIndex: 1, - delta: "Need the current directory.", - partial: { - content: [ - { type: "text", text: rawToolText }, - { type: "thinking", thinking: "Need the current directory." }, - ], - }, - }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "thinking_delta", - "done", - ]); - expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); - expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0); - expect((result.content as Array>).map((block) => block.type)).toEqual([ - "toolCall", - "thinking", - ]); - }); - - it("preserves intervening thinking when promoting multiple text blocks", async () => { - const firstRawToolText = [ - "[tool:exec]", - "", - "pwd", - "", - "", - ].join("\n"); - const secondRawToolText = [ - "[tool:exec]", - "", - "whoami", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [ - { type: "text", text: firstRawToolText }, - { type: "thinking", thinking: "Need one more check." }, - { type: "text", text: secondRawToolText }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: firstRawToolText }, - { - type: "thinking_delta", - contentIndex: 1, - delta: "Need one more check.", - partial: { - content: [ - { type: "text", text: firstRawToolText }, - { type: "thinking", thinking: "Need one more check." }, - { type: "text", text: secondRawToolText }, - ], - }, - }, - { type: "text_delta", contentIndex: 2, delta: secondRawToolText }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "thinking_delta", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); - expect(requireRecord(events[1], "first toolcall start").contentIndex).toBe(0); - expect(requireRecord(events[5], "second toolcall start").contentIndex).toBe(2); - expect((result.content as Array>).map((block) => block.type)).toEqual([ - "toolCall", - "thinking", - "toolCall", - ]); - expect(requireRecord((result.content as unknown[])[0], "first tool call")).toMatchObject({ - name: "exec", - arguments: { command: "pwd" }, - }); - expect(requireRecord((result.content as unknown[])[2], "second tool call")).toMatchObject({ - name: "exec", - arguments: { command: "whoami" }, - }); - }); - - it("promotes serialized tool calls split across adjacent text blocks", async () => { - const resultMessage = { - role: "assistant", - content: [ - { type: "text", text: "[tool:exec]\n\n" }, - { type: "text", text: "pwd\n\n" }, - { type: "thinking", thinking: "Checking location." }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: "[tool:exec]\n\n" }, - { type: "text_delta", contentIndex: 1, delta: "pwd\n\n" }, - { - type: "thinking_delta", - contentIndex: 2, - delta: "Checking location.", - partial: { content: resultMessage.content }, - }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "thinking_delta", - "done", - ]); - expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1); - expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0); - expect((result.content as Array>).map((block) => block.type)).toEqual([ - "toolCall", - "thinking", - ]); - expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ - name: "exec", - arguments: { command: "pwd" }, - }); - }); - - it("buffers case-insensitive tool-name prefixes until final promotion", async () => { - const rawToolText = [ - "[tool:read]", - "", - "src/index.ts", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: "[tool:rea" }, - { type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:rea".length) }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["Read"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - expect(result.stopReason).toBe("toolUse"); - expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ - type: "toolCall", - name: "Read", - arguments: { path: "src/index.ts" }, - }); - }); - - it("buffers normalized alias tool-name prefixes until final promotion", async () => { - const rawToolText = [ - "[tool:bash]", - "", - "pwd", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: "[tool:ba" }, - { type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:ba".length) }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({ - type: "toolCall", - name: "exec", - arguments: { command: "pwd" }, - }); - }); - - it.each([ - { - label: "case-insensitive name", - allowedToolName: "Read", - emittedToolName: "READ", - expectedToolName: "Read", - parameterName: "path", - parameterValue: "src/index.ts", - }, - { - label: "normalized alias", - allowedToolName: "exec", - emittedToolName: "bash", - expectedToolName: "exec", - parameterName: "command", - parameterValue: "pwd", - }, - ])( - "promotes $label XML consistently when the terminal reason is toolUse", - async ({ - allowedToolName, - emittedToolName, - expectedToolName, - parameterName, - parameterValue, - }) => { - const rawToolText = [ - ``, - ``, - parameterValue, - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "toolUse", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawToolText }, - { type: "done", reason: "toolUse", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( - baseFn as never, - new Set([allowedToolName]), - ); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - const expectedArguments = { [parameterName]: parameterValue }; - const expectedContent = [ - { - type: "toolCall", - id: expect.stringMatching(/^call_[a-f0-9]{24}$/), - name: expectedToolName, - arguments: expectedArguments, - partialArgs: JSON.stringify(expectedArguments), - }, - ]; - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - expect(requireRecord(events[2], "toolcall delta").delta).toBe( - JSON.stringify(expectedArguments), - ); - const doneEvent = requireRecord(events[4], "done event"); - expect(doneEvent.reason).toBe("toolUse"); - expect(requireRecord(doneEvent.message, "done message").content).toEqual(expectedContent); - expect(result).toMatchObject({ role: "assistant", stopReason: "toolUse" }); - expect(result.content).toEqual(expectedContent); - }, - ); - - it("keeps possible tool-call text buffered across interleaved non-text events", async () => { - const rawToolText = [ - "[tool:exec]", - "", - "pwd", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [ - { type: "thinking", thinking: "Need shell state." }, - { type: "text", text: rawToolText }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 1, delta: rawToolText }, - { - type: "thinking_delta", - contentIndex: 0, - delta: "Need shell state.", - partial: { - content: [ - { type: "thinking", thinking: "Need shell state." }, - { type: "text", text: rawToolText }, - ], - }, - }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "thinking_delta", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - const thinkingEvent = requireRecord(events[1], "thinking event"); - expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ - { type: "thinking", thinking: "Need shell state." }, - expect.objectContaining({ - type: "toolCall", - name: "exec", - arguments: { command: "pwd" }, - }), - ]); - expect(JSON.stringify(events)).not.toContain(rawToolText); - }); - - it("preserves interleaved event content indexes when buffered text is scrubbed first", async () => { - const rawToolText = [ - "[tool:exec]", - "", - "pwd", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [ - { type: "text", text: rawToolText }, - { type: "thinking", thinking: "Need shell state." }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawToolText }, - { - type: "thinking_delta", - contentIndex: 1, - delta: "Need shell state.", - partial: { - content: [ - { type: "text", text: rawToolText }, - { type: "thinking", thinking: "Need shell state." }, - ], - }, - }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "thinking_delta", - "done", - ]); - const thinkingEvent = requireRecord(events[4], "thinking event"); - expect(thinkingEvent.contentIndex).toBe(1); - expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ - expect.objectContaining({ - type: "toolCall", - name: "exec", - arguments: { command: "pwd" }, - }), - { type: "thinking", thinking: "Need shell state." }, - ]); - expect(JSON.stringify(events)).not.toContain(rawToolText); - }); - - it("closes the underlying stream iterator when consumers stop early", async () => { - const returnIterator = vi.fn(async () => ({ done: true, value: undefined })); - const nextIterator = vi - .fn() - .mockResolvedValueOnce({ done: false, value: { type: "start", partial: { content: [] } } }) - .mockResolvedValue({ done: true, value: undefined }); - const baseFn = vi.fn(() => ({ - async result() { - return { role: "assistant", content: [], stopReason: "stop" }; - }, - [Symbol.asyncIterator]() { - return { - next: nextIterator, - return: returnIterator, - }; - }, - })); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - const iterator = stream[Symbol.asyncIterator](); - - expect(await iterator.next()).toEqual({ - done: false, - value: { type: "start", partial: { content: [] } }, - }); - await iterator.return?.(); - - expect(returnIterator).toHaveBeenCalledTimes(1); - }); - - it("fails closed on buffered known-tool text before terminal errors", async () => { - const rawToolText = "[tool:exec]"; - const errorEvent = { type: "error", error: new Error("stream failed") }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [{ type: "text_delta", contentIndex: 0, delta: rawToolText }, errorEvent], - resultMessage: { role: "assistant", content: [], stopReason: "stop" }, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events).toEqual([errorEvent]); - }); - - it("buffers split XML function markers until final promotion", async () => { - const rawToolText = [ - "", - "", - "pwd", - "", - "", - ].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: "<" }, - { type: "text_delta", contentIndex: 0, delta: rawToolText.slice(1) }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "toolcall_start", - "toolcall_delta", - "toolcall_end", - "done", - ]); - }); - - it.each([ - { - label: "bracketed XML text over the character cap", - marker: "[tool:exec]", - rawToolText: [ - "[tool:exec]", - "", - "x".repeat(256_001), - "", - "", - ].join("\n"), - }, - { - label: "zero-argument XML text over the byte cap", - marker: "", - rawToolText: `${"\u00a0".repeat(128_001)}`, - }, - { - label: "incomplete XML text over the byte cap", - marker: "", - rawToolText: `${"\u00a0".repeat(128_001)}`, - }, - ])("suppresses $label instead of flushing it", async ({ marker, rawToolText }) => { - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "start", partial: { content: [] } }, - { - type: "text_start", - contentIndex: 0, - partial: { content: [{ type: "text", text: "" }] }, - }, - { type: "text_delta", contentIndex: 0, delta: rawToolText }, - { - type: "thinking_delta", - contentIndex: 1, - delta: "still thinking", - partial: { - content: [ - { type: "text", text: rawToolText }, - { type: "thinking", thinking: "still thinking" }, - ], - }, - }, - { type: "text_end", contentIndex: 0, content: rawToolText }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "start", - "thinking_delta", - "done", - ]); - const thinkingEvent = requireRecord(events[1], "thinking event"); - expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([ - { type: "text", text: "" }, - { type: "thinking", thinking: "still thinking" }, - ]); - const doneEvent = requireRecord(events[2], "done event"); - expect(doneEvent.reason).toBe("stop"); - expect(doneEvent.message).toMatchObject({ - role: "assistant", - content: [], - stopReason: "stop", - }); - expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" }); - expect(JSON.stringify(events)).not.toContain(marker); - expect(JSON.stringify(result)).not.toContain(marker); - }); - - it("scrubs split over-cap serialized XMLish text blocks from done messages", async () => { - const rawToolTextParts = [ - "[tool:exec]\n", - ["x".repeat(256_001), "", ""].join("\n"), - ]; - const resultMessage = { - role: "assistant", - content: rawToolTextParts.map((text) => ({ type: "text", text })), - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [{ type: "done", reason: "stop", message: resultMessage }], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - - expect(requireRecord(events[0], "done event").message).toMatchObject({ - role: "assistant", - content: [], - stopReason: "stop", - }); - expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" }); - expect(JSON.stringify(events)).not.toContain("[tool:exec]"); - expect(JSON.stringify(result)).not.toContain(""); - }); - - it("scrubs an over-cap whitespace-only XML body split into its own text block", async () => { - const resultMessage = { - role: "assistant", - content: [ - { type: "text", text: "" }, - { type: "text", text: "\u00a0".repeat(128_001) }, - { type: "text", text: "" }, - ], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [{ type: "done", reason: "stop", message: resultMessage }], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - const result = requireRecord(await stream.result(), "result message"); - const expectedMessage = { role: "assistant", content: [], stopReason: "stop" }; - - expect(events).toHaveLength(1); - const doneEvent = requireRecord(events[0], "done event"); - expect(doneEvent.type).toBe("done"); - expect(doneEvent.reason).toBe("stop"); - expect(doneEvent.message).toEqual(expectedMessage); - expect(result).toEqual(expectedMessage); - }); - - it.each(["error", "aborted"])( - "scrubs over-cap XML from stream.result() when stopReason is %s", - async (stopReason) => { - const rawToolText = `${"\u00a0".repeat(128_001)}`; - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason, - }; - const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls( - baseFn as never, - new Set(["exec"]), - ); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - - expect(result).toEqual({ role: "assistant", content: [], stopReason }); - expect(JSON.stringify(result)).not.toContain(""); - }, - ); - - it("scrubs an incomplete named call from stream.result()", async () => { - const rawToolText = "SECRET"; - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawToolText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage })); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - - expect(result).toEqual({ role: "assistant", content: [], stopReason: "stop" }); - }); - - it("preserves visible suffix text after an over-cap JSON tool payload", async () => { - const visibleSuffix = "Visible answer after oversized JSON."; - const rawText = [`[tool:exec] {"command":"${"x".repeat(256_001)}"}`, visibleSuffix].join("\n"); - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: rawText }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawText }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "text_delta", - "done", - ]); - const textEvent = requireRecord(events[0], "text event"); - expect(String(textEvent.delta)).toBe(visibleSuffix); - expect(requireRecord(textEvent.partial, "text partial").content).toEqual([ - { type: "text", text: visibleSuffix }, - ]); - expect(JSON.stringify(events)).not.toContain("[tool:exec]"); - }); - - it("scrubs mixed under-cap calls from pre-iteration results and multi-block done events", async () => { - const rawCall = ""; - const visibleText = "Visible answer after the leaked call."; - const rawText = `${rawCall}\n${visibleText}`; - const createMessage = () => ({ - role: "assistant", - content: [ - { type: "text", text: rawCall }, - { type: "text", text: visibleText }, - ], - stopReason: "stop", - }); - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: rawText }, - { type: "done", reason: "stop", message: createMessage() }, - ], - resultMessage: createMessage(), - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const result = requireRecord(await stream.result(), "result message"); - const events = await collectStreamEvents(stream); - const expectedContent = [{ type: "text", text: visibleText }]; - - expect(result.content).toEqual(expectedContent); - expect(events.map((event) => requireRecord(event, "event").type)).toEqual([ - "text_delta", - "done", - ]); - expect(requireRecord(events[0], "text event").delta).toBe(visibleText); - expect( - requireRecord(requireRecord(events[1], "done event").message, "done message").content, - ).toEqual(expectedContent); - expect(JSON.stringify({ events, result })).not.toContain(""); - }); - - it("does not buffer normal prose that starts like a final answer", async () => { - const resultMessage = { - role: "assistant", - content: [{ type: "text", text: "Finally, the audit is done." }], - stopReason: "stop", - }; - const baseFn = vi.fn(() => - createFakeStream({ - events: [ - { type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." }, - { type: "done", reason: "stop", message: resultMessage }, - ], - resultMessage, - }), - ); - const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"])); - const stream = (await Promise.resolve( - wrapped({} as never, {} as never, {} as never), - )) as FakeWrappedStream; - - const events = await collectStreamEvents(stream); - - expect(events).toEqual([ - { type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." }, - { type: "done", reason: "stop", message: resultMessage }, - ]); - }); -}); - -describe("sanitizeReplayToolCallIdsForStream", () => { - it("skips strict stream id sanitization when provider policy opts out", () => { - expect( - shouldApplyReplayToolCallIdSanitizer({ - sanitizeToolCallIds: false, - isOpenAIResponsesApi: false, - }), - ).toBe(false); - expect( - shouldApplyReplayToolCallIdSanitizer({ - sanitizeToolCallIds: true, - toolCallIdMode: "strict", - isOpenAIResponsesApi: false, - }), - ).toBe(true); - expect( - shouldApplyReplayToolCallIdSanitizer({ - sanitizeToolCallIds: true, - toolCallIdMode: "strict", - isOpenAIResponsesApi: true, - }), - ).toBe(false); - }); - - it("drops orphaned tool results after strict id sanitization", () => { - const messages: AgentMessage[] = [ - { - role: "toolResult", - toolCallId: "call_function_av7cbkigmk7x1", - toolUseId: "call_function_av7cbkigmk7x1", - toolName: "read", - content: [{ type: "text", text: "stale" }], - isError: false, - } as never, - ]; - - expect( - sanitizeReplayToolCallIdsForStream({ - messages, - mode: "strict", - repairToolUseResultPairing: true, - }), - ).toStrictEqual([]); - }); - - it("keeps matched assistant and tool-result ids aligned", () => { - const rawId = "call_function_av7cbkigmk7x1"; - const messages: AgentMessage[] = [ - { - role: "assistant", - content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "read", - content: [{ type: "text", text: "ok" }], - isError: false, - } as never, - ]; - - const out = sanitizeReplayToolCallIdsForStream({ - messages, - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); - expect(assistantToolUseSummaries(out[0])).toEqual([ - { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, - ]); - expect(toolResultSummary(out[1])).toEqual({ - role: "toolResult", - toolCallId: "callfunctionav7cbkigmk7x1", - toolUseId: "callfunctionav7cbkigmk7x1", - toolName: "read", - isError: false, - }); - }); - - it("pairs repeated raw ids before assigning provider-safe occurrence ids", () => { - const rawId = "exec_0"; - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } }], - } as never, - { - role: "assistant", - content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } }], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "exec", - content: [{ type: "text", text: "second result" }], - isError: false, - } as never, - ], - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual([ - "assistant", - "toolResult", - "assistant", - "toolResult", - ]); - expect(assistantToolUseSummaries(out[0])).toEqual([ - { type: "toolUse", id: "exec0", name: "exec" }, - ]); - expect(toolResultSummary(out[1])).toMatchObject({ - toolCallId: "exec0", - isError: true, - }); - expect(assistantToolUseSummaries(out[2])).toEqual([ - { type: "toolUse", id: "exec02", name: "exec" }, - ]); - expect(toolResultSummary(out[3])).toEqual({ - role: "toolResult", - toolCallId: "exec02", - toolUseId: "exec02", - toolName: "exec", - isError: false, - }); - expect(requireToolResultMessage(out[3]).content).toEqual([ - { type: "text", text: "second result" }, - ]); - }); - - it("keeps same-turn repeated calls and results aligned after id rewriting", () => { - const rawId = "exec_0"; - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - content: [ - { type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } }, - { type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } }, - ], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "exec", - content: [{ type: "text", text: "first result" }], - isError: false, - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "exec", - content: [{ type: "text", text: "second result" }], - isError: false, - } as never, - ], - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]); - expect(assistantToolUseSummaries(out[0])).toEqual([ - { type: "toolUse", id: "exec0", name: "exec" }, - { type: "toolUse", id: "exec02", name: "exec" }, - ]); - expect(toolResultSummary(out[1])).toMatchObject({ - toolCallId: "exec0", - toolUseId: "exec0", - isError: false, - }); - expect(toolResultSummary(out[2])).toMatchObject({ - toolCallId: "exec02", - toolUseId: "exec02", - isError: false, - }); - }); - - it("preserves signed-thinking replay ids when requested by provider policy", () => { - const rawId = "call_1"; - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - content: [ - { type: "thinking", thinking: "internal", thinkingSignature: "sig_1" }, - { type: "toolUse", id: rawId, name: "read", input: { path: "." } }, - ], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "read", - content: [{ type: "text", text: "ok" }], - isError: false, - } as never, - ], - mode: "strict", - preserveReplaySafeThinkingToolCallIds: true, - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); - expect(requireAssistantMessage(out[0]).content[1]).toMatchObject({ - type: "toolUse", - id: "call_1", - name: "read", - }); - expect(toolResultSummary(out[1])).toEqual({ - role: "toolResult", - toolCallId: "call_1", - toolUseId: "call_1", - toolName: "read", - isError: false, - }); - }); - - it("synthesizes missing tool results after strict id sanitization", () => { - const rawId = "call_function_av7cbkigmk7x1"; - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - content: [ - { type: "toolUse", id: rawId, name: "read", input: { path: "." } }, - { type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } }, - ], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "read", - content: [{ type: "text", text: "ok" }], - isError: false, - } as never, - ], - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]); - expect(assistantToolUseSummaries(out[0])).toEqual([ - { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, - { type: "toolUse", id: "callmissing", name: "exec" }, - ]); - expect(toolResultSummary(out[1])).toEqual({ - role: "toolResult", - toolCallId: "callfunctionav7cbkigmk7x1", - toolUseId: "callfunctionav7cbkigmk7x1", - toolName: "read", - isError: false, - }); - expect(toolResultSummary(out[2])).toEqual({ - role: "toolResult", - toolCallId: "callmissing", - toolUseId: undefined, - toolName: "exec", - isError: true, - }); - }); - - it("synthesizes missing tool results when repair is enabled", () => { - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - content: [{ type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } }], - } as never, - ], - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]); - expect(toolResultSummary(out[1])).toEqual({ - role: "toolResult", - toolCallId: "callmissing", - toolUseId: undefined, - toolName: "exec", - isError: true, - }); - }); - - it("keeps real tool results for aborted assistant spans", () => { - const rawId = "call_function_av7cbkigmk7x1"; - const out = sanitizeReplayToolCallIdsForStream({ - messages: [ - { - role: "assistant", - stopReason: "aborted", - content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }], - } as never, - { - role: "toolResult", - toolCallId: rawId, - toolUseId: rawId, - toolName: "read", - content: [{ type: "text", text: "partial" }], - isError: false, - } as never, - { - role: "user", - content: [{ type: "text", text: "retry" }], - } as never, - ], - mode: "strict", - repairToolUseResultPairing: true, - }); - - expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "user"]); - expect(requireAssistantMessage(out[0]).stopReason).toBe("aborted"); - expect(assistantToolUseSummaries(out[0])).toEqual([ - { type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" }, - ]); - expect(toolResultSummary(out[1])).toEqual({ - role: "toolResult", - toolCallId: "callfunctionav7cbkigmk7x1", - toolUseId: "callfunctionav7cbkigmk7x1", - toolName: "read", - isError: false, - }); - }); -}); - -describe("wrapStreamFnSanitizeMalformedToolCalls", () => { - it("keeps valid non-Responses replay inputs pass-through", () => { - const messages: AgentMessage[] = [ - { - role: "assistant", - stopReason: "toolUse", - content: [ - { - type: "toolCall", - id: "call_1", - name: "image_generate", - arguments: { prompt: "QA lighthouse" }, - }, - ], - } as never, - ]; - const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) => - createFakeStream({ - events: [], - resultMessage: { role: "assistant", content: "ok" }, - }), - ); - const wrapped = wrapStreamFnSanitizeMalformedToolCalls( - baseFn as never, - new Set(["image_generate"]), - undefined, - "openai", - ); - - void wrapped({ api: "openai" } as never, { messages } as never, {} as never); - - const forwardedContext = baseFn.mock.calls[0]?.[1] as { - messages?: AgentMessage[]; - }; - expect(forwardedContext.messages).toBe(messages); - }); - - it("repairs OpenAI Responses pairing even when replay inputs do not change", () => { - const messages: AgentMessage[] = [ - { - role: "assistant", - stopReason: "toolUse", - content: [ - { - type: "toolCall", - id: "call_mock_image_generate_2", - name: "image_generate", - arguments: { prompt: "QA lighthouse" }, - }, - ], - } as never, - { - role: "assistant", - stopReason: "stop", - content: "Worked: the QA lighthouse image completed.", - } as never, - ]; - const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) => - createFakeStream({ - events: [], - resultMessage: { role: "assistant", content: "ok" }, - }), - ); - const wrapped = wrapStreamFnSanitizeMalformedToolCalls( - baseFn as never, - new Set(["image_generate"]), - undefined, - "openai", - ); - - void wrapped({ api: "openai-responses" } as never, { messages } as never, {} as never); - - const forwardedContext = baseFn.mock.calls[0]?.[1] as { - messages?: AgentMessage[]; - }; - expect(forwardedContext.messages?.map((message) => message.role)).toEqual([ - "assistant", - "toolResult", - "assistant", - ]); - expect(forwardedContext.messages?.[1]).toMatchObject({ - role: "toolResult", - toolCallId: "call_mock_image_generate_2", - toolName: "image_generate", - isError: true, - content: [{ type: "text", text: "aborted" }], - }); - }); -}); - -describe("sanitizeOpenAIResponsesReplayForStream", () => { - it("normalizes live responses continuations before pi-ai splits ids", () => { - const longCallId = `call_${"x".repeat(120)}`; - const longItemId = `notfc_${"y".repeat(120)}`; - const rawToolCallId = `${longCallId}|${longItemId}`; - const messages: AgentMessage[] = [ - { - role: "assistant", - content: [{ type: "toolCall", id: rawToolCallId, name: "noop", arguments: {} }], - } as never, - { - role: "toolResult", - toolCallId: rawToolCallId, - toolName: "noop", - content: [{ type: "text", text: "ok" }], - isError: false, - } as never, - ]; - - const out = sanitizeOpenAIResponsesReplayForStream(messages); - const assistant = out[0] as Extract; - const toolCall = assistant.content.find( - (block) => - Boolean(block) && - typeof block === "object" && - (block as { type?: unknown }).type === "toolCall" && - typeof (block as { id?: unknown }).id === "string", - ) as { id: string } | undefined; - - expect(toolCall?.id).toMatch(/^call_[A-Za-z0-9_-]{1,59}$/); - expect(toolCall?.id).not.toBe(rawToolCallId); - expect(toolCall?.id).not.toContain("|"); - expect((out[1] as Extract).toolCallId).toBe(toolCall?.id); - }); - - it("preserves canonical same-model reasoning pairs", () => { - const messages: AgentMessage[] = [ - { - role: "assistant", - content: [ - { - type: "thinking", - thinking: "internal", - thinkingSignature: JSON.stringify({ id: "rs_123", type: "reasoning" }), - }, - { type: "toolCall", id: "call_123|fc_123", name: "noop", arguments: {} }, - ], - } as never, - { - role: "toolResult", - toolCallId: "call_123|fc_123", - toolName: "noop", - content: [{ type: "text", text: "ok" }], - isError: false, - } as never, - ]; - - expect(sanitizeOpenAIResponsesReplayForStream(messages)).toBe(messages); - }); - - it("repairs dangling OpenAI Responses tool calls from async resume replay", () => { - const messages: AgentMessage[] = [ - { - role: "user", - content: "Image generation check. Generate an image of a QA lighthouse.", - } as never, - { - role: "assistant", - stopReason: "toolUse", - content: [ - { - type: "toolCall", - id: "call_mock_image_generate_1", - name: "image_generate", - arguments: { prompt: "QA lighthouse" }, - }, - ], - } as never, - { - role: "toolResult", - toolCallId: "call_mock_image_generate_1", - toolName: "image_generate", - content: [{ type: "text", text: "Background task started for image generation." }], - isError: false, - } as never, - { - role: "custom", - content: "Image generation started; wait for completion.", - } as never, - { - role: "user", - content: "The image is ready for the original chat.", - } as never, - { - role: "assistant", - stopReason: "toolUse", - content: [ - { - type: "toolCall", - id: "call_mock_image_generate_2", - name: "image_generate", - arguments: { prompt: "QA lighthouse" }, - }, - ], - } as never, - { - role: "assistant", - stopReason: "stop", - content: "Worked: the QA lighthouse image completed.", - } as never, - ]; - - const out = sanitizeOpenAIResponsesReplayForStream(messages); - const danglingAssistant = out[5] as AssistantMessage; - const danglingToolCall = danglingAssistant.content.find( - (block) => - Boolean(block) && - typeof block === "object" && - (block as { type?: unknown }).type === "toolCall", - ) as { id?: string } | undefined; - const danglingResult = out[6] as Extract; - - expect(out.map((message) => message.role)).toEqual([ - "user", - "assistant", - "toolResult", - "custom", - "user", - "assistant", - "toolResult", - "assistant", - ]); - expect(danglingResult.toolCallId).toBe(danglingToolCall?.id); - expect(danglingResult.toolName).toBe("image_generate"); - expect(danglingResult.isError).toBe(true); - expect(danglingResult.content).toEqual([{ type: "text", text: "aborted" }]); - }); -}); -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts b/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts deleted file mode 100644 index e375f3eb740f..000000000000 --- a/src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts +++ /dev/null @@ -1,1234 +0,0 @@ -/** - * Normalizes tool-call names, ids, and standalone text calls for providers. - */ -import { randomUUID } from "node:crypto"; -import { - hasNonEmptyString as replayToolCallNonEmptyString, - normalizeLowercaseStringOrEmpty, -} from "../../../../packages/normalization-core/src/string-coerce.js"; -import { normalizeStringEntries } from "../../../../packages/normalization-core/src/string-normalization.js"; -import { - createPromotedPlainTextToolCallEvents, - normalizePlainTextToolCallStreamEvents, - projectScrubbedPlainTextToolCallMessage, - projectStandalonePlainTextToolCallMessage as projectPlainTextToolCallMessage, - type PlainTextToolCallBlock, - type PlainTextToolCallMessageNormalization, - type PlainTextToolCallNameMatcher, -} from "../../../../packages/tool-call-repair/src/index.js"; -import { visitObjectContentBlocks } from "../../../shared/message-content-blocks.js"; -import { findCodeRegions } from "../../../shared/text/code-regions.js"; -import { - downgradeOpenAIFunctionCallReasoningPairs, - downgradeOpenAIReasoningBlocks, - normalizeOpenAIResponsesToolCallIds, - validateAnthropicTurns, - validateGeminiTurns, -} from "../../embedded-agent-helpers.js"; -import type { AgentMessage, StreamFn } from "../../runtime/index.js"; -import { sanitizeToolUseResultPairing } from "../../session-transcript-repair.js"; -import { - extractToolCallsFromAssistant, - extractToolResultIds, - sanitizeToolCallIdsForCloudCodeAssist, - type ToolCallIdMode, -} from "../../tool-call-id.js"; -import { - couldNormalizeToolNamePrefixToAllowedTool, - normalizeToolPolicyName, -} from "../../tool-policy.js"; -import { shouldAllowProviderOwnedThinkingReplay } from "../../transcript-policy.js"; -import type { TranscriptPolicy } from "../../transcript-policy.js"; -import { isRunnerToolCallBlockType } from "./attempt-tool-call-block-type.js"; -import { wrapStreamObjectEvents } from "./stream-wrapper.js"; - -const BLANK_TOOL_CALL_NAME_DESCRIPTION = "blank tool name"; - -type UnknownToolLoopGuardState = { - lastUnknownToolName?: string; - count: number; - countedMessages: WeakSet; -}; -type AssistantStream = Awaited>; - -function resolveCaseInsensitiveAllowedToolName( - rawName: string, - allowedToolNames?: Set, -): string | null { - if (!allowedToolNames || allowedToolNames.size === 0) { - return null; - } - const folded = normalizeLowercaseStringOrEmpty(rawName); - let caseInsensitiveMatch: string | null = null; - for (const name of allowedToolNames) { - if (normalizeLowercaseStringOrEmpty(name) !== folded) { - continue; - } - if (caseInsensitiveMatch && caseInsensitiveMatch !== name) { - return null; - } - caseInsensitiveMatch = name; - } - return caseInsensitiveMatch; -} - -function resolveExactAllowedToolName( - rawName: string, - allowedToolNames?: Set, -): string | null { - if (!allowedToolNames || allowedToolNames.size === 0) { - return null; - } - if (allowedToolNames.has(rawName)) { - return rawName; - } - const normalized = normalizeToolPolicyName(rawName); - if (allowedToolNames.has(normalized)) { - return normalized; - } - return ( - resolveCaseInsensitiveAllowedToolName(rawName, allowedToolNames) ?? - resolveCaseInsensitiveAllowedToolName(normalized, allowedToolNames) - ); -} - -function buildStructuredToolNameCandidates(rawName: string): string[] { - const trimmed = rawName.trim(); - if (!trimmed) { - return []; - } - - const candidates: string[] = []; - const seen = new Set(); - const addCandidate = (value: string) => { - const candidate = value.trim(); - if (!candidate || seen.has(candidate)) { - return; - } - seen.add(candidate); - candidates.push(candidate); - }; - - addCandidate(trimmed); - addCandidate(normalizeToolPolicyName(trimmed)); - const structuredSeeds = [trimmed]; - - const xmlFragmentOffset = ['"', "'", "<"] - .map((separator) => trimmed.indexOf(separator)) - .filter((offset) => offset > 0) - .reduce( - (earliest, offset) => (earliest === undefined || offset < earliest ? offset : earliest), - undefined, - ); - if (xmlFragmentOffset !== undefined) { - const prefix = trimmed.slice(0, xmlFragmentOffset); - addCandidate(prefix); - addCandidate(normalizeToolPolicyName(prefix)); - structuredSeeds.push(prefix); - } - - for (const seed of structuredSeeds) { - const normalizedDelimiter = seed.replace(/\//g, "."); - addCandidate(normalizedDelimiter); - addCandidate(normalizeToolPolicyName(normalizedDelimiter)); - - const segments = normalizeStringEntries(normalizedDelimiter.split(".")); - if (segments.length > 1) { - for (let index = 1; index < segments.length; index += 1) { - const suffix = segments.slice(index).join("."); - addCandidate(suffix); - addCandidate(normalizeToolPolicyName(suffix)); - } - } - } - - return candidates; -} - -function resolveStructuredAllowedToolName( - rawName: string, - allowedToolNames?: Set, -): string | null { - if (!allowedToolNames || allowedToolNames.size === 0) { - return null; - } - - const candidateNames = buildStructuredToolNameCandidates(rawName); - for (const candidate of candidateNames) { - if (allowedToolNames.has(candidate)) { - return candidate; - } - } - - for (const candidate of candidateNames) { - const caseInsensitiveMatch = resolveCaseInsensitiveAllowedToolName(candidate, allowedToolNames); - if (caseInsensitiveMatch) { - return caseInsensitiveMatch; - } - } - - return null; -} - -function inferToolNameFromToolCallId( - rawId: string | undefined, - allowedToolNames?: Set, -): string | null { - if (!rawId || !allowedToolNames || allowedToolNames.size === 0) { - return null; - } - const id = rawId.trim(); - if (!id) { - return null; - } - - const candidateTokens = new Set(); - const addToken = (value: string) => { - const trimmed = value.trim(); - if (!trimmed) { - return; - } - candidateTokens.add(trimmed); - candidateTokens.add(trimmed.replace(/[:._/-]\d+$/, "")); - candidateTokens.add(trimmed.replace(/\d+$/, "")); - - const normalizedDelimiter = trimmed.replace(/\//g, "."); - candidateTokens.add(normalizedDelimiter); - candidateTokens.add(normalizedDelimiter.replace(/[:._-]\d+$/, "")); - candidateTokens.add(normalizedDelimiter.replace(/\d+$/, "")); - - for (const prefixPattern of [/^functions?[._-]?/i, /^tools?[._-]?/i]) { - const stripped = normalizedDelimiter.replace(prefixPattern, ""); - if (stripped !== normalizedDelimiter) { - candidateTokens.add(stripped); - candidateTokens.add(stripped.replace(/[:._-]\d+$/, "")); - candidateTokens.add(stripped.replace(/\d+$/, "")); - } - } - }; - - const preColon = id.split(":")[0] ?? id; - for (const seed of [id, preColon]) { - addToken(seed); - } - - let singleMatch: string | null = null; - for (const candidate of candidateTokens) { - const matched = resolveStructuredAllowedToolName(candidate, allowedToolNames); - if (!matched) { - continue; - } - if (singleMatch && singleMatch !== matched) { - return null; - } - singleMatch = matched; - } - - return singleMatch; -} - -function looksLikeMalformedToolNameCounter(rawName: string): boolean { - const normalizedDelimiter = rawName.trim().replace(/\//g, "."); - return ( - /^(?:functions?|tools?)[._-]?/i.test(normalizedDelimiter) && - /(?:[:._-]\d+|\d+)$/.test(normalizedDelimiter) - ); -} - -function normalizeToolCallNameForDispatch( - rawName: string, - allowedToolNames?: Set, - rawToolCallId?: string, -): string { - const trimmed = rawName.trim(); - if (!trimmed) { - return inferToolNameFromToolCallId(rawToolCallId, allowedToolNames) ?? rawName; - } - if (!allowedToolNames || allowedToolNames.size === 0) { - return trimmed; - } - - const exact = resolveExactAllowedToolName(trimmed, allowedToolNames); - if (exact) { - return exact; - } - const inferredFromName = inferToolNameFromToolCallId(trimmed, allowedToolNames); - if (inferredFromName) { - return inferredFromName; - } - - if (looksLikeMalformedToolNameCounter(trimmed)) { - return trimmed; - } - - return resolveStructuredAllowedToolName(trimmed, allowedToolNames) ?? trimmed; -} - -const REPLAY_TOOL_CALL_NAME_MAX_CHARS = 64; - -type ReplayToolCallBlock = { - type?: unknown; - id?: unknown; - name?: unknown; - input?: unknown; - arguments?: unknown; -}; - -type ReplayToolCallSanitizeReport = { - messages: AgentMessage[]; - droppedAssistantMessages: number; -}; - -type AnthropicToolResultContentBlock = { - type?: unknown; - toolUseId?: unknown; - toolCallId?: unknown; - tool_use_id?: unknown; - tool_call_id?: unknown; -}; - -function isThinkingLikeReplayBlock(block: unknown): boolean { - if (!block || typeof block !== "object") { - return false; - } - const type = (block as { type?: unknown }).type; - return type === "thinking" || type === "redacted_thinking"; -} - -function isReplaySafeThinkingTurn(content: unknown[], allowedToolNames?: Set): boolean { - const seenToolCallIds = new Set(); - for (const block of content) { - if (!isReplayToolCallBlock(block)) { - continue; - } - const replayBlock = block; - const toolCallId = typeof replayBlock.id === "string" ? replayBlock.id.trim() : ""; - if (!replayToolCallHasInput(replayBlock) || !toolCallId || seenToolCallIds.has(toolCallId)) { - return false; - } - seenToolCallIds.add(toolCallId); - const rawName = typeof replayBlock.name === "string" ? replayBlock.name : ""; - const resolvedName = resolveReplayToolCallName(rawName, toolCallId, allowedToolNames); - if (!resolvedName || replayBlock.name !== resolvedName) { - return false; - } - } - return true; -} - -function isReplayToolCallBlock(block: unknown): block is ReplayToolCallBlock { - if (!block || typeof block !== "object") { - return false; - } - return isRunnerToolCallBlockType((block as { type?: unknown }).type); -} - -function replayToolCallHasInput(block: ReplayToolCallBlock): boolean { - const hasInput = "input" in block ? block.input !== undefined && block.input !== null : false; - const hasArguments = - "arguments" in block ? block.arguments !== undefined && block.arguments !== null : false; - return hasInput || hasArguments; -} - -function collectFollowingToolResults( - messages: AgentMessage[], - index: number, -): { ids: Set; displaced: boolean } { - const ids = new Set(); - let sawNonToolResult = false; - let displaced = false; - for (let nextIndex = index + 1; nextIndex < messages.length; nextIndex += 1) { - const message = messages[nextIndex]; - if (!message || typeof message !== "object") { - sawNonToolResult = true; - continue; - } - if (message.role === "assistant" && assistantTurnHasReplayToolCall(message)) { - break; - } - if (message.role === "toolResult") { - const resultIds = extractToolResultIds(message); - for (const id of resultIds) { - ids.add(id); - } - displaced ||= resultIds.length > 0 && sawNonToolResult; - continue; - } - sawNonToolResult = true; - } - return { ids, displaced }; -} - -function resolveReplayToolCallName( - rawName: string, - rawId: string, - allowedToolNames?: Set, -): string | null { - if (rawName.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS * 2) { - return null; - } - const normalized = normalizeToolCallNameForDispatch(rawName, allowedToolNames, rawId); - const trimmed = normalized.trim(); - if (!trimmed || trimmed.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS || /\s/.test(trimmed)) { - return null; - } - if (!allowedToolNames || allowedToolNames.size === 0) { - return trimmed; - } - return resolveExactAllowedToolName(trimmed, allowedToolNames); -} - -function sanitizeReplayToolCallInputs( - messages: AgentMessage[], - allowedToolNames?: Set, - allowProviderOwnedThinkingReplay?: boolean, -): ReplayToolCallSanitizeReport { - let changed = false; - let droppedAssistantMessages = 0; - const out: AgentMessage[] = []; - const preservedThinkingToolCallIds = new Set(); - const priorToolCallIds = new Set(); - - for (const [index, message] of messages.entries()) { - if (!message) { - changed = true; - continue; - } - if (typeof message !== "object" || message.role !== "assistant") { - out.push(message); - continue; - } - if (!Array.isArray(message.content)) { - out.push(message); - continue; - } - if ( - allowProviderOwnedThinkingReplay && - message.content.some((block) => isThinkingLikeReplayBlock(block)) && - message.content.some((block) => isReplayToolCallBlock(block)) - ) { - const replaySafeToolCalls = extractToolCallsFromAssistant(message); - const followingToolResults = collectFollowingToolResults(messages, index); - if ( - isReplaySafeThinkingTurn(message.content, allowedToolNames) && - replaySafeToolCalls.every( - (toolCall) => - !preservedThinkingToolCallIds.has(toolCall.id) && - (!followingToolResults.displaced || !priorToolCallIds.has(toolCall.id)) && - followingToolResults.ids.has(toolCall.id), - ) - ) { - for (const toolCall of replaySafeToolCalls) { - preservedThinkingToolCallIds.add(toolCall.id); - priorToolCallIds.add(toolCall.id); - } - changed ||= followingToolResults.displaced; - out.push(message); - } else { - changed = true; - droppedAssistantMessages += 1; - } - continue; - } - - const nextContent: typeof message.content = []; - let messageChanged = false; - - for (const block of message.content) { - if (!isReplayToolCallBlock(block)) { - nextContent.push(block); - continue; - } - const replayBlock = block as ReplayToolCallBlock; - - if (!replayToolCallHasInput(replayBlock) || !replayToolCallNonEmptyString(replayBlock.id)) { - changed = true; - messageChanged = true; - continue; - } - - const rawName = typeof replayBlock.name === "string" ? replayBlock.name : ""; - const resolvedName = resolveReplayToolCallName(rawName, replayBlock.id, allowedToolNames); - if (!resolvedName) { - changed = true; - messageChanged = true; - continue; - } - - if (replayBlock.name !== resolvedName) { - nextContent.push({ ...(block as object), name: resolvedName } as typeof block); - changed = true; - messageChanged = true; - continue; - } - nextContent.push(block); - } - - if (messageChanged) { - changed = true; - if (nextContent.length > 0) { - const nextMessage = { ...message, content: nextContent }; - for (const toolCall of extractToolCallsFromAssistant(nextMessage)) { - priorToolCallIds.add(toolCall.id); - } - out.push(nextMessage); - } else { - droppedAssistantMessages += 1; - } - continue; - } - - for (const toolCall of extractToolCallsFromAssistant(message)) { - priorToolCallIds.add(toolCall.id); - } - out.push(message); - } - - return { - messages: changed ? out : messages, - droppedAssistantMessages, - }; -} - -function extractAnthropicReplayToolResultIds(block: AnthropicToolResultContentBlock): string[] { - const ids: string[] = []; - for (const value of [block.toolUseId, block.toolCallId, block.tool_use_id, block.tool_call_id]) { - if (typeof value !== "string") { - continue; - } - const trimmed = value.trim(); - if (!trimmed || ids.includes(trimmed)) { - continue; - } - ids.push(trimmed); - } - return ids; -} - -function isSignedThinkingReplayAssistantSpan(message: AgentMessage | undefined): boolean { - if (!message || typeof message !== "object" || message.role !== "assistant") { - return false; - } - const content = (message as { content?: unknown }).content; - if (!Array.isArray(content)) { - return false; - } - return ( - content.some((block) => isThinkingLikeReplayBlock(block)) && - content.some((block) => isReplayToolCallBlock(block)) - ); -} - -function sanitizeAnthropicReplayToolResults( - messages: AgentMessage[], - options?: { - disallowEmbeddedUserToolResultsForSignedThinkingReplay?: boolean; - }, -): AgentMessage[] { - let changed = false; - const out: AgentMessage[] = []; - const disallowEmbeddedUserToolResultsForSignedThinkingReplay = - options?.disallowEmbeddedUserToolResultsForSignedThinkingReplay === true; - - for (const [index, message] of messages.entries()) { - if (!message) { - changed = true; - continue; - } - if (typeof message !== "object" || message.role !== "user") { - out.push(message); - continue; - } - if (!Array.isArray(message.content)) { - out.push(message); - continue; - } - - const previous = messages[index - 1]; - const shouldStripEmbeddedToolResults = - disallowEmbeddedUserToolResultsForSignedThinkingReplay && - isSignedThinkingReplayAssistantSpan(previous); - const validToolUseIds = new Set(); - if (previous && typeof previous === "object" && previous.role === "assistant") { - const previousContent = (previous as { content?: unknown }).content; - if (Array.isArray(previousContent)) { - for (const block of previousContent) { - if (!block || typeof block !== "object") { - continue; - } - const typedBlock = block as { type?: unknown; id?: unknown }; - if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") { - continue; - } - const trimmedId = typedBlock.id.trim(); - if (trimmedId) { - validToolUseIds.add(trimmedId); - } - } - } - } - - const nextContent = message.content.filter((block) => { - if (!block || typeof block !== "object") { - return true; - } - const typedBlock = block as AnthropicToolResultContentBlock; - if (typedBlock.type !== "toolResult" && typedBlock.type !== "tool") { - return true; - } - if (shouldStripEmbeddedToolResults) { - changed = true; - return false; - } - const resultIds = extractAnthropicReplayToolResultIds(typedBlock); - if (resultIds.length === 0) { - changed = true; - return false; - } - return validToolUseIds.size > 0 && resultIds.some((id) => validToolUseIds.has(id)); - }); - - if (nextContent.length === message.content.length) { - out.push(message); - continue; - } - - changed = true; - if (nextContent.length > 0) { - out.push({ ...message, content: nextContent }); - continue; - } - - out.push({ - ...message, - content: [{ type: "text", text: "[tool results omitted]" }], - } as AgentMessage); - } - - return changed ? out : messages; -} - -function assistantTurnHasReplayToolCall(message: AgentMessage): boolean { - if (!message || typeof message !== "object" || message.role !== "assistant") { - return false; - } - const content = (message as { content?: unknown }).content; - if (!Array.isArray(content)) { - return false; - } - return content.some((block) => isReplayToolCallBlock(block)); -} - -function stripTrailingAssistantPrefillTurns(messages: AgentMessage[]): AgentMessage[] { - let end = messages.length; - while (end > 0) { - const message = messages[end - 1]; - if (!message || typeof message !== "object" || message.role !== "assistant") { - break; - } - if (assistantTurnHasReplayToolCall(message)) { - break; - } - end -= 1; - } - return end === messages.length ? messages : messages.slice(0, end); -} - -function createStandaloneTextToolCallId(): string { - return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`; -} - -function normalizeToolCallIdsInMessage(message: unknown, fallbackIdByContentIndex: string[]): void { - if (!message || typeof message !== "object") { - return; - } - const content = (message as { content?: unknown }).content; - if (!Array.isArray(content)) { - return; - } - - const usedIds = new Set(); - for (const block of content) { - if (!block || typeof block !== "object") { - continue; - } - const typedBlock = block as { type?: unknown; id?: unknown }; - if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") { - continue; - } - const trimmedId = typedBlock.id.trim(); - if (!trimmedId) { - continue; - } - usedIds.add(trimmedId); - } - - const assignedIds = new Set(); - for (const [contentIndex, block] of content.entries()) { - if (!block || typeof block !== "object") { - continue; - } - const typedBlock = block as { type?: unknown; id?: unknown }; - if (!isRunnerToolCallBlockType(typedBlock.type)) { - continue; - } - if (typeof typedBlock.id === "string") { - const trimmedId = typedBlock.id.trim(); - if (trimmedId) { - if (!assignedIds.has(trimmedId)) { - if (typedBlock.id !== trimmedId) { - typedBlock.id = trimmedId; - } - assignedIds.add(trimmedId); - continue; - } - } - } - - let fallbackId = fallbackIdByContentIndex[contentIndex]; - while (!fallbackId || usedIds.has(fallbackId) || assignedIds.has(fallbackId)) { - fallbackId = createStandaloneTextToolCallId(); - } - fallbackIdByContentIndex[contentIndex] = fallbackId; - typedBlock.id = fallbackId; - usedIds.add(fallbackId); - assignedIds.add(fallbackId); - } -} - -function trimWhitespaceFromToolCallNamesInMessage( - message: unknown, - allowedToolNames: Set | undefined, - fallbackIdByContentIndex: string[], -): void { - visitObjectContentBlocks(message, (block) => { - const typedBlock = block as { type?: unknown; name?: unknown; id?: unknown }; - if (!isRunnerToolCallBlockType(typedBlock.type)) { - return; - } - const rawId = typeof typedBlock.id === "string" ? typedBlock.id : undefined; - if (typeof typedBlock.name === "string") { - const normalized = normalizeToolCallNameForDispatch(typedBlock.name, allowedToolNames, rawId); - if (normalized !== typedBlock.name) { - typedBlock.name = normalized; - } - return; - } - const inferred = inferToolNameFromToolCallId(rawId, allowedToolNames); - if (inferred) { - typedBlock.name = inferred; - } - }); - normalizeToolCallIdsInMessage(message, fallbackIdByContentIndex); -} - -function classifyToolCallMessage( - message: unknown, - allowedToolNames?: Set, -): - | { kind: "none" } - | { kind: "allowed" } - | { kind: "incomplete" } - | { kind: "malformed"; toolName: string } - | { kind: "unknown"; toolName: string } { - if (!message || typeof message !== "object") { - return { kind: "none" }; - } - const content = (message as { content?: unknown }).content; - if (!Array.isArray(content)) { - return { kind: "none" }; - } - - let unknownToolName: string | undefined; - let sawToolCall = false; - let sawAllowedToolCall = false; - let sawIncompleteToolCall = false; - let sawBlankStringToolCall = false; - const hasAllowedToolNames = Boolean(allowedToolNames && allowedToolNames.size > 0); - for (const block of content) { - if (!block || typeof block !== "object") { - continue; - } - const typedBlock = block as { type?: unknown; name?: unknown }; - if (!isRunnerToolCallBlockType(typedBlock.type)) { - continue; - } - sawToolCall = true; - const rawBlockName = typedBlock.name; - const hasStringName = typeof rawBlockName === "string"; - const rawName = hasStringName ? rawBlockName.trim() : ""; - if (!rawName) { - if (hasStringName) { - sawBlankStringToolCall = true; - } else { - sawIncompleteToolCall = true; - } - continue; - } - if (!hasAllowedToolNames) { - continue; - } - if (resolveExactAllowedToolName(rawName, allowedToolNames)) { - sawAllowedToolCall = true; - continue; - } - const normalizedUnknownToolName = normalizeToolPolicyName(rawName); - if (!unknownToolName) { - unknownToolName = normalizedUnknownToolName; - continue; - } - if (unknownToolName !== normalizedUnknownToolName) { - sawIncompleteToolCall = true; - } - } - - if (!sawToolCall) { - return { kind: "none" }; - } - if (!hasAllowedToolNames) { - return sawBlankStringToolCall - ? { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION } - : { kind: "none" }; - } - if (sawAllowedToolCall) { - return { kind: "allowed" }; - } - if (sawBlankStringToolCall && !sawIncompleteToolCall && unknownToolName === undefined) { - return { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION }; - } - if (sawIncompleteToolCall) { - return { kind: "incomplete" }; - } - return unknownToolName ? { kind: "unknown", toolName: unknownToolName } : { kind: "incomplete" }; -} - -function rewriteUnknownToolLoopMessage(message: unknown, toolName: string): void { - if (!message || typeof message !== "object") { - return; - } - (message as { content?: unknown }).content = [ - { - type: "text", - text: `I can't use the tool "${toolName}" here because it isn't available. I need to stop retrying it and answer without that tool.`, - }, - ]; -} - -function guardUnknownToolLoopInMessage( - message: unknown, - state: UnknownToolLoopGuardState, - params: { - allowedToolNames?: Set; - threshold?: number; - countAttempt: boolean; - resetOnAllowedTool?: boolean; - resetOnMissingUnknownTool?: boolean; - rewriteMalformedBlankToolName?: boolean; - }, -): boolean { - const toolCallState = classifyToolCallMessage(message, params.allowedToolNames); - if (toolCallState.kind === "allowed") { - if (params.resetOnAllowedTool === true) { - state.lastUnknownToolName = undefined; - state.count = 0; - } - return false; - } - if (toolCallState.kind === "malformed") { - if (params.rewriteMalformedBlankToolName === true) { - rewriteUnknownToolLoopMessage(message, toolCallState.toolName); - return true; - } - if (params.countAttempt && params.resetOnMissingUnknownTool !== false) { - state.lastUnknownToolName = undefined; - state.count = 0; - } - return false; - } - const threshold = params.threshold; - if (threshold === undefined || threshold <= 0) { - return false; - } - if (toolCallState.kind !== "unknown") { - if (params.countAttempt && params.resetOnMissingUnknownTool !== false) { - state.lastUnknownToolName = undefined; - state.count = 0; - } - return false; - } - const unknownToolName = toolCallState.toolName; - - if (!params.countAttempt) { - // Partial stream events can rewrite after the threshold, but only final - // messages advance the loop counter. - if (state.lastUnknownToolName === unknownToolName && state.count > threshold) { - rewriteUnknownToolLoopMessage(message, unknownToolName); - } - return false; - } - - if (message && typeof message === "object") { - if (state.countedMessages.has(message)) { - if (state.lastUnknownToolName === unknownToolName && state.count > threshold) { - rewriteUnknownToolLoopMessage(message, unknownToolName); - } - return true; - } - state.countedMessages.add(message); - } - - if (state.lastUnknownToolName === unknownToolName) { - state.count += 1; - } else { - state.lastUnknownToolName = unknownToolName; - state.count = 1; - } - - if (state.count > threshold) { - rewriteUnknownToolLoopMessage(message, unknownToolName); - } - return true; -} - -function isRetainableNonVisibleBlock(block: Record): boolean { - return block.type === "thinking" || block.type === "redacted_thinking"; -} - -const STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS = new Set(["stop", "toolUse"]); - -function createStandaloneToolCallNameMatcher( - allowedToolNames: Set, -): PlainTextToolCallNameMatcher { - return { - hasExactName: (name) => Boolean(resolveExactAllowedToolName(name, allowedToolNames)), - hasNamePrefix: (prefix) => couldNormalizeToolNamePrefixToAllowedTool(prefix, allowedToolNames), - }; -} - -function wrapStreamPromoteStandaloneTextToolCalls( - stream: AssistantStream, - allowedToolNames: Set, -): AssistantStream { - const matcher = createStandaloneToolCallNameMatcher(allowedToolNames); - const promotedIdBySource = new Map(); - const normalizeTerminalMessage = (params: { - allowPromotion: boolean; - message: unknown; - preserveEmptyTextBlocks?: boolean; - }): PlainTextToolCallMessageNormalization => { - const scrubbed = projectScrubbedPlainTextToolCallMessage({ - forceIncompleteCandidates: true, - matcher, - message: params.message, - preserveEmptyTextBlocks: params.preserveEmptyTextBlocks, - resolveProtectedRanges: findCodeRegions, - requireAssistantRole: true, - }); - if (scrubbed) { - return { kind: "scrubbed", ...scrubbed }; - } - if (!params.allowPromotion) { - return undefined; - } - let ordinal = 0; - const createStableToolCallBlock = ( - block: PlainTextToolCallBlock, - name: string, - ): Record => { - const sourceKey = `${ordinal}:${block.start}:${block.end}`; - ordinal += 1; - let id = promotedIdBySource.get(sourceKey); - if (!id) { - id = createStandaloneTextToolCallId(); - promotedIdBySource.set(sourceKey, id); - } - return { - type: "toolCall", - id, - name, - arguments: block.arguments, - partialArgs: JSON.stringify(block.arguments), - }; - }; - const promoted = projectPlainTextToolCallMessage({ - allowedStopReasons: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS, - allowedToolNames, - createToolCallBlock: createStableToolCallBlock, - isRetainableNonTextBlock: isRetainableNonVisibleBlock, - message: params.message, - requireAssistantRole: true, - resolveProtectedRanges: findCodeRegions, - resolveToolName: resolveExactAllowedToolName, - }); - return promoted ? { kind: "promoted", ...promoted } : undefined; - }; - - const originalResult = stream.result.bind(stream); - stream.result = async () => { - const message = await originalResult(); - const reason = - message && typeof message === "object" - ? (message as { stopReason?: unknown }).stopReason - : undefined; - return (normalizeTerminalMessage({ - allowPromotion: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS.has(reason), - message, - })?.message ?? message) as Awaited>; - }; - - const originalAsyncIterator = stream[Symbol.asyncIterator].bind(stream); - (stream as unknown as { [Symbol.asyncIterator]: () => AsyncIterator })[ - Symbol.asyncIterator - ] = async function* () { - const source = { - [Symbol.asyncIterator]: originalAsyncIterator, - } as AsyncIterable; - yield* normalizePlainTextToolCallStreamEvents(source, { - createPromotedToolCallEvents: createPromotedPlainTextToolCallEvents, - matcher, - normalizeTerminalMessage, - resolveProtectedRanges: findCodeRegions, - }); - }; - - return stream; -} - -/** Promotes standalone plain-text tool-call replies into structured toolCall blocks when safe. */ -export function wrapStreamFnPromoteStandaloneTextToolCalls( - baseFn: StreamFn, - allowedToolNames?: Set, -): StreamFn { - if (!allowedToolNames || allowedToolNames.size === 0) { - return baseFn; - } - return (model, context, streamOptions) => { - const maybeStream = baseFn(model, context, streamOptions); - if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) { - return Promise.resolve(maybeStream).then((stream) => - wrapStreamPromoteStandaloneTextToolCalls(stream, allowedToolNames), - ); - } - return wrapStreamPromoteStandaloneTextToolCalls(maybeStream, allowedToolNames); - }; -} - -function wrapStreamTrimToolCallNames( - stream: AssistantStream, - allowedToolNames?: Set, - options?: { unknownToolThreshold?: number; state?: UnknownToolLoopGuardState }, -): AssistantStream { - const unknownToolGuardState = options?.state ?? { - count: 0, - countedMessages: new WeakSet(), - }; - // Provider-omitted ids are only message-local. Reuse one generated id per - // content position across this response's partial/final projections, while a - // later assistant response gets a fresh namespace and cannot alias it. - const fallbackIdByContentIndex: string[] = []; - let streamAttemptAlreadyCounted = false; - const originalResult = stream.result.bind(stream); - stream.result = async () => { - const message = await originalResult(); - trimWhitespaceFromToolCallNamesInMessage(message, allowedToolNames, fallbackIdByContentIndex); - guardUnknownToolLoopInMessage(message, unknownToolGuardState, { - allowedToolNames, - threshold: options?.unknownToolThreshold, - countAttempt: !streamAttemptAlreadyCounted, - resetOnAllowedTool: true, - rewriteMalformedBlankToolName: true, - }); - return message; - }; - - wrapStreamObjectEvents(stream, (event) => { - trimWhitespaceFromToolCallNamesInMessage( - event.partial, - allowedToolNames, - fallbackIdByContentIndex, - ); - trimWhitespaceFromToolCallNamesInMessage( - event.message, - allowedToolNames, - fallbackIdByContentIndex, - ); - if (event.message && typeof event.message === "object") { - const countedStreamAttempt = guardUnknownToolLoopInMessage( - event.message, - unknownToolGuardState, - { - allowedToolNames, - threshold: options?.unknownToolThreshold, - countAttempt: !streamAttemptAlreadyCounted, - resetOnAllowedTool: true, - resetOnMissingUnknownTool: false, - }, - ); - streamAttemptAlreadyCounted ||= countedStreamAttempt; - } - guardUnknownToolLoopInMessage(event.partial, unknownToolGuardState, { - allowedToolNames, - threshold: options?.unknownToolThreshold, - countAttempt: false, - }); - }); - - return stream; -} - -/** Normalizes streamed tool-call names and guards repeated unknown-tool loops. */ -export function wrapStreamFnTrimToolCallNames( - baseFn: StreamFn, - allowedToolNames?: Set, - guardOptions?: { unknownToolThreshold?: number }, -): StreamFn { - const unknownToolGuardState: UnknownToolLoopGuardState = { - count: 0, - countedMessages: new WeakSet(), - }; - return (model, context, streamOptions) => { - const maybeStream = baseFn(model, context, streamOptions); - if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) { - return Promise.resolve(maybeStream).then((stream) => - wrapStreamTrimToolCallNames(stream, allowedToolNames, { - unknownToolThreshold: guardOptions?.unknownToolThreshold, - state: unknownToolGuardState, - }), - ); - } - return wrapStreamTrimToolCallNames(maybeStream, allowedToolNames, { - unknownToolThreshold: guardOptions?.unknownToolThreshold, - state: unknownToolGuardState, - }); - }; -} - -type ReplayToolCallIdSanitizerDecision = { - sanitizeToolCallIds: boolean; - toolCallIdMode?: ToolCallIdMode; - isOpenAIResponsesApi: boolean; -}; - -/** Returns whether replayed tool-call ids should be sanitized for non-Responses providers. */ -export function shouldApplyReplayToolCallIdSanitizer( - params: ReplayToolCallIdSanitizerDecision, -): params is ReplayToolCallIdSanitizerDecision & { toolCallIdMode: ToolCallIdMode } { - return ( - params.sanitizeToolCallIds && Boolean(params.toolCallIdMode) && !params.isOpenAIResponsesApi - ); -} - -/** Rewrites replayed tool-call ids into provider-safe ids and optionally repairs result pairing. */ -export function sanitizeReplayToolCallIdsForStream(params: { - messages: AgentMessage[]; - mode: ToolCallIdMode; - allowedToolNames?: Set; - preserveNativeAnthropicToolUseIds?: boolean; - duplicateToolCallIdStyle?: "openai"; - preserveReplaySafeThinkingToolCallIds?: boolean; - repairToolUseResultPairing?: boolean; -}): AgentMessage[] { - const paired = params.repairToolUseResultPairing - ? sanitizeToolUseResultPairing(params.messages) - : params.messages; - return sanitizeToolCallIdsForCloudCodeAssist(paired, params.mode, { - preserveNativeAnthropicToolUseIds: params.preserveNativeAnthropicToolUseIds, - duplicateToolCallIdStyle: params.duplicateToolCallIdStyle, - preserveReplaySafeThinkingToolCallIds: params.preserveReplaySafeThinkingToolCallIds, - allowedToolNames: params.allowedToolNames, - }); -} - -/** Downgrades OpenAI Responses replay turns into the stream format expected by runtime callers. */ -export function sanitizeOpenAIResponsesReplayForStream(messages: AgentMessage[]): AgentMessage[] { - const repaired = sanitizeToolUseResultPairing(messages, { - erroredAssistantResultPolicy: "drop", - missingToolResultText: "aborted", - }); - return downgradeOpenAIFunctionCallReasoningPairs( - normalizeOpenAIResponsesToolCallIds(downgradeOpenAIReasoningBlocks(repaired)), - ); -} - -/** - * Sanitizes malformed replay tool calls before provider submission. The wrapper - * drops invalid assistant tool calls, repairs adjacent tool results when needed, - * strips trailing assistant prefill turns for strict providers, and revalidates - * Anthropic/Gemini transcripts after mutations. - */ -export function wrapStreamFnSanitizeMalformedToolCalls( - baseFn: StreamFn, - allowedToolNames?: Set, - transcriptPolicy?: Pick< - TranscriptPolicy, - "validateGeminiTurns" | "validateAnthropicTurns" | "preserveSignatures" | "dropThinkingBlocks" - >, - provider?: string | null, -): StreamFn { - return (model, context, options) => { - const ctx = context as unknown as { messages?: unknown }; - const messages = ctx?.messages; - if (!Array.isArray(messages)) { - return baseFn(model, context, options); - } - const allowProviderOwnedThinkingReplay = shouldAllowProviderOwnedThinkingReplay({ - modelApi: (model as { api?: unknown })?.api as string | null | undefined, - provider, - policy: { - validateAnthropicTurns: transcriptPolicy?.validateAnthropicTurns === true, - preserveSignatures: transcriptPolicy?.preserveSignatures === true, - dropThinkingBlocks: transcriptPolicy?.dropThinkingBlocks === true, - }, - }); - const sanitized = sanitizeReplayToolCallInputs( - messages as AgentMessage[], - allowedToolNames, - allowProviderOwnedThinkingReplay, - ); - const isOpenAIResponsesApi = - (model as { api?: unknown }).api === "openai-responses" || - (model as { api?: unknown }).api === "openai-chatgpt-responses" || - (model as { api?: unknown }).api === "azure-openai-responses"; - const replayInputsChanged = sanitized.messages !== messages; - let nextMessages = isOpenAIResponsesApi - ? sanitizeToolUseResultPairing(sanitized.messages, { - erroredAssistantResultPolicy: "drop", - missingToolResultText: "aborted", - }) - : replayInputsChanged - ? sanitizeToolUseResultPairing(sanitized.messages) - : sanitized.messages; - let strippedTrailingAssistantPrefill = false; - if (transcriptPolicy?.validateAnthropicTurns) { - nextMessages = sanitizeAnthropicReplayToolResults(nextMessages, { - disallowEmbeddedUserToolResultsForSignedThinkingReplay: allowProviderOwnedThinkingReplay, - }); - } - if (transcriptPolicy?.validateAnthropicTurns || transcriptPolicy?.validateGeminiTurns) { - const beforeStrip = nextMessages; - nextMessages = stripTrailingAssistantPrefillTurns(nextMessages); - strippedTrailingAssistantPrefill ||= nextMessages !== beforeStrip; - } - if (nextMessages === messages) { - return baseFn(model, context, options); - } - if ( - sanitized.droppedAssistantMessages > 0 || - transcriptPolicy?.validateAnthropicTurns || - strippedTrailingAssistantPrefill - ) { - if (transcriptPolicy?.validateGeminiTurns) { - nextMessages = validateGeminiTurns(nextMessages); - } - if (transcriptPolicy?.validateAnthropicTurns) { - nextMessages = validateAnthropicTurns(nextMessages); - } - } - const nextContext = { - ...(context as unknown as Record), - messages: nextMessages, - } as unknown; - return baseFn(model, nextContext as typeof context, options); - }; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts b/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts new file mode 100644 index 000000000000..fc532f8578a0 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts @@ -0,0 +1,249 @@ +/** Classifies terminal assistant visibility and provider retry eligibility. */ +import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; +import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { isSilentReplyPayloadText, SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js"; +import { + isStrictAgenticSupportedProviderModel, + stripProviderPrefix, +} from "../../execution-contract.js"; +import type { AgentMessage } from "../../runtime/index.js"; +import { assessLastAssistantMessage } from "../thinking.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +export type IncompleteTurnAttempt = Pick< + EmbeddedRunAttemptResult, + | "assistantTexts" + | "clientToolCalls" + | "currentAttemptAssistant" + | "yieldDetected" + | "didSendDeterministicApprovalPrompt" + | "heartbeatToolResponse" + | "toolMediaUrls" + | "toolAudioAsVoice" + | "toolTrustedLocalMedia" + | "hasToolMediaBlockReply" + | "didDeliverSourceReplyViaMessageTool" + | "messagingToolSourceReplyPayloads" + | "didSendViaMessagingTool" + | "messagingToolSentTexts" + | "messagingToolSentMediaUrls" + | "messagingToolSentTargets" + | "lastToolError" + | "lastAssistant" + | "itemLifecycle" + | "messagesSnapshot" + | "replayMetadata" + | "terminal" + | "toolMetas" +> & + Partial>; + +export function hasPositiveOutputTokenUsage(message: AgentMessage | null): boolean { + if (!message || typeof message !== "object") { + return false; + } + const usage = (message as { usage?: unknown }).usage; + if (!usage || typeof usage !== "object") { + return false; + } + const output = asFiniteNumber((usage as { output?: unknown }).output); + return output !== undefined && output > 0; +} + +export function isIncompleteTerminalAssistantTurn(params: { + hasAssistantVisibleText: boolean; + hasTerminalOutput?: boolean; + lastAssistant?: { stopReason?: string } | null; +}): boolean { + const stopReason = params.lastAssistant?.stopReason; + // Tool-use expects a post-tool continuation; length means the output budget + // ended before a complete final answer. Partial visible text completes neither. + return stopReason === "toolUse" || (stopReason === "length" && !params.hasTerminalOutput); +} + +const GEMINI_INCOMPLETE_TURN_PROVIDER_IDS = new Set([ + "google", + "google-vertex", + "google-antigravity", + "google-gemini-cli", +]); +const GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN = /^gemini(?:[.-]|$)/; +// Ollama native `/api/chat` can finish with only thinking/internal blocks when constrained. +const OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN = /^ollama(?:-|$)/; + +export function isOllamaIncompleteTurnProvider(provider?: string): boolean { + return OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test( + normalizeLowercaseStringOrEmpty(provider ?? ""), + ); +} +// Model APIs eligible for the non-visible turn retry guard. OpenAI Responses +// family can produce reasoning-only turns where usage.output > 0 but no visible +// text is emitted; without the guard these pass through as successful. (#85364) +const RETRY_GUARD_MODEL_APIS = new Set([ + "openai-completions", + "anthropic-messages", + "bedrock-converse-stream", + "openai-responses", + "openai-chatgpt-responses", + "azure-openai-responses", + "openclaw-openai-responses-transport", + "openclaw-openai-chatgpt-responses-transport", + "openclaw-azure-openai-responses-transport", +]); +export function joinAssistantTexts(assistantTexts?: readonly string[]): string { + return (assistantTexts ?? []).join("\n\n").trim(); +} + +export function hasOnlySilentAssistantReply(assistantTexts?: readonly string[]): boolean { + const nonEmptyTexts = (assistantTexts ?? []).filter((text) => text.trim().length > 0); + return ( + nonEmptyTexts.length > 0 && + nonEmptyTexts.every((text) => isSilentReplyPayloadText(text, SILENT_REPLY_TOKEN)) + ); +} + +export function isReasoningOnlyAssistantTurn(message: unknown): boolean { + if (!message || typeof message !== "object") { + return false; + } + return assessLastAssistantMessage(message as AgentMessage) === "incomplete-text"; +} + +// Unsigned thinking blocks have no cryptographic signature; assessLastAssistantMessage +// returns "incomplete-thinking" for them. Empty content also returns "incomplete-thinking", +// so the content.length > 0 guard is required to distinguish the two cases. +export function isUnsignedThinkingOnlyAssistantTurn(message: unknown): boolean { + if (message == null || typeof message !== "object") { + return false; + } + const content = (message as { content?: unknown }).content; + if (!Array.isArray(content) || content.length === 0) { + return false; + } + return assessLastAssistantMessage(message as AgentMessage) === "incomplete-thinking"; +} + +export function isEmptyResponseAssistantTurn(params: { + payloadCount: number; + attempt: Pick< + IncompleteTurnAttempt, + "assistantTexts" | "currentAttemptAssistant" | "lastAssistant" + >; +}): boolean { + if (params.payloadCount !== 0) { + return false; + } + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { + return false; + } + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + if (!assistant) { + return true; + } + if (assistant.stopReason === "error") { + return false; + } + if ( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: false, + lastAssistant: assistant, + }) || + isReasoningOnlyAssistantTurn(assistant) + ) { + return false; + } + return true; +} + +export function isNonVisibleAssistantTurnEligibleForSilentReply(params: { + payloadCount: number; + attempt: Pick< + IncompleteTurnAttempt, + "assistantTexts" | "currentAttemptAssistant" | "lastAssistant" + >; +}): boolean { + if (isEmptyResponseAssistantTurn(params)) { + return true; + } + if (params.payloadCount !== 0) { + return false; + } + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { + return false; + } + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + if (!assistant || assistant.stopReason === "error") { + return false; + } + if ( + isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: false, + lastAssistant: assistant, + }) + ) { + return false; + } + return isReasoningOnlyAssistantTurn(assistant); +} + +export function shouldApplyNonVisibleTurnRetryGuard(params: { + provider?: string; + modelId?: string; + modelApi?: string; + executionContract?: string; +}): boolean { + if ( + params.executionContract === "strict-agentic" || + isIncompleteTurnRecoverySupportedProviderModel({ + provider: params.provider, + modelId: params.modelId, + }) + ) { + return true; + } + if (RETRY_GUARD_MODEL_APIS.has(normalizeLowercaseStringOrEmpty(params.modelApi ?? ""))) { + return true; + } + // This path uses provider output structure only: no user or assistant prose classification. + return isOllamaIncompleteTurnProvider(params.provider); +} + +function isIncompleteTurnRecoverySupportedProviderModel(params: { + provider?: string; + modelId?: string; +}): boolean { + if ( + isStrictAgenticSupportedProviderModel({ + provider: params.provider, + modelId: params.modelId, + }) + ) { + return true; + } + const provider = normalizeLowercaseStringOrEmpty(params.provider ?? ""); + if (!GEMINI_INCOMPLETE_TURN_PROVIDER_IDS.has(provider)) { + return false; + } + const modelId = typeof params.modelId === "string" ? params.modelId : ""; + return GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN.test(stripProviderPrefix(modelId)); +} + +export function classifyAssistantTurn(params: { + payloadCount: number; + attempt: Pick< + IncompleteTurnAttempt, + "assistantTexts" | "currentAttemptAssistant" | "lastAssistant" + >; +}) { + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + return { + assistant, + visibleText: joinAssistantTexts(params.attempt.assistantTexts), + onlySilentReply: hasOnlySilentAssistantReply(params.attempt.assistantTexts), + reasoningOnly: isReasoningOnlyAssistantTurn(assistant), + unsignedThinkingOnly: isUnsignedThinkingOnlyAssistantTurn(assistant), + emptyResponse: isEmptyResponseAssistantTurn(params), + nonVisibleEligibleForSilentReply: isNonVisibleAssistantTurnEligibleForSilentReply(params), + hasPositiveOutputTokenUsage: hasPositiveOutputTokenUsage(assistant ?? null), + }; +} diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts b/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts new file mode 100644 index 000000000000..10f4b24ed93d --- /dev/null +++ b/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts @@ -0,0 +1,370 @@ +/** Owns side-effect-sensitive retry and silent-reply recovery policy. */ +import { isReplayUnsafeAssistantError } from "../../../llm/utils/retry.js"; +import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js"; +import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js"; +import { + hasCommittedMessagingToolDeliveryEvidence, + hasCompletedMessagingToolDeliveryEvidence, +} from "../delivery-evidence.js"; +import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js"; +import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js"; +import { + hasOnlySilentAssistantReply, + hasPositiveOutputTokenUsage, + isEmptyResponseAssistantTurn, + isNonVisibleAssistantTurnEligibleForSilentReply, + isOllamaIncompleteTurnProvider, + isReasoningOnlyAssistantTurn, + isUnsignedThinkingOnlyAssistantTurn, + joinAssistantTexts, + shouldApplyNonVisibleTurnRetryGuard, + type IncompleteTurnAttempt, +} from "./incomplete-turn-classification.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +// Allow one immediate continuation plus one follow-up continuation before +// surfacing the existing incomplete-turn error path. +export const DEFAULT_REASONING_ONLY_RETRY_LIMIT = 2; +export const DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT = 1; +const REASONING_ONLY_RETRY_INSTRUCTION = + "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; +const EMPTY_RESPONSE_RETRY_INSTRUCTION = + "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; +const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = + "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; + +export function shouldRetrySilentErrorAssistantTurn(params: { + attempt: Pick< + EmbeddedRunAttemptResult, + | "assistantTexts" + | "clientToolCalls" + | "yieldDetected" + | "didSendDeterministicApprovalPrompt" + | "heartbeatToolResponse" + | "lastToolError" + | "toolMediaUrls" + | "toolAudioAsVoice" + | "toolTrustedLocalMedia" + | "didDeliverSourceReplyViaMessageTool" + | "messagingToolSourceReplyPayloads" + | "replayMetadata" + | "currentAttemptReplayMetadata" + >; + assistant: EmbeddedRunAttemptResult["lastAssistant"] | null | undefined; +}): boolean { + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { + return false; + } + if (hasAttemptTerminalState(params.attempt)) { + return false; + } + // Current-attempt evidence avoids blocking on prior committed effects; older + // harnesses retain the cumulative, fail-closed behavior. + const retryReplayMetadata = + params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata; + if (retryReplayMetadata.hadPotentialSideEffects) { + return false; + } + + const assistant = params.assistant; + if (!assistant || assistant.stopReason !== "error" || isReplayUnsafeAssistantError(assistant)) { + return false; + } + + const content = (assistant as { content?: unknown }).content; + if (!Array.isArray(content)) { + return false; + } + if (content.length === 0) { + return !hasPositiveOutputTokenUsage(assistant); + } + + return hasOnlyAssistantReasoningContent(assistant); +} + +function shouldSkipNonVisibleTurnRetry(params: { + aborted: boolean; + timedOut: boolean; + attempt: IncompleteTurnAttempt; + /** Reply-optional silent classification tolerates committed side effects; retries never can. */ + tolerateSideEffects?: boolean; +}): boolean { + return Boolean( + params.aborted || + params.timedOut || + params.attempt.clientToolCalls || + params.attempt.yieldDetected || + params.attempt.didSendDeterministicApprovalPrompt || + params.attempt.lastToolError || + hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) || + (params.tolerateSideEffects !== true && params.attempt.replayMetadata.hadPotentialSideEffects), + ); +} + +/** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */ +export function shouldTreatEmptyAssistantReplyAsSilent(params: { + allowEmptyAssistantReplyAsSilent?: boolean; + onlyExplicitSilentReply?: boolean; + terminalReplyExpectation?: "required" | "optional"; + payloadCount: number; + aborted: boolean; + timedOut: boolean; + attempt: IncompleteTurnAttempt; +}): boolean { + // "optional" is the run consumer's declaration that no user-facing reply is + // owed (e.g. cron without a delivery route). Silence after side-effecting + // tools is intentional there; retry is replay-unsafe, so erroring would mark + // successful tool-only runs as failures. + const terminalReplyOptional = params.terminalReplyExpectation === "optional"; + if ( + !params.allowEmptyAssistantReplyAsSilent || + shouldSkipNonVisibleTurnRetry({ ...params, tolerateSideEffects: terminalReplyOptional }) + ) { + return false; + } + if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { + return false; + } + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + if ( + params.payloadCount === 0 && + assistant?.stopReason !== "error" && + hasOnlySilentAssistantReply(params.attempt.assistantTexts) + ) { + return true; + } + // A visible turn owes a reply unless the model explicitly chose NO_REPLY. + // Bare empty and reasoning-only stops are provider failures, even when the + // conversation policy permits deliberate silence. + if (params.onlyExplicitSilentReply || !terminalReplyOptional) { + return false; + } + return isNonVisibleAssistantTurnEligibleForSilentReply({ + payloadCount: params.payloadCount, + attempt: params.attempt, + }); +} + +/** + * Builds the retry instruction for reasoning-only turns that consumed provider + * output budget but produced no visible assistant text. + */ +export function resolveReasoningOnlyRetryInstruction(params: { + provider?: string; + modelId?: string; + modelApi?: string; + executionContract?: string; + aborted: boolean; + timedOut: boolean; + attempt: IncompleteTurnAttempt; +}): string | null { + if (shouldSkipNonVisibleTurnRetry(params)) { + return null; + } + + if ( + !shouldApplyNonVisibleTurnRetryGuard({ + provider: params.provider, + modelId: params.modelId, + modelApi: params.modelApi, + executionContract: params.executionContract, + }) + ) { + return null; + } + + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { + return null; + } + if (assistant?.stopReason === "error") { + return null; + } + if (!isReasoningOnlyAssistantTurn(assistant) && !isUnsignedThinkingOnlyAssistantTurn(assistant)) { + return null; + } + + return REASONING_ONLY_RETRY_INSTRUCTION; +} + +/** Builds one fresh continuation after settled tools ended without a visible final answer. */ +export function resolveSettledToolTerminalContinuationInstruction(params: { + provider?: string; + modelId?: string; + modelApi?: string; + executionContract?: string; + allowEmptyStopContinuation?: boolean; + payloadCount: number; + hasTerminalToolPresentation?: boolean; + aborted: boolean; + promptError?: unknown; + timedOut: boolean; + attempt: IncompleteTurnAttempt; +}): string | null { + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + const currentAttemptAssistant = params.attempt.currentAttemptAssistant; + const emptyStopAfterSettledTools = Boolean( + params.allowEmptyStopContinuation && + currentAttemptAssistant?.stopReason === "stop" && + params.attempt.toolMetas.length > 0 && + params.attempt.toolMetas.every((tool) => tool.isError !== true && tool.asyncStarted !== true) && + params.attempt.itemLifecycle.startedCount > 0 && + params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount && + params.attempt.itemLifecycle.activeCount === 0 && + !hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) && + isEmptyResponseAssistantTurn({ + payloadCount: params.payloadCount, + attempt: params.attempt, + }), + ); + // Idle is not proof of settlement: skipped or partially dispatched tools must + // never be described as completed. Match each terminal call's id and owner to + // its own current-batch result; a reported failure is settled, not successful. + const requestedToolCalls = Array.isArray(assistant?.content) + ? assistant.content.flatMap((item) => { + const block = item as { type?: unknown; id?: unknown; name?: unknown } | null; + return block?.type === "toolCall" + ? [ + { + id: typeof block.id === "string" ? block.id : null, + name: typeof block.name === "string" ? block.name : null, + }, + ] + : []; + }) + : []; + // Scan only results AFTER the terminal assistant: the snapshot spans the whole + // session, and a prior turn's toolResult with a model-reused id would otherwise + // prove "completion" for a batch that never dispatched. Assistant not found in + // the snapshot fails closed to the existing incomplete-turn error. + const snapshot = params.attempt.messagesSnapshot ?? []; + const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1; + const settledToolResults = new Map( + (assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => { + const result = message as { + role?: unknown; + toolCallId?: unknown; + toolName?: unknown; + isError?: unknown; + }; + return result.role === "toolResult" && + typeof result.toolCallId === "string" && + typeof result.toolName === "string" + ? [ + [ + result.toolCallId, + { toolName: result.toolName, isError: result.isError === true }, + ] as const, + ] + : []; + }), + ); + const allToolsProvenSettled = + params.attempt.itemLifecycle?.activeCount === 0 && + requestedToolCalls.length > 0 && + requestedToolCalls.every( + ({ id, name }) => + id !== null && name !== null && settledToolResults.get(id)?.toolName === name, + ); + const failedTerminalToolNames = new Set( + requestedToolCalls.flatMap(({ id, name }) => + id !== null && name !== null && settledToolResults.get(id)?.isError === true ? [name] : [], + ), + ); + const hasSettledTerminalToolFailure = allToolsProvenSettled && failedTerminalToolNames.size > 0; + // ToolErrorSummary has no call id: its owner must match a failed result in the + // proven terminal batch, or a stale/unrelated error could authorize finalization. + const hasUnsettledToolError = Boolean( + params.attempt.lastToolError && + (assistant?.stopReason !== "toolUse" || + !hasSettledTerminalToolFailure || + !failedTerminalToolNames.has(params.attempt.lastToolError.toolName)), + ); + if ( + params.payloadCount !== 0 || + params.hasTerminalToolPresentation || + params.aborted || + params.promptError != null || + params.timedOut || + (assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) || + hasUnsettledToolError || + (hasSettledTerminalToolFailure && + (hasAsyncActivity(params.attempt.toolMetas) || + hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) || + params.attempt.clientToolCalls || + params.attempt.yieldDetected || + params.attempt.didSendDeterministicApprovalPrompt + ) { + return null; + } + if (hasCompletedMessagingToolDeliveryEvidence(params.attempt)) { + return null; + } + if ( + !shouldApplyNonVisibleTurnRetryGuard({ + provider: params.provider, + modelId: params.modelId, + modelApi: params.modelApi, + executionContract: params.executionContract, + }) + ) { + return null; + } + return hasSettledTerminalToolFailure + ? `${SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION} If any tool failed, state that failure plainly and do not claim it succeeded.` + : SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION; +} + +/** + * Builds the retry instruction for empty assistant turns when the provider/model + * is eligible for non-visible turn recovery. + */ +export function resolveEmptyResponseRetryInstruction(params: { + provider?: string; + modelId?: string; + modelApi?: string; + executionContract?: string; + payloadCount: number; + aborted: boolean; + timedOut: boolean; + attempt: IncompleteTurnAttempt; +}): string | null { + if (shouldSkipNonVisibleTurnRetry(params)) { + return null; + } + + if ( + !isEmptyResponseAssistantTurn({ + payloadCount: params.payloadCount, + attempt: params.attempt, + }) + ) { + return null; + } + + const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null; + if ( + assistant?.stopReason === "stop" && + isOllamaIncompleteTurnProvider(params.provider) && + !hasPositiveOutputTokenUsage(assistant) + ) { + return null; + } + + if ( + shouldApplyNonVisibleTurnRetryGuard({ + provider: params.provider, + modelId: params.modelId, + modelApi: params.modelApi, + executionContract: params.executionContract, + }) || + // Keep the generic zero-usage stop retry for providers that expose a + // provider-neutral "nothing was generated" signal, even outside the + // provider allowlist above. + isZeroUsageEmptyStopAssistantTurn(assistant) + ) { + return EMPTY_RESPONSE_RETRY_INSTRUCTION; + } + + return null; +} diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts b/src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts new file mode 100644 index 000000000000..67938eb48cf8 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/incomplete-turn-resolution.ts @@ -0,0 +1,328 @@ +/** Resolves incomplete-turn payloads, continuation evidence, and run liveness. */ +import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js"; +import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js"; +import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; +import { collectTextContentBlocks } from "../../content-blocks.js"; +import type { MessagingToolSend } from "../../embedded-agent-messaging.types.js"; +import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js"; +import type { AgentMessage } from "../../runtime/index.js"; +import { hasCommittedMessagingToolDeliveryEvidence } from "../delivery-evidence.js"; +import type { EmbeddedRunLivenessState } from "../types.js"; +import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js"; +import { + classifyAssistantTurn, + hasOnlySilentAssistantReply, + isEmptyResponseAssistantTurn, + isIncompleteTerminalAssistantTurn, + isReasoningOnlyAssistantTurn, + joinAssistantTexts, + type IncompleteTurnAttempt, +} from "./incomplete-turn-classification.js"; +import type { EmbeddedRunAttemptResult } from "./types.js"; + +type SilentToolResultAttempt = Pick< + EmbeddedRunAttemptResult, + | "clientToolCalls" + | "yieldDetected" + | "didSendDeterministicApprovalPrompt" + | "lastToolError" + | "messagesSnapshot" + | "toolMetas" +>; + +type RunLivenessAttempt = Pick< + EmbeddedRunAttemptResult, + "lastAssistant" | "replayMetadata" | "terminal" +>; + +/** + * Builds the user-visible incomplete-turn warning when a terminal attempt did + * not produce a safe final assistant response and no committed delivery/progress + * already completed the task. + */ +export function resolveIncompleteTurnPayloadText(params: { + payloadCount: number; + aborted: boolean; + externalAbort: boolean; + timedOut: boolean; + hadPotentialSideEffects?: boolean; + attempt: IncompleteTurnAttempt; +}): string | null { + // Prefer the current attempt's terminal message. The session fallback can + // still point at the pre-tool turn after a post-tool answer completes. (#80918) + const assistantState = classifyAssistantTurn(params); + const assistant = assistantState.assistant; + const hasTerminalOutput = hasAttemptTerminalState(params.attempt); + // Tool-use expects a post-tool continuation, while length means the output + // budget ended. Partial visible text completes neither. (#76477) + const incompleteTerminalAssistant = isIncompleteTerminalAssistantTurn({ + hasAssistantVisibleText: params.payloadCount > 0, + hasTerminalOutput, + lastAssistant: assistant, + }); + // Thinking payloads can count toward payloadCount but carry no user-visible + // content; bypass the visible-text guard when thinking was the only output + // so that incomplete-turn stall detection fires below. (#89787, #91953) + const thinkingOnlyTerminal = + params.payloadCount !== 0 && + !assistantState.visibleText.length && + !hasTerminalOutput && + Boolean(assistant && hasOnlyAssistantReasoningContent(assistant)); + + if ( + (params.payloadCount !== 0 && !incompleteTerminalAssistant && !thinkingOnlyTerminal) || + (params.aborted && params.externalAbort) || + params.timedOut || + params.attempt.clientToolCalls || + params.attempt.yieldDetected || + params.attempt.didSendDeterministicApprovalPrompt || + params.attempt.lastToolError + ) { + return null; + } + + if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) { + return null; + } + + if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { + return null; + } + + if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) { + return null; + } + + if (hasAsyncActivity(params.attempt.toolMetas)) { + return null; + } + + const stopReason = assistant?.stopReason; + const reasoningOnlyAssistant = isReasoningOnlyAssistantTurn(assistant); + const emptyResponseAssistant = isEmptyResponseAssistantTurn({ + payloadCount: params.payloadCount, + attempt: params.attempt, + }); + if ( + !incompleteTerminalAssistant && + !reasoningOnlyAssistant && + !thinkingOnlyTerminal && + !emptyResponseAssistant && + stopReason !== "error" + ) { + return null; + } + + return params.hadPotentialSideEffects || params.attempt.replayMetadata.hadPotentialSideEffects + ? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying." + : "⚠️ Agent couldn't generate a response. Please try again."; +} + +/** + * Allows one retry when the provider returned no assistant turn at all and the + * attempt has no side effects, active lifecycle items, delivery, or terminal + * assistant/tool state. + */ +export function shouldRetryMissingAssistantTurn(params: { + payloadCount: number; + aborted: boolean; + promptError?: unknown; + timedOut: boolean; + attempt: IncompleteTurnAttempt; +}): boolean { + if ( + params.payloadCount !== 0 || + params.aborted || + Boolean(params.promptError) || + params.timedOut || + params.attempt.clientToolCalls || + params.attempt.currentAttemptAssistant || + params.attempt.lastAssistant || + params.attempt.yieldDetected || + params.attempt.didSendDeterministicApprovalPrompt || + params.attempt.lastToolError + ) { + return false; + } + + if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) { + return false; + } + + if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { + return false; + } + + if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { + return false; + } + + if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) { + return false; + } + + if (hasAsyncActivity(params.attempt.toolMetas)) { + return false; + } + + if ( + (params.attempt.itemLifecycle?.startedCount ?? 0) > 0 || + (params.attempt.itemLifecycle?.activeCount ?? 0) > 0 + ) { + return false; + } + + return !params.attempt.replayMetadata.hadPotentialSideEffects; +} + +/** Fields needed to determine whether a yielded turn already delivered or can continue. */ +interface YieldContinuationAttempt { + clientToolCalls?: readonly unknown[]; + didSendDeterministicApprovalPrompt?: boolean; + successfulCronAdds?: number; + acceptedSessionSpawns?: readonly { runId: string; childSessionKey: string }[]; + messagingToolSentTexts?: readonly string[]; + messagingToolSentMediaUrls?: readonly string[]; + messagingToolSentTargets?: readonly MessagingToolSend[]; + toolMetas?: readonly { asyncStarted?: boolean }[]; +} + +/** Continuation evidence for a yielded turn — sources that will produce future output. */ +export function hasYieldContinuationEvidence(attempt: YieldContinuationAttempt): boolean { + // Only same-attempt evidence is causal here. Session-wide active descendants may be + // stale or unrelated and must not suppress the diagnostic for this yielded turn. + return ( + (attempt.clientToolCalls?.length ?? 0) > 0 || + attempt.didSendDeterministicApprovalPrompt === true || + hasCommittedMessagingToolDeliveryEvidence({ + messagingToolSentTexts: attempt.messagingToolSentTexts ?? [], + messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [], + messagingToolSentTargets: attempt.messagingToolSentTargets ?? [], + }) || + hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) || + hasAsyncActivity(attempt.toolMetas) || + (attempt.successfulCronAdds ?? 0) > 0 + ); +} + +export const YIELD_DIAGNOSTIC_TEXT = + "⚠️ Turn yielded without a continuation source. Send a message to resume."; + +function isToolResultRole(role: string): boolean { + return role === "toolresult" || role === "tool_result" || role === "tool"; +} + +function readMessageTextContent(message: AgentMessage): string | undefined { + const content = (message as { content?: unknown }).content; + if (typeof content === "string") { + const trimmed = content.trim(); + return trimmed || undefined; + } + const text = collectTextContentBlocks(content) + .map((item) => item.trim()) + .filter((item) => item.length > 0) + .join("\n"); + return text || undefined; +} + +function readToolResultAggregatedText(message: AgentMessage): string | undefined { + const aggregated = (message as { details?: { aggregated?: unknown } }).details?.aggregated; + if (typeof aggregated !== "string") { + return undefined; + } + const trimmed = aggregated.trim(); + return trimmed || undefined; +} + +function hasTrailingSilentToolResult(messages: readonly AgentMessage[]): boolean { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const message = messages[i]; + if (!message) { + continue; + } + const role = normalizeLowercaseStringOrEmpty(message?.role); + if (isToolResultRole(role)) { + if ((message as { isError?: boolean }).isError === true) { + return false; + } + const text = readMessageTextContent(message) ?? readToolResultAggregatedText(message); + return isSilentReplyText(text, SILENT_REPLY_TOKEN); + } + if (role === "assistant" && !readMessageTextContent(message)) { + continue; + } + return false; + } + return false; +} + +/** Emits the silent-reply token for cron turns whose last successful tool result is silent. */ +export function resolveSilentToolResultReplyPayload(params: { + isCronTrigger: boolean; + payloadCount: number; + aborted: boolean; + timedOut: boolean; + attempt: SilentToolResultAttempt; +}): { text: typeof SILENT_REPLY_TOKEN } | null { + if ( + !params.isCronTrigger || + params.payloadCount !== 0 || + params.aborted || + params.timedOut || + (params.attempt.toolMetas?.length ?? 0) === 0 || + params.attempt.clientToolCalls || + params.attempt.yieldDetected || + params.attempt.didSendDeterministicApprovalPrompt || + params.attempt.lastToolError || + (params.attempt.messagesSnapshot?.length ?? 0) === 0 + ) { + return null; + } + + return hasTrailingSilentToolResult(params.attempt.messagesSnapshot) + ? { text: SILENT_REPLY_TOKEN } + : null; +} + +/** + * Marks replay invalid whenever the recorded attempt might not be safe to + * replay or the current run ended in a compaction/incomplete-turn state that + * needs a fresh prompt boundary. + */ +export function resolveReplayInvalidFlag(params: { + attempt: RunLivenessAttempt; + incompleteTurnText?: string | null; +}): boolean { + const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal); + return ( + !params.attempt.replayMetadata.replaySafe || + terminal.promptErrorSource === "compaction" || + terminal.timedOutDuringCompaction || + Boolean(params.incompleteTurnText) + ); +} + +/** Classifies the persisted run state used by session recovery and resume logic. */ +export function resolveRunLivenessState(params: { + payloadCount: number; + aborted: boolean; + timedOut: boolean; + attempt: RunLivenessAttempt; + incompleteTurnText?: string | null; +}): EmbeddedRunLivenessState { + if (params.incompleteTurnText) { + return "abandoned"; + } + const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal); + if (terminal.promptErrorSource === "compaction" || terminal.timedOutDuringCompaction) { + return "paused"; + } + if ((params.aborted || params.timedOut) && params.payloadCount === 0) { + return "blocked"; + } + if (params.attempt.lastAssistant?.stopReason === "error") { + return "blocked"; + } + return "working"; +} diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn.ts b/src/agents/embedded-agent-runner/run/incomplete-turn.ts deleted file mode 100644 index e0d004225521..000000000000 --- a/src/agents/embedded-agent-runner/run/incomplete-turn.ts +++ /dev/null @@ -1,986 +0,0 @@ -/** - * Classifies incomplete terminal assistant turns and retry instructions. - */ -import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion"; -import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; -import { - isSilentReplyPayloadText, - isSilentReplyText, - SILENT_REPLY_TOKEN, -} from "../../../auto-reply/tokens.js"; -import { isReplayUnsafeAssistantError } from "../../../llm/utils/retry.js"; -import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js"; -import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; -import { collectTextContentBlocks } from "../../content-blocks.js"; -import type { MessagingToolSend } from "../../embedded-agent-messaging.types.js"; -import { - isStrictAgenticSupportedProviderModel, - stripProviderPrefix, -} from "../../execution-contract.js"; -import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js"; -import type { AgentMessage } from "../../runtime/index.js"; -import { - hasCommittedMessagingToolDeliveryEvidence, - hasCompletedMessagingToolDeliveryEvidence, - hasMessagingToolDeliveryEvidence, -} from "../delivery-evidence.js"; -import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js"; -import { assessLastAssistantMessage } from "../thinking.js"; -import type { EmbeddedRunLivenessState } from "../types.js"; -import type { EmbeddedRunAttemptResult } from "./types.js"; - -type ReplayMetadataAttempt = Pick< - EmbeddedRunAttemptResult, - | "toolMetas" - | "didSendViaMessagingTool" - | "messagingToolSentTexts" - | "messagingToolSentMediaUrls" - | "successfulCronAdds" -> & - Partial>; - -type IncompleteTurnAttempt = Pick< - EmbeddedRunAttemptResult, - | "assistantTexts" - | "clientToolCalls" - | "currentAttemptAssistant" - | "yieldDetected" - | "didSendDeterministicApprovalPrompt" - | "heartbeatToolResponse" - | "toolMediaUrls" - | "toolAudioAsVoice" - | "toolTrustedLocalMedia" - | "hasToolMediaBlockReply" - | "didDeliverSourceReplyViaMessageTool" - | "messagingToolSourceReplyPayloads" - | "didSendViaMessagingTool" - | "messagingToolSentTexts" - | "messagingToolSentMediaUrls" - | "messagingToolSentTargets" - | "lastToolError" - | "lastAssistant" - | "itemLifecycle" - | "messagesSnapshot" - | "replayMetadata" - | "terminal" - | "toolMetas" -> & - Partial>; - -function hasPositiveOutputTokenUsage(message: AgentMessage | null): boolean { - if (!message || typeof message !== "object") { - return false; - } - const usage = (message as { usage?: unknown }).usage; - if (!usage || typeof usage !== "object") { - return false; - } - const output = asFiniteNumber((usage as { output?: unknown }).output); - return output !== undefined && output > 0; -} - -type SilentToolResultAttempt = Pick< - EmbeddedRunAttemptResult, - | "clientToolCalls" - | "yieldDetected" - | "didSendDeterministicApprovalPrompt" - | "lastToolError" - | "messagesSnapshot" - | "toolMetas" ->; - -type RunLivenessAttempt = Pick< - EmbeddedRunAttemptResult, - "lastAssistant" | "replayMetadata" | "terminal" ->; - -export function isIncompleteTerminalAssistantTurn(params: { - hasAssistantVisibleText: boolean; - hasTerminalOutput?: boolean; - lastAssistant?: { stopReason?: string } | null; -}): boolean { - const stopReason = params.lastAssistant?.stopReason; - // Tool-use expects a post-tool continuation; length means the output budget - // ended before a complete final answer. Partial visible text completes neither. - return stopReason === "toolUse" || (stopReason === "length" && !params.hasTerminalOutput); -} - -const GEMINI_INCOMPLETE_TURN_PROVIDER_IDS = new Set([ - "google", - "google-vertex", - "google-antigravity", - "google-gemini-cli", -]); -const GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN = /^gemini(?:[.-]|$)/; -// Ollama native `/api/chat` can finish with only thinking/internal blocks when constrained. -const OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN = /^ollama(?:-|$)/; -// Model APIs eligible for the non-visible turn retry guard. OpenAI Responses -// family can produce reasoning-only turns where usage.output > 0 but no visible -// text is emitted; without the guard these pass through as successful. (#85364) -const RETRY_GUARD_MODEL_APIS = new Set([ - "openai-completions", - "anthropic-messages", - "bedrock-converse-stream", - "openai-responses", - "openai-chatgpt-responses", - "azure-openai-responses", - "openclaw-openai-responses-transport", - "openclaw-openai-chatgpt-responses-transport", - "openclaw-azure-openai-responses-transport", -]); -// Allow one immediate continuation plus one follow-up continuation before -// surfacing the existing incomplete-turn error path. -export const DEFAULT_REASONING_ONLY_RETRY_LIMIT = 2; -export const DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT = 1; -const REASONING_ONLY_RETRY_INSTRUCTION = - "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; -const EMPTY_RESPONSE_RETRY_INSTRUCTION = - "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; -const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = - "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; - -/** - * Marks whether retrying the attempt can safely replay the prompt. Concrete - * tool-instance policy, async work, committed delivery, spawned sessions, and - * cron writes all contribute side-effect evidence. - */ -export function buildAttemptReplayMetadata( - params: ReplayMetadataAttempt, -): EmbeddedRunAttemptResult["replayMetadata"] { - const hadUnsafeTools = params.toolMetas.some((entry) => entry.replaySafe !== true); - const hadAsyncStartedTool = params.toolMetas.some((t) => t.asyncStarted === true); - const hadPotentialSideEffects = - hadUnsafeTools || - hadAsyncStartedTool || - hasMessagingToolDeliveryEvidence(params) || - hasAcceptedSessionSpawn(params.acceptedSessionSpawns) || - (params.successfulCronAdds ?? 0) > 0; - return { - hadPotentialSideEffects, - replaySafe: !hadPotentialSideEffects, - }; -} - -type TerminalAttemptState = Pick< - EmbeddedRunAttemptResult, - | "clientToolCalls" - | "yieldDetected" - | "didSendDeterministicApprovalPrompt" - | "heartbeatToolResponse" - | "lastToolError" - | "toolMediaUrls" - | "toolAudioAsVoice" - | "toolTrustedLocalMedia" - | "hasToolMediaBlockReply" - | "didDeliverSourceReplyViaMessageTool" - | "messagingToolSourceReplyPayloads" - | "successfulCronAdds" -> & - Partial< - Pick< - EmbeddedRunAttemptResult, - | "acceptedSessionSpawns" - | "messagingToolSentTexts" - | "messagingToolSentMediaUrls" - | "messagingToolSentTargets" - > - > & { - toolMetas?: readonly { asyncStarted?: boolean }[]; - }; - -export function hasAttemptTerminalState(attempt: TerminalAttemptState): boolean { - return Boolean( - attempt.clientToolCalls || - attempt.yieldDetected || - attempt.didSendDeterministicApprovalPrompt || - attempt.heartbeatToolResponse || - attempt.lastToolError || - attempt.toolMediaUrls?.some((url) => url.trim().length > 0) || - attempt.toolAudioAsVoice || - attempt.toolTrustedLocalMedia || - attempt.hasToolMediaBlockReply || - attempt.didDeliverSourceReplyViaMessageTool || - attempt.messagingToolSourceReplyPayloads?.length || - hasCommittedMessagingToolDeliveryEvidence({ - messagingToolSentTexts: attempt.messagingToolSentTexts ?? [], - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [], - messagingToolSentTargets: attempt.messagingToolSentTargets ?? [], - }) || - hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) || - hasAsyncStartedToolActivity(attempt.toolMetas) || - (attempt.successfulCronAdds ?? 0) > 0, - ); -} - -/** - * Builds the user-visible incomplete-turn warning when a terminal attempt did - * not produce a safe final assistant response and no committed delivery/progress - * already completed the task. - */ -export function resolveIncompleteTurnPayloadText(params: { - payloadCount: number; - aborted: boolean; - externalAbort: boolean; - timedOut: boolean; - hadPotentialSideEffects?: boolean; - attempt: IncompleteTurnAttempt; -}): string | null { - // Prefer the current attempt's terminal message. The session fallback can - // still point at the pre-tool turn after a post-tool answer completes. (#80918) - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - const hasTerminalOutput = hasAttemptTerminalState(params.attempt); - // Tool-use expects a post-tool continuation, while length means the output - // budget ended. Partial visible text completes neither. (#76477) - const incompleteTerminalAssistant = isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: params.payloadCount > 0, - hasTerminalOutput, - lastAssistant: assistant, - }); - // Thinking payloads can count toward payloadCount but carry no user-visible - // content; bypass the visible-text guard when thinking was the only output - // so that incomplete-turn stall detection fires below. (#89787, #91953) - const thinkingOnlyTerminal = - params.payloadCount !== 0 && - !joinAssistantTexts(params.attempt.assistantTexts).length && - !hasTerminalOutput && - Boolean(assistant && hasOnlyAssistantReasoningContent(assistant)); - - if ( - (params.payloadCount !== 0 && !incompleteTerminalAssistant && !thinkingOnlyTerminal) || - (params.aborted && params.externalAbort) || - params.timedOut || - params.attempt.clientToolCalls || - params.attempt.yieldDetected || - params.attempt.didSendDeterministicApprovalPrompt || - params.attempt.lastToolError - ) { - return null; - } - - if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) { - return null; - } - - if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { - return null; - } - - if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) { - return null; - } - - if (hasAsyncStartedToolActivity(params.attempt.toolMetas)) { - return null; - } - - const stopReason = assistant?.stopReason; - const reasoningOnlyAssistant = isReasoningOnlyAssistantTurn(assistant); - const emptyResponseAssistant = isEmptyResponseAssistantTurn({ - payloadCount: params.payloadCount, - attempt: params.attempt, - }); - if ( - !incompleteTerminalAssistant && - !reasoningOnlyAssistant && - !thinkingOnlyTerminal && - !emptyResponseAssistant && - stopReason !== "error" - ) { - return null; - } - - return params.hadPotentialSideEffects || params.attempt.replayMetadata.hadPotentialSideEffects - ? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying." - : "⚠️ Agent couldn't generate a response. Please try again."; -} - -/** - * Allows one retry when the provider returned no assistant turn at all and the - * attempt has no side effects, active lifecycle items, delivery, or terminal - * assistant/tool state. - */ -export function shouldRetryMissingAssistantTurn(params: { - payloadCount: number; - aborted: boolean; - promptError?: unknown; - timedOut: boolean; - attempt: IncompleteTurnAttempt; -}): boolean { - if ( - params.payloadCount !== 0 || - params.aborted || - Boolean(params.promptError) || - params.timedOut || - params.attempt.clientToolCalls || - params.attempt.currentAttemptAssistant || - params.attempt.lastAssistant || - params.attempt.yieldDetected || - params.attempt.didSendDeterministicApprovalPrompt || - params.attempt.lastToolError - ) { - return false; - } - - if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) { - return false; - } - - if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { - return false; - } - - if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { - return false; - } - - if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) { - return false; - } - - if (hasAsyncStartedToolActivity(params.attempt.toolMetas)) { - return false; - } - - if ( - (params.attempt.itemLifecycle?.startedCount ?? 0) > 0 || - (params.attempt.itemLifecycle?.activeCount ?? 0) > 0 - ) { - return false; - } - - return !params.attempt.replayMetadata.hadPotentialSideEffects; -} - -function joinAssistantTexts(assistantTexts?: readonly string[]): string { - return (assistantTexts ?? []).join("\n\n").trim(); -} - -function hasOnlySilentAssistantReply(assistantTexts?: readonly string[]): boolean { - const nonEmptyTexts = (assistantTexts ?? []).filter((text) => text.trim().length > 0); - return ( - nonEmptyTexts.length > 0 && - nonEmptyTexts.every((text) => isSilentReplyPayloadText(text, SILENT_REPLY_TOKEN)) - ); -} - -function hasAsyncStartedToolActivity(toolMetas?: readonly { asyncStarted?: boolean }[]): boolean { - return (toolMetas ?? []).some((entry) => entry.asyncStarted === true); -} - -/** Fields needed to determine whether a yielded turn already delivered or can continue. */ -interface YieldContinuationAttempt { - clientToolCalls?: readonly unknown[]; - didSendDeterministicApprovalPrompt?: boolean; - successfulCronAdds?: number; - acceptedSessionSpawns?: readonly { runId: string; childSessionKey: string }[]; - messagingToolSentTexts?: readonly string[]; - messagingToolSentMediaUrls?: readonly string[]; - messagingToolSentTargets?: readonly MessagingToolSend[]; - toolMetas?: readonly { asyncStarted?: boolean }[]; -} - -/** Continuation evidence for a yielded turn — sources that will produce future output. */ -export function hasYieldContinuationEvidence(attempt: YieldContinuationAttempt): boolean { - // Only same-attempt evidence is causal here. Session-wide active descendants may be - // stale or unrelated and must not suppress the diagnostic for this yielded turn. - return ( - (attempt.clientToolCalls?.length ?? 0) > 0 || - attempt.didSendDeterministicApprovalPrompt === true || - hasCommittedMessagingToolDeliveryEvidence({ - messagingToolSentTexts: attempt.messagingToolSentTexts ?? [], - messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [], - messagingToolSentTargets: attempt.messagingToolSentTargets ?? [], - }) || - hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) || - hasAsyncStartedToolActivity(attempt.toolMetas) || - (attempt.successfulCronAdds ?? 0) > 0 - ); -} - -export const YIELD_DIAGNOSTIC_TEXT = - "⚠️ Turn yielded without a continuation source. Send a message to resume."; - -function isToolResultRole(role: string): boolean { - return role === "toolresult" || role === "tool_result" || role === "tool"; -} - -function readMessageTextContent(message: AgentMessage): string | undefined { - const content = (message as { content?: unknown }).content; - if (typeof content === "string") { - const trimmed = content.trim(); - return trimmed || undefined; - } - const text = collectTextContentBlocks(content) - .map((item) => item.trim()) - .filter((item) => item.length > 0) - .join("\n"); - return text || undefined; -} - -function readToolResultAggregatedText(message: AgentMessage): string | undefined { - const aggregated = (message as { details?: { aggregated?: unknown } }).details?.aggregated; - if (typeof aggregated !== "string") { - return undefined; - } - const trimmed = aggregated.trim(); - return trimmed || undefined; -} - -function hasTrailingSilentToolResult(messages: readonly AgentMessage[]): boolean { - for (let i = messages.length - 1; i >= 0; i -= 1) { - const message = messages[i]; - if (!message) { - continue; - } - const role = normalizeLowercaseStringOrEmpty(message?.role); - if (isToolResultRole(role)) { - if ((message as { isError?: boolean }).isError === true) { - return false; - } - const text = readMessageTextContent(message) ?? readToolResultAggregatedText(message); - return isSilentReplyText(text, SILENT_REPLY_TOKEN); - } - if (role === "assistant" && !readMessageTextContent(message)) { - continue; - } - return false; - } - return false; -} - -/** Emits the silent-reply token for cron turns whose last successful tool result is silent. */ -export function resolveSilentToolResultReplyPayload(params: { - isCronTrigger: boolean; - payloadCount: number; - aborted: boolean; - timedOut: boolean; - attempt: SilentToolResultAttempt; -}): { text: typeof SILENT_REPLY_TOKEN } | null { - if ( - !params.isCronTrigger || - params.payloadCount !== 0 || - params.aborted || - params.timedOut || - (params.attempt.toolMetas?.length ?? 0) === 0 || - params.attempt.clientToolCalls || - params.attempt.yieldDetected || - params.attempt.didSendDeterministicApprovalPrompt || - params.attempt.lastToolError || - (params.attempt.messagesSnapshot?.length ?? 0) === 0 - ) { - return null; - } - - return hasTrailingSilentToolResult(params.attempt.messagesSnapshot) - ? { text: SILENT_REPLY_TOKEN } - : null; -} - -/** - * Marks replay invalid whenever the recorded attempt might not be safe to - * replay or the current run ended in a compaction/incomplete-turn state that - * needs a fresh prompt boundary. - */ -export function resolveReplayInvalidFlag(params: { - attempt: RunLivenessAttempt; - incompleteTurnText?: string | null; -}): boolean { - const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal); - return ( - !params.attempt.replayMetadata.replaySafe || - terminal.promptErrorSource === "compaction" || - terminal.timedOutDuringCompaction || - Boolean(params.incompleteTurnText) - ); -} - -/** Classifies the persisted run state used by session recovery and resume logic. */ -export function resolveRunLivenessState(params: { - payloadCount: number; - aborted: boolean; - timedOut: boolean; - attempt: RunLivenessAttempt; - incompleteTurnText?: string | null; -}): EmbeddedRunLivenessState { - if (params.incompleteTurnText) { - return "abandoned"; - } - const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal); - if (terminal.promptErrorSource === "compaction" || terminal.timedOutDuringCompaction) { - return "paused"; - } - if ((params.aborted || params.timedOut) && params.payloadCount === 0) { - return "blocked"; - } - if (params.attempt.lastAssistant?.stopReason === "error") { - return "blocked"; - } - return "working"; -} - -function isReasoningOnlyAssistantTurn(message: unknown): boolean { - if (!message || typeof message !== "object") { - return false; - } - return assessLastAssistantMessage(message as AgentMessage) === "incomplete-text"; -} - -// Unsigned thinking blocks have no cryptographic signature; assessLastAssistantMessage -// returns "incomplete-thinking" for them. Empty content also returns "incomplete-thinking", -// so the content.length > 0 guard is required to distinguish the two cases. -function isUnsignedThinkingOnlyAssistantTurn(message: unknown): boolean { - if (message == null || typeof message !== "object") { - return false; - } - const content = (message as { content?: unknown }).content; - if (!Array.isArray(content) || content.length === 0) { - return false; - } - return assessLastAssistantMessage(message as AgentMessage) === "incomplete-thinking"; -} - -export function shouldRetrySilentErrorAssistantTurn(params: { - attempt: Pick< - EmbeddedRunAttemptResult, - | "assistantTexts" - | "clientToolCalls" - | "yieldDetected" - | "didSendDeterministicApprovalPrompt" - | "heartbeatToolResponse" - | "lastToolError" - | "toolMediaUrls" - | "toolAudioAsVoice" - | "toolTrustedLocalMedia" - | "didDeliverSourceReplyViaMessageTool" - | "messagingToolSourceReplyPayloads" - | "replayMetadata" - | "currentAttemptReplayMetadata" - >; - assistant: EmbeddedRunAttemptResult["lastAssistant"] | null | undefined; -}): boolean { - if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { - return false; - } - if (hasAttemptTerminalState(params.attempt)) { - return false; - } - // Current-attempt evidence avoids blocking on prior committed effects; older - // harnesses retain the cumulative, fail-closed behavior. - const retryReplayMetadata = - params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata; - if (retryReplayMetadata.hadPotentialSideEffects) { - return false; - } - - const assistant = params.assistant; - if (!assistant || assistant.stopReason !== "error" || isReplayUnsafeAssistantError(assistant)) { - return false; - } - - const content = (assistant as { content?: unknown }).content; - if (!Array.isArray(content)) { - return false; - } - if (content.length === 0) { - return !hasPositiveOutputTokenUsage(assistant); - } - - return hasOnlyAssistantReasoningContent(assistant); -} - -function isEmptyResponseAssistantTurn(params: { - payloadCount: number; - attempt: Pick< - IncompleteTurnAttempt, - "assistantTexts" | "currentAttemptAssistant" | "lastAssistant" - >; -}): boolean { - if (params.payloadCount !== 0) { - return false; - } - if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { - return false; - } - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - if (!assistant) { - return true; - } - if (assistant.stopReason === "error") { - return false; - } - if ( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: false, - lastAssistant: assistant, - }) || - isReasoningOnlyAssistantTurn(assistant) - ) { - return false; - } - return true; -} - -function isNonVisibleAssistantTurnEligibleForSilentReply(params: { - payloadCount: number; - attempt: Pick< - IncompleteTurnAttempt, - "assistantTexts" | "currentAttemptAssistant" | "lastAssistant" - >; -}): boolean { - if (isEmptyResponseAssistantTurn(params)) { - return true; - } - if (params.payloadCount !== 0) { - return false; - } - if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { - return false; - } - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - if (!assistant || assistant.stopReason === "error") { - return false; - } - if ( - isIncompleteTerminalAssistantTurn({ - hasAssistantVisibleText: false, - lastAssistant: assistant, - }) - ) { - return false; - } - return isReasoningOnlyAssistantTurn(assistant); -} - -function shouldSkipNonVisibleTurnRetry(params: { - aborted: boolean; - timedOut: boolean; - attempt: IncompleteTurnAttempt; - /** Reply-optional silent classification tolerates committed side effects; retries never can. */ - tolerateSideEffects?: boolean; -}): boolean { - return Boolean( - params.aborted || - params.timedOut || - params.attempt.clientToolCalls || - params.attempt.yieldDetected || - params.attempt.didSendDeterministicApprovalPrompt || - params.attempt.lastToolError || - hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) || - (params.tolerateSideEffects !== true && params.attempt.replayMetadata.hadPotentialSideEffects), - ); -} - -/** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */ -export function shouldTreatEmptyAssistantReplyAsSilent(params: { - allowEmptyAssistantReplyAsSilent?: boolean; - onlyExplicitSilentReply?: boolean; - terminalReplyExpectation?: "required" | "optional"; - payloadCount: number; - aborted: boolean; - timedOut: boolean; - attempt: IncompleteTurnAttempt; -}): boolean { - // "optional" is the run consumer's declaration that no user-facing reply is - // owed (e.g. cron without a delivery route). Silence after side-effecting - // tools is intentional there; retry is replay-unsafe, so erroring would mark - // successful tool-only runs as failures. - const terminalReplyOptional = params.terminalReplyExpectation === "optional"; - if ( - !params.allowEmptyAssistantReplyAsSilent || - shouldSkipNonVisibleTurnRetry({ ...params, tolerateSideEffects: terminalReplyOptional }) - ) { - return false; - } - if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) { - return false; - } - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - if ( - params.payloadCount === 0 && - assistant?.stopReason !== "error" && - hasOnlySilentAssistantReply(params.attempt.assistantTexts) - ) { - return true; - } - // A visible turn owes a reply unless the model explicitly chose NO_REPLY. - // Bare empty and reasoning-only stops are provider failures, even when the - // conversation policy permits deliberate silence. - if (params.onlyExplicitSilentReply || !terminalReplyOptional) { - return false; - } - return isNonVisibleAssistantTurnEligibleForSilentReply({ - payloadCount: params.payloadCount, - attempt: params.attempt, - }); -} - -/** - * Builds the retry instruction for reasoning-only turns that consumed provider - * output budget but produced no visible assistant text. - */ -export function resolveReasoningOnlyRetryInstruction(params: { - provider?: string; - modelId?: string; - modelApi?: string; - executionContract?: string; - aborted: boolean; - timedOut: boolean; - attempt: IncompleteTurnAttempt; -}): string | null { - if (shouldSkipNonVisibleTurnRetry(params)) { - return null; - } - - if ( - !shouldApplyNonVisibleTurnRetryGuard({ - provider: params.provider, - modelId: params.modelId, - modelApi: params.modelApi, - executionContract: params.executionContract, - }) - ) { - return null; - } - - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) { - return null; - } - if (assistant?.stopReason === "error") { - return null; - } - if (!isReasoningOnlyAssistantTurn(assistant) && !isUnsignedThinkingOnlyAssistantTurn(assistant)) { - return null; - } - - return REASONING_ONLY_RETRY_INSTRUCTION; -} - -/** Builds one fresh continuation after settled tools ended without a visible final answer. */ -export function resolveSettledToolTerminalContinuationInstruction(params: { - provider?: string; - modelId?: string; - modelApi?: string; - executionContract?: string; - allowEmptyStopContinuation?: boolean; - payloadCount: number; - hasTerminalToolPresentation?: boolean; - aborted: boolean; - promptError?: unknown; - timedOut: boolean; - attempt: IncompleteTurnAttempt; -}): string | null { - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; - const currentAttemptAssistant = params.attempt.currentAttemptAssistant; - const emptyStopAfterSettledTools = Boolean( - params.allowEmptyStopContinuation && - currentAttemptAssistant?.stopReason === "stop" && - params.attempt.toolMetas.length > 0 && - params.attempt.toolMetas.every((tool) => tool.isError !== true && tool.asyncStarted !== true) && - params.attempt.itemLifecycle.startedCount > 0 && - params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount && - params.attempt.itemLifecycle.activeCount === 0 && - !hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) && - isEmptyResponseAssistantTurn({ - payloadCount: params.payloadCount, - attempt: params.attempt, - }), - ); - // Idle is not proof of settlement: skipped or partially dispatched tools must - // never be described as completed. Match each terminal call's id and owner to - // its own current-batch result; a reported failure is settled, not successful. - const requestedToolCalls = Array.isArray(assistant?.content) - ? assistant.content.flatMap((item) => { - const block = item as { type?: unknown; id?: unknown; name?: unknown } | null; - return block?.type === "toolCall" - ? [ - { - id: typeof block.id === "string" ? block.id : null, - name: typeof block.name === "string" ? block.name : null, - }, - ] - : []; - }) - : []; - // Scan only results AFTER the terminal assistant: the snapshot spans the whole - // session, and a prior turn's toolResult with a model-reused id would otherwise - // prove "completion" for a batch that never dispatched. Assistant not found in - // the snapshot fails closed to the existing incomplete-turn error. - const snapshot = params.attempt.messagesSnapshot ?? []; - const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1; - const settledToolResults = new Map( - (assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => { - const result = message as { - role?: unknown; - toolCallId?: unknown; - toolName?: unknown; - isError?: unknown; - }; - return result.role === "toolResult" && - typeof result.toolCallId === "string" && - typeof result.toolName === "string" - ? [ - [ - result.toolCallId, - { toolName: result.toolName, isError: result.isError === true }, - ] as const, - ] - : []; - }), - ); - const allToolsProvenSettled = - params.attempt.itemLifecycle?.activeCount === 0 && - requestedToolCalls.length > 0 && - requestedToolCalls.every( - ({ id, name }) => - id !== null && name !== null && settledToolResults.get(id)?.toolName === name, - ); - const failedTerminalToolNames = new Set( - requestedToolCalls.flatMap(({ id, name }) => - id !== null && name !== null && settledToolResults.get(id)?.isError === true ? [name] : [], - ), - ); - const hasSettledTerminalToolFailure = allToolsProvenSettled && failedTerminalToolNames.size > 0; - // ToolErrorSummary has no call id: its owner must match a failed result in the - // proven terminal batch, or a stale/unrelated error could authorize finalization. - const hasUnsettledToolError = Boolean( - params.attempt.lastToolError && - (assistant?.stopReason !== "toolUse" || - !hasSettledTerminalToolFailure || - !failedTerminalToolNames.has(params.attempt.lastToolError.toolName)), - ); - if ( - params.payloadCount !== 0 || - params.hasTerminalToolPresentation || - params.aborted || - params.promptError != null || - params.timedOut || - (assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) || - hasUnsettledToolError || - (hasSettledTerminalToolFailure && - (hasAsyncStartedToolActivity(params.attempt.toolMetas) || - hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) || - params.attempt.clientToolCalls || - params.attempt.yieldDetected || - params.attempt.didSendDeterministicApprovalPrompt - ) { - return null; - } - if (hasCompletedMessagingToolDeliveryEvidence(params.attempt)) { - return null; - } - if ( - !shouldApplyNonVisibleTurnRetryGuard({ - provider: params.provider, - modelId: params.modelId, - modelApi: params.modelApi, - executionContract: params.executionContract, - }) - ) { - return null; - } - return hasSettledTerminalToolFailure - ? `${SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION} If any tool failed, state that failure plainly and do not claim it succeeded.` - : SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION; -} - -/** - * Builds the retry instruction for empty assistant turns when the provider/model - * is eligible for non-visible turn recovery. - */ -export function resolveEmptyResponseRetryInstruction(params: { - provider?: string; - modelId?: string; - modelApi?: string; - executionContract?: string; - payloadCount: number; - aborted: boolean; - timedOut: boolean; - attempt: IncompleteTurnAttempt; -}): string | null { - if (shouldSkipNonVisibleTurnRetry(params)) { - return null; - } - - if ( - !isEmptyResponseAssistantTurn({ - payloadCount: params.payloadCount, - attempt: params.attempt, - }) - ) { - return null; - } - - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null; - if ( - assistant?.stopReason === "stop" && - OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test( - normalizeLowercaseStringOrEmpty(params.provider ?? ""), - ) && - !hasPositiveOutputTokenUsage(assistant) - ) { - return null; - } - - if ( - shouldApplyNonVisibleTurnRetryGuard({ - provider: params.provider, - modelId: params.modelId, - modelApi: params.modelApi, - executionContract: params.executionContract, - }) || - // Keep the generic zero-usage stop retry for providers that expose a - // provider-neutral "nothing was generated" signal, even outside the - // provider allowlist above. - isZeroUsageEmptyStopAssistantTurn(assistant) - ) { - return EMPTY_RESPONSE_RETRY_INSTRUCTION; - } - - return null; -} - -function shouldApplyNonVisibleTurnRetryGuard(params: { - provider?: string; - modelId?: string; - modelApi?: string; - executionContract?: string; -}): boolean { - if ( - params.executionContract === "strict-agentic" || - isIncompleteTurnRecoverySupportedProviderModel({ - provider: params.provider, - modelId: params.modelId, - }) - ) { - return true; - } - if (RETRY_GUARD_MODEL_APIS.has(normalizeLowercaseStringOrEmpty(params.modelApi ?? ""))) { - return true; - } - // This path uses provider output structure only: no user or assistant prose classification. - return OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test( - normalizeLowercaseStringOrEmpty(params.provider ?? ""), - ); -} - -function isIncompleteTurnRecoverySupportedProviderModel(params: { - provider?: string; - modelId?: string; -}): boolean { - if ( - isStrictAgenticSupportedProviderModel({ - provider: params.provider, - modelId: params.modelId, - }) - ) { - return true; - } - const provider = normalizeLowercaseStringOrEmpty(params.provider ?? ""); - if (!GEMINI_INCOMPLETE_TURN_PROVIDER_IDS.has(provider)) { - return false; - } - const modelId = typeof params.modelId === "string" ? params.modelId : ""; - return GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN.test(stripProviderPrefix(modelId)); -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/src/agents/embedded-agent-runner/run/terminal-resolution.ts b/src/agents/embedded-agent-runner/run/terminal-resolution.ts index 808ccccbdfdc..4e0756b762f5 100644 --- a/src/agents/embedded-agent-runner/run/terminal-resolution.ts +++ b/src/agents/embedded-agent-runner/run/terminal-resolution.ts @@ -15,24 +15,26 @@ import type { EmbeddedRunFailureSignal, TraceAttempt, } from "../types.js"; +import { hasAttemptTerminalState } from "./attempt-terminal-evidence.js"; import { markEmbeddedRunAuthProfileSuccess, reportEmbeddedRunSuccessfulAuthBinding, } from "./auth-profile-success.js"; import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js"; import { - hasAttemptTerminalState, - hasYieldContinuationEvidence, resolveEmptyResponseRetryInstruction, - resolveIncompleteTurnPayloadText, resolveReasoningOnlyRetryInstruction, + resolveSettledToolTerminalContinuationInstruction, + shouldTreatEmptyAssistantReplyAsSilent, +} from "./incomplete-turn-recovery.js"; +import { + hasYieldContinuationEvidence, + resolveIncompleteTurnPayloadText, resolveRunLivenessState, resolveSilentToolResultReplyPayload, - resolveSettledToolTerminalContinuationInstruction, shouldRetryMissingAssistantTurn, - shouldTreatEmptyAssistantReplyAsSilent, YIELD_DIAGNOSTIC_TEXT, -} from "./incomplete-turn.js"; +} from "./incomplete-turn-resolution.js"; import type { RunEmbeddedAgentParams } from "./params.js"; import { isEmbeddedRunTerminalAbort, diff --git a/src/agents/embedded-agent-runner/run/terminal-timeout.ts b/src/agents/embedded-agent-runner/run/terminal-timeout.ts index 75aad944567c..406b223ff66d 100644 --- a/src/agents/embedded-agent-runner/run/terminal-timeout.ts +++ b/src/agents/embedded-agent-runner/run/terminal-timeout.ts @@ -1,7 +1,7 @@ import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import { hasMessagingToolDeliveryEvidence } from "../delivery-evidence.js"; import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; -import { resolveRunLivenessState } from "./incomplete-turn.js"; +import { resolveRunLivenessState } from "./incomplete-turn-resolution.js"; import { isEmbeddedRunTerminalAbort, isEmbeddedRunTerminalTimeout, diff --git a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts index c086150c49db..2e1b1d658584 100644 --- a/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts +++ b/src/agents/embedded-agent-subscribe.handlers.lifecycle.ts @@ -17,10 +17,8 @@ import { GENERIC_ASSISTANT_ERROR_TEXT, } from "./embedded-agent-helpers.js"; import { hasCommittedMessagingToolDeliveryEvidence } from "./embedded-agent-runner/delivery-evidence.js"; -import { - hasAttemptTerminalState, - isIncompleteTerminalAssistantTurn, -} from "./embedded-agent-runner/run/incomplete-turn.js"; +import { hasAttemptTerminalState } from "./embedded-agent-runner/run/attempt-terminal-evidence.js"; +import { isIncompleteTerminalAssistantTurn } from "./embedded-agent-runner/run/incomplete-turn-classification.js"; import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js"; import { consumePendingToolMediaReply, diff --git a/src/agents/test-helpers/embedded-agent-runner-e2e-fixtures.ts b/src/agents/test-helpers/embedded-agent-runner-e2e-fixtures.ts index 6ee9cf3a83b2..657c3d04fdd2 100644 --- a/src/agents/test-helpers/embedded-agent-runner-e2e-fixtures.ts +++ b/src/agents/test-helpers/embedded-agent-runner-e2e-fixtures.ts @@ -8,7 +8,7 @@ import os from "node:os"; import path from "node:path"; import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { buildAttemptReplayMetadata } from "../embedded-agent-runner/run/incomplete-turn.js"; +import { buildAttemptReplayMetadata } from "../embedded-agent-runner/run/attempt-terminal-evidence.js"; import type { EmbeddedRunAttemptResult } from "../embedded-agent-runner/run/types.js"; export type EmbeddedAgentRunnerTestWorkspace = { diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index bd02a733bbf7..2bd452d55b9e 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -992,10 +992,6 @@ describe("scripts/test-projects changed-target routing", () => { "src/agents/embedded-agent-runner/run.before-agent-reply-cron.test.ts", "test/vitest/vitest.agents-embedded-agent.config.ts", ], - [ - "src/agents/embedded-agent-runner/run.incomplete-turn.test.ts", - "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts", - ], [ "src/agents/embedded-agent-runner/run.overflow-compaction.test.ts", "test/vitest/vitest.agents-embedded-agent-overflow-compaction.config.ts", @@ -1021,6 +1017,28 @@ describe("scripts/test-projects changed-target routing", () => { ]); }); + it("routes every split incomplete-turn test to its dedicated serial shard", () => { + const root = "src/agents/embedded-agent-runner"; + const discovered = fs + .readdirSync(root) + .filter((name) => name.startsWith("run.incomplete-turn.") && name.endsWith(".test.ts")) + .map((name) => `${root}/${name}`) + .toSorted(); + const owned = agentVitestProjectOwners.embeddedIncompleteTurn.include.toSorted(); + + expect(owned).toEqual(discovered); + for (const testFile of discovered) { + expect(buildVitestRunPlans([testFile])).toEqual([ + { + config: "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts", + forwardedArgs: [], + includePatterns: [testFile], + watchMode: false, + }, + ]); + } + }); + it.each([ [ "src/agents/embedded-agent-runner/run", @@ -1061,7 +1079,7 @@ describe("scripts/test-projects changed-target routing", () => { { config: "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts", forwardedArgs: [], - includePatterns: [`${root}/run.incomplete-turn.test.ts`], + includePatterns: agentVitestProjectOwners.embeddedIncompleteTurn.include, watchMode: false, }, { diff --git a/test/vitest/vitest.agents-paths.mjs b/test/vitest/vitest.agents-paths.mjs index 32bdd913acac..f7e1c4f9781d 100644 --- a/test/vitest/vitest.agents-paths.mjs +++ b/test/vitest/vitest.agents-paths.mjs @@ -17,7 +17,19 @@ const coreIsolatedFiles = [ "src/agents/subagents/registry/subagent-registry.announce-loop-guard.test.ts", "src/agents/subagents/registry/subagent-registry-restart-recovery.test.ts", ]; -const incompleteTurnFiles = [`${embeddedRoot}/run.incomplete-turn.test.ts`]; +const incompleteTurnFiles = [ + `${embeddedRoot}/run.incomplete-turn.attempt-lifecycle.test.ts`, + `${embeddedRoot}/run.incomplete-turn.classification.test.ts`, + `${embeddedRoot}/run.incomplete-turn.delivery-resolution.test.ts`, + `${embeddedRoot}/run.incomplete-turn.empty-response-recovery.test.ts`, + `${embeddedRoot}/run.incomplete-turn.error-recovery.test.ts`, + `${embeddedRoot}/run.incomplete-turn.payload-resolution.test.ts`, + `${embeddedRoot}/run.incomplete-turn.reasoning-recovery.test.ts`, + `${embeddedRoot}/run.incomplete-turn.settled-tool-continuation.test.ts`, + `${embeddedRoot}/run.incomplete-turn.settled-tool-recovery.test.ts`, + `${embeddedRoot}/run.incomplete-turn.silent-reply.test.ts`, + `${embeddedRoot}/run.incomplete-turn.terminal-evidence.test.ts`, +]; const overflowCompactionFiles = [ `${embeddedRoot}/run.overflow-compaction.test.ts`, `${embeddedRoot}/run.prepared-harness-source-delivery.integration.test.ts`, diff --git a/test/vitest/vitest.unit-fast-paths.mjs b/test/vitest/vitest.unit-fast-paths.mjs index ed2a1dc4681e..d2a211cd5354 100644 --- a/test/vitest/vitest.unit-fast-paths.mjs +++ b/test/vitest/vitest.unit-fast-paths.mjs @@ -167,6 +167,7 @@ const broadUnitFastCandidateGlobs = [ ]; const ownerRoutedUnitTestPatterns = [ ...cliProcessTestFiles, + "src/agents/embedded-agent-runner/run.incomplete-turn.*.test.ts", "src/agents/embedded-agent-runner/run/attempt.abort-race.test.ts", "src/agents/openai-transport-stream.*.test.ts", "src/agents/embedded-agent-runner/run.shared-integration.test.ts",