diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts index 046b690b3110..c115d3b7b99a 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-continuation.test.ts @@ -236,7 +236,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); }); - it("records silent success when the settled-tool finalization completes empty", async () => { + it("surfaces an incomplete turn when a required settled-tool finalizer completes empty", async () => { const emptyStopAssistant = makeLastAssistant(); mockedClassifyFailoverReason.mockReturnValue(null); mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { @@ -261,16 +261,19 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { const result = await runEmbeddedAgent( makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", { allowEmptyAssistantReplyAsSilent: true, + terminalReplyExpectation: "required", }), ); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); - expect(result.payloads).toBeUndefined(); - expect(result.meta.error).toBeUndefined(); + expect(result.payloads?.[0]).toMatchObject({ isError: true }); + expect(result.payloads?.[0]?.text).toContain( + "some tool actions may have already been executed", + ); + expect(result.meta.error?.kind).toBe("incomplete_turn"); expect(result.meta.terminalReplyKind).toBeUndefined(); expect(result.meta.finalAssistantVisibleText).toBeUndefined(); expect(result.meta.finalAssistantRawText).toBeUndefined(); - expect(result.meta.stopReason).toBe("stop"); expectNoWarnMessageWith("empty response detected"); expectWarnMessageWith("settled-turn finalization completed without a visible answer"); }); diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts index b3170e6e6e5a..c4d652bb38cc 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.settled-tool-recovery.test.ts @@ -404,32 +404,17 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { }); it("continues once after settled side-effecting tools finish without a final answer", async () => { - const acceptedSessionSpawns = [ - { runId: "child-run", childSessionKey: "agent:main:subagent:child" }, - ]; const toolUseAssistant = makeLastAssistant({ stopReason: "toolUse", content: [ { type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } }, { type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } }, - { - type: "toolCall", - id: "tool_spawn", - name: "sessions_spawn", - arguments: { task: "follow up" }, - }, ], }); const settledToolResults = [ toolUseAssistant, { role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false }, { role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false }, - { - role: "toolResult", - toolCallId: "tool_spawn", - toolName: "sessions_spawn", - isError: false, - }, ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"]; mockedClassifyFailoverReason.mockReturnValue(null); mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { @@ -437,15 +422,10 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { return makeAttemptResult({ assistantTexts: [], latestMcpAppChannelView: { viewId: "view-after-tools" }, - toolMetas: [ - { toolName: "write", meta: "path=note.txt" }, - { toolName: "cron" }, - { toolName: "sessions_spawn" }, - ], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }, { toolName: "cron" }], successfulNestedToolNames: ["read"], - acceptedSessionSpawns, successfulCronAdds: 1, - itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 }, + itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 }, messagesSnapshot: settledToolResults, lastAssistant: toolUseAssistant, currentAttemptAssistant: toolUseAssistant, @@ -475,10 +455,9 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" }); expect(result.successfulCronAdds).toBe(1); - expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns); expect(result.meta.toolSummary).toEqual({ - calls: 3, - tools: ["write", "cron", "sessions_spawn"], + calls: 2, + tools: ["write", "cron"], failures: 0, }); expect(result.meta.agentMeta).toMatchObject({ diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts index 6b6f6e45c759..8d3fc0cb9e7b 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.terminal-evidence.test.ts @@ -17,6 +17,35 @@ import { } from "./run/incomplete-turn-resolution.js"; import type { EmbeddedRunAttemptResult } from "./run/types.js"; +function makeSettledIdleWriteAttempt(options?: { + terminal?: EmbeddedRunAttemptResult["terminal"]; + stalePriorTurn?: boolean; +}) { + const toolUseAssistant = makeLastAssistant({ + stopReason: "toolUse", + content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }], + }); + const abortedAssistant = makeLastAssistant({ stopReason: "aborted", content: [] }); + return makeAttemptResult({ + terminal: options?.terminal ?? { kind: "timeout", phase: "prompt", source: "idle" }, + assistantTexts: [], + toolMetas: [{ toolName: "write", replaySafe: false }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + { role: "user", content: [{ type: "text", text: "old turn" }] }, + toolUseAssistant, + { role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false }, + ...(options?.stalePriorTurn + ? [{ role: "user", content: [{ type: "text", text: "current turn" }] }] + : []), + abortedAssistant, + ] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"], + lastAssistant: abortedAssistant, + currentAttemptAssistant: abortedAssistant, + currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false }, + }); +} + describe("runEmbeddedAgent incomplete-turn safety", () => { beforeEach(() => { resetRunIncompleteTurnOwnerMocks(); @@ -87,26 +116,81 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { ).toBe(true); }); - it.each([ - { label: "aborted", aborted: true, timedOut: false, promptError: null }, - { label: "timed out", aborted: false, timedOut: true, promptError: null }, - { label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") }, - ])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => { - const toolUseAssistant = makeLastAssistant({ - stopReason: "toolUse", - content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }], - }); + it("continues an exactly settled current-turn tool batch after an idle prompt timeout", () => { const instruction = resolveSettledToolTerminalContinuationInstruction( - makeSettledContinuationParams( - { - assistantTexts: [], - toolMetas: [{ toolName: "bash" }], - itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, - lastAssistant: toolUseAssistant, - currentAttemptAssistant: toolUseAssistant, - }, - { aborted, timedOut, promptError }, - ), + makeSettledContinuationParams(makeSettledIdleWriteAttempt(), { + timedOut: true, + promptError: new Error("LLM idle timeout"), + }), + ); + + expect(instruction).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + }); + + it.each([ + { + label: "external abort", + terminal: { kind: "timeout", phase: "prompt", source: "external" } as const, + aborted: true, + timedOut: true, + }, + { + label: "runtime timeout", + terminal: { kind: "timeout", phase: "prompt", source: "runtime" } as const, + aborted: false, + timedOut: true, + }, + { + label: "run budget timeout", + terminal: { kind: "timeout", phase: "prompt", source: "run_budget" } as const, + aborted: false, + timedOut: true, + }, + { + label: "compaction timeout", + terminal: { kind: "timeout", phase: "compaction", source: "idle" } as const, + aborted: false, + timedOut: true, + }, + { + label: "tool execution timeout", + terminal: { kind: "timeout", phase: "tool_execution", source: "idle" } as const, + aborted: false, + timedOut: true, + }, + { + label: "timeout observation", + terminal: { kind: "timeout", phase: "tool_execution", source: "observation" } as const, + aborted: false, + timedOut: false, + }, + { + label: "prompt error without idle timeout", + terminal: { kind: "ok" } as const, + aborted: false, + timedOut: false, + promptError: new Error("closed"), + }, + ])( + "does not finalize settled tools after a $label", + ({ terminal, aborted, timedOut, promptError }) => { + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams(makeSettledIdleWriteAttempt({ terminal }), { + aborted, + timedOut, + promptError, + }), + ); + + expect(instruction).toBeNull(); + }, + ); + + it("does not use a settled prior-turn batch to authorize idle-timeout finalization", () => { + const instruction = resolveSettledToolTerminalContinuationInstruction( + makeSettledContinuationParams(makeSettledIdleWriteAttempt({ stalePriorTurn: true }), { + timedOut: true, + }), ); expect(instruction).toBeNull(); diff --git a/src/agents/embedded-agent-runner/run.prompt-timeout-fallback.test-support.ts b/src/agents/embedded-agent-runner/run.prompt-timeout-fallback.test-support.ts index 89651ae2d386..5fc40ac91108 100644 --- a/src/agents/embedded-agent-runner/run.prompt-timeout-fallback.test-support.ts +++ b/src/agents/embedded-agent-runner/run.prompt-timeout-fallback.test-support.ts @@ -4,7 +4,9 @@ import { makeModelFallbackCfg } from "../test-helpers/model-fallback-config-fixt import { makeAttemptResult } from "./run.overflow-compaction.fixture.js"; import { MockedFailoverError, + mockedBuildEmbeddedRunPayloads, mockedClassifyFailoverReason, + mockedGetApiKeyForModel, mockedRunEmbeddedAttempt, overflowBaseRunParams, resetSharedRunIntegrationHarnessMocks, @@ -58,4 +60,102 @@ describe("runEmbeddedAgent prompt timeout fallback handoff", () => { await expect(promise).rejects.toThrow("LLM request timed out."); expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); }); + + it("finalizes a settled write after an idle timeout without replaying the prompt", async () => { + const toolUseAssistant = { + role: "assistant" as const, + stopReason: "toolUse" as const, + provider: "openai", + model: "gpt-5.4", + content: [ + { + type: "toolCall", + id: "tool_write", + name: "write", + arguments: { path: "note.txt", content: "done" }, + }, + ], + }; + const abortedAssistant = { + role: "assistant" as const, + stopReason: "aborted" as const, + provider: "openai", + model: "gpt-5.4", + content: [], + }; + const finalAssistant = { + role: "assistant" as const, + stopReason: "stop" as const, + provider: "openai", + model: "gpt-5.4", + content: [{ type: "text", text: "The note was written once." }], + }; + mockedClassifyFailoverReason.mockReturnValue("timeout"); + mockedRunEmbeddedAttempt + .mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + terminal: { kind: "timeout", phase: "prompt", source: "idle" }, + toolMetas: [{ toolName: "write", replaySafe: false }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + messagesSnapshot: [ + { role: "user", content: [{ type: "text", text: "Write note.txt" }] }, + toolUseAssistant, + { + role: "toolResult", + toolCallId: "tool_write", + toolName: "write", + isError: false, + }, + abortedAssistant, + ] as never, + lastAssistant: abortedAssistant as never, + currentAttemptAssistant: abortedAssistant as never, + currentAttemptReplayMetadata: { + hadPotentialSideEffects: true, + replaySafe: false, + }, + }), + ) + .mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: ["The note was written once."], + lastAssistant: finalAssistant as never, + currentAttemptAssistant: finalAssistant as never, + currentAttemptCompletedAssistant: finalAssistant as never, + }), + ); + mockedBuildEmbeddedRunPayloads + .mockReturnValueOnce([]) + .mockReturnValueOnce([{ text: "The note was written once." }]); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.4", + runId: "run-post-tool-idle-finalization", + config: makeModelFallbackCfg({ + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-opus-4-6"], + }, + }, + }, + }), + }); + + expect(result.payloads).toEqual([{ text: "The note was written once." }]); + expect(result.meta.executionTrace?.fallbackUsed).toBe(false); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(mockedRunEmbeddedAttempt.mock.calls[1]?.[0]).toMatchObject({ + operation: "settled-tool-finalization", + disableTools: true, + skipPreparedUserTurnMessage: true, + prompt: + "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.", + }); + expect(mockedGetApiKeyForModel).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.test.ts b/src/agents/embedded-agent-runner/run/assistant-failure.test.ts index c7b3765c207c..87d1828f7d17 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failure.test.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failure.test.ts @@ -110,6 +110,37 @@ function makeExhaustedCredentialFailureInput(options?: { replaySafe?: boolean }) }; } +function makeIdleTimeoutFailureInput(options?: { replaySafe?: boolean }) { + const fixture = makeExhaustedCredentialFailureInput(); + const replaySafe = options?.replaySafe === true; + const assistant = buildEmbeddedRunnerAssistant({ + provider: "anthropic", + model: "mock-1", + stopReason: "aborted", + }); + const replayMetadata = { + hadPotentialSideEffects: !replaySafe, + replaySafe, + }; + const attempt = makeEmbeddedRunnerAttempt({ + terminal: { kind: "timeout", phase: "prompt", source: "idle" }, + lastAssistant: assistant, + currentAttemptAssistant: assistant, + toolMetas: replaySafe ? [] : [{ toolName: "write", replaySafe: false }], + replayMetadata, + currentAttemptReplayMetadata: replayMetadata, + }); + fixture.input.attempt = attempt; + fixture.input.attemptAssistant = assistant; + fixture.input.currentAttemptAssistant = assistant; + fixture.input.terminalState = resolveEmbeddedRunAttemptTerminalState({ attempt, assistant }); + fixture.input.emptyErrorRetries = 0; + fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => true); + fixture.input.maybeRetrySameModelRateLimit = vi.fn(async () => true); + fixture.input.advanceRateLimitAuthProfile = vi.fn(async () => true); + return fixture; +} + describe("handleEmbeddedAssistantFailure", () => { it("uses prepared OpenRouter ownership for custom-provider billing failures", async () => { const fixture = makeExhaustedCredentialFailureInput(); @@ -165,10 +196,11 @@ describe("handleEmbeddedAssistantFailure", () => { } fixture.input.attemptAssistant.errorCode = PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE; fixture.input.attemptAssistant.errorMessage = "reasoning is required"; + fixture.input.resolveAuthProfileFailureReason = vi.fn(() => "timeout" as const); const outcome = await handleEmbeddedAssistantFailure(fixture.input); - expect(outcome.action).toBe("proceed"); + expect(outcome).toMatchObject({ action: "proceed", assistantProfileFailureReason: null }); expect(fixture.advanceAuthProfile).not.toHaveBeenCalled(); expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled(); expect(fixture.traceAttempts).toEqual([]); @@ -212,6 +244,37 @@ describe("handleEmbeddedAssistantFailure", () => { expect(fixture.traceAttempts).toEqual([]); }); + it("closes every failover retry after an idle timeout commits a write", async () => { + const fixture = makeIdleTimeoutFailureInput(); + + const outcome = await handleEmbeddedAssistantFailure(fixture.input); + + expect(outcome.action).toBe("proceed"); + expect(fixture.input.maybeRefreshRuntimeAuthForAuthError).not.toHaveBeenCalled(); + expect(fixture.input.maybeRetrySameModelRateLimit).not.toHaveBeenCalled(); + expect(fixture.advanceAuthProfile).not.toHaveBeenCalled(); + expect(fixture.input.advanceRateLimitAuthProfile).not.toHaveBeenCalled(); + expect(fixture.traceAttempts).toEqual([]); + }); + + it("keeps replay-safe idle timeout profile rotation available", async () => { + const fixture = makeIdleTimeoutFailureInput({ replaySafe: true }); + fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => false); + + const outcome = await handleEmbeddedAssistantFailure(fixture.input); + + expect(outcome).toMatchObject({ action: "retry", lastRetryFailoverReason: "timeout" }); + expect(fixture.advanceAuthProfile).toHaveBeenCalledOnce(); + expect(fixture.traceAttempts).toEqual([ + { + provider: "anthropic", + model: "mock-1", + result: "rotate_profile", + stage: "assistant", + }, + ]); + }); + it("does not cache an exact credential-file failure from a fallback candidate", async () => { const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS; process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000"; diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.ts b/src/agents/embedded-agent-runner/run/assistant-failure.ts index b9ce09fff744..8c721e4a94a3 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failure.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failure.ts @@ -23,6 +23,7 @@ import { import { log } from "../logger.js"; import type { TraceAttempt } from "../types.js"; import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js"; +import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js"; import { createFailoverDecisionLogger } from "./failover-observation.js"; import { resolveRunFailoverDecision } from "./failover-policy.js"; import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn-recovery.js"; @@ -100,28 +101,10 @@ export async function handleEmbeddedAssistantFailure(input: { projectAgentRunAttemptTerminal(input.attempt.terminal); const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome); const { signalOwnedInterruption } = input.terminalState; - if (isReplayUnsafeAssistantError(input.attemptAssistant)) { - return buildOutcome(input, { - action: "proceed", - assistantProfileFailureReason: null, - }); - } const fallbackThinking = pickFallbackThinkingLevel({ message: input.attemptAssistant?.errorMessage, attempted: input.attemptedThinking, }); - if (fallbackThinking && !terminalInterrupted) { - log.warn( - `unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`, - ); - return buildOutcome(input, { - action: "retry", - thinkLevel: fallbackThinking, - preserveSameModelRateLimitRetryCount: true, - assistantProfileFailureReason: null, - }); - } - const authFailure = isAuthAssistantError(input.attemptAssistant); const rateLimitFailure = isRateLimitAssistantError(input.attemptAssistant); const billingFailure = isBillingAssistantError(input.attemptAssistant); @@ -144,6 +127,26 @@ export async function handleEmbeddedAssistantFailure(input: { isShortWindowRateLimitMessage(input.attemptAssistant?.errorMessage), }, ); + const replayUnsafeAssistantError = isReplayUnsafeAssistantError(input.attemptAssistant); + if (replayUnsafeAssistantError || !isCurrentAttemptReplaySafe(input.attempt)) { + return buildOutcome(input, { + action: "proceed", + assistantProfileFailureReason: replayUnsafeAssistantError + ? null + : assistantProfileFailureReason, + }); + } + if (fallbackThinking && !terminalInterrupted) { + log.warn( + `unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`, + ); + return buildOutcome(input, { + action: "retry", + thinkLevel: fallbackThinking, + preserveSameModelRateLimitRetryCount: true, + assistantProfileFailureReason, + }); + } const cloudCodeAssistFormatError = input.attempt.cloudCodeAssistFormatError; const imageDimensionError = parseImageDimensionError(input.attemptAssistant?.errorMessage ?? ""); const genericUnknownReasoningError = diff --git a/src/agents/embedded-agent-runner/run/attempt-recovery.ts b/src/agents/embedded-agent-runner/run/attempt-recovery.ts index b5034dcd076e..31696d7b0cb5 100644 --- a/src/agents/embedded-agent-runner/run/attempt-recovery.ts +++ b/src/agents/embedded-agent-runner/run/attempt-recovery.ts @@ -11,6 +11,7 @@ import type { EmbeddedAgentRunResult, TraceAttempt } from "../types.js"; import type { createUsageAccumulator } from "../usage-accumulator.js"; import type { prepareAndDispatchEmbeddedRunAttempt } from "./attempt-dispatch-preparation.js"; import type { normalizeEmbeddedRunAttempt } from "./attempt-normalization.js"; +import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js"; import { buildEmbeddedRunBlockedResult } from "./blocked-run-result.js"; import { resolveCodexAppServerRecoveryRetry } from "./codex-app-server-recovery.js"; import { resolveCompactionLiveModelSelection } from "./compaction-live-model-selection.js"; @@ -110,6 +111,7 @@ export async function recoverEmbeddedRunAttempt(input: { timedOutByRunBudget, } = projectAgentRunAttemptTerminal(attempt.terminal); const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalState.outcome); + const currentAttemptReplaySafe = isCurrentAttemptReplaySafe(attempt); const { signalOwnedInterruption } = terminalState; const assistantOverflowCandidate = currentAttemptCompletedAssistant !== undefined @@ -137,6 +139,40 @@ export async function recoverEmbeddedRunAttempt(input: { thinkLevel: updates?.thinkLevel ?? runtime.thinkLevel, }); + if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) { + const errorText = formatErrorMessage(promptError); + const replayInvalid = resolveReplayInvalidForAttempt(); + setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" }); + return { + action: "complete", + result: buildEmbeddedRunBlockedResult({ + text: errorText, + errorKind: "hook_block", + errorMessage: errorText, + durationMs: Date.now() - runInput.startedAtMs, + agentMeta: buildErrorAgentMeta({ + sessionId: sessionIdUsed, + sessionFile: sessionPromptState.sessionFile, + provider: preparedRuntime.provider, + model: preparedRuntime.model.id, + ...runtime.outerContextTokenMeta, + usageAccumulator: input.usageAccumulator, + lastRunPromptUsage: input.lastRunPromptUsage, + currentAttemptAssistant, + }), + attempt, + replayInvalid, + }), + }; + } + if (!currentAttemptReplaySafe) { + return { + action: "proceed", + shouldSurfaceCodexCompletionTimeout: + attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && timedOut, + }; + } + const requestedSelection = shouldSwitchToLiveModel({ cfg: params.config, sessionKey: runInput.resolvedSessionKey, @@ -255,32 +291,6 @@ export async function recoverEmbeddedRunAttempt(input: { }), }; } - if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) { - const errorText = formatErrorMessage(promptError); - const replayInvalid = resolveReplayInvalidForAttempt(); - setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" }); - return { - action: "complete", - result: buildEmbeddedRunBlockedResult({ - text: errorText, - errorKind: "hook_block", - errorMessage: errorText, - durationMs: Date.now() - runInput.startedAtMs, - agentMeta: buildErrorAgentMeta({ - sessionId: sessionIdUsed, - sessionFile: sessionPromptState.sessionFile, - provider: preparedRuntime.provider, - model: preparedRuntime.model.id, - ...runtime.outerContextTokenMeta, - usageAccumulator: input.usageAccumulator, - lastRunPromptUsage: input.lastRunPromptUsage, - currentAttemptAssistant, - }), - attempt, - replayInvalid, - }), - }; - } const hasRecoverableCodexAppServerTimeoutOutcome = Boolean( attempt.codexAppServerFailure && attempt.promptTimeoutOutcome, ); diff --git a/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts b/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts index 8f9852b4f63e..ea8b87ca5639 100644 --- a/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts +++ b/src/agents/embedded-agent-runner/run/attempt-terminal-evidence.ts @@ -16,6 +16,14 @@ type ReplayMetadataAttempt = Pick< > & Partial>; +/** Uses current-attempt evidence when available and otherwise preserves fail-closed legacy state. */ +export function isCurrentAttemptReplaySafe( + attempt: Pick, +): boolean { + const replayMetadata = attempt.currentAttemptReplayMetadata ?? attempt.replayMetadata; + return replayMetadata.replaySafe && !replayMetadata.hadPotentialSideEffects; +} + /** * Marks whether retrying the attempt can safely replay the prompt. Concrete * tool-instance policy, async work, committed delivery, spawned sessions, and diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts b/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts index fc532f8578a0..51e1b49a245f 100644 --- a/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts +++ b/src/agents/embedded-agent-runner/run/incomplete-turn-classification.ts @@ -33,6 +33,7 @@ export type IncompleteTurnAttempt = Pick< | "itemLifecycle" | "messagesSnapshot" | "replayMetadata" + | "currentAttemptReplayMetadata" | "terminal" | "toolMetas" > & diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts b/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts index 10f4b24ed93d..dcaff3f76d58 100644 --- a/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts +++ b/src/agents/embedded-agent-runner/run/incomplete-turn-recovery.ts @@ -7,7 +7,11 @@ import { hasCompletedMessagingToolDeliveryEvidence, } from "../delivery-evidence.js"; import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js"; -import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js"; +import { + hasAsyncActivity, + hasAttemptTerminalState, + isCurrentAttemptReplaySafe, +} from "./attempt-terminal-evidence.js"; import { hasOnlySilentAssistantReply, hasPositiveOutputTokenUsage, @@ -60,9 +64,7 @@ export function shouldRetrySilentErrorAssistantTurn(params: { } // Current-attempt evidence avoids blocking on prior committed effects; older // harnesses retain the cumulative, fail-closed behavior. - const retryReplayMetadata = - params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata; - if (retryReplayMetadata.hadPotentialSideEffects) { + if (!isCurrentAttemptReplaySafe(params.attempt)) { return false; } @@ -187,6 +189,27 @@ export function resolveReasoningOnlyRetryInstruction(params: { return REASONING_ONLY_RETRY_INSTRUCTION; } +type SettledToolCall = { id: string | null; name: string | null }; + +function readSettledToolCalls( + message: EmbeddedRunAttemptResult["currentAttemptAssistant"] | null | undefined, +): SettledToolCall[] { + if (!Array.isArray(message?.content)) { + return []; + } + return message.content.flatMap((item) => { + const block = item as { type?: unknown; id?: unknown; name?: unknown } | null; + return block?.type === "toolCall" + ? [ + { + id: typeof block.id === "string" ? block.id : null, + name: typeof block.name === "string" ? block.name : null, + }, + ] + : []; + }); +} + /** Builds one fresh continuation after settled tools ended without a visible final answer. */ export function resolveSettledToolTerminalContinuationInstruction(params: { provider?: string; @@ -201,8 +224,27 @@ export function resolveSettledToolTerminalContinuationInstruction(params: { timedOut: boolean; attempt: IncompleteTurnAttempt; }): string | null { - const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; const currentAttemptAssistant = params.attempt.currentAttemptAssistant; + const snapshot = params.attempt.messagesSnapshot ?? []; + const latestUserIndex = snapshot.findLastIndex((message) => message.role === "user"); + let assistant: EmbeddedRunAttemptResult["currentAttemptAssistant"] = currentAttemptAssistant; + let assistantIndex = assistant ? snapshot.indexOf(assistant) : -1; + if (assistantIndex <= latestUserIndex || readSettledToolCalls(assistant).length === 0) { + assistantIndex = snapshot.findLastIndex( + (message, index) => + index > latestUserIndex && + message.role === "assistant" && + readSettledToolCalls(message).length > 0, + ); + const assistantCandidate = assistantIndex >= 0 ? snapshot[assistantIndex] : undefined; + assistant = assistantCandidate?.role === "assistant" ? assistantCandidate : undefined; + } + const terminal = params.attempt.terminal; + const idlePromptTimeout = + terminal.kind === "timeout" && + terminal.phase === "prompt" && + terminal.source === "idle" && + params.attempt.currentAttemptReplayMetadata?.hadPotentialSideEffects === true; const emptyStopAfterSettledTools = Boolean( params.allowEmptyStopContinuation && currentAttemptAssistant?.stopReason === "stop" && @@ -220,25 +262,11 @@ export function resolveSettledToolTerminalContinuationInstruction(params: { // Idle is not proof of settlement: skipped or partially dispatched tools must // never be described as completed. Match each terminal call's id and owner to // its own current-batch result; a reported failure is settled, not successful. - const requestedToolCalls = Array.isArray(assistant?.content) - ? assistant.content.flatMap((item) => { - const block = item as { type?: unknown; id?: unknown; name?: unknown } | null; - return block?.type === "toolCall" - ? [ - { - id: typeof block.id === "string" ? block.id : null, - name: typeof block.name === "string" ? block.name : null, - }, - ] - : []; - }) - : []; + const requestedToolCalls = readSettledToolCalls(assistant); // Scan only results AFTER the terminal assistant: the snapshot spans the whole // session, and a prior turn's toolResult with a model-reused id would otherwise // prove "completion" for a batch that never dispatched. Assistant not found in // the snapshot fails closed to the existing incomplete-turn error. - const snapshot = params.attempt.messagesSnapshot ?? []; - const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1; const settledToolResults = new Map( (assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => { const result = message as { @@ -260,7 +288,9 @@ export function resolveSettledToolTerminalContinuationInstruction(params: { }), ); const allToolsProvenSettled = - params.attempt.itemLifecycle?.activeCount === 0 && + params.attempt.itemLifecycle.startedCount > 0 && + params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount && + params.attempt.itemLifecycle.activeCount === 0 && requestedToolCalls.length > 0 && requestedToolCalls.every( ({ id, name }) => @@ -284,13 +314,14 @@ export function resolveSettledToolTerminalContinuationInstruction(params: { params.payloadCount !== 0 || params.hasTerminalToolPresentation || params.aborted || - params.promptError != null || - params.timedOut || + ((params.promptError != null || + params.timedOut || + params.attempt.terminal.kind === "timeout") && + !idlePromptTimeout) || (assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) || hasUnsettledToolError || - (hasSettledTerminalToolFailure && - (hasAsyncActivity(params.attempt.toolMetas) || - hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) || + hasAsyncActivity(params.attempt.toolMetas) || + hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) || params.attempt.clientToolCalls || params.attempt.yieldDetected || params.attempt.didSendDeterministicApprovalPrompt diff --git a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts index 3aea8ff8d73c..192a0e5aeff8 100644 --- a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts +++ b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts @@ -107,34 +107,16 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { prompt, noteLaneTaskProgress: input.finalization.noteLaneTaskProgress, }); - if (finalization.outcome === "empty") { - mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, finalization.result.usage); - lastRunPromptUsage = finalization.result.usage ?? lastRunPromptUsage; - log.warn( - `settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + - `provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`, - ); - const emptyAssistant = finalization.result.assistant; - const completedEmptyAttempt = { - ...initial.attempt, - lastAssistant: emptyAssistant, - currentAttemptAssistant: emptyAssistant, - currentAttemptCompletedAssistant: emptyAssistant, - }; - return { - ...initial, - attempt: completedEmptyAttempt, - attemptAssistant: emptyAssistant, - currentAttemptCompletedAssistant: emptyAssistant, - prepared, - lastRunPromptUsage, - finalizationOutcome: "completed-empty" as const, - }; - } attempt = finalization.attempt; mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage); mergeAttemptRunStatsIntoAccumulator(input.terminalBase.usageAccumulator, attempt); lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage; + if (finalization.outcome === "empty") { + log.warn( + `settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`, + ); + } // Successful isolated finalization owns a fresh terminal, never the original abort signal. const terminalState: EmbeddedRunTerminalState = { outcome: resolveEmbeddedRunAttemptTerminalOutcome({ @@ -164,7 +146,8 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { sessionFileUsed: attempt.sessionFileUsed, prepared, lastRunPromptUsage, - finalizationOutcome: "answered" as const, + finalizationOutcome: + finalization.outcome === "empty" ? ("completed-empty" as const) : ("answered" as const), }; } catch (error) { log.warn( @@ -186,13 +169,7 @@ async function runPreparedSettledTurnFinalization(input: { harness: AgentHarness; prompt: string; noteLaneTaskProgress: () => void; -}): Promise< - | { outcome: "answered"; attempt: EmbeddedRunAttemptWithReceiptEvidence } - | { - outcome: "empty"; - result: AgentHarnessSettledTurnFinalizationResult; - } -> { +}): Promise<{ outcome: "answered" | "empty"; attempt: EmbeddedRunAttemptWithReceiptEvidence }> { return await withEmbeddedRunLaneProgressHeartbeat(input.noteLaneTaskProgress, async () => { const finalization = await runEmbeddedSettledTurnFinalizationWithBackend( { @@ -206,12 +183,10 @@ async function runPreparedSettledTurnFinalization(input: { input.settledAttempt, input.harness, ); - if (finalization.outcome === "empty") { - return finalization; - } return { - outcome: "answered", + outcome: finalization.outcome, attempt: buildSettledTurnFinalizationAttemptResult({ + outcome: finalization.outcome, result: finalization.result, settledAttempt: input.settledAttempt, prompt: input.prompt, @@ -222,13 +197,14 @@ async function runPreparedSettledTurnFinalization(input: { } function buildSettledTurnFinalizationAttemptResult(input: { + outcome: "answered" | "empty"; result: AgentHarnessSettledTurnFinalizationResult; settledAttempt: EmbeddedRunAttemptWithReceiptEvidence; prompt: string; agentHarnessId?: string; }): EmbeddedRunAttemptWithReceiptEvidence { const { result, settledAttempt } = input; - const text = resolveSettledTurnFinalizationText(result); + const text = input.outcome === "empty" ? "" : resolveSettledTurnFinalizationText(result); // Finalization replaces terminal ownership, not host-private facts from settled tools. // Keep those facts while replay, abort, and lifecycle state remain finalizer-local. return { diff --git a/src/agents/embedded-agent-runner/run/terminal-resolution.ts b/src/agents/embedded-agent-runner/run/terminal-resolution.ts index 9fc501936976..0e78982afa0d 100644 --- a/src/agents/embedded-agent-runner/run/terminal-resolution.ts +++ b/src/agents/embedded-agent-runner/run/terminal-resolution.ts @@ -63,6 +63,14 @@ type TerminalResolution = | { action: "retry" } | { action: "complete"; result: EmbeddedAgentRunResult }; +function requiresVisibleTerminalReply(runParams: TerminalRunParams): boolean { + return ( + runParams.terminalReplyExpectation === "required" || + (runParams.terminalReplyExpectation == null && + (runParams.trigger == null || runParams.trigger === "user" || runParams.trigger === "manual")) + ); +} + export function resolveSettledTurnFinalizationRequest(input: { runParams: TerminalRunParams; attempt: EmbeddedRunAttemptResult; @@ -131,12 +139,7 @@ export function resolveSettledTurnFinalizationRequest(input: { modelId: input.activeErrorContext.model, modelApi: input.modelApi, executionContract: input.executionContract, - allowEmptyStopContinuation: - input.runParams.terminalReplyExpectation === "required" || - (input.runParams.terminalReplyExpectation == null && - (input.runParams.trigger == null || - input.runParams.trigger === "user" || - input.runParams.trigger === "manual")), + allowEmptyStopContinuation: requiresVisibleTerminalReply(input.runParams), payloadCount, hasTerminalToolPresentation: input.hasTerminalToolPresentation, aborted: terminalAborted, @@ -311,8 +314,10 @@ export async function resolveEmbeddedRunTerminal(input: { ); return { action: "retry" }; } + const completedEmptyFinalization = input.settledTurnFinalizationOutcome === "completed-empty"; const incompleteTurnText = - emptyAssistantReplyIsSilent || input.settledTurnFinalizationOutcome === "completed-empty" + emptyAssistantReplyIsSilent || + (completedEmptyFinalization && !requiresVisibleTerminalReply(runParams)) ? null : resolveIncompleteTurnPayloadText({ payloadCount,