From df5d85a5b2ff5249d1bbfbe95579251edfe8adb1 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Jul 2026 01:21:07 -0400 Subject: [PATCH] refactor(agents): centralize terminal outcome handling (#114310) --- src/agents/embedded-agent-runner/run-loop.ts | 66 ++------- .../run/assistant-failure.ts | 80 ++++++----- .../run/attempt-normalization.ts | 23 +--- .../run/attempt-recovery.ts | 26 ++-- .../run/settled-turn-finalization.ts | 49 +++---- .../run/terminal-outcome.test.ts | 129 +++++++++++++++++- .../run/terminal-outcome.ts | 17 +++ .../run/terminal-preparation.ts | 20 ++- .../run/terminal-resolution.ts | 98 +++++++------ .../run/terminal-timeout.ts | 30 ++-- 10 files changed, 324 insertions(+), 214 deletions(-) diff --git a/src/agents/embedded-agent-runner/run-loop.ts b/src/agents/embedded-agent-runner/run-loop.ts index d32021ca1f71..72ffe94d03b9 100644 --- a/src/agents/embedded-agent-runner/run-loop.ts +++ b/src/agents/embedded-agent-runner/run-loop.ts @@ -376,26 +376,12 @@ export async function runPreparedEmbeddedLoop( accumulatedReplayState = normalizedAttempt.replayState; const { attempt, - terminalProjection: { - aborted, - externalAbort, - promptError, - timedOut, - idleTimedOut, - timedOutDuringCompaction, - timedOutDuringToolExecution, - timedOutByRunBudget, - }, sessionIdUsed, sessionFileUsed, currentAttemptAssistant, currentAttemptCompletedAssistant, attemptAssistant, - terminalOutcome, - terminalAborted, - terminalTimedOut, - terminalInterrupted, - signalOwnedInterruption, + terminalState, setTerminalLifecycleMeta, attemptCompactionCount, activeErrorContext, @@ -445,9 +431,7 @@ export async function runPreparedEmbeddedLoop( attempt, attemptAssistant, currentAttemptAssistant, - terminalProviderStarted: terminalOutcome.providerStarted === true, - terminalInterrupted, - promptError, + terminalState, activeErrorContext, provider, modelId, @@ -455,14 +439,6 @@ export async function runPreparedEmbeddedLoop( thinkLevel, getThinkLevel: () => preparedRuntime.snapshot().thinkLevel, attemptedThinking, - timedOut, - idleTimedOut, - timedOutDuringCompaction, - timedOutDuringToolExecution, - timedOutByRunBudget, - signalOwnedInterruption, - externalAbort, - aborted, fallbackConfigured, pluginHarnessOwnsTransport, canRestartForLiveSwitch, @@ -506,22 +482,15 @@ export async function runPreparedEmbeddedLoop( } let assistantProfileFailureReason = assistantFailureOutcome.assistantProfileFailureReason; const terminalToolPresentation = readAttemptTerminalToolPresentation(); - const terminalState = await prepareTerminalWithSettledTurnFinalization({ + const finalizedTerminal = await prepareTerminalWithSettledTurnFinalization({ initial: { attempt, attemptAssistant, currentAttemptCompletedAssistant, sessionIdUsed, sessionFileUsed, - terminalAborted, - terminalTimedOut, - terminalInterrupted, - externalAbort, - signalOwnedInterruption, - promptError, + terminalState, attemptCompactionCount, - timedOutDuringCompaction, - timedOutDuringToolExecution, }, terminalBase: { runParams: params, @@ -549,19 +518,14 @@ export async function runPreparedEmbeddedLoop( const { attempt: terminalAttempt, attemptAssistant: terminalAttemptAssistant, - terminalAborted: terminalAbortedState, - terminalTimedOut: terminalTimedOutState, - terminalInterrupted: terminalInterruptedState, - externalAbort: terminalExternalAbort, - signalOwnedInterruption: terminalSignalOwnedInterruption, - promptError: terminalPromptError, + terminalState: resolvedTerminalState, attemptCompactionCount: terminalAttemptCompactionCount, prepared: terminalPrepared, finalizationAttempted: settledTurnFinalizationAttempted, - } = terminalState; - lastRunPromptUsage = terminalState.lastRunPromptUsage; - lastTurnTotal = terminalState.lastTurnTotal; - if (terminalState.finalizationSucceeded) { + } = finalizedTerminal; + lastRunPromptUsage = finalizedTerminal.lastRunPromptUsage; + lastTurnTotal = finalizedTerminal.lastTurnTotal; + if (finalizedTerminal.finalizationSucceeded) { assistantProfileFailureReason = null; } @@ -584,14 +548,11 @@ export async function runPreparedEmbeddedLoop( timedOutDuringPrompt, hasSuccessfulFinalAssistantAfterPromptTimeout, shouldSurfaceCodexCompletionTimeout, - idleTimedOut, attempt: terminalAttempt, hasPartialAssistantTextAfterPromptTimeout, payloads, payloadsWithToolMedia, - terminalAborted: terminalAbortedState, - terminalTimedOut: terminalTimedOutState, - terminalOutcome, + terminalState: resolvedTerminalState, resolveReplayInvalid: resolveReplayInvalidForAttempt, setTerminalLifecycleMeta, startedAtMs: started, @@ -613,12 +574,7 @@ export async function runPreparedEmbeddedLoop( activeErrorContext, modelApi: effectiveModel.api, executionContract, - terminalAborted: terminalAbortedState, - terminalTimedOut: terminalTimedOutState, - terminalInterrupted: terminalInterruptedState, - externalAbort: terminalExternalAbort, - signalOwnedInterruption: terminalSignalOwnedInterruption, - promptError: terminalPromptError, + terminalState: resolvedTerminalState, payloadsWithToolMedia, recoveredFinalAssistantPayloadsAfterPromptTimeout, finalAssistantVisibleText, diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.ts b/src/agents/embedded-agent-runner/run/assistant-failure.ts index 3a58996f9c9e..69a208f86d3f 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failure.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failure.ts @@ -1,5 +1,6 @@ import type { ThinkLevel } from "../../../auto-reply/thinking.js"; import type { AssistantMessage } from "../../../llm/types.js"; +import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js"; import { classifyAssistantFailoverReason, @@ -24,6 +25,10 @@ import { createFailoverDecisionLogger } from "./failover-observation.js"; import { resolveRunFailoverDecision } from "./failover-policy.js"; import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn.js"; import type { RunEmbeddedAgentParams } from "./params.js"; +import { + isEmbeddedRunTerminalInterrupted, + type EmbeddedRunTerminalState, +} from "./terminal-outcome.js"; import type { EmbeddedRunAttemptResult } from "./types.js"; const MAX_EMPTY_ERROR_RETRIES = 3; @@ -46,9 +51,7 @@ export async function handleEmbeddedAssistantFailure(input: { attempt: EmbeddedRunAttemptResult; attemptAssistant?: AssistantMessage; currentAttemptAssistant?: AssistantMessage; - terminalProviderStarted: boolean; - terminalInterrupted: boolean; - promptError: unknown; + terminalState: EmbeddedRunTerminalState; activeErrorContext: { provider: string; model: string }; provider: string; modelId: string; @@ -57,14 +60,6 @@ export async function handleEmbeddedAssistantFailure(input: { // Profile rotation resets thinking inside the runtime; read it after advancing. getThinkLevel: () => ThinkLevel; attemptedThinking: Set; - timedOut: boolean; - idleTimedOut: boolean; - timedOutDuringCompaction: boolean; - timedOutDuringToolExecution: boolean; - timedOutByRunBudget: boolean; - signalOwnedInterruption: boolean; - externalAbort: boolean; - aborted: boolean; fallbackConfigured: boolean; pluginHarnessOwnsTransport: boolean; canRestartForLiveSwitch: boolean; @@ -100,11 +95,23 @@ export async function handleEmbeddedAssistantFailure(input: { agentDir: string; isProbeSession: boolean; }): Promise { + const { + aborted, + externalAbort, + idleTimedOut, + promptError, + timedOut, + timedOutByRunBudget, + timedOutDuringCompaction, + timedOutDuringToolExecution, + } = projectAgentRunAttemptTerminal(input.attempt.terminal); + const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome); + const { signalOwnedInterruption } = input.terminalState; const fallbackThinking = pickFallbackThinkingLevel({ message: input.attemptAssistant?.errorMessage, attempted: input.attemptedThinking, }); - if (fallbackThinking && !input.terminalInterrupted) { + if (fallbackThinking && !terminalInterrupted) { log.warn( `unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`, ); @@ -122,10 +129,11 @@ export async function handleEmbeddedAssistantFailure(input: { const failoverFailure = isFailoverAssistantError(input.attemptAssistant); const assistantFailoverReason = classifyAssistantFailoverReason(input.attemptAssistant); const assistantProviderStarted = - Boolean(input.currentAttemptAssistant?.provider) || input.terminalProviderStarted; + Boolean(input.currentAttemptAssistant?.provider) || + input.terminalState.outcome.providerStarted === true; const assistantProfileFailoverReason = assistantFailoverReason ?? - (assistantProviderStarted && (input.timedOut || input.idleTimedOut) ? "timeout" : null); + (assistantProviderStarted && (timedOut || idleTimedOut) ? "timeout" : null); const assistantProfileFailureReason = input.resolveAuthProfileFailureReason( assistantProfileFailoverReason, { @@ -154,8 +162,8 @@ export async function handleEmbeddedAssistantFailure(input: { !billingFailure && !cloudCodeAssistFormatError && !imageDimensionError && - !input.terminalInterrupted && - !input.promptError && + !terminalInterrupted && + !promptError && silentErrorRetryReason && shouldRetrySilentErrorAssistantTurn({ attempt: input.attempt, @@ -192,11 +200,11 @@ export async function handleEmbeddedAssistantFailure(input: { sourceModel: input.attemptAssistant?.model ?? input.modelId, profileId: failedProfileId, fallbackConfigured: input.fallbackConfigured, - timedOut: input.timedOut, - aborted: input.aborted, + timedOut, + aborted, }); if ( - !input.signalOwnedInterruption && + !signalOwnedInterruption && authFailure && (await input.maybeRefreshRuntimeAuthForAuthError( input.attemptAssistant?.errorMessage ?? "", @@ -232,35 +240,35 @@ export async function handleEmbeddedAssistantFailure(input: { const initialDecision = resolveRunFailoverDecision({ stage: "assistant", allowFormatRetry: cloudCodeAssistFormatError, - aborted: input.aborted, - externalAbort: input.externalAbort || input.signalOwnedInterruption, + aborted, + externalAbort: externalAbort || signalOwnedInterruption, fallbackConfigured: input.fallbackConfigured, failoverFailure, failoverReason: assistantFailoverReason, - timedOut: input.timedOut, - idleTimedOut: input.idleTimedOut, - timedOutDuringCompaction: input.timedOutDuringCompaction, - timedOutDuringToolExecution: input.timedOutDuringToolExecution, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, harnessOwnsTransport: input.pluginHarnessOwnsTransport, - timedOutByRunBudget: input.timedOutByRunBudget, + timedOutByRunBudget, profileRotated: false, }); const outcome = await handleAssistantFailover({ initialDecision, - aborted: input.aborted, - externalAbort: input.externalAbort || input.signalOwnedInterruption, + aborted, + externalAbort: externalAbort || signalOwnedInterruption, fallbackConfigured: input.fallbackConfigured, failoverFailure, failoverReason: assistantFailoverReason, - timedOut: input.timedOut, - idleTimedOut: input.idleTimedOut, - timedOutDuringCompaction: input.timedOutDuringCompaction, - timedOutDuringToolExecution: input.timedOutDuringToolExecution, - timedOutByRunBudget: input.timedOutByRunBudget, + timedOut, + idleTimedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, allowSameModelIdleTimeoutRetry: - input.timedOut && - input.idleTimedOut && - !input.timedOutDuringCompaction && + timedOut && + idleTimedOut && + !timedOutDuringCompaction && !input.fallbackConfigured && input.canRestartForLiveSwitch && input.sameModelIdleTimeoutRetries < MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES, diff --git a/src/agents/embedded-agent-runner/run/attempt-normalization.ts b/src/agents/embedded-agent-runner/run/attempt-normalization.ts index 0b2d8c25fb47..4a8b18f8a800 100644 --- a/src/agents/embedded-agent-runner/run/attempt-normalization.ts +++ b/src/agents/embedded-agent-runner/run/attempt-normalization.ts @@ -36,7 +36,7 @@ import { isEmbeddedRunTerminalAbort, isEmbeddedRunTerminalInterrupted, isEmbeddedRunTerminalTimeout, - resolveEmbeddedRunAttemptTerminalOutcome, + resolveEmbeddedRunAttemptTerminalState, } from "./terminal-outcome.js"; type PreparedRuntime = Awaited>; @@ -74,7 +74,6 @@ export async function normalizeEmbeddedRunAttempt(input: { lastTurnTotal: number | undefined; replayState: ReplayState; attempt: ReturnType; - terminalProjection: ReturnType; sessionIdUsed: string; sessionFileUsed: string | undefined; currentAttemptAssistant: ReturnType< @@ -86,11 +85,7 @@ export async function normalizeEmbeddedRunAttempt(input: { attemptAssistant: ReturnType< typeof normalizeEmbeddedRunAttemptResult >["currentAttemptAssistant"]; - terminalOutcome: ReturnType; - terminalAborted: boolean; - terminalTimedOut: boolean; - terminalInterrupted: boolean; - signalOwnedInterruption: boolean; + terminalState: ReturnType; setTerminalLifecycleMeta: NonNullable< ReturnType["setTerminalLifecycleMeta"] >; @@ -121,8 +116,7 @@ export async function normalizeEmbeddedRunAttempt(input: { currentAttemptAssistant, currentAttemptCompletedAssistant, } = attempt; - const terminalProjection = projectAgentRunAttemptTerminal(terminal); - const { idleTimedOut } = terminalProjection; + const { idleTimedOut } = projectAgentRunAttemptTerminal(terminal); const sessionAssistantForCandidate = !currentAttemptAssistant && !isAssistantForModelRef(sessionLastAssistant, { @@ -132,15 +126,15 @@ export async function normalizeEmbeddedRunAttempt(input: { ? undefined : sessionLastAssistant; const attemptAssistant = currentAttemptAssistant ?? sessionAssistantForCandidate; - const terminalOutcome = resolveEmbeddedRunAttemptTerminalOutcome({ + const terminalState = resolveEmbeddedRunAttemptTerminalState({ attempt, assistant: currentAttemptAssistant, abortSignal: params.abortSignal, }); + const { outcome: terminalOutcome, signalOwnedInterruption } = terminalState; const terminalAborted = isEmbeddedRunTerminalAbort(terminalOutcome); const terminalTimedOut = isEmbeddedRunTerminalTimeout(terminalOutcome); const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalOutcome); - const signalOwnedInterruption = terminalInterrupted && params.abortSignal?.aborted === true; const setTerminalLifecycleMeta: NonNullable = (meta) => { const { stopReason, ...remainingMeta } = meta; const terminalStopReason = terminalInterrupted ? terminalOutcome.stopReason : stopReason; @@ -299,17 +293,12 @@ export async function normalizeEmbeddedRunAttempt(input: { lastTurnTotal, replayState, attempt, - terminalProjection, sessionIdUsed, sessionFileUsed, currentAttemptAssistant, currentAttemptCompletedAssistant, attemptAssistant, - terminalOutcome, - terminalAborted, - terminalTimedOut, - terminalInterrupted, - signalOwnedInterruption, + terminalState, setTerminalLifecycleMeta, attemptCompactionCount, activeErrorContext, diff --git a/src/agents/embedded-agent-runner/run/attempt-recovery.ts b/src/agents/embedded-agent-runner/run/attempt-recovery.ts index 65cc5812d3de..878fb135cabe 100644 --- a/src/agents/embedded-agent-runner/run/attempt-recovery.ts +++ b/src/agents/embedded-agent-runner/run/attempt-recovery.ts @@ -22,6 +22,7 @@ import { recoverEmbeddedRunOverflow } from "./overflow-context-recovery.js"; import { handleEmbeddedPromptFailure } from "./prompt-failure.js"; import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js"; import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js"; +import { isEmbeddedRunTerminalInterrupted } from "./terminal-outcome.js"; import { recoverEmbeddedRunTimeout } from "./timeout-context-recovery.js"; type PreparedRuntime = Awaited>; @@ -86,21 +87,10 @@ export async function recoverEmbeddedRunAttempt(input: { const runtime = preparedRuntime.snapshot(); const { attempt, - terminalProjection: { - aborted, - externalAbort, - promptError, - promptErrorSource, - timedOut, - timedOutDuringCompaction, - timedOutDuringToolExecution, - timedOutByRunBudget, - }, sessionIdUsed, attemptAssistant, currentAttemptCompletedAssistant, - terminalInterrupted, - signalOwnedInterruption, + terminalState, setTerminalLifecycleMeta, attemptCompactionCount, activeErrorContext, @@ -108,6 +98,18 @@ export async function recoverEmbeddedRunAttempt(input: { assistantErrorText, canRestartForLiveSwitch, } = normalizedAttempt; + const { + aborted, + externalAbort, + promptError, + promptErrorSource, + timedOut, + timedOutDuringCompaction, + timedOutDuringToolExecution, + timedOutByRunBudget, + } = projectAgentRunAttemptTerminal(attempt.terminal); + const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalState.outcome); + const { signalOwnedInterruption } = terminalState; const assistantOverflowCandidate = currentAttemptCompletedAssistant !== undefined ? currentAttemptCompletedAssistant.stopReason === "error" || diff --git a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts index bbb80dbb0629..269d9b2ec83a 100644 --- a/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts +++ b/src/agents/embedded-agent-runner/run/settled-turn-finalization.ts @@ -8,6 +8,10 @@ import { log } from "../logger.js"; import { mergeUsageIntoAccumulator } from "../usage-accumulator.js"; import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js"; import { EMBEDDED_RUN_LANE_HEARTBEAT_MS } from "./lane-runtime.js"; +import { + resolveEmbeddedRunAttemptTerminalOutcome, + type EmbeddedRunTerminalState, +} from "./terminal-outcome.js"; import { prepareEmbeddedRunTerminal } from "./terminal-preparation.js"; import { resolveSettledTurnFinalizationRequest } from "./terminal-resolution.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js"; @@ -21,10 +25,7 @@ type TerminalPreparationBase = Omit< | "sessionFileUsed" | "lastRunPromptUsage" | "lastTurnTotal" - | "terminalInterrupted" - | "terminalTimedOut" - | "timedOutDuringCompaction" - | "timedOutDuringToolExecution" + | "terminalState" >; export async function prepareTerminalWithSettledTurnFinalization(input: { @@ -34,15 +35,8 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"]; sessionIdUsed: string; sessionFileUsed?: string; - terminalAborted: boolean; - terminalTimedOut: boolean; - terminalInterrupted: boolean; - externalAbort: boolean; - signalOwnedInterruption: boolean; - promptError: unknown; + terminalState: EmbeddedRunTerminalState; attemptCompactionCount: number; - timedOutDuringCompaction: boolean; - timedOutDuringToolExecution: boolean; }; terminalBase: TerminalPreparationBase; lastRunPromptUsage: TerminalPreparationInput["lastRunPromptUsage"]; @@ -70,10 +64,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { sessionFileUsed: initial.sessionFileUsed, lastRunPromptUsage, lastTurnTotal, - terminalInterrupted: initial.terminalInterrupted, - terminalTimedOut: initial.terminalTimedOut, - timedOutDuringCompaction: initial.timedOutDuringCompaction, - timedOutDuringToolExecution: initial.timedOutDuringToolExecution, + terminalState: initial.terminalState, }); const prompt = resolveSettledTurnFinalizationRequest({ runParams: input.terminalBase.runParams, @@ -85,9 +76,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { recoveredFinalAssistantPayloadsAfterPromptTimeout: prepared.recoveredFinalAssistantPayloadsAfterPromptTimeout, hasTerminalToolPresentation: input.finalization.hasTerminalToolPresentation, - terminalAborted: initial.terminalAborted, - terminalTimedOut: initial.terminalTimedOut, - promptError: initial.promptError, + terminalState: initial.terminalState, settledTurnFinalizationAvailable: typeof input.finalization.harness.finalizeSettledTurn === "function", }); @@ -119,6 +108,14 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage); lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage; lastTurnTotal = attempt.attemptUsage?.total ?? lastTurnTotal; + // Successful isolated finalization owns a fresh terminal, never the original abort signal. + const terminalState: EmbeddedRunTerminalState = { + outcome: resolveEmbeddedRunAttemptTerminalOutcome({ + attempt, + assistant: attempt.currentAttemptAssistant, + }), + signalOwnedInterruption: false, + }; prepared = prepareEmbeddedRunTerminal({ ...input.terminalBase, attempt, @@ -127,24 +124,14 @@ export async function prepareTerminalWithSettledTurnFinalization(input: { sessionFileUsed: attempt.sessionFileUsed, lastRunPromptUsage, lastTurnTotal, - terminalInterrupted: false, - terminalTimedOut: false, - timedOutDuringCompaction: false, - timedOutDuringToolExecution: false, + terminalState, }); return { attempt, attemptAssistant: attempt.currentAttemptAssistant, currentAttemptCompletedAssistant: attempt.currentAttemptCompletedAssistant, - terminalAborted: false, - terminalTimedOut: false, - terminalInterrupted: false, - externalAbort: false, - signalOwnedInterruption: false, - promptError: null, + terminalState, attemptCompactionCount: 0, - timedOutDuringCompaction: false, - timedOutDuringToolExecution: false, sessionIdUsed: attempt.sessionIdUsed, sessionFileUsed: attempt.sessionFileUsed, prepared, diff --git a/src/agents/embedded-agent-runner/run/terminal-outcome.test.ts b/src/agents/embedded-agent-runner/run/terminal-outcome.test.ts index c134d308bf53..d42a589fe004 100644 --- a/src/agents/embedded-agent-runner/run/terminal-outcome.test.ts +++ b/src/agents/embedded-agent-runner/run/terminal-outcome.test.ts @@ -1,7 +1,13 @@ import type { AssistantMessage } from "openclaw/plugin-sdk/llm"; import { describe, expect, it } from "vitest"; import { createAgentRunRestartAbortError } from "../../run-termination.js"; -import { resolveEmbeddedRunAttemptTerminalOutcome } from "./terminal-outcome.js"; +import { + isEmbeddedRunTerminalAbort, + isEmbeddedRunTerminalInterrupted, + isEmbeddedRunTerminalTimeout, + resolveEmbeddedRunAttemptTerminalOutcome, + resolveEmbeddedRunAttemptTerminalState, +} from "./terminal-outcome.js"; type EmbeddedRunAttemptTerminalInput = Parameters< typeof resolveEmbeddedRunAttemptTerminalOutcome @@ -38,6 +44,127 @@ function makeAssistant(stopReason: AssistantMessage["stopReason"]): AssistantMes } describe("embedded run attempt terminal outcome", () => { + it.each([ + { + name: "run-budget prompt timeout", + terminal: { kind: "timeout", phase: "prompt", source: "run_budget" }, + reason: "hard_timeout", + aborted: false, + timedOut: true, + }, + { + name: "idle prompt timeout", + terminal: { kind: "timeout", phase: "prompt", source: "idle" }, + reason: "hard_timeout", + aborted: false, + timedOut: true, + }, + { + name: "recovered compaction observation", + terminal: { kind: "timeout", phase: "compaction", source: "observation" }, + reason: "completed", + aborted: false, + timedOut: false, + }, + { + name: "yield-only cleanup", + terminal: { kind: "aborted", source: "yield_cleanup" }, + reason: "completed", + aborted: false, + timedOut: false, + }, + { + name: "external attempt cancellation", + terminal: { kind: "aborted", source: "external" }, + reason: "aborted", + aborted: true, + timedOut: false, + }, + ] as const)("carries $name through canonical interruption predicates", (testCase) => { + const outcome = resolveEmbeddedRunAttemptTerminalOutcome({ + attempt: makeAttempt({ terminal: testCase.terminal }), + assistant: makeAssistant("stop"), + }); + + expect(outcome.reason).toBe(testCase.reason); + expect(isEmbeddedRunTerminalAbort(outcome)).toBe(testCase.aborted); + expect(isEmbeddedRunTerminalTimeout(outcome)).toBe(testCase.timedOut); + expect(isEmbeddedRunTerminalInterrupted(outcome)).toBe(testCase.aborted || testCase.timedOut); + }); + + it("keeps user-signal cancellation authoritative before assistant completion", () => { + const controller = new AbortController(); + controller.abort(); + + const outcome = resolveEmbeddedRunAttemptTerminalOutcome({ + attempt: makeAttempt(), + assistant: undefined, + abortSignal: controller.signal, + }); + + expect(outcome).toMatchObject({ + reason: "aborted", + status: "error", + stopReason: "aborted", + }); + expect(isEmbeddedRunTerminalAbort(outcome)).toBe(true); + expect(isEmbeddedRunTerminalTimeout(outcome)).toBe(false); + expect(isEmbeddedRunTerminalInterrupted(outcome)).toBe(true); + }); + + it("captures signal ownership before a later cancellation can reclassify completion", () => { + const controller = new AbortController(); + const terminal = resolveEmbeddedRunAttemptTerminalState({ + attempt: makeAttempt(), + assistant: makeAssistant("stop"), + abortSignal: controller.signal, + }); + + controller.abort(); + + expect(terminal).toEqual({ + outcome: { reason: "completed", status: "ok", stopReason: "stop" }, + signalOwnedInterruption: false, + }); + }); + + it("captures user cancellation with the same terminal that owns the interruption", () => { + const controller = new AbortController(); + controller.abort(); + + expect( + resolveEmbeddedRunAttemptTerminalState({ + attempt: makeAttempt(), + assistant: undefined, + abortSignal: controller.signal, + }), + ).toMatchObject({ + outcome: { reason: "aborted", status: "error", stopReason: "aborted" }, + signalOwnedInterruption: true, + }); + }); + + it("starts successful settled finalization without inheriting the original abort signal", () => { + const controller = new AbortController(); + controller.abort(); + const originalTerminal = resolveEmbeddedRunAttemptTerminalState({ + attempt: makeAttempt(), + assistant: undefined, + abortSignal: controller.signal, + }); + + expect(originalTerminal.signalOwnedInterruption).toBe(true); + expect( + resolveEmbeddedRunAttemptTerminalState({ + attempt: makeAttempt(), + assistant: makeAssistant("stop"), + }), + ).toEqual({ + outcome: { reason: "completed", status: "ok", stopReason: "stop" }, + signalOwnedInterruption: false, + }); + }); + it("keeps prompt timeout ownership ahead of generic abort metadata", () => { const outcome = resolveEmbeddedRunAttemptTerminalOutcome({ attempt: makeAttempt({ diff --git a/src/agents/embedded-agent-runner/run/terminal-outcome.ts b/src/agents/embedded-agent-runner/run/terminal-outcome.ts index 0efa9eb8f7df..ece07017b70a 100644 --- a/src/agents/embedded-agent-runner/run/terminal-outcome.ts +++ b/src/agents/embedded-agent-runner/run/terminal-outcome.ts @@ -9,6 +9,11 @@ type EmbeddedRunAttemptTerminalInput = Pick< "terminal" | "promptTimeoutOutcome" >; +export type EmbeddedRunTerminalState = { + outcome: AgentRunTerminalOutcome; + signalOwnedInterruption: boolean; +}; + /** Projects private attempt metadata into the canonical agent terminal outcome. */ export function resolveEmbeddedRunAttemptTerminalOutcome(params: { attempt: EmbeddedRunAttemptTerminalInput; @@ -34,3 +39,15 @@ export function isEmbeddedRunTerminalAbort(outcome: AgentRunTerminalOutcome): bo export function isEmbeddedRunTerminalInterrupted(outcome: AgentRunTerminalOutcome): boolean { return isEmbeddedRunTerminalTimeout(outcome) || isEmbeddedRunTerminalAbort(outcome); } + +/** Captures signal ownership with the outcome before async recovery can change the signal. */ +export function resolveEmbeddedRunAttemptTerminalState( + params: Parameters[0], +): EmbeddedRunTerminalState { + const outcome = resolveEmbeddedRunAttemptTerminalOutcome(params); + return { + outcome, + signalOwnedInterruption: + isEmbeddedRunTerminalInterrupted(outcome) && params.abortSignal?.aborted === true, + }; +} diff --git a/src/agents/embedded-agent-runner/run/terminal-preparation.ts b/src/agents/embedded-agent-runner/run/terminal-preparation.ts index d28dcd230842..d1207bea28af 100644 --- a/src/agents/embedded-agent-runner/run/terminal-preparation.ts +++ b/src/agents/embedded-agent-runner/run/terminal-preparation.ts @@ -1,5 +1,6 @@ import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js"; import type { AssistantMessage } from "../../../llm/types.js"; +import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import type { AuthProfileStore } from "../../auth-profiles.js"; import type { NormalizedUsage, UsageLike } from "../../usage.js"; import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js"; @@ -15,6 +16,11 @@ import { import type { RunEmbeddedAgentParams } from "./params.js"; import { buildEmbeddedRunPayloads } from "./payloads.js"; import { buildTraceToolSummary } from "./run-attempt-result.js"; +import { + isEmbeddedRunTerminalInterrupted, + isEmbeddedRunTerminalTimeout, + type EmbeddedRunTerminalState, +} from "./terminal-outcome.js"; import { mergeAttemptToolMediaPayloads } from "./tool-media-payloads.js"; import type { EmbeddedRunAttemptResult } from "./types.js"; @@ -35,10 +41,7 @@ export function prepareEmbeddedRunTerminal(input: { lastTurnTotal?: number; contextRecoveryState: EmbeddedRunContextRecoveryState; resolvedToolResultFormat: NonNullable; - terminalInterrupted: boolean; - terminalTimedOut: boolean; - timedOutDuringCompaction: boolean; - timedOutDuringToolExecution: boolean; + terminalState: EmbeddedRunTerminalState; }): { agentMeta: EmbeddedAgentMeta; reportedModelRef: { provider: string; model: string }; @@ -54,8 +57,13 @@ export function prepareEmbeddedRunTerminal(input: { failureSignal: ReturnType; } { const { runParams, attempt } = input; + const { timedOutDuringCompaction, timedOutDuringToolExecution } = projectAgentRunAttemptTerminal( + attempt.terminal, + ); const timedOutDuringPrompt = - input.terminalTimedOut && !input.timedOutDuringCompaction && !input.timedOutDuringToolExecution; + isEmbeddedRunTerminalTimeout(input.terminalState.outcome) && + !timedOutDuringCompaction && + !timedOutDuringToolExecution; // Session transcript fallbacks can reference an earlier rewritten turn. // Terminal delivery and metadata must stay scoped to this model attempt. const terminalAssistant = input.currentAttemptCompletedAssistant; @@ -132,7 +140,7 @@ export function prepareEmbeddedRunTerminal(input: { sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode, agentId: runParams.agentId, runId: runParams.runId, - runAborted: input.terminalInterrupted, + runAborted: isEmbeddedRunTerminalInterrupted(input.terminalState.outcome), didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt, heartbeatToolResponse: attempt.heartbeatToolResponse, }); diff --git a/src/agents/embedded-agent-runner/run/terminal-resolution.ts b/src/agents/embedded-agent-runner/run/terminal-resolution.ts index 7fbb37b978dd..4da5d0419186 100644 --- a/src/agents/embedded-agent-runner/run/terminal-resolution.ts +++ b/src/agents/embedded-agent-runner/run/terminal-resolution.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import { SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js"; import { freezeDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js"; import type { AssistantMessage } from "../../../llm/types.js"; +import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js"; import type { AgentExecutionAuthBinding } from "../../execution-auth-binding.js"; import type { ResolvedProviderAuth } from "../../model-auth.js"; @@ -33,6 +34,12 @@ import { YIELD_DIAGNOSTIC_TEXT, } from "./incomplete-turn.js"; import type { RunEmbeddedAgentParams } from "./params.js"; +import { + isEmbeddedRunTerminalAbort, + isEmbeddedRunTerminalInterrupted, + isEmbeddedRunTerminalTimeout, + type EmbeddedRunTerminalState, +} from "./terminal-outcome.js"; import { MAX_BEFORE_AGENT_FINALIZE_REVISIONS, type EmbeddedRunTerminalRetryState, @@ -65,19 +72,20 @@ export function resolveSettledTurnFinalizationRequest(input: { payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"]; recoveredFinalAssistantPayloadsAfterPromptTimeout?: EmbeddedAgentRunResult["payloads"]; hasTerminalToolPresentation: boolean; - terminalAborted: boolean; - terminalTimedOut: boolean; - promptError: unknown; + terminalState: EmbeddedRunTerminalState; settledTurnFinalizationAvailable: boolean; }): string | null { if (!input.settledTurnFinalizationAvailable) { return null; } + const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome); + const { promptError } = projectAgentRunAttemptTerminal(input.attempt.terminal); const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({ isCronTrigger: input.runParams.trigger === "cron", payloadCount: input.payloadsWithToolMedia?.length ?? 0, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt: input.attempt, }); const payloadCount = input.recoveredFinalAssistantPayloadsAfterPromptTimeout @@ -91,8 +99,8 @@ export function resolveSettledTurnFinalizationRequest(input: { allowEmptyAssistantReplyAsSilent: input.runParams.allowEmptyAssistantReplyAsSilent, onlyExplicitSilentReply: false, payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt: input.attempt, }); if (emptyAssistantReplyIsSilent) { @@ -111,9 +119,9 @@ export function resolveSettledTurnFinalizationRequest(input: { input.runParams.trigger === "manual")), payloadCount, hasTerminalToolPresentation: input.hasTerminalToolPresentation, - aborted: input.terminalAborted, - promptError: input.promptError, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + promptError, + timedOut: terminalTimedOut, attempt: input.attempt, }); } @@ -128,12 +136,7 @@ export async function resolveEmbeddedRunTerminal(input: { executionContract: Parameters< typeof resolveReasoningOnlyRetryInstruction >[0]["executionContract"]; - terminalAborted: boolean; - terminalTimedOut: boolean; - terminalInterrupted: boolean; - externalAbort: boolean; - signalOwnedInterruption: boolean; - promptError: unknown; + terminalState: EmbeddedRunTerminalState; payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"]; recoveredFinalAssistantPayloadsAfterPromptTimeout?: EmbeddedAgentRunResult["payloads"]; finalAssistantVisibleText?: string; @@ -178,11 +181,16 @@ export async function resolveEmbeddedRunTerminal(input: { contextRecoveryState: EmbeddedRunContextRecoveryState; }): Promise { const { runParams, attempt, retryState } = input; + const { externalAbort, promptError } = projectAgentRunAttemptTerminal(attempt.terminal); + const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome); + const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome); + const { signalOwnedInterruption } = input.terminalState; const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({ isCronTrigger: runParams.trigger === "cron", payloadCount: input.payloadsWithToolMedia?.length ?? 0, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt, }); const payloadsForTerminalPath = input.recoveredFinalAssistantPayloadsAfterPromptTimeout @@ -200,8 +208,8 @@ export async function resolveEmbeddedRunTerminal(input: { allowEmptyAssistantReplyAsSilent: runParams.allowEmptyAssistantReplyAsSilent, onlyExplicitSilentReply: settledTurnFinalizationAttempted, payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt, }); const nextReasoningOnlyRetryInstruction = @@ -212,8 +220,8 @@ export async function resolveEmbeddedRunTerminal(input: { modelId: input.activeErrorContext.model, modelApi: input.modelApi, executionContract: input.executionContract, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt, }); const nextEmptyResponseRetryInstruction = @@ -225,8 +233,8 @@ export async function resolveEmbeddedRunTerminal(input: { modelApi: input.modelApi, executionContract: input.executionContract, payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt, }); if ( @@ -250,9 +258,9 @@ export async function resolveEmbeddedRunTerminal(input: { !settledTurnFinalizationAttempted && shouldRetryMissingAssistantTurn({ payloadCount, - aborted: input.terminalAborted, - promptError: input.promptError, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + promptError, + timedOut: terminalTimedOut, attempt, }) && retryState.missingAssistantAttempts < MAX_MISSING_ASSISTANT_RETRIES @@ -284,16 +292,16 @@ export async function resolveEmbeddedRunTerminal(input: { ? null : resolveIncompleteTurnPayloadText({ payloadCount, - aborted: input.terminalAborted, - externalAbort: input.externalAbort || input.signalOwnedInterruption, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + externalAbort: externalAbort || signalOwnedInterruption, + timedOut: terminalTimedOut, hadPotentialSideEffects: input.replayState.hadPotentialSideEffects, attempt, }); const incompleteTurnFallbackSafe = Boolean( incompleteTurnText && - !input.terminalInterrupted && - !input.promptError && + !terminalInterrupted && + !promptError && !attempt.lastToolError && !hasAttemptTerminalState(attempt) && !input.replayState.hadPotentialSideEffects, @@ -306,8 +314,8 @@ export async function resolveEmbeddedRunTerminal(input: { !settledTurnFinalizationAttempted && input.attemptCompactionCount > 0 && payloadCount === 0 && - !input.terminalInterrupted && - !input.promptError && + !terminalInterrupted && + !promptError && !attempt.clientToolCalls && !attempt.yieldDetected && !attempt.didSendDeterministicApprovalPrompt && @@ -380,8 +388,8 @@ export async function resolveEmbeddedRunTerminal(input: { if ( beforeFinalizeRevisionReason && !settledTurnFinalizationAttempted && - !input.terminalInterrupted && - !input.promptError && + !terminalInterrupted && + !promptError && !attempt.clientToolCalls && !attempt.yieldDetected && !emptyAssistantReplyIsSilent @@ -416,11 +424,13 @@ async function surfaceIncompleteTurn( terminalToolPresentation?: string; }, ): Promise { + const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome); const replayInvalid = input.resolveReplayInvalid(input.text); const livenessState = resolveRunLivenessState({ payloadCount: input.payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt: input.attempt, incompleteTurnText: input.text, }); @@ -446,7 +456,7 @@ async function surfaceIncompleteTurn( meta: { durationMs: Date.now() - input.startedAtMs, agentMeta: input.agentMeta, - aborted: input.terminalAborted, + aborted: terminalAborted, systemPromptReport: input.attempt.systemPromptReport, finalPromptText: input.attempt.finalPromptText, finalAssistantVisibleText: input.finalAssistantVisibleText, @@ -475,8 +485,10 @@ function completeEmbeddedRun( emptyAssistantReplyIsSilent: boolean; }, ): TerminalResolution { + const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome); log.debug( - `embedded run done: runId=${input.runParams.runId} sessionId=${input.runParams.sessionId} durationMs=${Date.now() - input.startedAtMs} aborted=${input.terminalAborted}`, + `embedded run done: runId=${input.runParams.runId} sessionId=${input.runParams.sessionId} durationMs=${Date.now() - input.startedAtMs} aborted=${terminalAborted}`, ); markEmbeddedRunAuthProfileSuccess({ authProfileStateMode: input.runParams.authProfileStateMode, @@ -507,8 +519,8 @@ function completeEmbeddedRun( ? "paused" : resolveRunLivenessState({ payloadCount: input.payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt: input.attempt, incompleteTurnText: null, }); @@ -542,7 +554,7 @@ function completeEmbeddedRun( meta: { durationMs: Date.now() - input.startedAtMs, agentMeta: input.agentMeta, - aborted: input.terminalAborted, + aborted: terminalAborted, systemPromptReport: input.attempt.systemPromptReport, finalPromptText: input.attempt.finalPromptText, finalAssistantVisibleText: input.finalAssistantVisibleText, diff --git a/src/agents/embedded-agent-runner/run/terminal-timeout.ts b/src/agents/embedded-agent-runner/run/terminal-timeout.ts index 8aaf57fa81b9..75aad944567c 100644 --- a/src/agents/embedded-agent-runner/run/terminal-timeout.ts +++ b/src/agents/embedded-agent-runner/run/terminal-timeout.ts @@ -1,6 +1,12 @@ +import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js"; import { hasMessagingToolDeliveryEvidence } from "../delivery-evidence.js"; import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js"; import { resolveRunLivenessState } from "./incomplete-turn.js"; +import { + isEmbeddedRunTerminalAbort, + isEmbeddedRunTerminalTimeout, + type EmbeddedRunTerminalState, +} from "./terminal-outcome.js"; import { copyAttemptDeliveryState } from "./terminal-resolution.js"; import type { EmbeddedRunAttemptResult } from "./types.js"; @@ -8,17 +14,11 @@ export function resolveEmbeddedRunTerminalTimeout(input: { timedOutDuringPrompt: boolean; hasSuccessfulFinalAssistantAfterPromptTimeout: boolean; shouldSurfaceCodexCompletionTimeout: boolean; - idleTimedOut: boolean; attempt: EmbeddedRunAttemptResult; hasPartialAssistantTextAfterPromptTimeout: boolean; payloads: EmbeddedAgentRunResult["payloads"]; payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"]; - terminalAborted: boolean; - terminalTimedOut: boolean; - terminalOutcome: { - timeoutPhase?: EmbeddedAgentRunResult["meta"]["timeoutPhase"]; - providerStarted?: boolean; - }; + terminalState: EmbeddedRunTerminalState; resolveReplayInvalid: (incompleteTurnText?: string | null) => boolean; setTerminalLifecycleMeta: NonNullable; startedAtMs: number; @@ -35,7 +35,10 @@ export function resolveEmbeddedRunTerminalTimeout(input: { ) { return undefined; } - const defaultTimeoutText = input.idleTimedOut + const { idleTimedOut } = projectAgentRunAttemptTerminal(input.attempt.terminal); + const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome); + const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome); + const defaultTimeoutText = idleTimedOut ? "The model did not produce a response before the model idle timeout. " + "Please try again, or increase `models.providers..timeoutSeconds` for slow local or self-hosted providers. " + "If `agents.defaults.timeoutSeconds` or a run-specific timeout is lower, raise that ceiling too; provider timeouts cannot extend the whole agent run." @@ -50,15 +53,16 @@ export function resolveEmbeddedRunTerminalTimeout(input: { payloadCount: input.hasPartialAssistantTextAfterPromptTimeout ? 0 : (input.payloads?.length ?? 0), - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, + aborted: terminalAborted, + timedOut: terminalTimedOut, attempt: input.attempt, incompleteTurnText: null, }); const timeoutPhase = - input.attempt.promptTimeoutOutcome?.timeoutPhase ?? input.terminalOutcome.timeoutPhase; + input.attempt.promptTimeoutOutcome?.timeoutPhase ?? input.terminalState.outcome.timeoutPhase; const providerStarted = - input.attempt.promptTimeoutOutcome?.providerStarted ?? input.terminalOutcome.providerStarted; + input.attempt.promptTimeoutOutcome?.providerStarted ?? + input.terminalState.outcome.providerStarted; const timeoutAttribution = { ...(timeoutPhase ? { timeoutPhase } : {}), ...(typeof providerStarted === "boolean" ? { providerStarted } : {}), @@ -72,7 +76,7 @@ export function resolveEmbeddedRunTerminalTimeout(input: { meta: { durationMs: Date.now() - input.startedAtMs, agentMeta: input.agentMeta, - aborted: input.terminalAborted, + aborted: terminalAborted, systemPromptReport: input.attempt.systemPromptReport, finalPromptText: input.attempt.finalPromptText, finalAssistantVisibleText: input.finalAssistantVisibleText,