ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 12:56:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(agents): centralize terminal outcome handling (#114310)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-27 01:21:07 -04:00
committed by GitHub
parent 2a8c0ca513
commit df5d85a5b2
10 changed files with 324 additions and 214 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agents/embedded-agent-runner/run-loop.ts
+11 -55
View File
@@ -376,26 +376,12 @@ export async function runPreparedEmbeddedLoop(
accumulatedReplayState = normalizedAttempt.replayState;
const {
attempt,
terminalProjection: {
aborted,
externalAbort,
promptError,
timedOut,
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
},
sessionIdUsed,
sessionFileUsed,
currentAttemptAssistant,
currentAttemptCompletedAssistant,
attemptAssistant,
terminalOutcome,
terminalAborted,
terminalTimedOut,
terminalInterrupted,
signalOwnedInterruption,
terminalState,
setTerminalLifecycleMeta,
attemptCompactionCount,
activeErrorContext,
@@ -445,9 +431,7 @@ export async function runPreparedEmbeddedLoop(
attempt,
attemptAssistant,
currentAttemptAssistant,
terminalProviderStarted: terminalOutcome.providerStarted === true,
terminalInterrupted,
promptError,
terminalState,
activeErrorContext,
provider,
modelId,
@@ -455,14 +439,6 @@ export async function runPreparedEmbeddedLoop(
thinkLevel,
getThinkLevel: () => preparedRuntime.snapshot().thinkLevel,
attemptedThinking,
timedOut,
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
signalOwnedInterruption,
externalAbort,
aborted,
fallbackConfigured,
pluginHarnessOwnsTransport,
canRestartForLiveSwitch,
@@ -506,22 +482,15 @@ export async function runPreparedEmbeddedLoop(
}
let assistantProfileFailureReason = assistantFailureOutcome.assistantProfileFailureReason;
const terminalToolPresentation = readAttemptTerminalToolPresentation();
const terminalState = await prepareTerminalWithSettledTurnFinalization({
const finalizedTerminal = await prepareTerminalWithSettledTurnFinalization({
initial: {
attempt,
attemptAssistant,
currentAttemptCompletedAssistant,
sessionIdUsed,
sessionFileUsed,
terminalAborted,
terminalTimedOut,
terminalInterrupted,
externalAbort,
signalOwnedInterruption,
promptError,
terminalState,
attemptCompactionCount,
timedOutDuringCompaction,
timedOutDuringToolExecution,
},
terminalBase: {
runParams: params,
@@ -549,19 +518,14 @@ export async function runPreparedEmbeddedLoop(
const {
attempt: terminalAttempt,
attemptAssistant: terminalAttemptAssistant,
terminalAborted: terminalAbortedState,
terminalTimedOut: terminalTimedOutState,
terminalInterrupted: terminalInterruptedState,
externalAbort: terminalExternalAbort,
signalOwnedInterruption: terminalSignalOwnedInterruption,
promptError: terminalPromptError,
terminalState: resolvedTerminalState,
attemptCompactionCount: terminalAttemptCompactionCount,
prepared: terminalPrepared,
finalizationAttempted: settledTurnFinalizationAttempted,
} = terminalState;
lastRunPromptUsage = terminalState.lastRunPromptUsage;
lastTurnTotal = terminalState.lastTurnTotal;
if (terminalState.finalizationSucceeded) {
} = finalizedTerminal;
lastRunPromptUsage = finalizedTerminal.lastRunPromptUsage;
lastTurnTotal = finalizedTerminal.lastTurnTotal;
if (finalizedTerminal.finalizationSucceeded) {
assistantProfileFailureReason = null;
}
@@ -584,14 +548,11 @@ export async function runPreparedEmbeddedLoop(
timedOutDuringPrompt,
hasSuccessfulFinalAssistantAfterPromptTimeout,
shouldSurfaceCodexCompletionTimeout,
idleTimedOut,
attempt: terminalAttempt,
hasPartialAssistantTextAfterPromptTimeout,
payloads,
payloadsWithToolMedia,
terminalAborted: terminalAbortedState,
terminalTimedOut: terminalTimedOutState,
terminalOutcome,
terminalState: resolvedTerminalState,
resolveReplayInvalid: resolveReplayInvalidForAttempt,
setTerminalLifecycleMeta,
startedAtMs: started,
@@ -613,12 +574,7 @@ export async function runPreparedEmbeddedLoop(
activeErrorContext,
modelApi: effectiveModel.api,
executionContract,
terminalAborted: terminalAbortedState,
terminalTimedOut: terminalTimedOutState,
terminalInterrupted: terminalInterruptedState,
externalAbort: terminalExternalAbort,
signalOwnedInterruption: terminalSignalOwnedInterruption,
promptError: terminalPromptError,
terminalState: resolvedTerminalState,
payloadsWithToolMedia,
recoveredFinalAssistantPayloadsAfterPromptTimeout,
finalAssistantVisibleText,
src/agents/embedded-agent-runner/run/assistant-failure.ts
+44 -36
View File
@@ -1,5 +1,6 @@
import type { ThinkLevel } from "../../../auto-reply/thinking.js";
import type { AssistantMessage } from "../../../llm/types.js";
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js";
import {
classifyAssistantFailoverReason,
@@ -24,6 +25,10 @@ import { createFailoverDecisionLogger } from "./failover-observation.js";
import { resolveRunFailoverDecision } from "./failover-policy.js";
import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn.js";
import type { RunEmbeddedAgentParams } from "./params.js";
import {
isEmbeddedRunTerminalInterrupted,
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import type { EmbeddedRunAttemptResult } from "./types.js";
const MAX_EMPTY_ERROR_RETRIES = 3;
@@ -46,9 +51,7 @@ export async function handleEmbeddedAssistantFailure(input: {
attempt: EmbeddedRunAttemptResult;
attemptAssistant?: AssistantMessage;
currentAttemptAssistant?: AssistantMessage;
terminalProviderStarted: boolean;
terminalInterrupted: boolean;
promptError: unknown;
terminalState: EmbeddedRunTerminalState;
activeErrorContext: { provider: string; model: string };
provider: string;
modelId: string;
@@ -57,14 +60,6 @@ export async function handleEmbeddedAssistantFailure(input: {
// Profile rotation resets thinking inside the runtime; read it after advancing.
getThinkLevel: () => ThinkLevel;
attemptedThinking: Set<ThinkLevel>;
timedOut: boolean;
idleTimedOut: boolean;
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
timedOutByRunBudget: boolean;
signalOwnedInterruption: boolean;
externalAbort: boolean;
aborted: boolean;
fallbackConfigured: boolean;
pluginHarnessOwnsTransport: boolean;
canRestartForLiveSwitch: boolean;
@@ -100,11 +95,23 @@ export async function handleEmbeddedAssistantFailure(input: {
agentDir: string;
isProbeSession: boolean;
}): Promise<EmbeddedRunAssistantFailureOutcome> {
const {
aborted,
externalAbort,
idleTimedOut,
promptError,
timedOut,
timedOutByRunBudget,
timedOutDuringCompaction,
timedOutDuringToolExecution,
} = projectAgentRunAttemptTerminal(input.attempt.terminal);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome);
const { signalOwnedInterruption } = input.terminalState;
const fallbackThinking = pickFallbackThinkingLevel({
message: input.attemptAssistant?.errorMessage,
attempted: input.attemptedThinking,
});
if (fallbackThinking && !input.terminalInterrupted) {
if (fallbackThinking && !terminalInterrupted) {
log.warn(
`unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`,
);
@@ -122,10 +129,11 @@ export async function handleEmbeddedAssistantFailure(input: {
const failoverFailure = isFailoverAssistantError(input.attemptAssistant);
const assistantFailoverReason = classifyAssistantFailoverReason(input.attemptAssistant);
const assistantProviderStarted =
Boolean(input.currentAttemptAssistant?.provider) || input.terminalProviderStarted;
Boolean(input.currentAttemptAssistant?.provider) ||
input.terminalState.outcome.providerStarted === true;
const assistantProfileFailoverReason =
assistantFailoverReason ??
(assistantProviderStarted && (input.timedOut || input.idleTimedOut) ? "timeout" : null);
(assistantProviderStarted && (timedOut || idleTimedOut) ? "timeout" : null);
const assistantProfileFailureReason = input.resolveAuthProfileFailureReason(
assistantProfileFailoverReason,
{
@@ -154,8 +162,8 @@ export async function handleEmbeddedAssistantFailure(input: {
!billingFailure &&
!cloudCodeAssistFormatError &&
!imageDimensionError &&
!input.terminalInterrupted &&
!input.promptError &&
!terminalInterrupted &&
!promptError &&
silentErrorRetryReason &&
shouldRetrySilentErrorAssistantTurn({
attempt: input.attempt,
@@ -192,11 +200,11 @@ export async function handleEmbeddedAssistantFailure(input: {
sourceModel: input.attemptAssistant?.model ?? input.modelId,
profileId: failedProfileId,
fallbackConfigured: input.fallbackConfigured,
timedOut: input.timedOut,
aborted: input.aborted,
timedOut,
aborted,
});
if (
!input.signalOwnedInterruption &&
!signalOwnedInterruption &&
authFailure &&
(await input.maybeRefreshRuntimeAuthForAuthError(
input.attemptAssistant?.errorMessage ?? "",
@@ -232,35 +240,35 @@ export async function handleEmbeddedAssistantFailure(input: {
const initialDecision = resolveRunFailoverDecision({
stage: "assistant",
allowFormatRetry: cloudCodeAssistFormatError,
aborted: input.aborted,
externalAbort: input.externalAbort || input.signalOwnedInterruption,
aborted,
externalAbort: externalAbort || signalOwnedInterruption,
fallbackConfigured: input.fallbackConfigured,
failoverFailure,
failoverReason: assistantFailoverReason,
timedOut: input.timedOut,
idleTimedOut: input.idleTimedOut,
timedOutDuringCompaction: input.timedOutDuringCompaction,
timedOutDuringToolExecution: input.timedOutDuringToolExecution,
timedOut,
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
harnessOwnsTransport: input.pluginHarnessOwnsTransport,
timedOutByRunBudget: input.timedOutByRunBudget,
timedOutByRunBudget,
profileRotated: false,
});
const outcome = await handleAssistantFailover({
initialDecision,
aborted: input.aborted,
externalAbort: input.externalAbort || input.signalOwnedInterruption,
aborted,
externalAbort: externalAbort || signalOwnedInterruption,
fallbackConfigured: input.fallbackConfigured,
failoverFailure,
failoverReason: assistantFailoverReason,
timedOut: input.timedOut,
idleTimedOut: input.idleTimedOut,
timedOutDuringCompaction: input.timedOutDuringCompaction,
timedOutDuringToolExecution: input.timedOutDuringToolExecution,
timedOutByRunBudget: input.timedOutByRunBudget,
timedOut,
idleTimedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
allowSameModelIdleTimeoutRetry:
input.timedOut &&
input.idleTimedOut &&
!input.timedOutDuringCompaction &&
timedOut &&
idleTimedOut &&
!timedOutDuringCompaction &&
!input.fallbackConfigured &&
input.canRestartForLiveSwitch &&
input.sameModelIdleTimeoutRetries < MAX_SAME_MODEL_IDLE_TIMEOUT_RETRIES,
src/agents/embedded-agent-runner/run/attempt-normalization.ts
+6 -17
View File
@@ -36,7 +36,7 @@ import {
isEmbeddedRunTerminalAbort,
isEmbeddedRunTerminalInterrupted,
isEmbeddedRunTerminalTimeout,
resolveEmbeddedRunAttemptTerminalOutcome,
resolveEmbeddedRunAttemptTerminalState,
} from "./terminal-outcome.js";
type PreparedRuntime = Awaited<ReturnType<typeof prepareEmbeddedRunRuntime>>;
@@ -74,7 +74,6 @@ export async function normalizeEmbeddedRunAttempt(input: {
lastTurnTotal: number | undefined;
replayState: ReplayState;
attempt: ReturnType<typeof normalizeEmbeddedRunAttemptResult>;
terminalProjection: ReturnType<typeof projectAgentRunAttemptTerminal>;
sessionIdUsed: string;
sessionFileUsed: string | undefined;
currentAttemptAssistant: ReturnType<
@@ -86,11 +85,7 @@ export async function normalizeEmbeddedRunAttempt(input: {
attemptAssistant: ReturnType<
typeof normalizeEmbeddedRunAttemptResult
>["currentAttemptAssistant"];
terminalOutcome: ReturnType<typeof resolveEmbeddedRunAttemptTerminalOutcome>;
terminalAborted: boolean;
terminalTimedOut: boolean;
terminalInterrupted: boolean;
signalOwnedInterruption: boolean;
terminalState: ReturnType<typeof resolveEmbeddedRunAttemptTerminalState>;
setTerminalLifecycleMeta: NonNullable<
ReturnType<typeof normalizeEmbeddedRunAttemptResult>["setTerminalLifecycleMeta"]
>;
@@ -121,8 +116,7 @@ export async function normalizeEmbeddedRunAttempt(input: {
currentAttemptAssistant,
currentAttemptCompletedAssistant,
} = attempt;
const terminalProjection = projectAgentRunAttemptTerminal(terminal);
const { idleTimedOut } = terminalProjection;
const { idleTimedOut } = projectAgentRunAttemptTerminal(terminal);
const sessionAssistantForCandidate =
!currentAttemptAssistant &&
!isAssistantForModelRef(sessionLastAssistant, {
@@ -132,15 +126,15 @@ export async function normalizeEmbeddedRunAttempt(input: {
? undefined
: sessionLastAssistant;
const attemptAssistant = currentAttemptAssistant ?? sessionAssistantForCandidate;
const terminalOutcome = resolveEmbeddedRunAttemptTerminalOutcome({
const terminalState = resolveEmbeddedRunAttemptTerminalState({
attempt,
assistant: currentAttemptAssistant,
abortSignal: params.abortSignal,
});
const { outcome: terminalOutcome, signalOwnedInterruption } = terminalState;
const terminalAborted = isEmbeddedRunTerminalAbort(terminalOutcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(terminalOutcome);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalOutcome);
const signalOwnedInterruption = terminalInterrupted && params.abortSignal?.aborted === true;
const setTerminalLifecycleMeta: NonNullable<typeof attempt.setTerminalLifecycleMeta> = (meta) => {
const { stopReason, ...remainingMeta } = meta;
const terminalStopReason = terminalInterrupted ? terminalOutcome.stopReason : stopReason;
@@ -299,17 +293,12 @@ export async function normalizeEmbeddedRunAttempt(input: {
lastTurnTotal,
replayState,
attempt,
terminalProjection,
sessionIdUsed,
sessionFileUsed,
currentAttemptAssistant,
currentAttemptCompletedAssistant,
attemptAssistant,
terminalOutcome,
terminalAborted,
terminalTimedOut,
terminalInterrupted,
signalOwnedInterruption,
terminalState,
setTerminalLifecycleMeta,
attemptCompactionCount,
activeErrorContext,
src/agents/embedded-agent-runner/run/attempt-recovery.ts
+14 -12
View File
@@ -22,6 +22,7 @@ import { recoverEmbeddedRunOverflow } from "./overflow-context-recovery.js";
import { handleEmbeddedPromptFailure } from "./prompt-failure.js";
import type { prepareEmbeddedRunRuntime } from "./runtime-preparation.js";
import type { createEmbeddedRunSessionPromptState } from "./session-prompt-state.js";
import { isEmbeddedRunTerminalInterrupted } from "./terminal-outcome.js";
import { recoverEmbeddedRunTimeout } from "./timeout-context-recovery.js";
type PreparedRuntime = Awaited<ReturnType<typeof prepareEmbeddedRunRuntime>>;
@@ -86,21 +87,10 @@ export async function recoverEmbeddedRunAttempt(input: {
const runtime = preparedRuntime.snapshot();
const {
attempt,
terminalProjection: {
aborted,
externalAbort,
promptError,
promptErrorSource,
timedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
},
sessionIdUsed,
attemptAssistant,
currentAttemptCompletedAssistant,
terminalInterrupted,
signalOwnedInterruption,
terminalState,
setTerminalLifecycleMeta,
attemptCompactionCount,
activeErrorContext,
@@ -108,6 +98,18 @@ export async function recoverEmbeddedRunAttempt(input: {
assistantErrorText,
canRestartForLiveSwitch,
} = normalizedAttempt;
const {
aborted,
externalAbort,
promptError,
promptErrorSource,
timedOut,
timedOutDuringCompaction,
timedOutDuringToolExecution,
timedOutByRunBudget,
} = projectAgentRunAttemptTerminal(attempt.terminal);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalState.outcome);
const { signalOwnedInterruption } = terminalState;
const assistantOverflowCandidate =
currentAttemptCompletedAssistant !== undefined
? currentAttemptCompletedAssistant.stopReason === "error" ||
src/agents/embedded-agent-runner/run/settled-turn-finalization.ts
+18 -31
View File
@@ -8,6 +8,10 @@ import { log } from "../logger.js";
import { mergeUsageIntoAccumulator } from "../usage-accumulator.js";
import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js";
import { EMBEDDED_RUN_LANE_HEARTBEAT_MS } from "./lane-runtime.js";
import {
resolveEmbeddedRunAttemptTerminalOutcome,
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import { prepareEmbeddedRunTerminal } from "./terminal-preparation.js";
import { resolveSettledTurnFinalizationRequest } from "./terminal-resolution.js";
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js";
@@ -21,10 +25,7 @@ type TerminalPreparationBase = Omit<
| "sessionFileUsed"
| "lastRunPromptUsage"
| "lastTurnTotal"
| "terminalInterrupted"
| "terminalTimedOut"
| "timedOutDuringCompaction"
| "timedOutDuringToolExecution"
| "terminalState"
>;
export async function prepareTerminalWithSettledTurnFinalization(input: {
@@ -34,15 +35,8 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"];
sessionIdUsed: string;
sessionFileUsed?: string;
terminalAborted: boolean;
terminalTimedOut: boolean;
terminalInterrupted: boolean;
externalAbort: boolean;
signalOwnedInterruption: boolean;
promptError: unknown;
terminalState: EmbeddedRunTerminalState;
attemptCompactionCount: number;
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
};
terminalBase: TerminalPreparationBase;
lastRunPromptUsage: TerminalPreparationInput["lastRunPromptUsage"];
@@ -70,10 +64,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
sessionFileUsed: initial.sessionFileUsed,
lastRunPromptUsage,
lastTurnTotal,
terminalInterrupted: initial.terminalInterrupted,
terminalTimedOut: initial.terminalTimedOut,
timedOutDuringCompaction: initial.timedOutDuringCompaction,
timedOutDuringToolExecution: initial.timedOutDuringToolExecution,
terminalState: initial.terminalState,
});
const prompt = resolveSettledTurnFinalizationRequest({
runParams: input.terminalBase.runParams,
@@ -85,9 +76,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
recoveredFinalAssistantPayloadsAfterPromptTimeout:
prepared.recoveredFinalAssistantPayloadsAfterPromptTimeout,
hasTerminalToolPresentation: input.finalization.hasTerminalToolPresentation,
terminalAborted: initial.terminalAborted,
terminalTimedOut: initial.terminalTimedOut,
promptError: initial.promptError,
terminalState: initial.terminalState,
settledTurnFinalizationAvailable:
typeof input.finalization.harness.finalizeSettledTurn === "function",
});
@@ -119,6 +108,14 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage);
lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage;
lastTurnTotal = attempt.attemptUsage?.total ?? lastTurnTotal;
// Successful isolated finalization owns a fresh terminal, never the original abort signal.
const terminalState: EmbeddedRunTerminalState = {
outcome: resolveEmbeddedRunAttemptTerminalOutcome({
attempt,
assistant: attempt.currentAttemptAssistant,
}),
signalOwnedInterruption: false,
};
prepared = prepareEmbeddedRunTerminal({
...input.terminalBase,
attempt,
@@ -127,24 +124,14 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
sessionFileUsed: attempt.sessionFileUsed,
lastRunPromptUsage,
lastTurnTotal,
terminalInterrupted: false,
terminalTimedOut: false,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
terminalState,
});
return {
attempt,
attemptAssistant: attempt.currentAttemptAssistant,
currentAttemptCompletedAssistant: attempt.currentAttemptCompletedAssistant,
terminalAborted: false,
terminalTimedOut: false,
terminalInterrupted: false,
externalAbort: false,
signalOwnedInterruption: false,
promptError: null,
terminalState,
attemptCompactionCount: 0,
timedOutDuringCompaction: false,
timedOutDuringToolExecution: false,
sessionIdUsed: attempt.sessionIdUsed,
sessionFileUsed: attempt.sessionFileUsed,
prepared,
src/agents/embedded-agent-runner/run/terminal-outcome.test.ts
+128 -1
View File
@@ -1,7 +1,13 @@
import type { AssistantMessage } from "openclaw/plugin-sdk/llm";
import { describe, expect, it } from "vitest";
import { createAgentRunRestartAbortError } from "../../run-termination.js";
import { resolveEmbeddedRunAttemptTerminalOutcome } from "./terminal-outcome.js";
import {
isEmbeddedRunTerminalAbort,
isEmbeddedRunTerminalInterrupted,
isEmbeddedRunTerminalTimeout,
resolveEmbeddedRunAttemptTerminalOutcome,
resolveEmbeddedRunAttemptTerminalState,
} from "./terminal-outcome.js";
type EmbeddedRunAttemptTerminalInput = Parameters<
typeof resolveEmbeddedRunAttemptTerminalOutcome
@@ -38,6 +44,127 @@ function makeAssistant(stopReason: AssistantMessage["stopReason"]): AssistantMes
}
describe("embedded run attempt terminal outcome", () => {
it.each([
{
name: "run-budget prompt timeout",
terminal: { kind: "timeout", phase: "prompt", source: "run_budget" },
reason: "hard_timeout",
aborted: false,
timedOut: true,
},
{
name: "idle prompt timeout",
terminal: { kind: "timeout", phase: "prompt", source: "idle" },
reason: "hard_timeout",
aborted: false,
timedOut: true,
},
{
name: "recovered compaction observation",
terminal: { kind: "timeout", phase: "compaction", source: "observation" },
reason: "completed",
aborted: false,
timedOut: false,
},
{
name: "yield-only cleanup",
terminal: { kind: "aborted", source: "yield_cleanup" },
reason: "completed",
aborted: false,
timedOut: false,
},
{
name: "external attempt cancellation",
terminal: { kind: "aborted", source: "external" },
reason: "aborted",
aborted: true,
timedOut: false,
},
] as const)("carries $name through canonical interruption predicates", (testCase) => {
const outcome = resolveEmbeddedRunAttemptTerminalOutcome({
attempt: makeAttempt({ terminal: testCase.terminal }),
assistant: makeAssistant("stop"),
});
expect(outcome.reason).toBe(testCase.reason);
expect(isEmbeddedRunTerminalAbort(outcome)).toBe(testCase.aborted);
expect(isEmbeddedRunTerminalTimeout(outcome)).toBe(testCase.timedOut);
expect(isEmbeddedRunTerminalInterrupted(outcome)).toBe(testCase.aborted || testCase.timedOut);
});
it("keeps user-signal cancellation authoritative before assistant completion", () => {
const controller = new AbortController();
controller.abort();
const outcome = resolveEmbeddedRunAttemptTerminalOutcome({
attempt: makeAttempt(),
assistant: undefined,
abortSignal: controller.signal,
});
expect(outcome).toMatchObject({
reason: "aborted",
status: "error",
stopReason: "aborted",
});
expect(isEmbeddedRunTerminalAbort(outcome)).toBe(true);
expect(isEmbeddedRunTerminalTimeout(outcome)).toBe(false);
expect(isEmbeddedRunTerminalInterrupted(outcome)).toBe(true);
});
it("captures signal ownership before a later cancellation can reclassify completion", () => {
const controller = new AbortController();
const terminal = resolveEmbeddedRunAttemptTerminalState({
attempt: makeAttempt(),
assistant: makeAssistant("stop"),
abortSignal: controller.signal,
});
controller.abort();
expect(terminal).toEqual({
outcome: { reason: "completed", status: "ok", stopReason: "stop" },
signalOwnedInterruption: false,
});
});
it("captures user cancellation with the same terminal that owns the interruption", () => {
const controller = new AbortController();
controller.abort();
expect(
resolveEmbeddedRunAttemptTerminalState({
attempt: makeAttempt(),
assistant: undefined,
abortSignal: controller.signal,
}),
).toMatchObject({
outcome: { reason: "aborted", status: "error", stopReason: "aborted" },
signalOwnedInterruption: true,
});
});
it("starts successful settled finalization without inheriting the original abort signal", () => {
const controller = new AbortController();
controller.abort();
const originalTerminal = resolveEmbeddedRunAttemptTerminalState({
attempt: makeAttempt(),
assistant: undefined,
abortSignal: controller.signal,
});
expect(originalTerminal.signalOwnedInterruption).toBe(true);
expect(
resolveEmbeddedRunAttemptTerminalState({
attempt: makeAttempt(),
assistant: makeAssistant("stop"),
}),
).toEqual({
outcome: { reason: "completed", status: "ok", stopReason: "stop" },
signalOwnedInterruption: false,
});
});
it("keeps prompt timeout ownership ahead of generic abort metadata", () => {
const outcome = resolveEmbeddedRunAttemptTerminalOutcome({
attempt: makeAttempt({
src/agents/embedded-agent-runner/run/terminal-outcome.ts
+17
View File
@@ -9,6 +9,11 @@ type EmbeddedRunAttemptTerminalInput = Pick<
"terminal" | "promptTimeoutOutcome"
>;
export type EmbeddedRunTerminalState = {
outcome: AgentRunTerminalOutcome;
signalOwnedInterruption: boolean;
};
/** Projects private attempt metadata into the canonical agent terminal outcome. */
export function resolveEmbeddedRunAttemptTerminalOutcome(params: {
attempt: EmbeddedRunAttemptTerminalInput;
@@ -34,3 +39,15 @@ export function isEmbeddedRunTerminalAbort(outcome: AgentRunTerminalOutcome): bo
export function isEmbeddedRunTerminalInterrupted(outcome: AgentRunTerminalOutcome): boolean {
return isEmbeddedRunTerminalTimeout(outcome) || isEmbeddedRunTerminalAbort(outcome);
}
/** Captures signal ownership with the outcome before async recovery can change the signal. */
export function resolveEmbeddedRunAttemptTerminalState(
params: Parameters<typeof resolveEmbeddedRunAttemptTerminalOutcome>[0],
): EmbeddedRunTerminalState {
const outcome = resolveEmbeddedRunAttemptTerminalOutcome(params);
return {
outcome,
signalOwnedInterruption:
isEmbeddedRunTerminalInterrupted(outcome) && params.abortSignal?.aborted === true,
};
}
src/agents/embedded-agent-runner/run/terminal-preparation.ts
+14 -6
View File
@@ -1,5 +1,6 @@
import { copyReplyPayloadMetadata } from "../../../auto-reply/reply-payload.js";
import type { AssistantMessage } from "../../../llm/types.js";
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
import type { AuthProfileStore } from "../../auth-profiles.js";
import type { NormalizedUsage, UsageLike } from "../../usage.js";
import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js";
@@ -15,6 +16,11 @@ import {
import type { RunEmbeddedAgentParams } from "./params.js";
import { buildEmbeddedRunPayloads } from "./payloads.js";
import { buildTraceToolSummary } from "./run-attempt-result.js";
import {
isEmbeddedRunTerminalInterrupted,
isEmbeddedRunTerminalTimeout,
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import { mergeAttemptToolMediaPayloads } from "./tool-media-payloads.js";
import type { EmbeddedRunAttemptResult } from "./types.js";
@@ -35,10 +41,7 @@ export function prepareEmbeddedRunTerminal(input: {
lastTurnTotal?: number;
contextRecoveryState: EmbeddedRunContextRecoveryState;
resolvedToolResultFormat: NonNullable<RunEmbeddedAgentParams["toolResultFormat"]>;
terminalInterrupted: boolean;
terminalTimedOut: boolean;
timedOutDuringCompaction: boolean;
timedOutDuringToolExecution: boolean;
terminalState: EmbeddedRunTerminalState;
}): {
agentMeta: EmbeddedAgentMeta;
reportedModelRef: { provider: string; model: string };
@@ -54,8 +57,13 @@ export function prepareEmbeddedRunTerminal(input: {
failureSignal: ReturnType<typeof resolveEmbeddedRunFailureSignal>;
} {
const { runParams, attempt } = input;
const { timedOutDuringCompaction, timedOutDuringToolExecution } = projectAgentRunAttemptTerminal(
attempt.terminal,
);
const timedOutDuringPrompt =
input.terminalTimedOut && !input.timedOutDuringCompaction && !input.timedOutDuringToolExecution;
isEmbeddedRunTerminalTimeout(input.terminalState.outcome) &&
!timedOutDuringCompaction &&
!timedOutDuringToolExecution;
// Session transcript fallbacks can reference an earlier rewritten turn.
// Terminal delivery and metadata must stay scoped to this model attempt.
const terminalAssistant = input.currentAttemptCompletedAssistant;
@@ -132,7 +140,7 @@ export function prepareEmbeddedRunTerminal(input: {
sourceReplyDeliveryMode: runParams.sourceReplyDeliveryMode,
agentId: runParams.agentId,
runId: runParams.runId,
runAborted: input.terminalInterrupted,
runAborted: isEmbeddedRunTerminalInterrupted(input.terminalState.outcome),
didSendDeterministicApprovalPrompt: attempt.didSendDeterministicApprovalPrompt,
heartbeatToolResponse: attempt.heartbeatToolResponse,
});
src/agents/embedded-agent-runner/run/terminal-resolution.ts
+55 -43
View File
@@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto";
import { SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js";
import { freezeDiagnosticTraceContext } from "../../../infra/diagnostic-trace-context.js";
import type { AssistantMessage } from "../../../llm/types.js";
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
import type { AuthProfileFailureReason, AuthProfileStore } from "../../auth-profiles.js";
import type { AgentExecutionAuthBinding } from "../../execution-auth-binding.js";
import type { ResolvedProviderAuth } from "../../model-auth.js";
@@ -33,6 +34,12 @@ import {
YIELD_DIAGNOSTIC_TEXT,
} from "./incomplete-turn.js";
import type { RunEmbeddedAgentParams } from "./params.js";
import {
isEmbeddedRunTerminalAbort,
isEmbeddedRunTerminalInterrupted,
isEmbeddedRunTerminalTimeout,
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import {
MAX_BEFORE_AGENT_FINALIZE_REVISIONS,
type EmbeddedRunTerminalRetryState,
@@ -65,19 +72,20 @@ export function resolveSettledTurnFinalizationRequest(input: {
payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"];
recoveredFinalAssistantPayloadsAfterPromptTimeout?: EmbeddedAgentRunResult["payloads"];
hasTerminalToolPresentation: boolean;
terminalAborted: boolean;
terminalTimedOut: boolean;
promptError: unknown;
terminalState: EmbeddedRunTerminalState;
settledTurnFinalizationAvailable: boolean;
}): string | null {
if (!input.settledTurnFinalizationAvailable) {
return null;
}
const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome);
const { promptError } = projectAgentRunAttemptTerminal(input.attempt.terminal);
const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({
isCronTrigger: input.runParams.trigger === "cron",
payloadCount: input.payloadsWithToolMedia?.length ?? 0,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt: input.attempt,
});
const payloadCount = input.recoveredFinalAssistantPayloadsAfterPromptTimeout
@@ -91,8 +99,8 @@ export function resolveSettledTurnFinalizationRequest(input: {
allowEmptyAssistantReplyAsSilent: input.runParams.allowEmptyAssistantReplyAsSilent,
onlyExplicitSilentReply: false,
payloadCount,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt: input.attempt,
});
if (emptyAssistantReplyIsSilent) {
@@ -111,9 +119,9 @@ export function resolveSettledTurnFinalizationRequest(input: {
input.runParams.trigger === "manual")),
payloadCount,
hasTerminalToolPresentation: input.hasTerminalToolPresentation,
aborted: input.terminalAborted,
promptError: input.promptError,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
promptError,
timedOut: terminalTimedOut,
attempt: input.attempt,
});
}
@@ -128,12 +136,7 @@ export async function resolveEmbeddedRunTerminal(input: {
executionContract: Parameters<
typeof resolveReasoningOnlyRetryInstruction
>[0]["executionContract"];
terminalAborted: boolean;
terminalTimedOut: boolean;
terminalInterrupted: boolean;
externalAbort: boolean;
signalOwnedInterruption: boolean;
promptError: unknown;
terminalState: EmbeddedRunTerminalState;
payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"];
recoveredFinalAssistantPayloadsAfterPromptTimeout?: EmbeddedAgentRunResult["payloads"];
finalAssistantVisibleText?: string;
@@ -178,11 +181,16 @@ export async function resolveEmbeddedRunTerminal(input: {
contextRecoveryState: EmbeddedRunContextRecoveryState;
}): Promise<TerminalResolution> {
const { runParams, attempt, retryState } = input;
const { externalAbort, promptError } = projectAgentRunAttemptTerminal(attempt.terminal);
const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome);
const { signalOwnedInterruption } = input.terminalState;
const silentToolResultReplyPayload = resolveSilentToolResultReplyPayload({
isCronTrigger: runParams.trigger === "cron",
payloadCount: input.payloadsWithToolMedia?.length ?? 0,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt,
});
const payloadsForTerminalPath = input.recoveredFinalAssistantPayloadsAfterPromptTimeout
@@ -200,8 +208,8 @@ export async function resolveEmbeddedRunTerminal(input: {
allowEmptyAssistantReplyAsSilent: runParams.allowEmptyAssistantReplyAsSilent,
onlyExplicitSilentReply: settledTurnFinalizationAttempted,
payloadCount,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt,
});
const nextReasoningOnlyRetryInstruction =
@@ -212,8 +220,8 @@ export async function resolveEmbeddedRunTerminal(input: {
modelId: input.activeErrorContext.model,
modelApi: input.modelApi,
executionContract: input.executionContract,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt,
});
const nextEmptyResponseRetryInstruction =
@@ -225,8 +233,8 @@ export async function resolveEmbeddedRunTerminal(input: {
modelApi: input.modelApi,
executionContract: input.executionContract,
payloadCount,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt,
});
if (
@@ -250,9 +258,9 @@ export async function resolveEmbeddedRunTerminal(input: {
!settledTurnFinalizationAttempted &&
shouldRetryMissingAssistantTurn({
payloadCount,
aborted: input.terminalAborted,
promptError: input.promptError,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
promptError,
timedOut: terminalTimedOut,
attempt,
}) &&
retryState.missingAssistantAttempts < MAX_MISSING_ASSISTANT_RETRIES
@@ -284,16 +292,16 @@ export async function resolveEmbeddedRunTerminal(input: {
? null
: resolveIncompleteTurnPayloadText({
payloadCount,
aborted: input.terminalAborted,
externalAbort: input.externalAbort || input.signalOwnedInterruption,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
externalAbort: externalAbort || signalOwnedInterruption,
timedOut: terminalTimedOut,
hadPotentialSideEffects: input.replayState.hadPotentialSideEffects,
attempt,
});
const incompleteTurnFallbackSafe = Boolean(
incompleteTurnText &&
!input.terminalInterrupted &&
!input.promptError &&
!terminalInterrupted &&
!promptError &&
!attempt.lastToolError &&
!hasAttemptTerminalState(attempt) &&
!input.replayState.hadPotentialSideEffects,
@@ -306,8 +314,8 @@ export async function resolveEmbeddedRunTerminal(input: {
!settledTurnFinalizationAttempted &&
input.attemptCompactionCount > 0 &&
payloadCount === 0 &&
!input.terminalInterrupted &&
!input.promptError &&
!terminalInterrupted &&
!promptError &&
!attempt.clientToolCalls &&
!attempt.yieldDetected &&
!attempt.didSendDeterministicApprovalPrompt &&
@@ -380,8 +388,8 @@ export async function resolveEmbeddedRunTerminal(input: {
if (
beforeFinalizeRevisionReason &&
!settledTurnFinalizationAttempted &&
!input.terminalInterrupted &&
!input.promptError &&
!terminalInterrupted &&
!promptError &&
!attempt.clientToolCalls &&
!attempt.yieldDetected &&
!emptyAssistantReplyIsSilent
@@ -416,11 +424,13 @@ async function surfaceIncompleteTurn(
terminalToolPresentation?: string;
},
): Promise<TerminalResolution> {
const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome);
const replayInvalid = input.resolveReplayInvalid(input.text);
const livenessState = resolveRunLivenessState({
payloadCount: input.payloadCount,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt: input.attempt,
incompleteTurnText: input.text,
});
@@ -446,7 +456,7 @@ async function surfaceIncompleteTurn(
meta: {
durationMs: Date.now() - input.startedAtMs,
agentMeta: input.agentMeta,
aborted: input.terminalAborted,
aborted: terminalAborted,
systemPromptReport: input.attempt.systemPromptReport,
finalPromptText: input.attempt.finalPromptText,
finalAssistantVisibleText: input.finalAssistantVisibleText,
@@ -475,8 +485,10 @@ function completeEmbeddedRun(
emptyAssistantReplyIsSilent: boolean;
},
): TerminalResolution {
const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome);
log.debug(
`embedded run done: runId=${input.runParams.runId} sessionId=${input.runParams.sessionId} durationMs=${Date.now() - input.startedAtMs} aborted=${input.terminalAborted}`,
`embedded run done: runId=${input.runParams.runId} sessionId=${input.runParams.sessionId} durationMs=${Date.now() - input.startedAtMs} aborted=${terminalAborted}`,
);
markEmbeddedRunAuthProfileSuccess({
authProfileStateMode: input.runParams.authProfileStateMode,
@@ -507,8 +519,8 @@ function completeEmbeddedRun(
? "paused"
: resolveRunLivenessState({
payloadCount: input.payloadCount,
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt: input.attempt,
incompleteTurnText: null,
});
@@ -542,7 +554,7 @@ function completeEmbeddedRun(
meta: {
durationMs: Date.now() - input.startedAtMs,
agentMeta: input.agentMeta,
aborted: input.terminalAborted,
aborted: terminalAborted,
systemPromptReport: input.attempt.systemPromptReport,
finalPromptText: input.attempt.finalPromptText,
finalAssistantVisibleText: input.finalAssistantVisibleText,
src/agents/embedded-agent-runner/run/terminal-timeout.ts
+17 -13
View File
@@ -1,6 +1,12 @@
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
import { hasMessagingToolDeliveryEvidence } from "../delivery-evidence.js";
import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js";
import { resolveRunLivenessState } from "./incomplete-turn.js";
import {
isEmbeddedRunTerminalAbort,
isEmbeddedRunTerminalTimeout,
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import { copyAttemptDeliveryState } from "./terminal-resolution.js";
import type { EmbeddedRunAttemptResult } from "./types.js";
@@ -8,17 +14,11 @@ export function resolveEmbeddedRunTerminalTimeout(input: {
timedOutDuringPrompt: boolean;
hasSuccessfulFinalAssistantAfterPromptTimeout: boolean;
shouldSurfaceCodexCompletionTimeout: boolean;
idleTimedOut: boolean;
attempt: EmbeddedRunAttemptResult;
hasPartialAssistantTextAfterPromptTimeout: boolean;
payloads: EmbeddedAgentRunResult["payloads"];
payloadsWithToolMedia: EmbeddedAgentRunResult["payloads"];
terminalAborted: boolean;
terminalTimedOut: boolean;
terminalOutcome: {
timeoutPhase?: EmbeddedAgentRunResult["meta"]["timeoutPhase"];
providerStarted?: boolean;
};
terminalState: EmbeddedRunTerminalState;
resolveReplayInvalid: (incompleteTurnText?: string | null) => boolean;
setTerminalLifecycleMeta: NonNullable<EmbeddedRunAttemptResult["setTerminalLifecycleMeta"]>;
startedAtMs: number;
@@ -35,7 +35,10 @@ export function resolveEmbeddedRunTerminalTimeout(input: {
) {
return undefined;
}
const defaultTimeoutText = input.idleTimedOut
const { idleTimedOut } = projectAgentRunAttemptTerminal(input.attempt.terminal);
const terminalAborted = isEmbeddedRunTerminalAbort(input.terminalState.outcome);
const terminalTimedOut = isEmbeddedRunTerminalTimeout(input.terminalState.outcome);
const defaultTimeoutText = idleTimedOut
? "The model did not produce a response before the model idle timeout. " +
"Please try again, or increase `models.providers.<id>.timeoutSeconds` for slow local or self-hosted providers. " +
"If `agents.defaults.timeoutSeconds` or a run-specific timeout is lower, raise that ceiling too; provider timeouts cannot extend the whole agent run."
@@ -50,15 +53,16 @@ export function resolveEmbeddedRunTerminalTimeout(input: {
payloadCount: input.hasPartialAssistantTextAfterPromptTimeout
? 0
: (input.payloads?.length ?? 0),
aborted: input.terminalAborted,
timedOut: input.terminalTimedOut,
aborted: terminalAborted,
timedOut: terminalTimedOut,
attempt: input.attempt,
incompleteTurnText: null,
});
const timeoutPhase =
input.attempt.promptTimeoutOutcome?.timeoutPhase ?? input.terminalOutcome.timeoutPhase;
input.attempt.promptTimeoutOutcome?.timeoutPhase ?? input.terminalState.outcome.timeoutPhase;
const providerStarted =
input.attempt.promptTimeoutOutcome?.providerStarted ?? input.terminalOutcome.providerStarted;
input.attempt.promptTimeoutOutcome?.providerStarted ??
input.terminalState.outcome.providerStarted;
const timeoutAttribution = {
...(timeoutPhase ? { timeoutPhase } : {}),
...(typeof providerStarted === "boolean" ? { providerStarted } : {}),
@@ -72,7 +76,7 @@ export function resolveEmbeddedRunTerminalTimeout(input: {
meta: {
durationMs: Date.now() - input.startedAtMs,
agentMeta: input.agentMeta,
aborted: input.terminalAborted,
aborted: terminalAborted,
systemPromptReport: input.attempt.systemPromptReport,
finalPromptText: input.attempt.finalPromptText,
finalAssistantVisibleText: input.finalAssistantVisibleText,