fix(agent): post-tool timeout does not replay completed tools (#122516)

* fix(agent): prevent replay after post-tool timeout

* fix(agent): narrow settled tool assistant evidence
This commit is contained in:
Peter Steinberger
2026-08-12 00:31:55 -07:00
committed by GitHub
parent 1da8fffbcb
commit b46181bfc0
12 changed files with 426 additions and 163 deletions
@@ -236,7 +236,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
});
it("records silent success when the settled-tool finalization completes empty", async () => {
it("surfaces an incomplete turn when a required settled-tool finalizer completes empty", async () => {
const emptyStopAssistant = makeLastAssistant();
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
@@ -261,16 +261,19 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
const result = await runEmbeddedAgent(
makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", {
allowEmptyAssistantReplyAsSilent: true,
terminalReplyExpectation: "required",
}),
);
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
expect(result.payloads).toBeUndefined();
expect(result.meta.error).toBeUndefined();
expect(result.payloads?.[0]).toMatchObject({ isError: true });
expect(result.payloads?.[0]?.text).toContain(
"some tool actions may have already been executed",
);
expect(result.meta.error?.kind).toBe("incomplete_turn");
expect(result.meta.terminalReplyKind).toBeUndefined();
expect(result.meta.finalAssistantVisibleText).toBeUndefined();
expect(result.meta.finalAssistantRawText).toBeUndefined();
expect(result.meta.stopReason).toBe("stop");
expectNoWarnMessageWith("empty response detected");
expectWarnMessageWith("settled-turn finalization completed without a visible answer");
});
@@ -404,32 +404,17 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
});
it("continues once after settled side-effecting tools finish without a final answer", async () => {
const acceptedSessionSpawns = [
{ runId: "child-run", childSessionKey: "agent:main:subagent:child" },
];
const toolUseAssistant = makeLastAssistant({
stopReason: "toolUse",
content: [
{ type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } },
{ type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } },
{
type: "toolCall",
id: "tool_spawn",
name: "sessions_spawn",
arguments: { task: "follow up" },
},
],
});
const settledToolResults = [
toolUseAssistant,
{ role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false },
{ role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false },
{
role: "toolResult",
toolCallId: "tool_spawn",
toolName: "sessions_spawn",
isError: false,
},
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"];
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
@@ -437,15 +422,10 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
return makeAttemptResult({
assistantTexts: [],
latestMcpAppChannelView: { viewId: "view-after-tools" },
toolMetas: [
{ toolName: "write", meta: "path=note.txt" },
{ toolName: "cron" },
{ toolName: "sessions_spawn" },
],
toolMetas: [{ toolName: "write", meta: "path=note.txt" }, { toolName: "cron" }],
successfulNestedToolNames: ["read"],
acceptedSessionSpawns,
successfulCronAdds: 1,
itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 },
itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 },
messagesSnapshot: settledToolResults,
lastAssistant: toolUseAssistant,
currentAttemptAssistant: toolUseAssistant,
@@ -475,10 +455,9 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer.");
expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" });
expect(result.successfulCronAdds).toBe(1);
expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns);
expect(result.meta.toolSummary).toEqual({
calls: 3,
tools: ["write", "cron", "sessions_spawn"],
calls: 2,
tools: ["write", "cron"],
failures: 0,
});
expect(result.meta.agentMeta).toMatchObject({
@@ -17,6 +17,35 @@ import {
} from "./run/incomplete-turn-resolution.js";
import type { EmbeddedRunAttemptResult } from "./run/types.js";
function makeSettledIdleWriteAttempt(options?: {
terminal?: EmbeddedRunAttemptResult["terminal"];
stalePriorTurn?: boolean;
}) {
const toolUseAssistant = makeLastAssistant({
stopReason: "toolUse",
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }],
});
const abortedAssistant = makeLastAssistant({ stopReason: "aborted", content: [] });
return makeAttemptResult({
terminal: options?.terminal ?? { kind: "timeout", phase: "prompt", source: "idle" },
assistantTexts: [],
toolMetas: [{ toolName: "write", replaySafe: false }],
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
messagesSnapshot: [
{ role: "user", content: [{ type: "text", text: "old turn" }] },
toolUseAssistant,
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
...(options?.stalePriorTurn
? [{ role: "user", content: [{ type: "text", text: "current turn" }] }]
: []),
abortedAssistant,
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
lastAssistant: abortedAssistant,
currentAttemptAssistant: abortedAssistant,
currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
});
}
describe("runEmbeddedAgent incomplete-turn safety", () => {
beforeEach(() => {
resetRunIncompleteTurnOwnerMocks();
@@ -87,26 +116,81 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
).toBe(true);
});
it.each([
{ label: "aborted", aborted: true, timedOut: false, promptError: null },
{ label: "timed out", aborted: false, timedOut: true, promptError: null },
{ label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") },
])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => {
const toolUseAssistant = makeLastAssistant({
stopReason: "toolUse",
content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }],
});
it("continues an exactly settled current-turn tool batch after an idle prompt timeout", () => {
const instruction = resolveSettledToolTerminalContinuationInstruction(
makeSettledContinuationParams(
{
assistantTexts: [],
toolMetas: [{ toolName: "bash" }],
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: toolUseAssistant,
currentAttemptAssistant: toolUseAssistant,
},
{ aborted, timedOut, promptError },
),
makeSettledContinuationParams(makeSettledIdleWriteAttempt(), {
timedOut: true,
promptError: new Error("LLM idle timeout"),
}),
);
expect(instruction).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
});
it.each([
{
label: "external abort",
terminal: { kind: "timeout", phase: "prompt", source: "external" } as const,
aborted: true,
timedOut: true,
},
{
label: "runtime timeout",
terminal: { kind: "timeout", phase: "prompt", source: "runtime" } as const,
aborted: false,
timedOut: true,
},
{
label: "run budget timeout",
terminal: { kind: "timeout", phase: "prompt", source: "run_budget" } as const,
aborted: false,
timedOut: true,
},
{
label: "compaction timeout",
terminal: { kind: "timeout", phase: "compaction", source: "idle" } as const,
aborted: false,
timedOut: true,
},
{
label: "tool execution timeout",
terminal: { kind: "timeout", phase: "tool_execution", source: "idle" } as const,
aborted: false,
timedOut: true,
},
{
label: "timeout observation",
terminal: { kind: "timeout", phase: "tool_execution", source: "observation" } as const,
aborted: false,
timedOut: false,
},
{
label: "prompt error without idle timeout",
terminal: { kind: "ok" } as const,
aborted: false,
timedOut: false,
promptError: new Error("closed"),
},
])(
"does not finalize settled tools after a $label",
({ terminal, aborted, timedOut, promptError }) => {
const instruction = resolveSettledToolTerminalContinuationInstruction(
makeSettledContinuationParams(makeSettledIdleWriteAttempt({ terminal }), {
aborted,
timedOut,
promptError,
}),
);
expect(instruction).toBeNull();
},
);
it("does not use a settled prior-turn batch to authorize idle-timeout finalization", () => {
const instruction = resolveSettledToolTerminalContinuationInstruction(
makeSettledContinuationParams(makeSettledIdleWriteAttempt({ stalePriorTurn: true }), {
timedOut: true,
}),
);
expect(instruction).toBeNull();
@@ -4,7 +4,9 @@ import { makeModelFallbackCfg } from "../test-helpers/model-fallback-config-fixt
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
import {
MockedFailoverError,
mockedBuildEmbeddedRunPayloads,
mockedClassifyFailoverReason,
mockedGetApiKeyForModel,
mockedRunEmbeddedAttempt,
overflowBaseRunParams,
resetSharedRunIntegrationHarnessMocks,
@@ -58,4 +60,102 @@ describe("runEmbeddedAgent prompt timeout fallback handoff", () => {
await expect(promise).rejects.toThrow("LLM request timed out.");
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
});
it("finalizes a settled write after an idle timeout without replaying the prompt", async () => {
const toolUseAssistant = {
role: "assistant" as const,
stopReason: "toolUse" as const,
provider: "openai",
model: "gpt-5.4",
content: [
{
type: "toolCall",
id: "tool_write",
name: "write",
arguments: { path: "note.txt", content: "done" },
},
],
};
const abortedAssistant = {
role: "assistant" as const,
stopReason: "aborted" as const,
provider: "openai",
model: "gpt-5.4",
content: [],
};
const finalAssistant = {
role: "assistant" as const,
stopReason: "stop" as const,
provider: "openai",
model: "gpt-5.4",
content: [{ type: "text", text: "The note was written once." }],
};
mockedClassifyFailoverReason.mockReturnValue("timeout");
mockedRunEmbeddedAttempt
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
terminal: { kind: "timeout", phase: "prompt", source: "idle" },
toolMetas: [{ toolName: "write", replaySafe: false }],
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
messagesSnapshot: [
{ role: "user", content: [{ type: "text", text: "Write note.txt" }] },
toolUseAssistant,
{
role: "toolResult",
toolCallId: "tool_write",
toolName: "write",
isError: false,
},
abortedAssistant,
] as never,
lastAssistant: abortedAssistant as never,
currentAttemptAssistant: abortedAssistant as never,
currentAttemptReplayMetadata: {
hadPotentialSideEffects: true,
replaySafe: false,
},
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: ["The note was written once."],
lastAssistant: finalAssistant as never,
currentAttemptAssistant: finalAssistant as never,
currentAttemptCompletedAssistant: finalAssistant as never,
}),
);
mockedBuildEmbeddedRunPayloads
.mockReturnValueOnce([])
.mockReturnValueOnce([{ text: "The note was written once." }]);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
provider: "openai",
model: "gpt-5.4",
runId: "run-post-tool-idle-finalization",
config: makeModelFallbackCfg({
agents: {
defaults: {
model: {
primary: "openai/gpt-5.4",
fallbacks: ["anthropic/claude-opus-4-6"],
},
},
},
}),
});
expect(result.payloads).toEqual([{ text: "The note was written once." }]);
expect(result.meta.executionTrace?.fallbackUsed).toBe(false);
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
expect(mockedRunEmbeddedAttempt.mock.calls[1]?.[0]).toMatchObject({
operation: "settled-tool-finalization",
disableTools: true,
skipPreparedUserTurnMessage: true,
prompt:
"The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.",
});
expect(mockedGetApiKeyForModel).toHaveBeenCalledTimes(1);
});
});
@@ -110,6 +110,37 @@ function makeExhaustedCredentialFailureInput(options?: { replaySafe?: boolean })
};
}
function makeIdleTimeoutFailureInput(options?: { replaySafe?: boolean }) {
const fixture = makeExhaustedCredentialFailureInput();
const replaySafe = options?.replaySafe === true;
const assistant = buildEmbeddedRunnerAssistant({
provider: "anthropic",
model: "mock-1",
stopReason: "aborted",
});
const replayMetadata = {
hadPotentialSideEffects: !replaySafe,
replaySafe,
};
const attempt = makeEmbeddedRunnerAttempt({
terminal: { kind: "timeout", phase: "prompt", source: "idle" },
lastAssistant: assistant,
currentAttemptAssistant: assistant,
toolMetas: replaySafe ? [] : [{ toolName: "write", replaySafe: false }],
replayMetadata,
currentAttemptReplayMetadata: replayMetadata,
});
fixture.input.attempt = attempt;
fixture.input.attemptAssistant = assistant;
fixture.input.currentAttemptAssistant = assistant;
fixture.input.terminalState = resolveEmbeddedRunAttemptTerminalState({ attempt, assistant });
fixture.input.emptyErrorRetries = 0;
fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => true);
fixture.input.maybeRetrySameModelRateLimit = vi.fn(async () => true);
fixture.input.advanceRateLimitAuthProfile = vi.fn(async () => true);
return fixture;
}
describe("handleEmbeddedAssistantFailure", () => {
it("uses prepared OpenRouter ownership for custom-provider billing failures", async () => {
const fixture = makeExhaustedCredentialFailureInput();
@@ -165,10 +196,11 @@ describe("handleEmbeddedAssistantFailure", () => {
}
fixture.input.attemptAssistant.errorCode = PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE;
fixture.input.attemptAssistant.errorMessage = "reasoning is required";
fixture.input.resolveAuthProfileFailureReason = vi.fn(() => "timeout" as const);
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
expect(outcome.action).toBe("proceed");
expect(outcome).toMatchObject({ action: "proceed", assistantProfileFailureReason: null });
expect(fixture.advanceAuthProfile).not.toHaveBeenCalled();
expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled();
expect(fixture.traceAttempts).toEqual([]);
@@ -212,6 +244,37 @@ describe("handleEmbeddedAssistantFailure", () => {
expect(fixture.traceAttempts).toEqual([]);
});
it("closes every failover retry after an idle timeout commits a write", async () => {
const fixture = makeIdleTimeoutFailureInput();
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
expect(outcome.action).toBe("proceed");
expect(fixture.input.maybeRefreshRuntimeAuthForAuthError).not.toHaveBeenCalled();
expect(fixture.input.maybeRetrySameModelRateLimit).not.toHaveBeenCalled();
expect(fixture.advanceAuthProfile).not.toHaveBeenCalled();
expect(fixture.input.advanceRateLimitAuthProfile).not.toHaveBeenCalled();
expect(fixture.traceAttempts).toEqual([]);
});
it("keeps replay-safe idle timeout profile rotation available", async () => {
const fixture = makeIdleTimeoutFailureInput({ replaySafe: true });
fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => false);
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
expect(outcome).toMatchObject({ action: "retry", lastRetryFailoverReason: "timeout" });
expect(fixture.advanceAuthProfile).toHaveBeenCalledOnce();
expect(fixture.traceAttempts).toEqual([
{
provider: "anthropic",
model: "mock-1",
result: "rotate_profile",
stage: "assistant",
},
]);
});
it("does not cache an exact credential-file failure from a fallback candidate", async () => {
const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS;
process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000";
@@ -23,6 +23,7 @@ import {
import { log } from "../logger.js";
import type { TraceAttempt } from "../types.js";
import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js";
import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js";
import { createFailoverDecisionLogger } from "./failover-observation.js";
import { resolveRunFailoverDecision } from "./failover-policy.js";
import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn-recovery.js";
@@ -100,28 +101,10 @@ export async function handleEmbeddedAssistantFailure(input: {
projectAgentRunAttemptTerminal(input.attempt.terminal);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome);
const { signalOwnedInterruption } = input.terminalState;
if (isReplayUnsafeAssistantError(input.attemptAssistant)) {
return buildOutcome(input, {
action: "proceed",
assistantProfileFailureReason: null,
});
}
const fallbackThinking = pickFallbackThinkingLevel({
message: input.attemptAssistant?.errorMessage,
attempted: input.attemptedThinking,
});
if (fallbackThinking && !terminalInterrupted) {
log.warn(
`unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`,
);
return buildOutcome(input, {
action: "retry",
thinkLevel: fallbackThinking,
preserveSameModelRateLimitRetryCount: true,
assistantProfileFailureReason: null,
});
}
const authFailure = isAuthAssistantError(input.attemptAssistant);
const rateLimitFailure = isRateLimitAssistantError(input.attemptAssistant);
const billingFailure = isBillingAssistantError(input.attemptAssistant);
@@ -144,6 +127,26 @@ export async function handleEmbeddedAssistantFailure(input: {
isShortWindowRateLimitMessage(input.attemptAssistant?.errorMessage),
},
);
const replayUnsafeAssistantError = isReplayUnsafeAssistantError(input.attemptAssistant);
if (replayUnsafeAssistantError || !isCurrentAttemptReplaySafe(input.attempt)) {
return buildOutcome(input, {
action: "proceed",
assistantProfileFailureReason: replayUnsafeAssistantError
? null
: assistantProfileFailureReason,
});
}
if (fallbackThinking && !terminalInterrupted) {
log.warn(
`unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`,
);
return buildOutcome(input, {
action: "retry",
thinkLevel: fallbackThinking,
preserveSameModelRateLimitRetryCount: true,
assistantProfileFailureReason,
});
}
const cloudCodeAssistFormatError = input.attempt.cloudCodeAssistFormatError;
const imageDimensionError = parseImageDimensionError(input.attemptAssistant?.errorMessage ?? "");
const genericUnknownReasoningError =
@@ -11,6 +11,7 @@ import type { EmbeddedAgentRunResult, TraceAttempt } from "../types.js";
import type { createUsageAccumulator } from "../usage-accumulator.js";
import type { prepareAndDispatchEmbeddedRunAttempt } from "./attempt-dispatch-preparation.js";
import type { normalizeEmbeddedRunAttempt } from "./attempt-normalization.js";
import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js";
import { buildEmbeddedRunBlockedResult } from "./blocked-run-result.js";
import { resolveCodexAppServerRecoveryRetry } from "./codex-app-server-recovery.js";
import { resolveCompactionLiveModelSelection } from "./compaction-live-model-selection.js";
@@ -110,6 +111,7 @@ export async function recoverEmbeddedRunAttempt(input: {
timedOutByRunBudget,
} = projectAgentRunAttemptTerminal(attempt.terminal);
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalState.outcome);
const currentAttemptReplaySafe = isCurrentAttemptReplaySafe(attempt);
const { signalOwnedInterruption } = terminalState;
const assistantOverflowCandidate =
currentAttemptCompletedAssistant !== undefined
@@ -137,6 +139,40 @@ export async function recoverEmbeddedRunAttempt(input: {
thinkLevel: updates?.thinkLevel ?? runtime.thinkLevel,
});
if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) {
const errorText = formatErrorMessage(promptError);
const replayInvalid = resolveReplayInvalidForAttempt();
setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" });
return {
action: "complete",
result: buildEmbeddedRunBlockedResult({
text: errorText,
errorKind: "hook_block",
errorMessage: errorText,
durationMs: Date.now() - runInput.startedAtMs,
agentMeta: buildErrorAgentMeta({
sessionId: sessionIdUsed,
sessionFile: sessionPromptState.sessionFile,
provider: preparedRuntime.provider,
model: preparedRuntime.model.id,
...runtime.outerContextTokenMeta,
usageAccumulator: input.usageAccumulator,
lastRunPromptUsage: input.lastRunPromptUsage,
currentAttemptAssistant,
}),
attempt,
replayInvalid,
}),
};
}
if (!currentAttemptReplaySafe) {
return {
action: "proceed",
shouldSurfaceCodexCompletionTimeout:
attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && timedOut,
};
}
const requestedSelection = shouldSwitchToLiveModel({
cfg: params.config,
sessionKey: runInput.resolvedSessionKey,
@@ -255,32 +291,6 @@ export async function recoverEmbeddedRunAttempt(input: {
}),
};
}
if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) {
const errorText = formatErrorMessage(promptError);
const replayInvalid = resolveReplayInvalidForAttempt();
setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" });
return {
action: "complete",
result: buildEmbeddedRunBlockedResult({
text: errorText,
errorKind: "hook_block",
errorMessage: errorText,
durationMs: Date.now() - runInput.startedAtMs,
agentMeta: buildErrorAgentMeta({
sessionId: sessionIdUsed,
sessionFile: sessionPromptState.sessionFile,
provider: preparedRuntime.provider,
model: preparedRuntime.model.id,
...runtime.outerContextTokenMeta,
usageAccumulator: input.usageAccumulator,
lastRunPromptUsage: input.lastRunPromptUsage,
currentAttemptAssistant,
}),
attempt,
replayInvalid,
}),
};
}
const hasRecoverableCodexAppServerTimeoutOutcome = Boolean(
attempt.codexAppServerFailure && attempt.promptTimeoutOutcome,
);
@@ -16,6 +16,14 @@ type ReplayMetadataAttempt = Pick<
> &
Partial<Pick<EmbeddedRunAttemptResult, "messagingToolSentTargets" | "acceptedSessionSpawns">>;
/** Uses current-attempt evidence when available and otherwise preserves fail-closed legacy state. */
export function isCurrentAttemptReplaySafe(
attempt: Pick<EmbeddedRunAttemptResult, "replayMetadata" | "currentAttemptReplayMetadata">,
): boolean {
const replayMetadata = attempt.currentAttemptReplayMetadata ?? attempt.replayMetadata;
return replayMetadata.replaySafe && !replayMetadata.hadPotentialSideEffects;
}
/**
* Marks whether retrying the attempt can safely replay the prompt. Concrete
* tool-instance policy, async work, committed delivery, spawned sessions, and
@@ -33,6 +33,7 @@ export type IncompleteTurnAttempt = Pick<
| "itemLifecycle"
| "messagesSnapshot"
| "replayMetadata"
| "currentAttemptReplayMetadata"
| "terminal"
| "toolMetas"
> &
@@ -7,7 +7,11 @@ import {
hasCompletedMessagingToolDeliveryEvidence,
} from "../delivery-evidence.js";
import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js";
import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js";
import {
hasAsyncActivity,
hasAttemptTerminalState,
isCurrentAttemptReplaySafe,
} from "./attempt-terminal-evidence.js";
import {
hasOnlySilentAssistantReply,
hasPositiveOutputTokenUsage,
@@ -60,9 +64,7 @@ export function shouldRetrySilentErrorAssistantTurn(params: {
}
// Current-attempt evidence avoids blocking on prior committed effects; older
// harnesses retain the cumulative, fail-closed behavior.
const retryReplayMetadata =
params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata;
if (retryReplayMetadata.hadPotentialSideEffects) {
if (!isCurrentAttemptReplaySafe(params.attempt)) {
return false;
}
@@ -187,6 +189,27 @@ export function resolveReasoningOnlyRetryInstruction(params: {
return REASONING_ONLY_RETRY_INSTRUCTION;
}
type SettledToolCall = { id: string | null; name: string | null };
function readSettledToolCalls(
message: EmbeddedRunAttemptResult["currentAttemptAssistant"] | null | undefined,
): SettledToolCall[] {
if (!Array.isArray(message?.content)) {
return [];
}
return message.content.flatMap((item) => {
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
return block?.type === "toolCall"
? [
{
id: typeof block.id === "string" ? block.id : null,
name: typeof block.name === "string" ? block.name : null,
},
]
: [];
});
}
/** Builds one fresh continuation after settled tools ended without a visible final answer. */
export function resolveSettledToolTerminalContinuationInstruction(params: {
provider?: string;
@@ -201,8 +224,27 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
timedOut: boolean;
attempt: IncompleteTurnAttempt;
}): string | null {
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
const currentAttemptAssistant = params.attempt.currentAttemptAssistant;
const snapshot = params.attempt.messagesSnapshot ?? [];
const latestUserIndex = snapshot.findLastIndex((message) => message.role === "user");
let assistant: EmbeddedRunAttemptResult["currentAttemptAssistant"] = currentAttemptAssistant;
let assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
if (assistantIndex <= latestUserIndex || readSettledToolCalls(assistant).length === 0) {
assistantIndex = snapshot.findLastIndex(
(message, index) =>
index > latestUserIndex &&
message.role === "assistant" &&
readSettledToolCalls(message).length > 0,
);
const assistantCandidate = assistantIndex >= 0 ? snapshot[assistantIndex] : undefined;
assistant = assistantCandidate?.role === "assistant" ? assistantCandidate : undefined;
}
const terminal = params.attempt.terminal;
const idlePromptTimeout =
terminal.kind === "timeout" &&
terminal.phase === "prompt" &&
terminal.source === "idle" &&
params.attempt.currentAttemptReplayMetadata?.hadPotentialSideEffects === true;
const emptyStopAfterSettledTools = Boolean(
params.allowEmptyStopContinuation &&
currentAttemptAssistant?.stopReason === "stop" &&
@@ -220,25 +262,11 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
// Idle is not proof of settlement: skipped or partially dispatched tools must
// never be described as completed. Match each terminal call's id and owner to
// its own current-batch result; a reported failure is settled, not successful.
const requestedToolCalls = Array.isArray(assistant?.content)
? assistant.content.flatMap((item) => {
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
return block?.type === "toolCall"
? [
{
id: typeof block.id === "string" ? block.id : null,
name: typeof block.name === "string" ? block.name : null,
},
]
: [];
})
: [];
const requestedToolCalls = readSettledToolCalls(assistant);
// Scan only results AFTER the terminal assistant: the snapshot spans the whole
// session, and a prior turn's toolResult with a model-reused id would otherwise
// prove "completion" for a batch that never dispatched. Assistant not found in
// the snapshot fails closed to the existing incomplete-turn error.
const snapshot = params.attempt.messagesSnapshot ?? [];
const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
const settledToolResults = new Map(
(assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => {
const result = message as {
@@ -260,7 +288,9 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
}),
);
const allToolsProvenSettled =
params.attempt.itemLifecycle?.activeCount === 0 &&
params.attempt.itemLifecycle.startedCount > 0 &&
params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount &&
params.attempt.itemLifecycle.activeCount === 0 &&
requestedToolCalls.length > 0 &&
requestedToolCalls.every(
({ id, name }) =>
@@ -284,13 +314,14 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
params.payloadCount !== 0 ||
params.hasTerminalToolPresentation ||
params.aborted ||
params.promptError != null ||
params.timedOut ||
((params.promptError != null ||
params.timedOut ||
params.attempt.terminal.kind === "timeout") &&
!idlePromptTimeout) ||
(assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) ||
hasUnsettledToolError ||
(hasSettledTerminalToolFailure &&
(hasAsyncActivity(params.attempt.toolMetas) ||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) ||
hasAsyncActivity(params.attempt.toolMetas) ||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) ||
params.attempt.clientToolCalls ||
params.attempt.yieldDetected ||
params.attempt.didSendDeterministicApprovalPrompt
@@ -107,34 +107,16 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
prompt,
noteLaneTaskProgress: input.finalization.noteLaneTaskProgress,
});
if (finalization.outcome === "empty") {
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, finalization.result.usage);
lastRunPromptUsage = finalization.result.usage ?? lastRunPromptUsage;
log.warn(
`settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
`provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`,
);
const emptyAssistant = finalization.result.assistant;
const completedEmptyAttempt = {
...initial.attempt,
lastAssistant: emptyAssistant,
currentAttemptAssistant: emptyAssistant,
currentAttemptCompletedAssistant: emptyAssistant,
};
return {
...initial,
attempt: completedEmptyAttempt,
attemptAssistant: emptyAssistant,
currentAttemptCompletedAssistant: emptyAssistant,
prepared,
lastRunPromptUsage,
finalizationOutcome: "completed-empty" as const,
};
}
attempt = finalization.attempt;
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage);
mergeAttemptRunStatsIntoAccumulator(input.terminalBase.usageAccumulator, attempt);
lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage;
if (finalization.outcome === "empty") {
log.warn(
`settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
`provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`,
);
}
// Successful isolated finalization owns a fresh terminal, never the original abort signal.
const terminalState: EmbeddedRunTerminalState = {
outcome: resolveEmbeddedRunAttemptTerminalOutcome({
@@ -164,7 +146,8 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
sessionFileUsed: attempt.sessionFileUsed,
prepared,
lastRunPromptUsage,
finalizationOutcome: "answered" as const,
finalizationOutcome:
finalization.outcome === "empty" ? ("completed-empty" as const) : ("answered" as const),
};
} catch (error) {
log.warn(
@@ -186,13 +169,7 @@ async function runPreparedSettledTurnFinalization(input: {
harness: AgentHarness;
prompt: string;
noteLaneTaskProgress: () => void;
}): Promise<
| { outcome: "answered"; attempt: EmbeddedRunAttemptWithReceiptEvidence }
| {
outcome: "empty";
result: AgentHarnessSettledTurnFinalizationResult;
}
> {
}): Promise<{ outcome: "answered" | "empty"; attempt: EmbeddedRunAttemptWithReceiptEvidence }> {
return await withEmbeddedRunLaneProgressHeartbeat(input.noteLaneTaskProgress, async () => {
const finalization = await runEmbeddedSettledTurnFinalizationWithBackend(
{
@@ -206,12 +183,10 @@ async function runPreparedSettledTurnFinalization(input: {
input.settledAttempt,
input.harness,
);
if (finalization.outcome === "empty") {
return finalization;
}
return {
outcome: "answered",
outcome: finalization.outcome,
attempt: buildSettledTurnFinalizationAttemptResult({
outcome: finalization.outcome,
result: finalization.result,
settledAttempt: input.settledAttempt,
prompt: input.prompt,
@@ -222,13 +197,14 @@ async function runPreparedSettledTurnFinalization(input: {
}
function buildSettledTurnFinalizationAttemptResult(input: {
outcome: "answered" | "empty";
result: AgentHarnessSettledTurnFinalizationResult;
settledAttempt: EmbeddedRunAttemptWithReceiptEvidence;
prompt: string;
agentHarnessId?: string;
}): EmbeddedRunAttemptWithReceiptEvidence {
const { result, settledAttempt } = input;
const text = resolveSettledTurnFinalizationText(result);
const text = input.outcome === "empty" ? "" : resolveSettledTurnFinalizationText(result);
// Finalization replaces terminal ownership, not host-private facts from settled tools.
// Keep those facts while replay, abort, and lifecycle state remain finalizer-local.
return {
@@ -63,6 +63,14 @@ type TerminalResolution =
| { action: "retry" }
| { action: "complete"; result: EmbeddedAgentRunResult };
function requiresVisibleTerminalReply(runParams: TerminalRunParams): boolean {
return (
runParams.terminalReplyExpectation === "required" ||
(runParams.terminalReplyExpectation == null &&
(runParams.trigger == null || runParams.trigger === "user" || runParams.trigger === "manual"))
);
}
export function resolveSettledTurnFinalizationRequest(input: {
runParams: TerminalRunParams;
attempt: EmbeddedRunAttemptResult;
@@ -131,12 +139,7 @@ export function resolveSettledTurnFinalizationRequest(input: {
modelId: input.activeErrorContext.model,
modelApi: input.modelApi,
executionContract: input.executionContract,
allowEmptyStopContinuation:
input.runParams.terminalReplyExpectation === "required" ||
(input.runParams.terminalReplyExpectation == null &&
(input.runParams.trigger == null ||
input.runParams.trigger === "user" ||
input.runParams.trigger === "manual")),
allowEmptyStopContinuation: requiresVisibleTerminalReply(input.runParams),
payloadCount,
hasTerminalToolPresentation: input.hasTerminalToolPresentation,
aborted: terminalAborted,
@@ -311,8 +314,10 @@ export async function resolveEmbeddedRunTerminal(input: {
);
return { action: "retry" };
}
const completedEmptyFinalization = input.settledTurnFinalizationOutcome === "completed-empty";
const incompleteTurnText =
emptyAssistantReplyIsSilent || input.settledTurnFinalizationOutcome === "completed-empty"
emptyAssistantReplyIsSilent ||
(completedEmptyFinalization && !requiresVisibleTerminalReply(runParams))
? null
: resolveIncompleteTurnPayloadText({
payloadCount,