mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
fix(agent): post-tool timeout does not replay completed tools (#122516)
* fix(agent): prevent replay after post-tool timeout * fix(agent): narrow settled tool assistant evidence
This commit is contained in:
committed by
GitHub
parent
1da8fffbcb
commit
b46181bfc0
+7
-4
@@ -236,7 +236,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("records silent success when the settled-tool finalization completes empty", async () => {
|
||||
it("surfaces an incomplete turn when a required settled-tool finalizer completes empty", async () => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
@@ -261,16 +261,19 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
terminalReplyExpectation: "required",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
expect(result.payloads?.[0]).toMatchObject({ isError: true });
|
||||
expect(result.payloads?.[0]?.text).toContain(
|
||||
"some tool actions may have already been executed",
|
||||
);
|
||||
expect(result.meta.error?.kind).toBe("incomplete_turn");
|
||||
expect(result.meta.terminalReplyKind).toBeUndefined();
|
||||
expect(result.meta.finalAssistantVisibleText).toBeUndefined();
|
||||
expect(result.meta.finalAssistantRawText).toBeUndefined();
|
||||
expect(result.meta.stopReason).toBe("stop");
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectWarnMessageWith("settled-turn finalization completed without a visible answer");
|
||||
});
|
||||
|
||||
@@ -404,32 +404,17 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
});
|
||||
|
||||
it("continues once after settled side-effecting tools finish without a final answer", async () => {
|
||||
const acceptedSessionSpawns = [
|
||||
{ runId: "child-run", childSessionKey: "agent:main:subagent:child" },
|
||||
];
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } },
|
||||
{ type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } },
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "tool_spawn",
|
||||
name: "sessions_spawn",
|
||||
arguments: { task: "follow up" },
|
||||
},
|
||||
],
|
||||
});
|
||||
const settledToolResults = [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false },
|
||||
{ role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false },
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_spawn",
|
||||
toolName: "sessions_spawn",
|
||||
isError: false,
|
||||
},
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"];
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
@@ -437,15 +422,10 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
latestMcpAppChannelView: { viewId: "view-after-tools" },
|
||||
toolMetas: [
|
||||
{ toolName: "write", meta: "path=note.txt" },
|
||||
{ toolName: "cron" },
|
||||
{ toolName: "sessions_spawn" },
|
||||
],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }, { toolName: "cron" }],
|
||||
successfulNestedToolNames: ["read"],
|
||||
acceptedSessionSpawns,
|
||||
successfulCronAdds: 1,
|
||||
itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 },
|
||||
itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 },
|
||||
messagesSnapshot: settledToolResults,
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
@@ -475,10 +455,9 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer.");
|
||||
expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" });
|
||||
expect(result.successfulCronAdds).toBe(1);
|
||||
expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns);
|
||||
expect(result.meta.toolSummary).toEqual({
|
||||
calls: 3,
|
||||
tools: ["write", "cron", "sessions_spawn"],
|
||||
calls: 2,
|
||||
tools: ["write", "cron"],
|
||||
failures: 0,
|
||||
});
|
||||
expect(result.meta.agentMeta).toMatchObject({
|
||||
|
||||
@@ -17,6 +17,35 @@ import {
|
||||
} from "./run/incomplete-turn-resolution.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
function makeSettledIdleWriteAttempt(options?: {
|
||||
terminal?: EmbeddedRunAttemptResult["terminal"];
|
||||
stalePriorTurn?: boolean;
|
||||
}) {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }],
|
||||
});
|
||||
const abortedAssistant = makeLastAssistant({ stopReason: "aborted", content: [] });
|
||||
return makeAttemptResult({
|
||||
terminal: options?.terminal ?? { kind: "timeout", phase: "prompt", source: "idle" },
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", replaySafe: false }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
{ role: "user", content: [{ type: "text", text: "old turn" }] },
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
...(options?.stalePriorTurn
|
||||
? [{ role: "user", content: [{ type: "text", text: "current turn" }] }]
|
||||
: []),
|
||||
abortedAssistant,
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: abortedAssistant,
|
||||
currentAttemptAssistant: abortedAssistant,
|
||||
currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
|
||||
});
|
||||
}
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
@@ -87,26 +116,81 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "aborted", aborted: true, timedOut: false, promptError: null },
|
||||
{ label: "timed out", aborted: false, timedOut: true, promptError: null },
|
||||
{ label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") },
|
||||
])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }],
|
||||
});
|
||||
it("continues an exactly settled current-turn tool batch after an idle prompt timeout", () => {
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "bash" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
},
|
||||
{ aborted, timedOut, promptError },
|
||||
),
|
||||
makeSettledContinuationParams(makeSettledIdleWriteAttempt(), {
|
||||
timedOut: true,
|
||||
promptError: new Error("LLM idle timeout"),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "external abort",
|
||||
terminal: { kind: "timeout", phase: "prompt", source: "external" } as const,
|
||||
aborted: true,
|
||||
timedOut: true,
|
||||
},
|
||||
{
|
||||
label: "runtime timeout",
|
||||
terminal: { kind: "timeout", phase: "prompt", source: "runtime" } as const,
|
||||
aborted: false,
|
||||
timedOut: true,
|
||||
},
|
||||
{
|
||||
label: "run budget timeout",
|
||||
terminal: { kind: "timeout", phase: "prompt", source: "run_budget" } as const,
|
||||
aborted: false,
|
||||
timedOut: true,
|
||||
},
|
||||
{
|
||||
label: "compaction timeout",
|
||||
terminal: { kind: "timeout", phase: "compaction", source: "idle" } as const,
|
||||
aborted: false,
|
||||
timedOut: true,
|
||||
},
|
||||
{
|
||||
label: "tool execution timeout",
|
||||
terminal: { kind: "timeout", phase: "tool_execution", source: "idle" } as const,
|
||||
aborted: false,
|
||||
timedOut: true,
|
||||
},
|
||||
{
|
||||
label: "timeout observation",
|
||||
terminal: { kind: "timeout", phase: "tool_execution", source: "observation" } as const,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
},
|
||||
{
|
||||
label: "prompt error without idle timeout",
|
||||
terminal: { kind: "ok" } as const,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
promptError: new Error("closed"),
|
||||
},
|
||||
])(
|
||||
"does not finalize settled tools after a $label",
|
||||
({ terminal, aborted, timedOut, promptError }) => {
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(makeSettledIdleWriteAttempt({ terminal }), {
|
||||
aborted,
|
||||
timedOut,
|
||||
promptError,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not use a settled prior-turn batch to authorize idle-timeout finalization", () => {
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(makeSettledIdleWriteAttempt({ stalePriorTurn: true }), {
|
||||
timedOut: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
|
||||
@@ -4,7 +4,9 @@ import { makeModelFallbackCfg } from "../test-helpers/model-fallback-config-fixt
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
MockedFailoverError,
|
||||
mockedBuildEmbeddedRunPayloads,
|
||||
mockedClassifyFailoverReason,
|
||||
mockedGetApiKeyForModel,
|
||||
mockedRunEmbeddedAttempt,
|
||||
overflowBaseRunParams,
|
||||
resetSharedRunIntegrationHarnessMocks,
|
||||
@@ -58,4 +60,102 @@ describe("runEmbeddedAgent prompt timeout fallback handoff", () => {
|
||||
await expect(promise).rejects.toThrow("LLM request timed out.");
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("finalizes a settled write after an idle timeout without replaying the prompt", async () => {
|
||||
const toolUseAssistant = {
|
||||
role: "assistant" as const,
|
||||
stopReason: "toolUse" as const,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "tool_write",
|
||||
name: "write",
|
||||
arguments: { path: "note.txt", content: "done" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const abortedAssistant = {
|
||||
role: "assistant" as const,
|
||||
stopReason: "aborted" as const,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
content: [],
|
||||
};
|
||||
const finalAssistant = {
|
||||
role: "assistant" as const,
|
||||
stopReason: "stop" as const,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "The note was written once." }],
|
||||
};
|
||||
mockedClassifyFailoverReason.mockReturnValue("timeout");
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
terminal: { kind: "timeout", phase: "prompt", source: "idle" },
|
||||
toolMetas: [{ toolName: "write", replaySafe: false }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
{ role: "user", content: [{ type: "text", text: "Write note.txt" }] },
|
||||
toolUseAssistant,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_write",
|
||||
toolName: "write",
|
||||
isError: false,
|
||||
},
|
||||
abortedAssistant,
|
||||
] as never,
|
||||
lastAssistant: abortedAssistant as never,
|
||||
currentAttemptAssistant: abortedAssistant as never,
|
||||
currentAttemptReplayMetadata: {
|
||||
hadPotentialSideEffects: true,
|
||||
replaySafe: false,
|
||||
},
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["The note was written once."],
|
||||
lastAssistant: finalAssistant as never,
|
||||
currentAttemptAssistant: finalAssistant as never,
|
||||
currentAttemptCompletedAssistant: finalAssistant as never,
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads
|
||||
.mockReturnValueOnce([])
|
||||
.mockReturnValueOnce([{ text: "The note was written once." }]);
|
||||
|
||||
const result = await runEmbeddedAgent({
|
||||
...overflowBaseRunParams,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
runId: "run-post-tool-idle-finalization",
|
||||
config: makeModelFallbackCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.4",
|
||||
fallbacks: ["anthropic/claude-opus-4-6"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "The note was written once." }]);
|
||||
expect(result.meta.executionTrace?.fallbackUsed).toBe(false);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(mockedRunEmbeddedAttempt.mock.calls[1]?.[0]).toMatchObject({
|
||||
operation: "settled-tool-finalization",
|
||||
disableTools: true,
|
||||
skipPreparedUserTurnMessage: true,
|
||||
prompt:
|
||||
"The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.",
|
||||
});
|
||||
expect(mockedGetApiKeyForModel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -110,6 +110,37 @@ function makeExhaustedCredentialFailureInput(options?: { replaySafe?: boolean })
|
||||
};
|
||||
}
|
||||
|
||||
function makeIdleTimeoutFailureInput(options?: { replaySafe?: boolean }) {
|
||||
const fixture = makeExhaustedCredentialFailureInput();
|
||||
const replaySafe = options?.replaySafe === true;
|
||||
const assistant = buildEmbeddedRunnerAssistant({
|
||||
provider: "anthropic",
|
||||
model: "mock-1",
|
||||
stopReason: "aborted",
|
||||
});
|
||||
const replayMetadata = {
|
||||
hadPotentialSideEffects: !replaySafe,
|
||||
replaySafe,
|
||||
};
|
||||
const attempt = makeEmbeddedRunnerAttempt({
|
||||
terminal: { kind: "timeout", phase: "prompt", source: "idle" },
|
||||
lastAssistant: assistant,
|
||||
currentAttemptAssistant: assistant,
|
||||
toolMetas: replaySafe ? [] : [{ toolName: "write", replaySafe: false }],
|
||||
replayMetadata,
|
||||
currentAttemptReplayMetadata: replayMetadata,
|
||||
});
|
||||
fixture.input.attempt = attempt;
|
||||
fixture.input.attemptAssistant = assistant;
|
||||
fixture.input.currentAttemptAssistant = assistant;
|
||||
fixture.input.terminalState = resolveEmbeddedRunAttemptTerminalState({ attempt, assistant });
|
||||
fixture.input.emptyErrorRetries = 0;
|
||||
fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => true);
|
||||
fixture.input.maybeRetrySameModelRateLimit = vi.fn(async () => true);
|
||||
fixture.input.advanceRateLimitAuthProfile = vi.fn(async () => true);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
describe("handleEmbeddedAssistantFailure", () => {
|
||||
it("uses prepared OpenRouter ownership for custom-provider billing failures", async () => {
|
||||
const fixture = makeExhaustedCredentialFailureInput();
|
||||
@@ -165,10 +196,11 @@ describe("handleEmbeddedAssistantFailure", () => {
|
||||
}
|
||||
fixture.input.attemptAssistant.errorCode = PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE;
|
||||
fixture.input.attemptAssistant.errorMessage = "reasoning is required";
|
||||
fixture.input.resolveAuthProfileFailureReason = vi.fn(() => "timeout" as const);
|
||||
|
||||
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
|
||||
|
||||
expect(outcome.action).toBe("proceed");
|
||||
expect(outcome).toMatchObject({ action: "proceed", assistantProfileFailureReason: null });
|
||||
expect(fixture.advanceAuthProfile).not.toHaveBeenCalled();
|
||||
expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled();
|
||||
expect(fixture.traceAttempts).toEqual([]);
|
||||
@@ -212,6 +244,37 @@ describe("handleEmbeddedAssistantFailure", () => {
|
||||
expect(fixture.traceAttempts).toEqual([]);
|
||||
});
|
||||
|
||||
it("closes every failover retry after an idle timeout commits a write", async () => {
|
||||
const fixture = makeIdleTimeoutFailureInput();
|
||||
|
||||
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
|
||||
|
||||
expect(outcome.action).toBe("proceed");
|
||||
expect(fixture.input.maybeRefreshRuntimeAuthForAuthError).not.toHaveBeenCalled();
|
||||
expect(fixture.input.maybeRetrySameModelRateLimit).not.toHaveBeenCalled();
|
||||
expect(fixture.advanceAuthProfile).not.toHaveBeenCalled();
|
||||
expect(fixture.input.advanceRateLimitAuthProfile).not.toHaveBeenCalled();
|
||||
expect(fixture.traceAttempts).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps replay-safe idle timeout profile rotation available", async () => {
|
||||
const fixture = makeIdleTimeoutFailureInput({ replaySafe: true });
|
||||
fixture.input.maybeRefreshRuntimeAuthForAuthError = vi.fn(async () => false);
|
||||
|
||||
const outcome = await handleEmbeddedAssistantFailure(fixture.input);
|
||||
|
||||
expect(outcome).toMatchObject({ action: "retry", lastRetryFailoverReason: "timeout" });
|
||||
expect(fixture.advanceAuthProfile).toHaveBeenCalledOnce();
|
||||
expect(fixture.traceAttempts).toEqual([
|
||||
{
|
||||
provider: "anthropic",
|
||||
model: "mock-1",
|
||||
result: "rotate_profile",
|
||||
stage: "assistant",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not cache an exact credential-file failure from a fallback candidate", async () => {
|
||||
const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS;
|
||||
process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000";
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import { log } from "../logger.js";
|
||||
import type { TraceAttempt } from "../types.js";
|
||||
import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js";
|
||||
import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js";
|
||||
import { createFailoverDecisionLogger } from "./failover-observation.js";
|
||||
import { resolveRunFailoverDecision } from "./failover-policy.js";
|
||||
import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn-recovery.js";
|
||||
@@ -100,28 +101,10 @@ export async function handleEmbeddedAssistantFailure(input: {
|
||||
projectAgentRunAttemptTerminal(input.attempt.terminal);
|
||||
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(input.terminalState.outcome);
|
||||
const { signalOwnedInterruption } = input.terminalState;
|
||||
if (isReplayUnsafeAssistantError(input.attemptAssistant)) {
|
||||
return buildOutcome(input, {
|
||||
action: "proceed",
|
||||
assistantProfileFailureReason: null,
|
||||
});
|
||||
}
|
||||
const fallbackThinking = pickFallbackThinkingLevel({
|
||||
message: input.attemptAssistant?.errorMessage,
|
||||
attempted: input.attemptedThinking,
|
||||
});
|
||||
if (fallbackThinking && !terminalInterrupted) {
|
||||
log.warn(
|
||||
`unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`,
|
||||
);
|
||||
return buildOutcome(input, {
|
||||
action: "retry",
|
||||
thinkLevel: fallbackThinking,
|
||||
preserveSameModelRateLimitRetryCount: true,
|
||||
assistantProfileFailureReason: null,
|
||||
});
|
||||
}
|
||||
|
||||
const authFailure = isAuthAssistantError(input.attemptAssistant);
|
||||
const rateLimitFailure = isRateLimitAssistantError(input.attemptAssistant);
|
||||
const billingFailure = isBillingAssistantError(input.attemptAssistant);
|
||||
@@ -144,6 +127,26 @@ export async function handleEmbeddedAssistantFailure(input: {
|
||||
isShortWindowRateLimitMessage(input.attemptAssistant?.errorMessage),
|
||||
},
|
||||
);
|
||||
const replayUnsafeAssistantError = isReplayUnsafeAssistantError(input.attemptAssistant);
|
||||
if (replayUnsafeAssistantError || !isCurrentAttemptReplaySafe(input.attempt)) {
|
||||
return buildOutcome(input, {
|
||||
action: "proceed",
|
||||
assistantProfileFailureReason: replayUnsafeAssistantError
|
||||
? null
|
||||
: assistantProfileFailureReason,
|
||||
});
|
||||
}
|
||||
if (fallbackThinking && !terminalInterrupted) {
|
||||
log.warn(
|
||||
`unsupported thinking level for ${input.provider}/${input.modelId}; retrying with ${fallbackThinking}`,
|
||||
);
|
||||
return buildOutcome(input, {
|
||||
action: "retry",
|
||||
thinkLevel: fallbackThinking,
|
||||
preserveSameModelRateLimitRetryCount: true,
|
||||
assistantProfileFailureReason,
|
||||
});
|
||||
}
|
||||
const cloudCodeAssistFormatError = input.attempt.cloudCodeAssistFormatError;
|
||||
const imageDimensionError = parseImageDimensionError(input.attemptAssistant?.errorMessage ?? "");
|
||||
const genericUnknownReasoningError =
|
||||
|
||||
@@ -11,6 +11,7 @@ import type { EmbeddedAgentRunResult, TraceAttempt } from "../types.js";
|
||||
import type { createUsageAccumulator } from "../usage-accumulator.js";
|
||||
import type { prepareAndDispatchEmbeddedRunAttempt } from "./attempt-dispatch-preparation.js";
|
||||
import type { normalizeEmbeddedRunAttempt } from "./attempt-normalization.js";
|
||||
import { isCurrentAttemptReplaySafe } from "./attempt-terminal-evidence.js";
|
||||
import { buildEmbeddedRunBlockedResult } from "./blocked-run-result.js";
|
||||
import { resolveCodexAppServerRecoveryRetry } from "./codex-app-server-recovery.js";
|
||||
import { resolveCompactionLiveModelSelection } from "./compaction-live-model-selection.js";
|
||||
@@ -110,6 +111,7 @@ export async function recoverEmbeddedRunAttempt(input: {
|
||||
timedOutByRunBudget,
|
||||
} = projectAgentRunAttemptTerminal(attempt.terminal);
|
||||
const terminalInterrupted = isEmbeddedRunTerminalInterrupted(terminalState.outcome);
|
||||
const currentAttemptReplaySafe = isCurrentAttemptReplaySafe(attempt);
|
||||
const { signalOwnedInterruption } = terminalState;
|
||||
const assistantOverflowCandidate =
|
||||
currentAttemptCompletedAssistant !== undefined
|
||||
@@ -137,6 +139,40 @@ export async function recoverEmbeddedRunAttempt(input: {
|
||||
thinkLevel: updates?.thinkLevel ?? runtime.thinkLevel,
|
||||
});
|
||||
|
||||
if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) {
|
||||
const errorText = formatErrorMessage(promptError);
|
||||
const replayInvalid = resolveReplayInvalidForAttempt();
|
||||
setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" });
|
||||
return {
|
||||
action: "complete",
|
||||
result: buildEmbeddedRunBlockedResult({
|
||||
text: errorText,
|
||||
errorKind: "hook_block",
|
||||
errorMessage: errorText,
|
||||
durationMs: Date.now() - runInput.startedAtMs,
|
||||
agentMeta: buildErrorAgentMeta({
|
||||
sessionId: sessionIdUsed,
|
||||
sessionFile: sessionPromptState.sessionFile,
|
||||
provider: preparedRuntime.provider,
|
||||
model: preparedRuntime.model.id,
|
||||
...runtime.outerContextTokenMeta,
|
||||
usageAccumulator: input.usageAccumulator,
|
||||
lastRunPromptUsage: input.lastRunPromptUsage,
|
||||
currentAttemptAssistant,
|
||||
}),
|
||||
attempt,
|
||||
replayInvalid,
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (!currentAttemptReplaySafe) {
|
||||
return {
|
||||
action: "proceed",
|
||||
shouldSurfaceCodexCompletionTimeout:
|
||||
attempt.codexAppServerFailure?.kind === "turn_completion_idle_timeout" && timedOut,
|
||||
};
|
||||
}
|
||||
|
||||
const requestedSelection = shouldSwitchToLiveModel({
|
||||
cfg: params.config,
|
||||
sessionKey: runInput.resolvedSessionKey,
|
||||
@@ -255,32 +291,6 @@ export async function recoverEmbeddedRunAttempt(input: {
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (promptErrorSource === "hook:before_agent_run" && !terminalInterrupted) {
|
||||
const errorText = formatErrorMessage(promptError);
|
||||
const replayInvalid = resolveReplayInvalidForAttempt();
|
||||
setTerminalLifecycleMeta({ replayInvalid, livenessState: "blocked" });
|
||||
return {
|
||||
action: "complete",
|
||||
result: buildEmbeddedRunBlockedResult({
|
||||
text: errorText,
|
||||
errorKind: "hook_block",
|
||||
errorMessage: errorText,
|
||||
durationMs: Date.now() - runInput.startedAtMs,
|
||||
agentMeta: buildErrorAgentMeta({
|
||||
sessionId: sessionIdUsed,
|
||||
sessionFile: sessionPromptState.sessionFile,
|
||||
provider: preparedRuntime.provider,
|
||||
model: preparedRuntime.model.id,
|
||||
...runtime.outerContextTokenMeta,
|
||||
usageAccumulator: input.usageAccumulator,
|
||||
lastRunPromptUsage: input.lastRunPromptUsage,
|
||||
currentAttemptAssistant,
|
||||
}),
|
||||
attempt,
|
||||
replayInvalid,
|
||||
}),
|
||||
};
|
||||
}
|
||||
const hasRecoverableCodexAppServerTimeoutOutcome = Boolean(
|
||||
attempt.codexAppServerFailure && attempt.promptTimeoutOutcome,
|
||||
);
|
||||
|
||||
@@ -16,6 +16,14 @@ type ReplayMetadataAttempt = Pick<
|
||||
> &
|
||||
Partial<Pick<EmbeddedRunAttemptResult, "messagingToolSentTargets" | "acceptedSessionSpawns">>;
|
||||
|
||||
/** Uses current-attempt evidence when available and otherwise preserves fail-closed legacy state. */
|
||||
export function isCurrentAttemptReplaySafe(
|
||||
attempt: Pick<EmbeddedRunAttemptResult, "replayMetadata" | "currentAttemptReplayMetadata">,
|
||||
): boolean {
|
||||
const replayMetadata = attempt.currentAttemptReplayMetadata ?? attempt.replayMetadata;
|
||||
return replayMetadata.replaySafe && !replayMetadata.hadPotentialSideEffects;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks whether retrying the attempt can safely replay the prompt. Concrete
|
||||
* tool-instance policy, async work, committed delivery, spawned sessions, and
|
||||
|
||||
@@ -33,6 +33,7 @@ export type IncompleteTurnAttempt = Pick<
|
||||
| "itemLifecycle"
|
||||
| "messagesSnapshot"
|
||||
| "replayMetadata"
|
||||
| "currentAttemptReplayMetadata"
|
||||
| "terminal"
|
||||
| "toolMetas"
|
||||
> &
|
||||
|
||||
@@ -7,7 +7,11 @@ import {
|
||||
hasCompletedMessagingToolDeliveryEvidence,
|
||||
} from "../delivery-evidence.js";
|
||||
import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js";
|
||||
import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js";
|
||||
import {
|
||||
hasAsyncActivity,
|
||||
hasAttemptTerminalState,
|
||||
isCurrentAttemptReplaySafe,
|
||||
} from "./attempt-terminal-evidence.js";
|
||||
import {
|
||||
hasOnlySilentAssistantReply,
|
||||
hasPositiveOutputTokenUsage,
|
||||
@@ -60,9 +64,7 @@ export function shouldRetrySilentErrorAssistantTurn(params: {
|
||||
}
|
||||
// Current-attempt evidence avoids blocking on prior committed effects; older
|
||||
// harnesses retain the cumulative, fail-closed behavior.
|
||||
const retryReplayMetadata =
|
||||
params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata;
|
||||
if (retryReplayMetadata.hadPotentialSideEffects) {
|
||||
if (!isCurrentAttemptReplaySafe(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -187,6 +189,27 @@ export function resolveReasoningOnlyRetryInstruction(params: {
|
||||
return REASONING_ONLY_RETRY_INSTRUCTION;
|
||||
}
|
||||
|
||||
type SettledToolCall = { id: string | null; name: string | null };
|
||||
|
||||
function readSettledToolCalls(
|
||||
message: EmbeddedRunAttemptResult["currentAttemptAssistant"] | null | undefined,
|
||||
): SettledToolCall[] {
|
||||
if (!Array.isArray(message?.content)) {
|
||||
return [];
|
||||
}
|
||||
return message.content.flatMap((item) => {
|
||||
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
|
||||
return block?.type === "toolCall"
|
||||
? [
|
||||
{
|
||||
id: typeof block.id === "string" ? block.id : null,
|
||||
name: typeof block.name === "string" ? block.name : null,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
|
||||
/** Builds one fresh continuation after settled tools ended without a visible final answer. */
|
||||
export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
provider?: string;
|
||||
@@ -201,8 +224,27 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
const currentAttemptAssistant = params.attempt.currentAttemptAssistant;
|
||||
const snapshot = params.attempt.messagesSnapshot ?? [];
|
||||
const latestUserIndex = snapshot.findLastIndex((message) => message.role === "user");
|
||||
let assistant: EmbeddedRunAttemptResult["currentAttemptAssistant"] = currentAttemptAssistant;
|
||||
let assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
|
||||
if (assistantIndex <= latestUserIndex || readSettledToolCalls(assistant).length === 0) {
|
||||
assistantIndex = snapshot.findLastIndex(
|
||||
(message, index) =>
|
||||
index > latestUserIndex &&
|
||||
message.role === "assistant" &&
|
||||
readSettledToolCalls(message).length > 0,
|
||||
);
|
||||
const assistantCandidate = assistantIndex >= 0 ? snapshot[assistantIndex] : undefined;
|
||||
assistant = assistantCandidate?.role === "assistant" ? assistantCandidate : undefined;
|
||||
}
|
||||
const terminal = params.attempt.terminal;
|
||||
const idlePromptTimeout =
|
||||
terminal.kind === "timeout" &&
|
||||
terminal.phase === "prompt" &&
|
||||
terminal.source === "idle" &&
|
||||
params.attempt.currentAttemptReplayMetadata?.hadPotentialSideEffects === true;
|
||||
const emptyStopAfterSettledTools = Boolean(
|
||||
params.allowEmptyStopContinuation &&
|
||||
currentAttemptAssistant?.stopReason === "stop" &&
|
||||
@@ -220,25 +262,11 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
// Idle is not proof of settlement: skipped or partially dispatched tools must
|
||||
// never be described as completed. Match each terminal call's id and owner to
|
||||
// its own current-batch result; a reported failure is settled, not successful.
|
||||
const requestedToolCalls = Array.isArray(assistant?.content)
|
||||
? assistant.content.flatMap((item) => {
|
||||
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
|
||||
return block?.type === "toolCall"
|
||||
? [
|
||||
{
|
||||
id: typeof block.id === "string" ? block.id : null,
|
||||
name: typeof block.name === "string" ? block.name : null,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
})
|
||||
: [];
|
||||
const requestedToolCalls = readSettledToolCalls(assistant);
|
||||
// Scan only results AFTER the terminal assistant: the snapshot spans the whole
|
||||
// session, and a prior turn's toolResult with a model-reused id would otherwise
|
||||
// prove "completion" for a batch that never dispatched. Assistant not found in
|
||||
// the snapshot fails closed to the existing incomplete-turn error.
|
||||
const snapshot = params.attempt.messagesSnapshot ?? [];
|
||||
const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
|
||||
const settledToolResults = new Map(
|
||||
(assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => {
|
||||
const result = message as {
|
||||
@@ -260,7 +288,9 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
}),
|
||||
);
|
||||
const allToolsProvenSettled =
|
||||
params.attempt.itemLifecycle?.activeCount === 0 &&
|
||||
params.attempt.itemLifecycle.startedCount > 0 &&
|
||||
params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount &&
|
||||
params.attempt.itemLifecycle.activeCount === 0 &&
|
||||
requestedToolCalls.length > 0 &&
|
||||
requestedToolCalls.every(
|
||||
({ id, name }) =>
|
||||
@@ -284,13 +314,14 @@ export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
params.payloadCount !== 0 ||
|
||||
params.hasTerminalToolPresentation ||
|
||||
params.aborted ||
|
||||
params.promptError != null ||
|
||||
params.timedOut ||
|
||||
((params.promptError != null ||
|
||||
params.timedOut ||
|
||||
params.attempt.terminal.kind === "timeout") &&
|
||||
!idlePromptTimeout) ||
|
||||
(assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) ||
|
||||
hasUnsettledToolError ||
|
||||
(hasSettledTerminalToolFailure &&
|
||||
(hasAsyncActivity(params.attempt.toolMetas) ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) ||
|
||||
hasAsyncActivity(params.attempt.toolMetas) ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt
|
||||
|
||||
@@ -107,34 +107,16 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
|
||||
prompt,
|
||||
noteLaneTaskProgress: input.finalization.noteLaneTaskProgress,
|
||||
});
|
||||
if (finalization.outcome === "empty") {
|
||||
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, finalization.result.usage);
|
||||
lastRunPromptUsage = finalization.result.usage ?? lastRunPromptUsage;
|
||||
log.warn(
|
||||
`settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
|
||||
`provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`,
|
||||
);
|
||||
const emptyAssistant = finalization.result.assistant;
|
||||
const completedEmptyAttempt = {
|
||||
...initial.attempt,
|
||||
lastAssistant: emptyAssistant,
|
||||
currentAttemptAssistant: emptyAssistant,
|
||||
currentAttemptCompletedAssistant: emptyAssistant,
|
||||
};
|
||||
return {
|
||||
...initial,
|
||||
attempt: completedEmptyAttempt,
|
||||
attemptAssistant: emptyAssistant,
|
||||
currentAttemptCompletedAssistant: emptyAssistant,
|
||||
prepared,
|
||||
lastRunPromptUsage,
|
||||
finalizationOutcome: "completed-empty" as const,
|
||||
};
|
||||
}
|
||||
attempt = finalization.attempt;
|
||||
mergeUsageIntoAccumulator(input.terminalBase.usageAccumulator, attempt.attemptUsage);
|
||||
mergeAttemptRunStatsIntoAccumulator(input.terminalBase.usageAccumulator, attempt);
|
||||
lastRunPromptUsage = attempt.attemptUsage ?? lastRunPromptUsage;
|
||||
if (finalization.outcome === "empty") {
|
||||
log.warn(
|
||||
`settled-turn finalization completed without a visible answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
|
||||
`provider=${errorContext.provider}/${errorContext.model} — recording completed-empty outcome`,
|
||||
);
|
||||
}
|
||||
// Successful isolated finalization owns a fresh terminal, never the original abort signal.
|
||||
const terminalState: EmbeddedRunTerminalState = {
|
||||
outcome: resolveEmbeddedRunAttemptTerminalOutcome({
|
||||
@@ -164,7 +146,8 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
|
||||
sessionFileUsed: attempt.sessionFileUsed,
|
||||
prepared,
|
||||
lastRunPromptUsage,
|
||||
finalizationOutcome: "answered" as const,
|
||||
finalizationOutcome:
|
||||
finalization.outcome === "empty" ? ("completed-empty" as const) : ("answered" as const),
|
||||
};
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
@@ -186,13 +169,7 @@ async function runPreparedSettledTurnFinalization(input: {
|
||||
harness: AgentHarness;
|
||||
prompt: string;
|
||||
noteLaneTaskProgress: () => void;
|
||||
}): Promise<
|
||||
| { outcome: "answered"; attempt: EmbeddedRunAttemptWithReceiptEvidence }
|
||||
| {
|
||||
outcome: "empty";
|
||||
result: AgentHarnessSettledTurnFinalizationResult;
|
||||
}
|
||||
> {
|
||||
}): Promise<{ outcome: "answered" | "empty"; attempt: EmbeddedRunAttemptWithReceiptEvidence }> {
|
||||
return await withEmbeddedRunLaneProgressHeartbeat(input.noteLaneTaskProgress, async () => {
|
||||
const finalization = await runEmbeddedSettledTurnFinalizationWithBackend(
|
||||
{
|
||||
@@ -206,12 +183,10 @@ async function runPreparedSettledTurnFinalization(input: {
|
||||
input.settledAttempt,
|
||||
input.harness,
|
||||
);
|
||||
if (finalization.outcome === "empty") {
|
||||
return finalization;
|
||||
}
|
||||
return {
|
||||
outcome: "answered",
|
||||
outcome: finalization.outcome,
|
||||
attempt: buildSettledTurnFinalizationAttemptResult({
|
||||
outcome: finalization.outcome,
|
||||
result: finalization.result,
|
||||
settledAttempt: input.settledAttempt,
|
||||
prompt: input.prompt,
|
||||
@@ -222,13 +197,14 @@ async function runPreparedSettledTurnFinalization(input: {
|
||||
}
|
||||
|
||||
function buildSettledTurnFinalizationAttemptResult(input: {
|
||||
outcome: "answered" | "empty";
|
||||
result: AgentHarnessSettledTurnFinalizationResult;
|
||||
settledAttempt: EmbeddedRunAttemptWithReceiptEvidence;
|
||||
prompt: string;
|
||||
agentHarnessId?: string;
|
||||
}): EmbeddedRunAttemptWithReceiptEvidence {
|
||||
const { result, settledAttempt } = input;
|
||||
const text = resolveSettledTurnFinalizationText(result);
|
||||
const text = input.outcome === "empty" ? "" : resolveSettledTurnFinalizationText(result);
|
||||
// Finalization replaces terminal ownership, not host-private facts from settled tools.
|
||||
// Keep those facts while replay, abort, and lifecycle state remain finalizer-local.
|
||||
return {
|
||||
|
||||
@@ -63,6 +63,14 @@ type TerminalResolution =
|
||||
| { action: "retry" }
|
||||
| { action: "complete"; result: EmbeddedAgentRunResult };
|
||||
|
||||
function requiresVisibleTerminalReply(runParams: TerminalRunParams): boolean {
|
||||
return (
|
||||
runParams.terminalReplyExpectation === "required" ||
|
||||
(runParams.terminalReplyExpectation == null &&
|
||||
(runParams.trigger == null || runParams.trigger === "user" || runParams.trigger === "manual"))
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveSettledTurnFinalizationRequest(input: {
|
||||
runParams: TerminalRunParams;
|
||||
attempt: EmbeddedRunAttemptResult;
|
||||
@@ -131,12 +139,7 @@ export function resolveSettledTurnFinalizationRequest(input: {
|
||||
modelId: input.activeErrorContext.model,
|
||||
modelApi: input.modelApi,
|
||||
executionContract: input.executionContract,
|
||||
allowEmptyStopContinuation:
|
||||
input.runParams.terminalReplyExpectation === "required" ||
|
||||
(input.runParams.terminalReplyExpectation == null &&
|
||||
(input.runParams.trigger == null ||
|
||||
input.runParams.trigger === "user" ||
|
||||
input.runParams.trigger === "manual")),
|
||||
allowEmptyStopContinuation: requiresVisibleTerminalReply(input.runParams),
|
||||
payloadCount,
|
||||
hasTerminalToolPresentation: input.hasTerminalToolPresentation,
|
||||
aborted: terminalAborted,
|
||||
@@ -311,8 +314,10 @@ export async function resolveEmbeddedRunTerminal(input: {
|
||||
);
|
||||
return { action: "retry" };
|
||||
}
|
||||
const completedEmptyFinalization = input.settledTurnFinalizationOutcome === "completed-empty";
|
||||
const incompleteTurnText =
|
||||
emptyAssistantReplyIsSilent || input.settledTurnFinalizationOutcome === "completed-empty"
|
||||
emptyAssistantReplyIsSilent ||
|
||||
(completedEmptyFinalization && !requiresVisibleTerminalReply(runParams))
|
||||
? null
|
||||
: resolveIncompleteTurnPayloadText({
|
||||
payloadCount,
|
||||
|
||||
Reference in New Issue
Block a user