From 8636bb6981844e4674ee2cdbc0d8d32aa2a8b816 Mon Sep 17 00:00:00 2001 From: joshavant <830519+joshavant@users.noreply.github.com> Date: Sat, 18 Jul 2026 02:35:19 -0500 Subject: [PATCH] fix(agents): continue settled empty post-tool turns --- .../mock-openai/mock-openai-contracts.ts | 4 + .../src/providers/mock-openai/server.test.ts | 43 +++- .../src/providers/mock-openai/server.ts | 18 +- ...m-empty-response-after-write-recovery.yaml | 80 +++++++ ...ty-response-recovery-replay-safe-read.yaml | 2 +- .../run.incomplete-turn.test.ts | 217 ++++++++++++++++-- .../run/attempt-dispatch-preparation.ts | 8 +- .../run/incomplete-turn.ts | 35 ++- .../run/terminal-resolution.ts | 124 +++++----- .../run/terminal-retry-state.ts | 4 +- 10 files changed, 451 insertions(+), 84 deletions(-) create mode 100644 qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index f02205a93d21..298fb3960220 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -160,6 +160,8 @@ export const QA_THINKING_VISIBILITY_OFF_PROMPT_RE = /qa thinking visibility chec export const QA_THINKING_VISIBILITY_MAX_PROMPT_RE = /qa thinking visibility check max/i; export const QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE = /empty response continuation qa check/i; export const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE = /empty response exhaustion qa check/i; +export const QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT_RE = + /empty response after write recovery qa check/i; export const QA_STREAMING_PROMPT_RE = /(?:partial|quiet) streaming qa check/i; export const QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE = /final-only marker streaming qa check/i; export const QA_BLOCK_STREAMING_PROMPT_RE = /block streaming qa check/i; @@ -232,6 +234,8 @@ export const QA_REASONING_ONLY_RETRY_NEEDLE = "recorded reasoning but did not produce a user-visible answer"; export const QA_EMPTY_RESPONSE_RETRY_NEEDLE = "The previous attempt did not produce a user-visible answer."; +export const QA_SETTLED_TOOL_TERMINAL_CONTINUATION_NEEDLE = + "The previous assistant turn completed its tool calls but did not produce a user-visible answer."; export const QA_SKILL_WORKSHOP_GIF_PROMPT_RE = /externally sourced animated GIF asset|animated GIF asset in a product UI/i; export const QA_SKILL_WORKSHOP_REVIEW_PROMPT_RE = /Review transcript for durable skill updates/i; diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index 6b3d15366f50..6cabae446cef 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -18,10 +18,14 @@ const QA_EMPTY_RESPONSE_RECOVERY_PROMPT = "Empty response continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-RECOVERED-OK."; const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT = "Empty response exhaustion QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-EXHAUSTED-OK."; +const QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT = + "Empty response after write recovery QA check: write qa-empty-response-side-effect.txt, then answer with exactly TELEGRAM-EMPTY-WRITE-RECOVERED-OK."; const QA_REASONING_ONLY_RETRY_INSTRUCTION = "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; const QA_EMPTY_RESPONSE_RETRY_INSTRUCTION = "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; +const QA_SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = + "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; afterEach(async () => { while (cleanups.length > 0) { @@ -6112,7 +6116,7 @@ describe("qa mock openai server", () => { model: "gpt-5.6-luna", input: [ makeUserInput(QA_EMPTY_RESPONSE_RECOVERY_PROMPT), - makeUserInput(QA_EMPTY_RESPONSE_RETRY_INSTRUCTION), + makeUserInput(QA_SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION), { type: "function_call_output", output: "QA mission: Understand this OpenClaw repo from source + docs before acting.", @@ -6162,6 +6166,43 @@ describe("qa mock openai server", () => { }); expect(secondEmpty.output?.[0]?.content?.[0]?.text).toBe(""); }); + + it("scripts settled continuation after an empty response from a side-effecting write", async () => { + const server = await startMockServer(); + + const toolPlan = await expectResponsesText(server, { + stream: true, + model: "gpt-5.6-luna", + input: [makeUserInput(QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT)], + }); + expect(toolPlan).toContain('"name":"write"'); + + const toolOutput = { + type: "function_call_output" as const, + output: "Successfully wrote 27 bytes to qa-empty-response-side-effect.txt", + }; + const emptyPayload = await expectResponsesJson<{ + output?: Array<{ content?: Array<{ text?: string }> }>; + }>(server, { + stream: false, + model: "gpt-5.6-luna", + input: [makeUserInput(QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT), toolOutput], + }); + expect(emptyPayload.output?.[0]?.content?.[0]?.text).toBe(""); + + const recoveredPayload = await expectResponsesJson<{ + output?: Array<{ content?: Array<{ text?: string }> }>; + }>(server, { + stream: false, + model: "gpt-5.6-luna", + input: [ + makeUserInput(QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT), + makeUserInput(QA_SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION), + toolOutput, + ], + }); + expect(outputText(recoveredPayload)).toBe("TELEGRAM-EMPTY-WRITE-RECOVERED-OK"); + }); }); describe("qa mock openai server provider variant tagging", () => { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index cab364f78c44..7a92855fbff1 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -21,6 +21,7 @@ import { QA_THINKING_VISIBILITY_MAX_PROMPT_RE, QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE, QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE, + QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT_RE, QA_STREAMING_PROMPT_RE, QA_FINAL_ONLY_MARKER_STREAMING_PROMPT_RE, QA_BLOCK_STREAMING_PROMPT_RE, @@ -49,6 +50,7 @@ import { QA_IMAGE_GENERATION_PROMPT_RE, QA_REASONING_ONLY_RETRY_NEEDLE, QA_EMPTY_RESPONSE_RETRY_NEEDLE, + QA_SETTLED_TOOL_TERMINAL_CONTINUATION_NEEDLE, QA_SKILL_WORKSHOP_GIF_PROMPT_RE, QA_SKILL_WORKSHOP_REVIEW_PROMPT_RE, QA_RELEASE_AUDIT_PROMPT_RE, @@ -192,7 +194,9 @@ async function buildResponsesPayload( const isGroupChat = allInputText.includes('"is_group_chat": true'); const isBaselineUnmentionedChannelChatter = /\bno bot ping here\b/i.test(prompt); const hasReasoningOnlyRetryInstruction = allInputText.includes(QA_REASONING_ONLY_RETRY_NEEDLE); - const hasEmptyResponseRetryInstruction = allInputText.includes(QA_EMPTY_RESPONSE_RETRY_NEEDLE); + const hasEmptyResponseRetryInstruction = + allInputText.includes(QA_EMPTY_RESPONSE_RETRY_NEEDLE) || + allInputText.includes(QA_SETTLED_TOOL_TERMINAL_CONTINUATION_NEEDLE); const canCallMockSubagentTool = QA_SUBAGENT_DIRECT_FALLBACK_PROMPT_RE.test(allInputText) || /subagent fanout synthesis check/i.test(allInputText) || @@ -435,6 +439,18 @@ async function buildResponsesPayload( } return buildAssistantEvents(""); } + if (QA_EMPTY_RESPONSE_SIDE_EFFECT_RECOVERY_PROMPT_RE.test(allInputText)) { + if (allInputText.includes(QA_SETTLED_TOOL_TERMINAL_CONTINUATION_NEEDLE)) { + return buildAssistantEvents("TELEGRAM-EMPTY-WRITE-RECOVERED-OK"); + } + if (!toolOutput) { + return buildToolCallEventsWithArgs("write", { + path: "qa-empty-response-side-effect.txt", + content: "side effect completed once\n", + }); + } + return buildAssistantEvents(""); + } if (QA_TELEGRAM_LONG_FINAL_THREE_CHUNK_PROMPT_RE.test(allInputText)) { const text = buildQaLongFinalText({ endMarker: "TELEGRAM-LONG-FINAL-3CHUNK-END", diff --git a/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml b/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml new file mode 100644 index 000000000000..867c4f87a9ad --- /dev/null +++ b/qa/scenarios/channels/telegram-empty-response-after-write-recovery.yaml @@ -0,0 +1,80 @@ +title: Telegram empty-response recovery after a side-effecting write + +scenario: + id: telegram-empty-response-after-write-recovery + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [runtime.empty-response-recovery] + secondary: [runtime.retry-policy] + regressionRefs: + - openclaw/openclaw#108738 + objective: Verify a settled side-effecting tool followed by an empty stop gets one fresh continuation and one visible Telegram final reply. + successCriteria: + - The model plans the side-effecting write exactly once. + - The runtime continues from settled tool results without replaying the write. + - Telegram receives the exact recovery marker. + codeRefs: + - src/agents/embedded-agent-runner/run/incomplete-turn.ts + - src/agents/embedded-agent-runner/run/terminal-resolution.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + execution: + kind: flow + channel: telegram + summary: Prove post-tool empty-stop recovery through real Telegram transport. + config: + requiredProviderMode: mock-openai + promptSnippet: Empty response after write recovery QA check + retryNeedle: The previous assistant turn completed its tool calls but did not produce a user-visible answer. + expectedMarker: TELEGRAM-EMPTY-WRITE-RECOVERED-OK + +flow: + steps: + - name: continues once after the settled write and delivers the final marker + actions: + - assert: + expr: "env.providerMode === 'mock-openai'" + message: this Telegram regression scenario is mock-openai only + - resetTransport: true + - set: requestCursorBefore + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: "@openclaw Empty response after write recovery QA check: write qa-empty-response-side-effect.txt with the text side effect completed once, then answer exactly TELEGRAM-EMPTY-WRITE-RECOVERED-OK." + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: { ref: config.expectedMarker } + timeoutMs: 60000 + saveAs: reply + - set: scenarioRequests + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)) : []" + - set: writeRequests + value: + expr: "scenarioRequests.filter((request) => request.plannedToolName === 'write')" + - set: continuationRequests + value: + expr: "scenarioRequests.filter((request) => String(request.allInputText ?? '').includes(config.retryNeedle))" + - assert: + expr: "writeRequests.length === 1" + message: + expr: "`expected one write plan, saw ${String(writeRequests.length)}`" + - assert: + expr: "continuationRequests.length === 1" + message: + expr: "`expected one settled-tool continuation, saw ${String(continuationRequests.length)}`" + - assert: + expr: "continuationRequests.every((request) => !Array.isArray(request.body?.tools) || request.body.tools.length === 0)" + message: settled-tool continuation exposed a model tool surface + - assert: + expr: "reply.text.trim() === config.expectedMarker" + message: + expr: "`Telegram reply did not exactly match the recovery marker: ${reply.text}`" + detailsExpr: "`${reply.text}\nrequests=${String(scenarioRequests.length)} writes=${String(writeRequests.length)} continuations=${String(continuationRequests.length)}`" diff --git a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml index 3013356d6382..3e4589c40d32 100644 --- a/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/empty-response-recovery-replay-safe-read.yaml @@ -27,7 +27,7 @@ scenario: promptSnippet: Empty response continuation QA check prompt: "Empty response continuation QA check: read QA_KICKOFF_TASK.md, then answer with exactly EMPTY-RECOVERED-OK." expectedReply: EMPTY-RECOVERED-OK - retryNeedle: The previous attempt did not produce a user-visible answer. + retryNeedle: The previous assistant turn completed its tool calls but did not produce a user-visible answer. flow: steps: diff --git a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts b/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts index 8edd1e90cb6a..db831a6ca7fa 100644 --- a/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts +++ b/src/agents/embedded-agent-runner/run.incomplete-turn.test.ts @@ -33,7 +33,7 @@ import { resolveReplayInvalidFlag, resolveRunLivenessState, resolveSilentToolResultReplyPayload, - resolveToolUseTerminalContinuationInstruction, + resolveSettledToolTerminalContinuationInstruction, shouldRetryMissingAssistantTurn, shouldRetrySilentErrorAssistantTurn, shouldTreatEmptyAssistantReplyAsSilent, @@ -44,7 +44,7 @@ const REASONING_ONLY_RETRY_INSTRUCTION = "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; const EMPTY_RESPONSE_RETRY_INSTRUCTION = "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; -const TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION = +const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; let runEmbeddedAgent: typeof import("./run.js").runEmbeddedAgent; @@ -85,6 +85,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { function runAttemptCall(index: number): { prompt?: string; + disableTools?: boolean; suppressNextUserMessagePersistence?: boolean; skipPreparedUserTurnMessage?: boolean; } { @@ -96,6 +97,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { } return call[0] as { prompt?: string; + disableTools?: boolean; suppressNextUserMessagePersistence?: boolean; skipPreparedUserTurnMessage?: boolean; }; @@ -1132,10 +1134,97 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toBe(TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION); + expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(secondCall.disableTools).toBe(true); expect(secondCall.suppressNextUserMessagePersistence).toBe(false); expect(secondCall.skipPreparedUserTurnMessage).toBe(true); - expectWarnMessageWith("tool-use terminal turn lacked a final answer"); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("continues from settled side-effecting tools after an empty stop without replaying them", async () => { + const emptyStopAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [], + } as unknown as NonNullable; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ assistantTexts: ["Write completed. Here is the final answer."] }), + ); + mockedBuildEmbeddedRunPayloads + .mockReturnValueOnce([]) + .mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + provider: "openai", + model: "gpt-5.5", + runId: "run-empty-stop-settled-tool-continuation", + }); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer."); + expect(runAttemptCall(1).prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); + expect(runAttemptCall(1).disableTools).toBe(true); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); + }); + + it("surfaces failure without cascading when the settled-tool continuation is also empty", async () => { + const emptyStopAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [], + } as unknown as NonNullable; + mockedClassifyFailoverReason.mockReturnValue(null); + mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => { + markUserMessagePersisted(attemptParams); + return makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", meta: "path=note.txt" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }); + }); + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ + assistantTexts: [], + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }), + ); + mockedBuildEmbeddedRunPayloads.mockReturnValue([]); + + const result = await runEmbeddedAgent({ + ...overflowBaseRunParams, + allowEmptyAssistantReplyAsSilent: true, + provider: "openai", + model: "gpt-5.5", + runId: "run-empty-stop-settled-tool-continuation-exhausted", + }); + + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); + expect(result.payloads?.[0]).toMatchObject({ isError: true }); + expect(result.payloads?.[0]?.text).toContain( + "some tool actions may have already been executed", + ); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settledToolContinuations=1/1"); }); it("surfaces the existing incomplete-turn error after one tool-use continuation", async () => { @@ -1173,7 +1262,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(result.payloads?.[0]?.text).toContain( "some tool actions may have already been executed", ); - expectWarnMessageWith("toolUseContinuations=1/1"); + expectWarnMessageWith("settledToolContinuations=1/1"); }); it("does not claim completion for a toolUse terminal whose tools never started", async () => { @@ -1203,9 +1292,11 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { }); for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain(TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION); + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); } - expectNoWarnMessageWith("tool-use terminal turn lacked a final answer"); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); }); it("ignores stale prior-turn tool results with colliding ids", async () => { @@ -1241,9 +1332,11 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { }); for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain(TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION); + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); } - expectNoWarnMessageWith("tool-use terminal turn lacked a final answer"); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); }); it("does not claim completion when only part of a multi-tool request dispatched", async () => { @@ -1280,9 +1373,11 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { }); for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) { - expect(runAttemptCall(call).prompt).not.toContain(TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION); + expect(runAttemptCall(call).prompt).not.toContain( + SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION, + ); } - expectNoWarnMessageWith("tool-use terminal turn lacked a final answer"); + expectNoWarnMessageWith("settled post-tool turn lacked a final answer"); }); it("returns NO_REPLY without retrying reasoning-only assistant turns when silence is allowed", async () => { @@ -2193,7 +2288,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { model: "gpt-5.5", content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }], } as unknown as NonNullable; - const instruction = resolveToolUseTerminalContinuationInstruction({ + const instruction = resolveSettledToolTerminalContinuationInstruction({ provider: "openai", modelId: "gpt-5.5", modelApi: "openai-chatgpt-responses", @@ -2213,6 +2308,88 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(instruction).toBeNull(); }); + it.each([ + { label: "background trigger", allowEmptyStopContinuation: false }, + { + label: "active tool", + allowEmptyStopContinuation: true, + completedCount: 0, + activeCount: 1, + }, + { + label: "partially completed tool batch", + allowEmptyStopContinuation: true, + startedCount: 2, + completedCount: 1, + }, + { label: "async tool", allowEmptyStopContinuation: true, asyncStarted: true }, + { label: "failed tool", allowEmptyStopContinuation: true, isError: true }, + ])( + "does not continue an empty stop after $label activity", + ({ + allowEmptyStopContinuation, + startedCount = 1, + completedCount = 1, + activeCount = 0, + asyncStarted, + isError, + }) => { + const emptyStopAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [], + } as unknown as NonNullable; + const instruction = resolveSettledToolTerminalContinuationInstruction({ + provider: "openai", + modelId: "gpt-5.5", + modelApi: "openai-chatgpt-responses", + allowEmptyStopContinuation, + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write", asyncStarted, isError }], + itemLifecycle: { startedCount, completedCount, activeCount }, + lastAssistant: emptyStopAssistant, + currentAttemptAssistant: emptyStopAssistant, + }), + }); + + expect(instruction).toBeNull(); + }, + ); + + it("does not use a stale prior-turn empty stop to prove a settled continuation", () => { + const staleEmptyStopAssistant = { + role: "assistant", + stopReason: "stop", + provider: "openai", + model: "gpt-5.5", + content: [], + } as unknown as NonNullable; + const instruction = resolveSettledToolTerminalContinuationInstruction({ + provider: "openai", + modelId: "gpt-5.5", + modelApi: "openai-chatgpt-responses", + allowEmptyStopContinuation: true, + payloadCount: 0, + aborted: false, + timedOut: false, + attempt: makeAttemptResult({ + assistantTexts: [], + toolMetas: [{ toolName: "write" }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, + lastAssistant: staleEmptyStopAssistant, + currentAttemptAssistant: undefined, + }), + }); + + expect(instruction).toBeNull(); + }); + it("does not flag stale lastAssistant=toolUse when currentAttemptAssistant=stop exists (#80918)", () => { const incompleteTurnText = resolveIncompleteTurnPayloadText({ payloadCount: 1, @@ -4157,7 +4334,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(result.meta.livenessState).toBe("working"); }); - it("retries post-tool openai-compatible empty stop turns even when empty silence is allowed", async () => { + it("continues post-tool openai-compatible empty stop turns even when silence is allowed", async () => { mockedClassifyFailoverReason.mockReturnValue(null); mockedResolveModelAsync.mockResolvedValue({ model: { @@ -4176,6 +4353,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { makeAttemptResult({ assistantTexts: [], toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }], + itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 }, lastAssistant: { role: "assistant", api: "openai-completions", @@ -4184,6 +4362,14 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { model: "step-router-v1", content: [], } as unknown as EmbeddedRunAttemptResult["lastAssistant"], + currentAttemptAssistant: { + role: "assistant", + api: "openai-completions", + stopReason: "stop", + provider: "stepfun", + model: "step-router-v1", + content: [], + } as unknown as EmbeddedRunAttemptResult["currentAttemptAssistant"], }), ); mockedRunEmbeddedAttempt.mockResolvedValueOnce( @@ -4210,10 +4396,11 @@ describe("runEmbeddedAgent incomplete-turn safety", () => { expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2); const secondCall = runAttemptCall(1); - expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION); + expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION); expect(result.meta.terminalReplyKind).toBeUndefined(); expect(result.meta.finalAssistantVisibleText).toBe("Visible StepFun answer."); - expectWarnMessageWith("empty response detected"); + expectNoWarnMessageWith("empty response detected"); + expectWarnMessageWith("settled post-tool turn lacked a final answer"); }); it("returns NO_REPLY without retrying post-tool exact silent assistant replies", async () => { diff --git a/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts b/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts index 44bee8fbd32e..7c0d2af9460e 100644 --- a/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts +++ b/src/agents/embedded-agent-runner/run/attempt-dispatch-preparation.ts @@ -155,6 +155,12 @@ export async function prepareAndDispatchEmbeddedRunAttempt(input: { workspaceDir, }) : undefined; + // Settled-tool recovery is final-answer-only. Remove every tool surface on + // the fresh continuation so a model cannot repeat already-completed effects. + const attemptParams = + terminalRetryState.settledToolContinuationAttempts > 0 && params.disableTools !== true + ? { ...params, disableTools: true } + : params; let startupStagesEmitted = input.startupStagesEmitted; if (!startupStagesEmitted) { startupStages.mark(EMBEDDED_RUN_ATTEMPT_DISPATCH_STAGE.runtimePlan); @@ -164,7 +170,7 @@ export async function prepareAndDispatchEmbeddedRunAttempt(input: { startupStagesEmitted = true; } const dispatchedAttempt = await dispatchEmbeddedRunAttempt({ - params, + params: attemptParams, runtime: { sessionId: sessionPromptState.sessionId, sessionFile: sessionPromptState.sessionFile, diff --git a/src/agents/embedded-agent-runner/run/incomplete-turn.ts b/src/agents/embedded-agent-runner/run/incomplete-turn.ts index 804a3c49b23a..e090b885b59f 100644 --- a/src/agents/embedded-agent-runner/run/incomplete-turn.ts +++ b/src/agents/embedded-agent-runner/run/incomplete-turn.ts @@ -139,7 +139,7 @@ const REASONING_ONLY_RETRY_INSTRUCTION = "The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch."; const EMPTY_RESPONSE_RETRY_INSTRUCTION = "The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch."; -const TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION = +const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION = "The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch."; /** @@ -232,6 +232,7 @@ export function resolveIncompleteTurnPayloadText(params: { aborted: boolean; externalAbort: boolean; timedOut: boolean; + hadPotentialSideEffects?: boolean; attempt: IncompleteTurnAttempt; }): string | null { // Prefer the current attempt's terminal message. The session fallback can @@ -298,7 +299,8 @@ export function resolveIncompleteTurnPayloadText(params: { return null; } - return resolveAttemptReplayMetadata(params.attempt).hadPotentialSideEffects + return params.hadPotentialSideEffects || + resolveAttemptReplayMetadata(params.attempt).hadPotentialSideEffects ? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying." : "⚠️ Agent couldn't generate a response. Please try again."; } @@ -682,6 +684,7 @@ function shouldSkipNonVisibleTurnRetry(params: { /** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */ export function shouldTreatEmptyAssistantReplyAsSilent(params: { allowEmptyAssistantReplyAsSilent?: boolean; + onlyExplicitSilentReply?: boolean; payloadCount: number; aborted: boolean; timedOut: boolean; @@ -701,6 +704,9 @@ export function shouldTreatEmptyAssistantReplyAsSilent(params: { ) { return true; } + if (params.onlyExplicitSilentReply) { + return false; + } // Post-tool empty stops are ambiguous provider failures, not intentional silence. // Let the retry/incomplete-turn paths decide whether replay is safe. if ( @@ -760,12 +766,13 @@ export function resolveReasoningOnlyRetryInstruction(params: { return REASONING_ONLY_RETRY_INSTRUCTION; } -/** Builds a fresh continuation for a clean tool-use terminal turn with settled tool activity. */ -export function resolveToolUseTerminalContinuationInstruction(params: { +/** Builds one fresh continuation after settled tools ended without a visible final answer. */ +export function resolveSettledToolTerminalContinuationInstruction(params: { provider?: string; modelId?: string; modelApi?: string; executionContract?: string; + allowEmptyStopContinuation?: boolean; payloadCount: number; hasTerminalToolPresentation?: boolean; aborted: boolean; @@ -774,6 +781,21 @@ export function resolveToolUseTerminalContinuationInstruction(params: { attempt: IncompleteTurnAttempt; }): string | null { const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant; + const currentAttemptAssistant = params.attempt.currentAttemptAssistant; + const emptyStopAfterSettledTools = Boolean( + params.allowEmptyStopContinuation && + currentAttemptAssistant?.stopReason === "stop" && + params.attempt.toolMetas.length > 0 && + params.attempt.toolMetas.every((tool) => tool.isError !== true && tool.asyncStarted !== true) && + params.attempt.itemLifecycle.startedCount > 0 && + params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount && + params.attempt.itemLifecycle.activeCount === 0 && + !hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) && + isEmptyResponseAssistantTurn({ + payloadCount: params.payloadCount, + attempt: params.attempt, + }), + ); // Idle is not proof of completion: a toolUse terminal whose requested tools never // (or only partially) dispatched must keep the incomplete-turn error, or the model // could claim skipped side effects succeeded. Lifecycle counts are attempt-cumulative @@ -811,8 +833,7 @@ export function resolveToolUseTerminalContinuationInstruction(params: { params.aborted || params.promptError != null || params.timedOut || - assistant?.stopReason !== "toolUse" || - !allToolsProvenComplete || + (assistant?.stopReason === "toolUse" ? !allToolsProvenComplete : !emptyStopAfterSettledTools) || params.attempt.lastToolError || params.attempt.clientToolCalls || params.attempt.yieldDetected || @@ -833,7 +854,7 @@ export function resolveToolUseTerminalContinuationInstruction(params: { ) { return null; } - return TOOL_USE_TERMINAL_CONTINUATION_INSTRUCTION; + return SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION; } /** diff --git a/src/agents/embedded-agent-runner/run/terminal-resolution.ts b/src/agents/embedded-agent-runner/run/terminal-resolution.ts index ec990cf0d620..218f7e47e0c8 100644 --- a/src/agents/embedded-agent-runner/run/terminal-resolution.ts +++ b/src/agents/embedded-agent-runner/run/terminal-resolution.ts @@ -27,7 +27,7 @@ import { resolveReasoningOnlyRetryInstruction, resolveRunLivenessState, resolveSilentToolResultReplyPayload, - resolveToolUseTerminalContinuationInstruction, + resolveSettledToolTerminalContinuationInstruction, shouldRetryMissingAssistantTurn, shouldTreatEmptyAssistantReplyAsSilent, YIELD_DIAGNOSTIC_TEXT, @@ -40,7 +40,7 @@ import { import type { EmbeddedRunAttemptResult } from "./types.js"; const MAX_MISSING_ASSISTANT_RETRIES = 1; -const MAX_TOOL_USE_TERMINAL_CONTINUATIONS = 1; +const MAX_SETTLED_TOOL_TERMINAL_CONTINUATIONS = 1; const COMPACTION_CONTINUATION_RETRY_INSTRUCTION = "The previous attempt compacted the conversation context before producing a final user-visible answer. Continue from the compacted transcript and produce the final answer now. Do not restart from scratch, do not repeat completed work, and do not rerun tools unless the transcript clearly lacks required evidence."; const BEFORE_AGENT_FINALIZE_RETRY_PROMPT_PREFIX = @@ -129,36 +129,42 @@ export async function resolveEmbeddedRunTerminal(input: { ? [silentToolResultReplyPayload] : input.payloadsWithToolMedia; const payloadCount = payloadsForTerminalPath?.length ?? 0; + // A settled-tool continuation is the final recovery attempt. Do not let its + // terminal shape cascade into another retry family and execute more work. + const afterSettledToolContinuation = retryState.settledToolContinuationAttempts > 0; const emptyAssistantReplyIsSilent = shouldTreatEmptyAssistantReplyAsSilent({ allowEmptyAssistantReplyAsSilent: runParams.allowEmptyAssistantReplyAsSilent, + onlyExplicitSilentReply: afterSettledToolContinuation, payloadCount, aborted: input.terminalAborted, timedOut: input.terminalTimedOut, attempt, }); - const nextReasoningOnlyRetryInstruction = emptyAssistantReplyIsSilent - ? null - : resolveReasoningOnlyRetryInstruction({ - provider: input.activeErrorContext.provider, - modelId: input.activeErrorContext.model, - modelApi: input.modelApi, - executionContract: input.executionContract, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, - attempt, - }); - const nextEmptyResponseRetryInstruction = emptyAssistantReplyIsSilent - ? null - : resolveEmptyResponseRetryInstruction({ - provider: input.activeErrorContext.provider, - modelId: input.activeErrorContext.model, - modelApi: input.modelApi, - executionContract: input.executionContract, - payloadCount, - aborted: input.terminalAborted, - timedOut: input.terminalTimedOut, - attempt, - }); + const nextReasoningOnlyRetryInstruction = + emptyAssistantReplyIsSilent || afterSettledToolContinuation + ? null + : resolveReasoningOnlyRetryInstruction({ + provider: input.activeErrorContext.provider, + modelId: input.activeErrorContext.model, + modelApi: input.modelApi, + executionContract: input.executionContract, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); + const nextEmptyResponseRetryInstruction = + emptyAssistantReplyIsSilent || afterSettledToolContinuation + ? null + : resolveEmptyResponseRetryInstruction({ + provider: input.activeErrorContext.provider, + modelId: input.activeErrorContext.model, + modelApi: input.modelApi, + executionContract: input.executionContract, + payloadCount, + aborted: input.terminalAborted, + timedOut: input.terminalTimedOut, + attempt, + }); if ( nextReasoningOnlyRetryInstruction && retryState.reasoningOnlyAttempts < input.maxReasoningOnlyRetryAttempts @@ -177,6 +183,7 @@ export async function resolveEmbeddedRunTerminal(input: { retryState.reasoningOnlyAttempts >= input.maxReasoningOnlyRetryAttempts; if ( !emptyAssistantReplyIsSilent && + !afterSettledToolContinuation && shouldRetryMissingAssistantTurn({ payloadCount, aborted: input.terminalAborted, @@ -194,6 +201,40 @@ export async function resolveEmbeddedRunTerminal(input: { ); return { action: "retry" }; } + const availableTerminalToolPresentation = input.readTerminalToolPresentation(); + const nextSettledToolTerminalContinuationInstruction = emptyAssistantReplyIsSilent + ? null + : resolveSettledToolTerminalContinuationInstruction({ + provider: input.activeErrorContext.provider, + modelId: input.activeErrorContext.model, + modelApi: input.modelApi, + executionContract: input.executionContract, + allowEmptyStopContinuation: + runParams.trigger == null || + runParams.trigger === "user" || + runParams.trigger === "manual", + payloadCount, + hasTerminalToolPresentation: Boolean(availableTerminalToolPresentation), + aborted: input.terminalAborted, + promptError: input.promptError, + timedOut: input.terminalTimedOut, + attempt, + }); + if ( + nextSettledToolTerminalContinuationInstruction && + retryState.settledToolContinuationAttempts < MAX_SETTLED_TOOL_TERMINAL_CONTINUATIONS + ) { + retryState.settledToolContinuationAttempts += 1; + // This starts a new persisted native-thread turn after settled tool results; it does not + // replay the failed prompt or completed tools. Therefore replaySafe does not apply. + input.activateInternalPrompt(nextSettledToolTerminalContinuationInstruction, false); + log.warn( + `settled post-tool turn lacked a final answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + + `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — continuing ${retryState.settledToolContinuationAttempts}/${MAX_SETTLED_TOOL_TERMINAL_CONTINUATIONS} ` + + `from settled tool results`, + ); + return { action: "retry" }; + } if ( !nextReasoningOnlyRetryInstruction && nextEmptyResponseRetryInstruction && @@ -208,36 +249,6 @@ export async function resolveEmbeddedRunTerminal(input: { ); return { action: "retry" }; } - const availableTerminalToolPresentation = input.readTerminalToolPresentation(); - const nextToolUseTerminalContinuationInstruction = emptyAssistantReplyIsSilent - ? null - : resolveToolUseTerminalContinuationInstruction({ - provider: input.activeErrorContext.provider, - modelId: input.activeErrorContext.model, - modelApi: input.modelApi, - executionContract: input.executionContract, - payloadCount, - hasTerminalToolPresentation: Boolean(availableTerminalToolPresentation), - aborted: input.terminalAborted, - promptError: input.promptError, - timedOut: input.terminalTimedOut, - attempt, - }); - if ( - nextToolUseTerminalContinuationInstruction && - retryState.toolUseContinuationAttempts < MAX_TOOL_USE_TERMINAL_CONTINUATIONS - ) { - retryState.toolUseContinuationAttempts += 1; - // This starts a new persisted native-thread turn after settled tool results; it does not - // replay the failed prompt or completed tools. Therefore replaySafe does not apply. - input.activateInternalPrompt(nextToolUseTerminalContinuationInstruction, false); - log.warn( - `tool-use terminal turn lacked a final answer: runId=${runParams.runId} sessionId=${runParams.sessionId} ` + - `provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — continuing ${retryState.toolUseContinuationAttempts}/${MAX_TOOL_USE_TERMINAL_CONTINUATIONS} ` + - `from settled tool results`, - ); - return { action: "retry" }; - } const incompleteTurnText = emptyAssistantReplyIsSilent ? null : resolveIncompleteTurnPayloadText({ @@ -245,6 +256,7 @@ export async function resolveEmbeddedRunTerminal(input: { aborted: input.terminalAborted, externalAbort: input.externalAbort || input.signalOwnedInterruption, timedOut: input.terminalTimedOut, + hadPotentialSideEffects: input.replayState.hadPotentialSideEffects, attempt, }); const incompleteTurnFallbackSafe = Boolean( @@ -319,7 +331,7 @@ export async function resolveEmbeddedRunTerminal(input: { `compactions=${input.attemptCompactionCount} reasoningRetries=${retryState.reasoningOnlyAttempts}/${input.maxReasoningOnlyRetryAttempts} ` + `emptyRetries=${retryState.emptyResponseAttempts}/${input.maxEmptyResponseRetryAttempts} ` + `missingAssistantRetries=${retryState.missingAssistantAttempts}/${MAX_MISSING_ASSISTANT_RETRIES} ` + - `toolUseContinuations=${retryState.toolUseContinuationAttempts}/${MAX_TOOL_USE_TERMINAL_CONTINUATIONS} — ` + + `settledToolContinuations=${retryState.settledToolContinuationAttempts}/${MAX_SETTLED_TOOL_TERMINAL_CONTINUATIONS} — ` + (terminalToolPresentation ? "surfacing tool-authored terminal presentation" : "surfacing error to user"), diff --git a/src/agents/embedded-agent-runner/run/terminal-retry-state.ts b/src/agents/embedded-agent-runner/run/terminal-retry-state.ts index 3f931199e9af..2fec02ad0d23 100644 --- a/src/agents/embedded-agent-runner/run/terminal-retry-state.ts +++ b/src/agents/embedded-agent-runner/run/terminal-retry-state.ts @@ -4,7 +4,7 @@ export type EmbeddedRunTerminalRetryState = { reasoningOnlyAttempts: number; emptyResponseAttempts: number; missingAssistantAttempts: number; - toolUseContinuationAttempts: number; + settledToolContinuationAttempts: number; compactionContinuationAttempts: number; compactionContinuationInstruction: string | null; beforeFinalizeRevisionAttempts: number; @@ -15,7 +15,7 @@ export function createEmbeddedRunTerminalRetryState(): EmbeddedRunTerminalRetryS reasoningOnlyAttempts: 0, emptyResponseAttempts: 0, missingAssistantAttempts: 0, - toolUseContinuationAttempts: 0, + settledToolContinuationAttempts: 0, compactionContinuationAttempts: 0, compactionContinuationInstruction: null, beforeFinalizeRevisionAttempts: 0,