mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(agent-runner): split run policy ownership (#122209)
* refactor(agent-runner): split tool-call normalization ownership * refactor(agent-runner): split incomplete-turn ownership * test(agent-runner): hide incomplete-turn test helper * test(ci): route split incomplete-turn tests * test(ci): enumerate split incomplete-turn tests * test(vitest): route split incomplete-turn tests to their serial shard
This commit is contained in:
committed by
GitHub
parent
09c39e0ec5
commit
a40df57bae
@@ -373,15 +373,11 @@ src/agents/embedded-agent-runner/extra-params.ts
|
||||
src/agents/embedded-agent-runner/model.provider-runtime.test-support.ts
|
||||
src/agents/embedded-agent-runner/model.test.ts
|
||||
src/agents/embedded-agent-runner/replay-history.ts
|
||||
src/agents/embedded-agent-runner/run.incomplete-turn.test.ts
|
||||
src/agents/embedded-agent-runner/run.overflow-compaction.harness.ts
|
||||
src/agents/embedded-agent-runner/run/attempt-spawn-workspace.test-support.ts
|
||||
src/agents/embedded-agent-runner/run/attempt.spawn-workspace.context-engine.test.ts
|
||||
src/agents/embedded-agent-runner/run/attempt.test.ts
|
||||
src/agents/embedded-agent-runner/run/attempt.tool-call-argument-repair.ts
|
||||
src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.test.ts
|
||||
src/agents/embedded-agent-runner/run/attempt.tool-call-normalization.ts
|
||||
src/agents/embedded-agent-runner/run/incomplete-turn.ts
|
||||
src/agents/embedded-agent-runner/runs.ts
|
||||
src/agents/embedded-agent-runner/thinking.test.ts
|
||||
src/agents/embedded-agent-runner/tool-result-context-guard.test.ts
|
||||
|
||||
@@ -3148,6 +3148,11 @@ function classifyTarget(arg: string, cwd: string) {
|
||||
if (relative.startsWith("src/plugins/contracts/")) {
|
||||
return "contractsPlugin";
|
||||
}
|
||||
// These tests share stateful runner mocks and must keep the dedicated serial
|
||||
// owner even when their contents also qualify for a unit-fast lane.
|
||||
if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) {
|
||||
return agentVitestProjectOwners.embeddedIncompleteTurn.kind;
|
||||
}
|
||||
if (resolveUnitFastTimerTestIncludePattern(relative)) {
|
||||
return "unitFastFakeTimers";
|
||||
}
|
||||
@@ -3333,9 +3338,6 @@ function classifyTarget(arg: string, cwd: string) {
|
||||
) {
|
||||
return agentVitestProjectOwners.all.kind;
|
||||
}
|
||||
if (agentVitestProjectOwners.embeddedIncompleteTurn.include.includes(relative)) {
|
||||
return agentVitestProjectOwners.embeddedIncompleteTurn.kind;
|
||||
}
|
||||
if (agentVitestProjectOwners.embeddedOverflowCompaction.include.includes(relative)) {
|
||||
return agentVitestProjectOwners.embeddedOverflowCompaction.kind;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Live checks for Anthropic replay transcript sanitization and tool-call history.
|
||||
import type { Message, Model } from "openclaw/plugin-sdk/llm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { wrapStreamFnSanitizeMalformedToolCalls } from "./embedded-agent-runner/run/attempt.tool-call-normalization.js";
|
||||
import { wrapStreamFnSanitizeMalformedToolCalls } from "./embedded-agent-runner/run/attempt-tool-call-replay-sanitization.js";
|
||||
import { extractEmbeddedAssistantText } from "./embedded-agent-utils.js";
|
||||
import { completeSimpleWithLiveTimeout, logLiveCache } from "./live-cache-test-support.js";
|
||||
import { isLiveTestEnabled } from "./live-test-helpers.js";
|
||||
|
||||
@@ -44,7 +44,7 @@ import { createIdleTimeoutBreakerState } from "./run/idle-timeout-breaker.js";
|
||||
import {
|
||||
DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT,
|
||||
DEFAULT_REASONING_ONLY_RETRY_LIMIT,
|
||||
} from "./run/incomplete-turn.js";
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
import { measureEmbeddedAgentPreparation } from "./run/preparation-timing.js";
|
||||
import {
|
||||
beginRunAttempt,
|
||||
|
||||
@@ -0,0 +1,521 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeBaseRunParams,
|
||||
makeRunParams,
|
||||
expectWarnMessageWith,
|
||||
expectNoWarnMessageWith,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedClassifyFailoverReason,
|
||||
mockedIsRateLimitAssistantError,
|
||||
mockedRunEmbeddedAttempt,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import { recoverEmbeddedRunAttempt } from "./run/attempt-recovery.js";
|
||||
import { resolveSilentToolResultReplyPayload } from "./run/incomplete-turn-resolution.js";
|
||||
import { resolveEmbeddedRunAttemptTerminalState } from "./run/terminal-outcome.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
import { createUsageAccumulator } from "./usage-accumulator.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("counts failed tool results in trace tool summaries", async () => {
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Done."],
|
||||
toolMetas: [
|
||||
{ toolName: "bash", meta: "exit=1", isError: true },
|
||||
{ toolName: "bash", meta: "exit=2", isError: true },
|
||||
{ toolName: "bash", meta: "exit=0" },
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeBaseRunParams("run-tool-summary-failure-count"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.meta?.toolSummary).toEqual({
|
||||
calls: 3,
|
||||
tools: ["bash"],
|
||||
failures: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("emits the before_agent_run hook block message as the agent payload", async () => {
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
promptError: new Error("Blocked by before-run policy."),
|
||||
promptErrorSource: "hook:before_agent_run",
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeBaseRunParams("run-before-agent-run-hook-block"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toEqual([{ text: "Blocked by before-run policy.", isError: true }]);
|
||||
expect(result.meta?.finalAssistantVisibleText).toBe("Blocked by before-run policy.");
|
||||
expect(result.meta?.finalAssistantRawText).toBe("Blocked by before-run policy.");
|
||||
expect(result.meta?.finalPromptText).toBeUndefined();
|
||||
expect(result.meta?.error).toEqual({
|
||||
kind: "hook_block",
|
||||
message: "Blocked by before-run policy.",
|
||||
});
|
||||
expect(result.meta?.livenessState).toBe("blocked");
|
||||
});
|
||||
|
||||
it("keeps carried usage ahead of transcript history on before_agent_run hook blocks", async () => {
|
||||
const historicalAssistant = makeLastAssistant({
|
||||
usage: { input: 128_814, output: 3_000, total: 131_814 },
|
||||
});
|
||||
const carriedUsage = { input: 42_000, output: 1_000, total: 43_000 };
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
promptError: new Error("Blocked by before-run policy."),
|
||||
promptErrorSource: "hook:before_agent_run",
|
||||
lastAssistant: historicalAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
});
|
||||
const terminalState = resolveEmbeddedRunAttemptTerminalState({
|
||||
attempt,
|
||||
assistant: historicalAssistant,
|
||||
});
|
||||
|
||||
const recovery = await recoverEmbeddedRunAttempt({
|
||||
runInput: {
|
||||
runParams: makeBaseRunParams("run-before-agent-run-hook-block-usage"),
|
||||
resolvedSessionKey: "agent:main:test-key",
|
||||
startedAtMs: Date.now(),
|
||||
},
|
||||
preparedRuntime: {
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.6-luna",
|
||||
model: { id: "gpt-5.6-luna" },
|
||||
genericCompactionRecoveryAllowed: false,
|
||||
snapshot: () => ({
|
||||
thinkLevel: "off",
|
||||
agentHarness: { id: "codex" },
|
||||
outerContextTokenMeta: {},
|
||||
}),
|
||||
},
|
||||
normalizedAttempt: {
|
||||
attempt,
|
||||
sessionIdUsed: attempt.sessionIdUsed,
|
||||
attemptAssistant: historicalAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
currentAttemptCompletedAssistant: undefined,
|
||||
terminalState,
|
||||
setTerminalLifecycleMeta: vi.fn(),
|
||||
attemptCompactionCount: 0,
|
||||
activeErrorContext: { provider: "openai", model: "gpt-5.6-luna" },
|
||||
resolveReplayInvalidForAttempt: () => false,
|
||||
canRestartForLiveSwitch: false,
|
||||
},
|
||||
runtimePlan: { auth: {} },
|
||||
sessionPromptState: { sessionFile: "/tmp/session.jsonl" },
|
||||
usageAccumulator: createUsageAccumulator(),
|
||||
lastRunPromptUsage: carriedUsage,
|
||||
} as never);
|
||||
|
||||
expect(recovery).toMatchObject({
|
||||
action: "complete",
|
||||
result: {
|
||||
meta: {
|
||||
agentMeta: { lastCallUsage: carriedUsage, promptTokens: 42_000 },
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("warns before retrying when an incomplete turn already sent a message", async () => {
|
||||
// Delivery evidence means retrying could duplicate user-visible output, so
|
||||
// the runner must surface a verify-before-retry payload instead.
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [],
|
||||
didSendViaMessagingTool: true,
|
||||
lastAssistant: {
|
||||
stopReason: "toolUse",
|
||||
errorMessage: "internal retry interrupted tool execution",
|
||||
provider: "openai",
|
||||
model: "mock-1",
|
||||
content: [],
|
||||
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-incomplete-turn-messaging-warning", { model: "gpt-4.1" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(mockedClassifyFailoverReason).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("verify before retrying");
|
||||
});
|
||||
|
||||
it("surfaces internal aborts after tool-use as visible incomplete-turn failures", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "web_search", meta: "query=next voice note" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-internal-abort-tool-use-incomplete"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toEqual([
|
||||
{ text: "⚠️ Agent couldn't generate a response. Please try again.", isError: true },
|
||||
]);
|
||||
expect(result.meta?.livenessState).toBe("abandoned");
|
||||
});
|
||||
|
||||
it("does not route caller timeouts through provider failover", async () => {
|
||||
const controller = new AbortController();
|
||||
const timeoutError = new Error("caller deadline elapsed");
|
||||
timeoutError.name = "TimeoutError";
|
||||
const setTerminalLifecycleMeta = vi.fn();
|
||||
const interruptedAssistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
errorMessage: "HTTP 429 Too Many Requests",
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue("rate_limit");
|
||||
mockedIsRateLimitAssistantError.mockReturnValue(true);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async () => {
|
||||
controller.abort(timeoutError);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: interruptedAssistant,
|
||||
currentAttemptAssistant: interruptedAssistant,
|
||||
setTerminalLifecycleMeta,
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeBaseRunParams("run-caller-timeout", { abortSignal: controller.signal }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.at(-1)?.text).toContain("timed out");
|
||||
expect(result.meta?.aborted).toBe(false);
|
||||
expect(result.meta?.timeoutPhase).toBeUndefined();
|
||||
expect(result.meta?.providerStarted).toBeUndefined();
|
||||
const lifecycleMeta = setTerminalLifecycleMeta.mock.lastCall?.[0];
|
||||
expect(lifecycleMeta).toMatchObject({
|
||||
aborted: false,
|
||||
livenessState: "blocked",
|
||||
stopReason: "timeout",
|
||||
});
|
||||
expect(lifecycleMeta).not.toHaveProperty("timeoutPhase");
|
||||
expect(lifecycleMeta).not.toHaveProperty("providerStarted");
|
||||
});
|
||||
|
||||
it("does not synthesize an incomplete turn for a caller abort before attempt flags settle", async () => {
|
||||
const controller = new AbortController();
|
||||
const abortError = new Error("caller cancelled");
|
||||
abortError.name = "AbortError";
|
||||
const setTerminalLifecycleMeta = vi.fn();
|
||||
const lateAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Late answer" }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async () => {
|
||||
controller.abort(abortError);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: ["Late answer"],
|
||||
lastAssistant: lateAssistant,
|
||||
currentAttemptAssistant: lateAssistant,
|
||||
setTerminalLifecycleMeta,
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeBaseRunParams("run-caller-abort", { abortSignal: controller.signal }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
expect(result.meta?.aborted).toBe(true);
|
||||
expect(result.meta?.error).toBeUndefined();
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({
|
||||
aborted: true,
|
||||
livenessState: "blocked",
|
||||
stopReason: "aborted",
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates canonical assistant aborts into terminal lifecycle metadata", async () => {
|
||||
const setTerminalLifecycleMeta = vi.fn();
|
||||
const abortedAssistant = makeLastAssistant({
|
||||
stopReason: "aborted",
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: abortedAssistant,
|
||||
currentAttemptAssistant: abortedAssistant,
|
||||
setTerminalLifecycleMeta,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeBaseRunParams("run-canonical-assistant-abort"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.meta?.aborted).toBe(true);
|
||||
expect(setTerminalLifecycleMeta.mock.lastCall?.[0]).toMatchObject({
|
||||
aborted: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("synthesizes a silent cron payload from a trailing current-attempt NO_REPLY tool result", () => {
|
||||
// Cron no-reply can be represented by a tool result rather than assistant
|
||||
// text, but only when it belongs to the current attempt.
|
||||
const payload = resolveSilentToolResultReplyPayload({
|
||||
isCronTrigger: true,
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec" }],
|
||||
messagesSnapshot: [
|
||||
{
|
||||
role: "toolResult",
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
details: { aggregated: "NO_REPLY" },
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(payload).toEqual({ text: "NO_REPLY" });
|
||||
});
|
||||
|
||||
it("does not reuse an older NO_REPLY tool result without current-attempt tool activity", () => {
|
||||
const payload = resolveSilentToolResultReplyPayload({
|
||||
isCronTrigger: true,
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [],
|
||||
messagesSnapshot: [
|
||||
{
|
||||
role: "toolResult",
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Current cron prompt" }],
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(payload).toBeNull();
|
||||
});
|
||||
|
||||
it("treats exact NO_REPLY tool output as a quiet cron success when the final assistant is empty", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec" }],
|
||||
messagesSnapshot: [
|
||||
{
|
||||
role: "toolResult",
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
details: { aggregated: "NO_REPLY" },
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-cron-no-reply-empty-final", { trigger: "cron", model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
});
|
||||
|
||||
it("surfaces the latest tool-authored presentation after a structured incomplete turn", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
(
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome?.({
|
||||
toolName: "web_fetch",
|
||||
argsHash: "args",
|
||||
resultHash: "result",
|
||||
terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200",
|
||||
});
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "web_fetch" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-structured-terminal-presentation", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
text:
|
||||
"Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" +
|
||||
"⚠️ Agent couldn't generate a response. Please try again.",
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
expect(result.meta.replayInvalid).toBe(true);
|
||||
expect(result.meta.livenessState).toBe("abandoned");
|
||||
expect(result.meta.error?.fallbackSafe).toBe(true);
|
||||
expect(result.meta.error?.terminalPresentation).toBe(true);
|
||||
expectWarnMessageWith("surfacing tool-authored terminal presentation");
|
||||
});
|
||||
|
||||
it("surfaces read-only cron presentation after a structured incomplete turn", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
(
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome?.({
|
||||
toolName: "cron",
|
||||
argsHash: "args",
|
||||
resultHash: "result",
|
||||
terminalPresentation: "Automations scheduler status.\nEnabled: yes",
|
||||
});
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "cron" }],
|
||||
replayMetadata: {
|
||||
hadPotentialSideEffects: false,
|
||||
replaySafe: true,
|
||||
},
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-read-only-cron-terminal-presentation", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
text:
|
||||
"Automations scheduler status.\nEnabled: yes\n\n" +
|
||||
"⚠️ Agent couldn't generate a response. Please try again.",
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
expect(result.meta.error?.fallbackSafe).toBe(true);
|
||||
expect(result.meta.error?.terminalPresentation).toBe(true);
|
||||
});
|
||||
|
||||
it("preserves a terminal tool presentation across an empty-response retry", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
(
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome?.({
|
||||
toolName: "web_fetch",
|
||||
argsHash: "args",
|
||||
resultHash: "result",
|
||||
terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200",
|
||||
});
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "web_fetch" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-preserved-terminal-presentation", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
text:
|
||||
"Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" +
|
||||
"⚠️ Agent couldn't generate a response. Please try again.",
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,347 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
EMPTY_RESPONSE_RETRY_INSTRUCTION,
|
||||
makeLastAssistant,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
makeIncompleteTurnParams,
|
||||
makeReasoningRetryParams,
|
||||
makeEmptyResponseRetryParams,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js";
|
||||
import {
|
||||
resolveEmptyResponseRetryInstruction,
|
||||
resolveReasoningOnlyRetryInstruction,
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("detects reasoning-only GPT turns from signed thinking blocks", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_helper", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("detects reasoning-only Gemini turns from signed thinking blocks", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "google",
|
||||
model: "gemini-2.5-pro",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "gemini_rs_helper", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ provider: "google", modelId: "gemini-2.5-pro" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries signed reasoning-only Bedrock Converse turns with a visible-answer continuation", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "amazon-bedrock",
|
||||
model: "openai.gpt-oss-120b-1:0",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: "bedrock-reasoning-signature",
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{
|
||||
provider: "amazon-bedrock",
|
||||
modelId: "openai.gpt-oss-120b-1:0",
|
||||
modelApi: "bedrock-converse-stream",
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries signed reasoning-only Ollama turns with a visible-answer continuation instruction", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "ollama",
|
||||
model: "gemma4:31b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "ollama_rs_helper", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "gemma4:31b" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries unsigned thinking-only turns via the reasoning-only path (openai-completions)", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "qwen3.6-35b-a3b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "let me plan the tool calls I need to make...",
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ modelId: "qwen3.6-35b-a3b", modelApi: "openai-completions" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries unsigned thinking-only Ollama turns via the reasoning-only path", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "ollama",
|
||||
model: "gemma4:31b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "gemma4:31b" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries unsigned-thinking Ollama turns via the empty-response path", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "ollama",
|
||||
model: "gemma4:31b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "gemma4:31b" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries generic empty Ollama turns without visible text", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "ollama",
|
||||
model: "gemma4:31b",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "gemma4:31b" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries empty Ollama stop turns when nonzero output tokens were generated", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "ollama",
|
||||
model: "minimax-m2.7:cloud",
|
||||
usage: { input: 100, output: 6, totalTokens: 106 },
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "minimax-m2.7:cloud" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("does not retry empty turns after an accepted sessions_spawn delivery", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
acceptedSessionSpawns: [
|
||||
{
|
||||
runId: "run-child",
|
||||
childSessionKey: "agent:claude:subagent:child",
|
||||
},
|
||||
],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "ollama",
|
||||
model: "gemma4:31b",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "gemma4:31b" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
});
|
||||
|
||||
it("retries empty openai-chatgpt-responses turns with non-zero output tokens (#85364)", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
usage: { input: 24794, output: 111, cacheRead: 4608, totalTokens: 29513 },
|
||||
}),
|
||||
},
|
||||
{ modelId: "gpt-5.5", modelApi: "openai-chatgpt-responses" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries empty openai-responses turns without visible text", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
usage: { input: 5000, output: 200, totalTokens: 5200 },
|
||||
}),
|
||||
},
|
||||
{ modelId: "gpt-5.5", modelApi: "openai-responses" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("retries generic empty OpenAI-compatible turns from custom endpoints", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "llama-cpp-local",
|
||||
model: "qwen3.6-27b",
|
||||
usage: { input: 950, output: 103, totalTokens: 1053 },
|
||||
}),
|
||||
},
|
||||
{
|
||||
provider: "llama-cpp-local",
|
||||
modelId: "qwen3.6-27b",
|
||||
modelApi: "openai-completions",
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("does not retry clean zero-token Ollama stop turns", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "ollama",
|
||||
model: "glm-5.1:cloud",
|
||||
usage: { input: 100, output: 0, totalTokens: 100 },
|
||||
}),
|
||||
},
|
||||
{ provider: "ollama", modelId: "glm-5.1:cloud" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
});
|
||||
|
||||
it("treats exact NO_REPLY as a deliberate silent assistant reply", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_no_reply", type: "reasoning" }),
|
||||
},
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "NO_REPLY" },
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,557 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
hasCommittedMessagingToolDeliveryEvidence,
|
||||
hasOutboundDeliveryEvidence,
|
||||
} from "./delivery-evidence.js";
|
||||
import {
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
makeRunParams,
|
||||
makeIncompleteTurnParams,
|
||||
makeReasoningRetryParams,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js";
|
||||
import {
|
||||
DEFAULT_REASONING_ONLY_RETRY_LIMIT,
|
||||
resolveReasoningOnlyRetryInstruction,
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("suppresses the incomplete-turn warning after committed messaging text delivery", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Delivered through the message tool."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "ollama",
|
||||
model: "kimi-k2.6:cloud",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("suppresses the incomplete-turn warning after committed messaging delivery before end_turn", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Delivered through the message tool."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "google",
|
||||
model: "gemini-2.5-pro",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_messaging_end_turn", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("suppresses the incomplete-turn warning after committed media-only messaging delivery", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: false,
|
||||
messagingToolSentMediaUrls: ["file:///tmp/render.png"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("suppresses the incomplete-turn warning after committed messaging delivery even when the provider errored", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Delivered before the provider error."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "ollama",
|
||||
model: "kimi-k2.6:cloud",
|
||||
errorMessage: "provider failed after delivery",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("suppresses the incomplete-turn warning after an accepted sessions_spawn terminal success", () => {
|
||||
const attemptWithAcceptedSpawn: Partial<EmbeddedRunAttemptResult> & {
|
||||
acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>;
|
||||
} = {
|
||||
assistantTexts: [],
|
||||
acceptedSessionSpawns: [
|
||||
{
|
||||
runId: "run-child",
|
||||
childSessionKey: "agent:claude:subagent:child",
|
||||
},
|
||||
],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
}),
|
||||
};
|
||||
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(attemptWithAcceptedSpawn),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("still returns a timeout payload when the parent prompt times out after an accepted sessions_spawn", async () => {
|
||||
const acceptedSessionSpawns = [
|
||||
{
|
||||
runId: "run-child",
|
||||
childSessionKey: "agent:claude:subagent:child",
|
||||
},
|
||||
];
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
acceptedSessionSpawns,
|
||||
timedOut: true,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-timeout-after-accepted-spawn", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
text: "Request timed out before a response was generated. Please try again, or increase `agents.defaults.timeoutSeconds` in your config.",
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns);
|
||||
});
|
||||
|
||||
it("still surfaces the incomplete-turn warning without an accepted sessions_spawn success", () => {
|
||||
const attemptWithMalformedSpawn: Partial<EmbeddedRunAttemptResult> & {
|
||||
acceptedSessionSpawns: Array<{ runId: string; childSessionKey: string }>;
|
||||
} = {
|
||||
assistantTexts: [],
|
||||
acceptedSessionSpawns: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
}),
|
||||
};
|
||||
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(attemptWithMalformedSpawn),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("still surfaces the incomplete-turn warning when no messaging delivery was committed", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "ollama",
|
||||
model: "kimi-k2.6:cloud",
|
||||
errorMessage: "provider failed mid-turn",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("verify before retrying");
|
||||
});
|
||||
|
||||
it("does not treat empty committed messaging arrays as delivery", () => {
|
||||
expect(
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: [" "],
|
||||
messagingToolSentMediaUrls: [],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats committed messaging media as delivery", () => {
|
||||
expect(
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: [],
|
||||
messagingToolSentMediaUrls: ["file:///tmp/render.png"],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("treats committed messaging targets as delivery", () => {
|
||||
expect(
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: [],
|
||||
messagingToolSentMediaUrls: [],
|
||||
messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
for (const { name, overrides } of [
|
||||
{
|
||||
name: "treats committed messaging text as replay-invalid side effect metadata",
|
||||
overrides: { messagingToolSentTexts: ["Delivered through the message tool."] },
|
||||
},
|
||||
{
|
||||
name: "treats async-started background tools as replay-invalid side effects",
|
||||
overrides: { toolMetas: [{ toolName: "image_generate", asyncStarted: true }] },
|
||||
},
|
||||
{
|
||||
name: "treats committed messaging media as replay-invalid side effect metadata",
|
||||
overrides: { messagingToolSentMediaUrls: ["file:///tmp/render.png"] },
|
||||
},
|
||||
{
|
||||
name: "treats committed messaging targets as replay-invalid side effect metadata",
|
||||
overrides: {
|
||||
messagingToolSentTargets: [{ tool: "message", provider: "slack", to: "channel-1" }],
|
||||
},
|
||||
},
|
||||
]) {
|
||||
it(name, () => {
|
||||
expect(
|
||||
buildAttemptReplayMetadata({
|
||||
toolMetas: [],
|
||||
didSendViaMessagingTool: false,
|
||||
messagingToolSentTexts: [],
|
||||
messagingToolSentMediaUrls: [],
|
||||
...overrides,
|
||||
}),
|
||||
).toEqual({ hadPotentialSideEffects: true, replaySafe: false });
|
||||
});
|
||||
}
|
||||
|
||||
it("treats accepted sessions_spawn as replay-invalid outbound delivery", () => {
|
||||
const acceptedSessionSpawns = [
|
||||
{
|
||||
runId: "run-child",
|
||||
childSessionKey: "agent:claude:subagent:child",
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
buildAttemptReplayMetadata({
|
||||
toolMetas: [],
|
||||
didSendViaMessagingTool: false,
|
||||
messagingToolSentTexts: [],
|
||||
messagingToolSentMediaUrls: [],
|
||||
acceptedSessionSpawns,
|
||||
}),
|
||||
).toEqual({ hadPotentialSideEffects: true, replaySafe: false });
|
||||
expect(hasOutboundDeliveryEvidence({ acceptedSessionSpawns })).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores malformed accepted sessions_spawn delivery evidence", () => {
|
||||
expect(
|
||||
hasOutboundDeliveryEvidence({
|
||||
acceptedSessionSpawns: [
|
||||
null,
|
||||
{
|
||||
runId: "run-child",
|
||||
childSessionKey: " ",
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("leaves committed delivery plus tool errors to the tool-error payload path", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Delivered through the message tool."],
|
||||
lastToolError: {
|
||||
toolName: "message",
|
||||
meta: "send",
|
||||
error: "delivery failed for second target",
|
||||
},
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("does not retry reasoning-only GPT turns after side effects", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_side_effect", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
expect(DEFAULT_REASONING_ONLY_RETRY_LIMIT).toBe(2);
|
||||
});
|
||||
|
||||
it("does not retry reasoning-only GPT turns when the assistant ended in error", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_helper_error", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
});
|
||||
|
||||
it("does not retry reasoning-only GPT turns when visible assistant text already exists", () => {
|
||||
const retryInstruction = resolveReasoningOnlyRetryInstruction(
|
||||
makeReasoningRetryParams({
|
||||
assistantTexts: ["Visible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({
|
||||
id: "rs_helper_visible_text",
|
||||
type: "reasoning",
|
||||
}),
|
||||
},
|
||||
{ type: "text", text: "" },
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces incomplete-turn text for errored signed-thinking-only turns with payloads", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-8",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning before provider error",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_error_payload", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("surfaces incomplete-turn text for token-limited partial answers", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Partial answer"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "length",
|
||||
provider: "ollama",
|
||||
model: "qwen3.5",
|
||||
content: [{ type: "text", text: "Partial answer" }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("keeps complete visible stop turns successful", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Complete answer"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "ollama",
|
||||
model: "qwen3.5",
|
||||
content: [{ type: "text", text: "Complete answer" }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves terminal tool media on token-limited turns", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Partial answer"],
|
||||
toolMediaUrls: ["file:///tmp/render.png"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "length",
|
||||
provider: "ollama",
|
||||
model: "qwen3.5",
|
||||
content: [{ type: "text", text: "Partial answer" }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves tool media already delivered through block replies", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Partial answer"],
|
||||
hasToolMediaBlockReply: true,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "length",
|
||||
provider: "ollama",
|
||||
model: "qwen3.5",
|
||||
content: [{ type: "text", text: "Partial answer" }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves successful cron progress on token-limited turns", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Partial answer"],
|
||||
successfulCronAdds: 1,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "length",
|
||||
provider: "ollama",
|
||||
model: "qwen3.5",
|
||||
content: [{ type: "text", text: "Partial answer" }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"heartbeat responses",
|
||||
{
|
||||
heartbeatToolResponse: {
|
||||
outcome: "progress" as const,
|
||||
notify: false,
|
||||
summary: "Still working",
|
||||
},
|
||||
},
|
||||
],
|
||||
["tool media", { toolMediaUrls: ["file:///tmp/render.png"] }],
|
||||
["voice media", { toolAudioAsVoice: true }],
|
||||
["trusted local media", { toolTrustedLocalMedia: true }],
|
||||
[
|
||||
"source reply payloads",
|
||||
{ messagingToolSourceReplyPayloads: [{ text: "Delivered through the source reply." }] },
|
||||
],
|
||||
["delivered source replies", { didDeliverSourceReplyViaMessageTool: true }],
|
||||
] satisfies Array<[string, Partial<EmbeddedRunAttemptResult>]>)(
|
||||
"does not replace terminal %s with an incomplete-turn warning",
|
||||
(_label, attemptState) => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
...attemptState,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-8",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning before provider error",
|
||||
thinkingSignature: JSON.stringify({
|
||||
id: "rs_terminal_payload",
|
||||
type: "reasoning",
|
||||
}),
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,316 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
EMPTY_RESPONSE_RETRY_INSTRUCTION,
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeRunParams,
|
||||
expectWarnMessageWith,
|
||||
runAttemptCall,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
mockedResolveModelAsync,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("retries zero-token empty Claude stop turns with a visible-answer continuation instruction", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4.7",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible Claude answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4.7",
|
||||
content: [{ type: "text", text: "Visible Claude answer." }],
|
||||
usage: {
|
||||
input: 100,
|
||||
output: 5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 105,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-zero-usage-claude-continuation", {
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4.7",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("empty response detected");
|
||||
});
|
||||
|
||||
it("retries empty openai-compatible stop turns even when the backend reports output tokens", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedResolveModelAsync.mockResolvedValue({
|
||||
model: {
|
||||
id: "qwen3.6-27b",
|
||||
provider: "llamacpp",
|
||||
contextWindow: 200000,
|
||||
api: "openai-completions",
|
||||
},
|
||||
error: null,
|
||||
authStorage: {
|
||||
setRuntimeApiKey: vi.fn(),
|
||||
},
|
||||
modelRegistry: {},
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "llamacpp",
|
||||
model: "qwen3.6-27b",
|
||||
usage: {
|
||||
input: 512,
|
||||
output: 103,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 615,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible local answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "llamacpp",
|
||||
model: "qwen3.6-27b",
|
||||
content: [{ type: "text", text: "Visible local answer." }],
|
||||
usage: {
|
||||
input: 640,
|
||||
output: 5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 645,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-openai-compatible-stop-continuation", {
|
||||
provider: "llamacpp",
|
||||
model: "qwen3.6-27b",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("empty response detected");
|
||||
});
|
||||
|
||||
it("retries empty Anthropic-compatible stop turns even when the provider is not Kimi", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedResolveModelAsync.mockResolvedValue({
|
||||
model: {
|
||||
id: "claude-opus-4-7",
|
||||
provider: "sub2api",
|
||||
contextWindow: 200000,
|
||||
api: "anthropic-messages",
|
||||
},
|
||||
error: null,
|
||||
authStorage: {
|
||||
setRuntimeApiKey: vi.fn(),
|
||||
},
|
||||
modelRegistry: {},
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "anthropic-messages",
|
||||
provider: "sub2api",
|
||||
model: "claude-opus-4-7",
|
||||
usage: {
|
||||
input: 2048,
|
||||
output: 3100,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 5148,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible Anthropic-compatible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "anthropic-messages",
|
||||
provider: "sub2api",
|
||||
model: "claude-opus-4-7",
|
||||
content: [{ type: "text", text: "Visible Anthropic-compatible answer." }],
|
||||
usage: {
|
||||
input: 2300,
|
||||
output: 8,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 2308,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-anthropic-compatible-stop-continuation", {
|
||||
provider: "sub2api",
|
||||
model: "claude-opus-4-7",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("empty response detected");
|
||||
});
|
||||
|
||||
it("surfaces an error after exhausting empty-response retries", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-response-exhausted", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("Please try again");
|
||||
expectWarnMessageWith("empty response retries exhausted");
|
||||
});
|
||||
|
||||
it("surfaces an error after exhausting reasoning-only retries without a visible answer", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({
|
||||
id: "rs_reasoning_exhausted",
|
||||
type: "reasoning",
|
||||
}),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-only-exhausted", {
|
||||
model: "gpt-5.4",
|
||||
reasoningLevel: "on",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("Please try again");
|
||||
expectWarnMessageWith("reasoning-only retries exhausted");
|
||||
});
|
||||
|
||||
it("preserves a terminal tool presentation after reasoning-only retries are exhausted", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const reasoningOnlyAttempt = async () =>
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({
|
||||
id: "rs_reasoning_terminal_presentation",
|
||||
type: "reasoning",
|
||||
}),
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
(
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome?.({
|
||||
toolName: "web_fetch",
|
||||
argsHash: "args",
|
||||
resultHash: "result",
|
||||
terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200",
|
||||
});
|
||||
return reasoningOnlyAttempt();
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockImplementation(reasoningOnlyAttempt);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-terminal-presentation", {
|
||||
model: "gpt-5.4",
|
||||
reasoningLevel: "on",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
text:
|
||||
"Web fetch completed.\nOrigin: https://example.com\nStatus: 200\n\n" +
|
||||
"⚠️ Agent couldn't generate a response. Please try again.",
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,379 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import { PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE } from "../../llm/types.js";
|
||||
import {
|
||||
EMPTY_RESPONSE_RETRY_INSTRUCTION,
|
||||
makeLastAssistant,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
makeIncompleteTurnParams,
|
||||
makeEmptyResponseRetryParams,
|
||||
makeSilentReplyParams,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT,
|
||||
resolveEmptyResponseRetryInstruction,
|
||||
shouldRetrySilentErrorAssistantTurn,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("retries replay-safe errored turns that only emitted thinking blocks", () => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-8",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning before provider error",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_error", type: "reasoning" }),
|
||||
},
|
||||
{ type: "redacted_thinking", data: "opaque" },
|
||||
{ type: "text", text: " " },
|
||||
],
|
||||
usage: { input: 100, output: 1120, totalTokens: 1220 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("does not retry an ambiguous post-dispatch provider outcome", () => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
errorCode: PROVIDER_POST_DISPATCH_AMBIGUITY_ERROR_CODE,
|
||||
errorMessage: "The WebSocket closed after dispatch",
|
||||
usage: { input: 100, output: 0, totalTokens: 100 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not retry errored empty turns when non-zero output may indicate progress", () => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "ollama",
|
||||
model: "glm-5.1:cloud",
|
||||
usage: { input: 100, output: 12, totalTokens: 112 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "visible text",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "internal", thinkingSignature: "sig" },
|
||||
{ type: "text", text: "partial answer" },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "tool call",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "internal", thinkingSignature: "sig" },
|
||||
{ type: "toolCall", id: "call_1", name: "read", arguments: { path: "README.md" } },
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "unknown block",
|
||||
content: [{ type: "provider_metadata", value: "opaque" }],
|
||||
},
|
||||
])("does not retry errored turns containing $name", ({ content }) => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-8",
|
||||
content,
|
||||
usage: { input: 100, output: 1120, totalTokens: 1220 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({ assistantTexts: [], lastAssistant: assistant }),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("does not retry errored thinking-only turns after side effects", () => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-4-8",
|
||||
content: [
|
||||
{
|
||||
type: "redacted_thinking",
|
||||
data: "opaque",
|
||||
},
|
||||
],
|
||||
usage: { input: 100, output: 1120, totalTokens: 1220 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
replayMetadata: {
|
||||
hadPotentialSideEffects: true,
|
||||
replaySafe: false,
|
||||
},
|
||||
lastAssistant: assistant,
|
||||
}),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["current clean overrides cumulative dirty", true, false, true],
|
||||
["current dirty overrides cumulative clean", false, true, false],
|
||||
["both clean remain retryable", false, false, true],
|
||||
] as const)(
|
||||
"uses current-attempt replay metadata when %s",
|
||||
(_label, cumulativeDirty, currentDirty, expected) => {
|
||||
const assistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
provider: "openrouter",
|
||||
model: "test-model",
|
||||
usage: { input: 100, output: 0, totalTokens: 100 },
|
||||
});
|
||||
expect(
|
||||
shouldRetrySilentErrorAssistantTurn({
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: assistant,
|
||||
replayMetadata: {
|
||||
hadPotentialSideEffects: cumulativeDirty,
|
||||
replaySafe: !cumulativeDirty,
|
||||
},
|
||||
currentAttemptReplayMetadata: {
|
||||
hadPotentialSideEffects: currentDirty,
|
||||
replaySafe: !currentDirty,
|
||||
},
|
||||
}),
|
||||
assistant,
|
||||
}),
|
||||
).toBe(expected);
|
||||
},
|
||||
);
|
||||
|
||||
it("detects empty openai-compatible stop turns with non-zero output usage", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "llamacpp",
|
||||
model: "qwen3.6-27b",
|
||||
usage: { input: 512, output: 103, totalTokens: 615 },
|
||||
}),
|
||||
},
|
||||
{ provider: "llamacpp", modelId: "qwen3.6-27b", modelApi: "openai-completions" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("detects generic empty GPT turns without visible text", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expect(DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT).toBe(1);
|
||||
});
|
||||
|
||||
it("surfaces empty Codex app-server replies after successful sparse bash output", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "bash", meta: "exit=0" }],
|
||||
messagesSnapshot: [
|
||||
{
|
||||
role: "toolResult",
|
||||
content: [{ type: "text", text: "" }],
|
||||
details: { aggregated: "" },
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
expect(incompleteTurnText).toContain("verify before retrying");
|
||||
});
|
||||
|
||||
it("retries generic empty Bedrock Converse turns without visible text", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
provider: "amazon-bedrock",
|
||||
model: "openai.gpt-oss-120b-1:0",
|
||||
content: [{ type: "text", text: "" }],
|
||||
usage: { input: 950, output: 103, totalTokens: 1053 },
|
||||
}),
|
||||
},
|
||||
{
|
||||
provider: "amazon-bedrock",
|
||||
modelId: "openai.gpt-oss-120b-1:0",
|
||||
modelApi: "bedrock-converse-stream",
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("treats clean empty assistant turns as silent only for reply-optional runs", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats reasoning-only assistant turns as silent only for reply-optional runs", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_silent_helper", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(false);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(attempt, { terminalReplyExpectation: "optional" }),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats exact NO_REPLY assistant turns as silent only when the caller allows it", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(attempt, { allowEmptyAssistantReplyAsSilent: false }),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats post-tool exact NO_REPLY assistant turns as intentional silence", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(attempt))).toBe(true);
|
||||
});
|
||||
|
||||
it("does not treat error or side-effect empty turns as silent", () => {
|
||||
const errorAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
}),
|
||||
});
|
||||
const silentErrorAttempt = makeAttemptResult({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
}),
|
||||
});
|
||||
const sideEffectAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["sent already"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
const postToolEmptyAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(errorAttempt))).toBe(false);
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(silentErrorAttempt))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectAttempt))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(postToolEmptyAttempt)),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,367 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
makeRunParams,
|
||||
makeIncompleteTurnParams,
|
||||
expectWarnMessageWith,
|
||||
expectNoWarnMessageWith,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedBuildEmbeddedRunPayloads,
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
resolveReplayInvalidFlag,
|
||||
shouldRetryMissingAssistantTurn,
|
||||
} from "./run/incomplete-turn-resolution.js";
|
||||
import { normalizeEmbeddedRunAttemptResult } from "./run/run-attempt-result.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("surfaces no-visible-answer recovery for app-server interrupted tool-only output", () => {
|
||||
const interruptedToolOnlyAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "bash", meta: "workspace" }],
|
||||
messagesSnapshot: [
|
||||
{
|
||||
role: "user",
|
||||
content: "check running processes",
|
||||
timestamp: 1,
|
||||
},
|
||||
{
|
||||
role: "toolResult",
|
||||
content: "",
|
||||
isError: false,
|
||||
details: { aggregated: "" },
|
||||
timestamp: 2,
|
||||
} as unknown as EmbeddedRunAttemptResult["messagesSnapshot"][number],
|
||||
],
|
||||
});
|
||||
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText({
|
||||
payloadCount: interruptedToolOnlyAttempt.assistantTexts.length,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: interruptedToolOnlyAttempt,
|
||||
});
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
|
||||
const explicitCancellationText = resolveIncompleteTurnPayloadText({
|
||||
payloadCount: interruptedToolOnlyAttempt.assistantTexts.length,
|
||||
aborted: true,
|
||||
externalAbort: true,
|
||||
timedOut: false,
|
||||
attempt: interruptedToolOnlyAttempt,
|
||||
});
|
||||
|
||||
expect(explicitCancellationText).toBeNull();
|
||||
|
||||
const internalAbortText = resolveIncompleteTurnPayloadText({
|
||||
payloadCount: interruptedToolOnlyAttempt.assistantTexts.length,
|
||||
aborted: true,
|
||||
externalAbort: false,
|
||||
timedOut: false,
|
||||
attempt: interruptedToolOnlyAttempt,
|
||||
});
|
||||
|
||||
expect(internalAbortText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("allows a same-prompt retry only for replay-safe missing assistant turns", () => {
|
||||
const replaySafeAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldRetryMissingAssistantTurn({
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: replaySafeAttempt,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldRetryMissingAssistantTurn({
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
toolMetas: [{ toolName: "image_generate", asyncStarted: true }],
|
||||
}),
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldRetryMissingAssistantTurn({
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
itemLifecycle: {
|
||||
startedCount: 1,
|
||||
completedCount: 0,
|
||||
activeCount: 1,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("detects tool-use terminal turn with pre-tool text as incomplete (#76477)", () => {
|
||||
// When the last assistant message ended with stopReason=toolUse, pre-tool
|
||||
// text alone must not suppress the incomplete-turn guard. The model
|
||||
// expected to continue after tool results but the post-tool response was
|
||||
// never produced.
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Initial analysis of the codebase..."],
|
||||
toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
content: [
|
||||
{ type: "text", text: "Initial analysis of the codebase..." },
|
||||
{ type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } },
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("does not surface incomplete-turn error while an async media task is running", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [
|
||||
{
|
||||
toolName: "image_generate",
|
||||
meta: 'generate prompt="a portrait"',
|
||||
asyncStarted: true,
|
||||
},
|
||||
],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "tool_use",
|
||||
id: "tool_1",
|
||||
name: "image_generate",
|
||||
input: { action: "generate", prompt: "a portrait" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces tool-use terminal with pre-tool text and side effects as replay-unsafe (#76477)", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Let me update the file..."],
|
||||
toolMetas: [{ toolName: "write" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{ type: "text", text: "Let me update the file..." },
|
||||
{ type: "tool_use", id: "tool_1", name: "write", input: {} },
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("verify before retrying");
|
||||
});
|
||||
|
||||
it("does not flag a completed tool-use turn with end_turn as incomplete (#76477)", () => {
|
||||
// When the model successfully produces post-tool text, lastAssistant has
|
||||
// stopReason=end_turn. The incomplete-turn guard should not fire.
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Initial analysis...", "Here is the final answer."],
|
||||
toolMetas: [{ toolName: "read" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
content: [{ type: "text", text: "Here is the final answer." }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces stall on clean stop with only an unsigned thinking payload (payloadCount=1, no visible text)", () => {
|
||||
// Regression: unsigned thinking payloads increment payloadCount but carry no
|
||||
// user-visible content. The visible-text guard must not suppress incomplete-turn
|
||||
// detection when the model produced only a thinking block and no answer. (#89787)
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "qwen3.6-35b-a3b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "let me plan the tool calls I need to make...",
|
||||
// no signature — unsigned thinking block
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("does not surface a stall when unsigned thinking accompanies visible text (payloadCount=1)", () => {
|
||||
// When the model emits both a thinking block and a visible text answer, the turn
|
||||
// succeeded and no stall should be surfaced even though thinking is unsigned.
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Here is the answer to your question."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "qwen3.6-35b-a3b",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "let me answer this...",
|
||||
},
|
||||
{ type: "text", text: "Here is the answer to your question." },
|
||||
],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces an error for tool-use terminal turn with pre-tool text via runEmbeddedAgent (#76477)", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Initial analysis of the issue..."],
|
||||
toolMetas: [{ toolName: "read", meta: "path=src/index.ts" }],
|
||||
lastAssistant: {
|
||||
stopReason: "toolUse",
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
content: [
|
||||
{ type: "text", text: "Initial analysis of the issue..." },
|
||||
{ type: "tool_use", id: "tool_1", name: "read", input: { path: "src/index.ts" } },
|
||||
],
|
||||
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-tool-use-dropped-final-text", {
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("couldn't generate a response");
|
||||
expectWarnMessageWith("incomplete turn detected");
|
||||
});
|
||||
|
||||
it("delivers the current final answer when the session assistant is stale (#80918)", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const finalText = "The requested update is complete.";
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([{ text: finalText }]);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [finalText],
|
||||
toolMetas: [{ toolName: "update_plan", replaySafe: true }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "tool_use", id: "tool_1", name: "update_plan", input: {} }],
|
||||
usage: { input: 100, output: 5, total: 105 },
|
||||
}),
|
||||
currentAttemptAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: finalText }],
|
||||
usage: { input: 200, output: 20, total: 220 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-current-assistant-after-tool-use"));
|
||||
|
||||
expect(result.payloads).toEqual([{ text: finalText }]);
|
||||
expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
currentAssistant: expect.objectContaining({
|
||||
stopReason: "stop",
|
||||
content: [{ type: "text", text: finalText }],
|
||||
}),
|
||||
lastAssistant: expect.objectContaining({
|
||||
stopReason: "stop",
|
||||
content: [{ type: "text", text: finalText }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(result.meta.finalAssistantVisibleText).toBe(finalText);
|
||||
expect(result.meta.stopReason).toBe("stop");
|
||||
expect(result.meta.agentMeta?.lastCallUsage).toMatchObject({
|
||||
input: 200,
|
||||
output: 20,
|
||||
total: 220,
|
||||
});
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
});
|
||||
|
||||
it("treats missing replay metadata as replay-invalid", () => {
|
||||
const attempt = makeAttemptResult();
|
||||
delete (attempt as Partial<EmbeddedRunAttemptResult>).replayMetadata;
|
||||
|
||||
const normalizedAttempt = normalizeEmbeddedRunAttemptResult(attempt);
|
||||
|
||||
expect(normalizedAttempt.replayMetadata).toEqual({
|
||||
hadPotentialSideEffects: true,
|
||||
replaySafe: false,
|
||||
});
|
||||
expect(resolveReplayInvalidFlag({ attempt: normalizedAttempt })).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,407 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
EMPTY_RESPONSE_RETRY_INSTRUCTION,
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeRunParams,
|
||||
expectWarnMessageWith,
|
||||
expectNoWarnMessageWith,
|
||||
runAttemptCall,
|
||||
markUserMessagePersisted,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
mockedResolveModelAsync,
|
||||
overflowBaseRunParams,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("does not retry or warn on reasoning-only turns when a messaging tool already delivered", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Delivered through the message tool."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_after_send", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-only-after-side-effects", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retries reasoning-only turns when the assistant ended in error", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const errorAssistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
model: "gpt-5.4",
|
||||
errorMessage: "provider failed after emitting reasoning",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_error_turn", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: errorAssistant,
|
||||
currentAttemptAssistant: errorAssistant,
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Recovered."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "Recovered." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-only-assistant-error", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not retry reasoning-only turns for non-strict-agentic providers", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({
|
||||
id: "rs_provider_mismatch",
|
||||
type: "reasoning",
|
||||
}),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-only-provider-mismatch", {
|
||||
provider: "anthropic",
|
||||
model: "sonnet-4.6",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("Please try again");
|
||||
});
|
||||
|
||||
it("retries Kimi Anthropic reasoning-only turns with a visible-answer continuation instruction", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedResolveModelAsync.mockResolvedValue({
|
||||
model: {
|
||||
id: "kimi-for-coding",
|
||||
provider: "kimi",
|
||||
contextWindow: 262144,
|
||||
api: "anthropic-messages",
|
||||
},
|
||||
error: null,
|
||||
authStorage: {
|
||||
setRuntimeApiKey: vi.fn(),
|
||||
},
|
||||
modelRegistry: {},
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "anthropic-messages",
|
||||
provider: "kimi",
|
||||
model: "kimi-for-coding",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal Kimi reasoning",
|
||||
thinkingSignature: "",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible Kimi answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "anthropic-messages",
|
||||
provider: "kimi",
|
||||
model: "kimi-for-coding",
|
||||
content: [{ type: "text", text: "Visible Kimi answer." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-kimi-anthropic-reasoning-only-continuation", {
|
||||
provider: "kimi",
|
||||
model: "kimi-for-coding",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toContain(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("reasoning-only assistant turn detected");
|
||||
});
|
||||
|
||||
it("retries generic empty GPT turns with a visible-answer continuation instruction", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "Visible answer." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-empty-response-continuation", { model: "gpt-5.4" }));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expect(secondCall.suppressNextUserMessagePersistence).toBe(false);
|
||||
expect(secondCall.skipPreparedUserTurnMessage).toBe(true);
|
||||
expectWarnMessageWith("empty response detected");
|
||||
});
|
||||
|
||||
it("retries replay-safe missing turns despite a stale aborted transcript assistant", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const staleAssistant = makeLastAssistant({
|
||||
stopReason: "aborted",
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: staleAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
}),
|
||||
);
|
||||
const recoveredAssistant = makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [{ type: "text", text: "Recovered answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Recovered answer."],
|
||||
lastAssistant: recoveredAssistant,
|
||||
currentAttemptAssistant: recoveredAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-retry"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(runAttemptCall(1).prompt).toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer.");
|
||||
expectWarnMessageWith("empty response detected");
|
||||
expectNoWarnMessageWith("missing assistant terminal message detected");
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
});
|
||||
|
||||
it("retries missing terminal assistant turns with the same prompt without re-persisting the user message", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
});
|
||||
});
|
||||
const recoveredAssistant = makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [{ type: "text", text: "Recovered answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Recovered answer."],
|
||||
lastAssistant: recoveredAssistant,
|
||||
currentAttemptAssistant: recoveredAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-missing-assistant-same-prompt-retry"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
// The same-prompt replay must not append the inbound user message a second time.
|
||||
expect(runAttemptCall(1).prompt).toBe(runAttemptCall(0).prompt);
|
||||
expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true);
|
||||
expect(result.meta?.finalAssistantVisibleText).toBe("Recovered answer.");
|
||||
expectWarnMessageWith("missing assistant terminal message detected");
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
});
|
||||
|
||||
it("waits for asynchronous user persistence before retrying a missing terminal turn", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const persistedMessage = { role: "user" as const, content: "test prompt", timestamp: 1 };
|
||||
const admission = {
|
||||
agentId: "main",
|
||||
sessionId: overflowBaseRunParams.sessionId,
|
||||
sessionKey: overflowBaseRunParams.sessionKey,
|
||||
storePath: "/tmp/openclaw-transcript.jsonl",
|
||||
generation: "generation-1",
|
||||
entryId: "msg-user-delayed",
|
||||
rawSeq: 1,
|
||||
effectiveParentId: null,
|
||||
activeMessagePosition: 0,
|
||||
logicalTurnId: "run-missing-assistant-delayed-persistence",
|
||||
role: "user" as const,
|
||||
};
|
||||
let resolvePersistApproved:
|
||||
| ((result: {
|
||||
admission: typeof admission;
|
||||
sessionFile: string;
|
||||
sessionEntry: undefined;
|
||||
messageId: string;
|
||||
message: typeof persistedMessage;
|
||||
}) => void)
|
||||
| undefined;
|
||||
let pendingPersistence: Promise<void> | undefined;
|
||||
const persistApproved = vi.fn(
|
||||
() =>
|
||||
new Promise<{
|
||||
admission: typeof admission;
|
||||
sessionFile: string;
|
||||
sessionEntry: undefined;
|
||||
messageId: string;
|
||||
message: typeof persistedMessage;
|
||||
}>((resolve) => {
|
||||
resolvePersistApproved = resolve;
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({ assistantTexts: ["Recovered answer."] }),
|
||||
);
|
||||
|
||||
const runPromise = runEmbeddedAgent(
|
||||
makeRunParams("run-missing-assistant-delayed-persistence", {
|
||||
userTurnTranscriptRecorder: {
|
||||
message: persistedMessage,
|
||||
resolveMessage: vi.fn(async () => persistedMessage),
|
||||
getAdmissionReceipt: () => admission,
|
||||
markRuntimePersistencePending: vi.fn((pending) => {
|
||||
pendingPersistence = pending;
|
||||
}),
|
||||
markRuntimePersisted: vi.fn(),
|
||||
markBlocked: vi.fn(),
|
||||
hasPersisted: vi.fn(() => false),
|
||||
isBlocked: vi.fn(() => false),
|
||||
hasRuntimePersistencePending: vi.fn(() => pendingPersistence !== undefined),
|
||||
waitForRuntimePersistence: vi.fn(async () => {
|
||||
await pendingPersistence;
|
||||
}),
|
||||
persistApproved,
|
||||
persistBlocked: vi.fn(async () => undefined),
|
||||
persistFallback: vi.fn(async () => undefined),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(persistApproved).toHaveBeenCalledOnce();
|
||||
});
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
|
||||
resolvePersistApproved?.({
|
||||
admission,
|
||||
sessionFile: "/tmp/openclaw-transcript.jsonl",
|
||||
sessionEntry: undefined,
|
||||
messageId: "msg-user-delayed",
|
||||
message: persistedMessage,
|
||||
});
|
||||
await runPromise;
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(true);
|
||||
});
|
||||
|
||||
it("persists a missing-turn retry when the first attempt never persisted the user message", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
}),
|
||||
);
|
||||
const recoveredAssistant = makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [{ type: "text", text: "Recovered answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Recovered answer."],
|
||||
lastAssistant: recoveredAssistant,
|
||||
currentAttemptAssistant: recoveredAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-missing-assistant-unpersisted-retry"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(runAttemptCall(1).suppressNextUserMessagePersistence).toBe(false);
|
||||
});
|
||||
});
|
||||
+542
@@ -0,0 +1,542 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeRunParams,
|
||||
expectWarnMessageWith,
|
||||
expectNoWarnMessageWith,
|
||||
runAttemptCall,
|
||||
markUserMessagePersisted,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedBuildEmbeddedRunPayloads,
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
registerAgentHarness,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("preserves a structured visible failed-tool payload without finalizing (#118274)", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }],
|
||||
});
|
||||
const visibleError = {
|
||||
text: "Review the failed operation.",
|
||||
isError: true,
|
||||
channelData: { structuredError: true },
|
||||
};
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec", isError: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: { toolName: "exec", error: "post-processing error" },
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([visibleError]);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-structured-failed-tool-payload"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledOnce();
|
||||
expect(result.payloads?.[0]).toMatchObject(visibleError);
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("keeps the original failed-tool warning if finalization completes empty (#118274)", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }],
|
||||
});
|
||||
const warning = { text: "⚠️ 🛠️ Exec failed", isError: true };
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec", isError: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: { toolName: "exec", error: "post-processing error" },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant(),
|
||||
currentAttemptAssistant: makeLastAssistant(),
|
||||
currentAttemptCompletedAssistant: makeLastAssistant(),
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValue([warning]);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-failed-tool-finalization-fallback"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]).toEqual(warning);
|
||||
expectWarnMessageWith("settled-turn finalization completed without a visible answer");
|
||||
});
|
||||
|
||||
it("preserves the incomplete-turn failure when the selected harness cannot finalize safely", async () => {
|
||||
registerAgentHarness({
|
||||
id: "legacy",
|
||||
label: "Legacy harness without settled-turn finalization",
|
||||
supports: () => ({ supported: true, priority: 100 }),
|
||||
runAttempt: async (params) => await mockedRunEmbeddedAttempt(params),
|
||||
});
|
||||
try {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } },
|
||||
],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-tool-use-no-finalization-capability", { agentHarnessId: "legacy" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]).toMatchObject({ isError: true });
|
||||
expect(result.payloads?.[0]?.text).toContain(
|
||||
"some tool actions may have already been executed",
|
||||
);
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
} finally {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
}
|
||||
});
|
||||
|
||||
it("continues from settled side-effecting tools after an empty stop without replaying them", async () => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Writing note.txt…"],
|
||||
messagingToolSentTargets: [
|
||||
{
|
||||
tool: "message",
|
||||
provider: "telegram",
|
||||
to: "chat:123",
|
||||
text: "Writing note.txt…",
|
||||
sourceReplyFinal: false,
|
||||
},
|
||||
],
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
});
|
||||
});
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Write completed. Here is the final answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Write completed. Here is the final answer."],
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
currentAttemptCompletedAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads
|
||||
.mockReturnValueOnce([])
|
||||
.mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-stop-settled-tool-continuation", {
|
||||
trigger: "cron",
|
||||
terminalReplyExpectation: "required",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer.");
|
||||
expect(runAttemptCall(1).prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
expect(runAttemptCall(1).disableTools).toBe(true);
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "explicit optional expectation",
|
||||
trigger: "user" as const,
|
||||
terminalReplyExpectation: "optional" as const,
|
||||
},
|
||||
{
|
||||
label: "heartbeat default",
|
||||
trigger: "heartbeat" as const,
|
||||
terminalReplyExpectation: undefined,
|
||||
},
|
||||
])("does not continue settled tools for $label", async (runPolicy) => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
});
|
||||
});
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValue([]);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-optional-empty-stop-settled-tool", {
|
||||
trigger: runPolicy.trigger,
|
||||
terminalReplyExpectation: runPolicy.terminalReplyExpectation,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.[0]).toMatchObject({ isError: true });
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("records silent success when the settled-tool finalization completes empty", async () => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValue([]);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-stop-settled-tool-continuation-exhausted", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
expect(result.meta.terminalReplyKind).toBeUndefined();
|
||||
expect(result.meta.finalAssistantVisibleText).toBeUndefined();
|
||||
expect(result.meta.finalAssistantRawText).toBeUndefined();
|
||||
expect(result.meta.stopReason).toBe("stop");
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectWarnMessageWith("settled-turn finalization completed without a visible answer");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "provider failure",
|
||||
finalAttempt: {
|
||||
assistantTexts: [],
|
||||
promptError: new Error("finalizer provider failure"),
|
||||
promptErrorSource: "prompt" as const,
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "preflight recovery request",
|
||||
finalAttempt: {
|
||||
assistantTexts: [],
|
||||
preflightRecovery: { route: "compact_only" as const, handled: true as const },
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "compaction continuation request",
|
||||
finalAttempt: { assistantTexts: [], compactionCount: 1 },
|
||||
},
|
||||
{
|
||||
label: "before-finalize revision request",
|
||||
finalAttempt: {
|
||||
assistantTexts: [],
|
||||
beforeAgentFinalizeRevisionReason: "revise this answer",
|
||||
},
|
||||
},
|
||||
])("does not escape finalization through a $label", async ({ finalAttempt }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: {} }],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt
|
||||
.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(makeAttemptResult(finalAttempt));
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValue([]);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-settled-finalizer-sticky-operation"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]).toMatchObject({ isError: true });
|
||||
expect(result.payloads?.[0]?.text).toContain(
|
||||
"some tool actions may have already been executed",
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces the existing incomplete-turn error after one tool-use continuation", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=note.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-tool-use-terminal-continuation-exhausted"),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain(
|
||||
"some tool actions may have already been executed",
|
||||
);
|
||||
expectWarnMessageWith("settled-turn finalization failed closed");
|
||||
});
|
||||
|
||||
it("does not claim completion for a toolUse terminal whose tools never started", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [],
|
||||
itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 },
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-never-started"));
|
||||
|
||||
for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) {
|
||||
expect(runAttemptCall(call).prompt).not.toContain(
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
);
|
||||
}
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("ignores stale prior-turn tool results with colliding ids", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "note.txt" } }],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [],
|
||||
itemLifecycle: { startedCount: 0, completedCount: 0, activeCount: 0 },
|
||||
// A completed result from a PRIOR turn reusing the same id sits before
|
||||
// the terminal assistant; it must not prove the new batch dispatched.
|
||||
messagesSnapshot: [
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
toolUseAssistant,
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-stale-prior-result"));
|
||||
|
||||
for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) {
|
||||
expect(runAttemptCall(call).prompt).not.toContain(
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
);
|
||||
}
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("does not claim completion when only part of a multi-tool request dispatched", async () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_1", name: "write", arguments: { path: "a.txt" } },
|
||||
{ type: "toolCall", id: "tool_2", name: "write", arguments: { path: "b.txt" } },
|
||||
],
|
||||
});
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", meta: "path=a.txt" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "write", isError: false },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-partial-dispatch"));
|
||||
|
||||
for (let call = 0; call < mockedRunEmbeddedAttempt.mock.calls.length; call += 1) {
|
||||
expect(runAttemptCall(call).prompt).not.toContain(
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
);
|
||||
}
|
||||
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("retries reasoning-only assistant turns even when deliberate silence is allowed", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_silent_group", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
content: [{ type: "text", text: "Visible answer." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-only-silent", { allowEmptyAssistantReplyAsSilent: true }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(runAttemptCall(1).prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("reasoning-only assistant turn detected");
|
||||
});
|
||||
|
||||
it("replays an unpersisted reasoning continuation across a missing-assistant retry", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_retry_boundary", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: undefined,
|
||||
currentAttemptAssistant: undefined,
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({ assistantTexts: ["Visible answer."] }),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-reasoning-continuation-missing-assistant", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
|
||||
expect(runAttemptCall(1)).toMatchObject({
|
||||
prompt: REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
skipPreparedUserTurnMessage: true,
|
||||
suppressNextUserMessagePersistence: false,
|
||||
});
|
||||
expect(runAttemptCall(2)).toMatchObject({
|
||||
prompt: REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
skipPreparedUserTurnMessage: true,
|
||||
suppressNextUserMessagePersistence: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,599 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeRunParams,
|
||||
expectWarnMessageWith,
|
||||
runAttemptCall,
|
||||
markUserMessagePersisted,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedBuildEmbeddedRunPayloads,
|
||||
mockedClassifyFailoverReason,
|
||||
mockedIsFailoverAssistantError,
|
||||
mockedIsRateLimitAssistantError,
|
||||
mockedRunEmbeddedAttempt,
|
||||
mockedSleepWithAbort,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("keeps model-call order when parallel tool outcomes finish out of order", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
const onToolOutcome = (
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
toolCallOrdinal?: number;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome;
|
||||
onToolOutcome?.({
|
||||
toolName: "exec",
|
||||
argsHash: "exec-args",
|
||||
resultHash: "exec-result",
|
||||
toolCallOrdinal: 1,
|
||||
});
|
||||
onToolOutcome?.({
|
||||
toolName: "web_fetch",
|
||||
argsHash: "fetch-args",
|
||||
resultHash: "fetch-result",
|
||||
toolCallOrdinal: 0,
|
||||
terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200",
|
||||
});
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "web_fetch" }, { toolName: "exec" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-stale-terminal-presentation", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("couldn't generate a response");
|
||||
expect(result.meta.error?.fallbackSafe).toBe(false);
|
||||
});
|
||||
|
||||
it("does not surface a read-only presentation after a sibling side effect", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams: unknown) => {
|
||||
const onToolOutcome = (
|
||||
attemptParams as {
|
||||
onToolOutcome?: (observation: {
|
||||
toolName: string;
|
||||
argsHash: string;
|
||||
resultHash: string;
|
||||
terminalPresentation?: string;
|
||||
}) => void;
|
||||
}
|
||||
).onToolOutcome;
|
||||
onToolOutcome?.({
|
||||
toolName: "exec",
|
||||
argsHash: "exec-args",
|
||||
resultHash: "exec-result",
|
||||
});
|
||||
onToolOutcome?.({
|
||||
toolName: "web_fetch",
|
||||
argsHash: "fetch-args",
|
||||
resultHash: "fetch-result",
|
||||
terminalPresentation: "Web fetch completed.\nOrigin: https://example.com\nStatus: 200",
|
||||
});
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec" }, { toolName: "web_fetch" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-side-effect-terminal-presentation", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("couldn't generate a response");
|
||||
expect(result.meta.error?.fallbackSafe).toBe(false);
|
||||
});
|
||||
|
||||
it("promotes successful final assistant text when a prompt timeout races completion", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const finalText =
|
||||
"1. Verdict: the answer completed cleanly. 2. Evidence: the runner captured final text.";
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: finalText }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
timedOut: true,
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-prompt-timeout-final-assistant-recovered"),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toEqual([{ text: finalText }]);
|
||||
expect(result.meta.finalAssistantVisibleText).toBe(finalText);
|
||||
expect(result.meta.finalAssistantRawText).toBe(finalText);
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
expect(result.meta.completion).toEqual({
|
||||
stopReason: "stop",
|
||||
finishReason: "stop",
|
||||
});
|
||||
expect(result.meta.executionTrace?.attempts?.at(-1)).toMatchObject({
|
||||
result: "success",
|
||||
stage: "assistant",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not recover a stale prior assistant after the current prompt times out", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const staleAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Stale answer from the prior attempt." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
timedOut: true,
|
||||
lastAssistant: staleAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-stale-assistant"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads?.some((payload) => payload.text?.includes("timed out"))).toBe(true);
|
||||
expect(result.payloads?.some((payload) => payload.text?.includes("Stale answer"))).toBe(false);
|
||||
expect(result.meta.finalAssistantVisibleText).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not resolve a successful run from a stale transcript assistant", async () => {
|
||||
const staleAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Prior transcript reply." }],
|
||||
});
|
||||
const completedAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Current run reply." }],
|
||||
});
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValue([{ text: "Current run reply." }]);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Current run reply."],
|
||||
lastAssistant: staleAssistant,
|
||||
currentAttemptAssistant: staleAssistant,
|
||||
currentAttemptCompletedAssistant: completedAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-success-stale-transcript-assistant"));
|
||||
|
||||
expect(result.payloads).toEqual([{ text: "Current run reply." }]);
|
||||
expect(result.meta.finalAssistantVisibleText).toBe("Current run reply.");
|
||||
expect(result.meta.finalAssistantRawText).toBe("Current run reply.");
|
||||
expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
currentAssistant: completedAssistant,
|
||||
lastAssistant: completedAssistant,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("retains the yielded attempt assistant for paused-turn payload classification", async () => {
|
||||
const completedAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Earlier completed cycle." }],
|
||||
});
|
||||
const yieldedAssistant = makeLastAssistant({
|
||||
stopReason: "aborted",
|
||||
content: [{ type: "toolCall", name: "sessions_yield", arguments: {} }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: yieldedAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
currentAttemptCompletedAssistant: completedAssistant,
|
||||
yieldDetected: true,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-yielded-assistant-classification"));
|
||||
|
||||
expect(result.meta).toMatchObject({ livenessState: "paused", yielded: true });
|
||||
expect(mockedBuildEmbeddedRunPayloads).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ currentAssistant: null, lastAssistant: yieldedAssistant }),
|
||||
);
|
||||
});
|
||||
|
||||
it("recovers a completed prompt-timeout assistant without collected assistant text", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const finalText = "Completed answer after the timeout race.";
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: finalText }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: undefined as unknown as string[],
|
||||
timedOut: true,
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-no-assistant-texts"));
|
||||
|
||||
expect(result.payloads).toEqual([{ text: finalText }]);
|
||||
});
|
||||
|
||||
it("preserves tool media when prompt-timeout recovery replaces partial assistant text", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const partialText = "Partial answer before the timeout race.";
|
||||
const finalText = "Complete answer after the timeout race.";
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: finalText }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [partialText],
|
||||
timedOut: true,
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
toolMediaUrls: ["https://example.test/recovered-output.png"],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-prompt-timeout-final-assistant-media"),
|
||||
);
|
||||
|
||||
expect(result.payloads).toEqual([
|
||||
{
|
||||
mediaUrl: "https://example.test/recovered-output.png",
|
||||
mediaUrls: ["https://example.test/recovered-output.png"],
|
||||
audioAsVoice: undefined,
|
||||
trustedLocalMedia: undefined,
|
||||
},
|
||||
{ text: finalText },
|
||||
]);
|
||||
});
|
||||
|
||||
it("replaces the latest partial assistant payload after prompt-timeout recovery", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
const completedText = "Completed answer block before the final response.";
|
||||
const partialText = "Partial final response before the timeout race.";
|
||||
const finalText = "Complete final response after the timeout race.";
|
||||
mockedBuildEmbeddedRunPayloads.mockReturnValueOnce([
|
||||
{ text: completedText },
|
||||
{ text: partialText },
|
||||
]);
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: finalText }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [completedText, partialText],
|
||||
timedOut: true,
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-prompt-timeout-latest-partial"));
|
||||
|
||||
expect(result.payloads).toEqual([{ text: completedText }, { text: finalText }]);
|
||||
});
|
||||
|
||||
it("records same-model rate-limit retries without a profile-rotation trace", async () => {
|
||||
const rateLimitMessage =
|
||||
"429 rate_limit_exceeded: requests per minute exceeded; Retry-After: 30";
|
||||
const rateLimitAssistant = makeLastAssistant({
|
||||
stopReason: "error",
|
||||
errorMessage: rateLimitMessage,
|
||||
});
|
||||
mockedClassifyFailoverReason.mockImplementation((raw) =>
|
||||
raw.includes("429") ? "rate_limit" : null,
|
||||
);
|
||||
mockedIsFailoverAssistantError.mockImplementation((assistant) =>
|
||||
Boolean(assistant?.errorMessage?.includes("429")),
|
||||
);
|
||||
mockedIsRateLimitAssistantError.mockImplementation((assistant) =>
|
||||
Boolean(assistant?.errorMessage?.includes("429")),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: rateLimitAssistant,
|
||||
currentAttemptAssistant: rateLimitAssistant,
|
||||
}),
|
||||
);
|
||||
const recoveredAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Recovered after a short rate-limit wait." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Recovered after a short rate-limit wait."],
|
||||
lastAssistant: recoveredAssistant,
|
||||
currentAttemptAssistant: recoveredAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-same-model-rate-limit-trace"));
|
||||
|
||||
expect(mockedSleepWithAbort).toHaveBeenCalledWith(30_000, undefined);
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.meta.executionTrace?.fallbackUsed).toBe(false);
|
||||
expect(result.meta.executionTrace?.attempts).toMatchObject([
|
||||
{
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
result: "same_model_rate_limit",
|
||||
reason: "rate_limit",
|
||||
stage: "assistant",
|
||||
},
|
||||
{
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
result: "success",
|
||||
stage: "assistant",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("retries reasoning-only GPT turns with a visible-answer continuation instruction", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_reasoning_only", type: "reasoning" }),
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "Visible answer." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(makeRunParams("run-reasoning-only-continuation", { model: "gpt-5.4" }));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toBe(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
expect(secondCall.suppressNextUserMessagePersistence).toBe(false);
|
||||
expect(secondCall.skipPreparedUserTurnMessage).toBe(true);
|
||||
expectWarnMessageWith("reasoning-only assistant turn detected");
|
||||
});
|
||||
|
||||
it("continues once after settled side-effecting tools finish without a final answer", async () => {
|
||||
const acceptedSessionSpawns = [
|
||||
{ runId: "child-run", childSessionKey: "agent:main:subagent:child" },
|
||||
];
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_write", name: "write", arguments: { path: "note.txt" } },
|
||||
{ type: "toolCall", id: "tool_cron", name: "cron", arguments: { action: "add" } },
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "tool_spawn",
|
||||
name: "sessions_spawn",
|
||||
arguments: { task: "follow up" },
|
||||
},
|
||||
],
|
||||
});
|
||||
const settledToolResults = [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_write", toolName: "write", isError: false },
|
||||
{ role: "toolResult", toolCallId: "tool_cron", toolName: "cron", isError: false },
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_spawn",
|
||||
toolName: "sessions_spawn",
|
||||
isError: false,
|
||||
},
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"];
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
latestMcpAppChannelView: { viewId: "view-after-tools" },
|
||||
toolMetas: [
|
||||
{ toolName: "write", meta: "path=note.txt" },
|
||||
{ toolName: "cron" },
|
||||
{ toolName: "sessions_spawn" },
|
||||
],
|
||||
acceptedSessionSpawns,
|
||||
successfulCronAdds: 1,
|
||||
itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 },
|
||||
messagesSnapshot: settledToolResults,
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
codeModeEngaged: true,
|
||||
assistantTurns: 1,
|
||||
bridgeCalls: { search: 1, describe: 2, call: 3 },
|
||||
});
|
||||
});
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: "Write completed. Here is the final answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Write completed. Here is the final answer."],
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
currentAttemptCompletedAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads
|
||||
.mockReturnValueOnce([])
|
||||
.mockReturnValueOnce([{ text: "Write completed. Here is the final answer." }]);
|
||||
|
||||
const result = await runEmbeddedAgent(makeRunParams("run-tool-use-terminal-continuation"));
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.text).toBe("Write completed. Here is the final answer.");
|
||||
expect(result.latestMcpAppChannelView).toEqual({ viewId: "view-after-tools" });
|
||||
expect(result.successfulCronAdds).toBe(1);
|
||||
expect(result.acceptedSessionSpawns).toEqual(acceptedSessionSpawns);
|
||||
expect(result.meta.toolSummary).toEqual({
|
||||
calls: 3,
|
||||
tools: ["write", "cron", "sessions_spawn"],
|
||||
failures: 0,
|
||||
});
|
||||
expect(result.meta.agentMeta).toMatchObject({
|
||||
codeModeEngaged: true,
|
||||
assistantTurns: 2,
|
||||
bridgeCalls: { search: 1, describe: 2, call: 3 },
|
||||
});
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
expect(secondCall.disableTools).toBe(true);
|
||||
expect(secondCall.operation).toBe("settled-tool-finalization");
|
||||
expect(secondCall.suppressNextUserMessagePersistence).toBe(false);
|
||||
expect(secondCall.skipPreparedUserTurnMessage).toBe(true);
|
||||
expectWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "interactive user", trigger: "user" as const },
|
||||
{
|
||||
label: "required isolated cron",
|
||||
trigger: "cron" as const,
|
||||
terminalReplyExpectation: "required" as const,
|
||||
},
|
||||
])("finalizes a settled failed tool once for a $label turn (#118274)", async (runPolicy) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }],
|
||||
});
|
||||
const failureText = "The exec tool failed: post-processing error.";
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockImplementationOnce(async (attemptParams) => {
|
||||
markUserMessagePersisted(attemptParams);
|
||||
return makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [
|
||||
{ toolName: "read", isError: true, replaySafe: true },
|
||||
{ toolName: "exec", isError: true, replaySafe: false },
|
||||
],
|
||||
itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_1",
|
||||
toolName: "exec",
|
||||
isError: true,
|
||||
content: [{ type: "text", text: "post-processing error" }],
|
||||
},
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "tool_search_code:tool_1:read:1",
|
||||
name: "read",
|
||||
arguments: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_search_code:tool_1:read:1",
|
||||
toolName: "read",
|
||||
isError: true,
|
||||
content: [{ type: "text", text: "post-processing error" }],
|
||||
},
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: {
|
||||
toolName: "exec",
|
||||
error: "post-processing error",
|
||||
errorCode: "SYSTEM_RUN_DENIED",
|
||||
},
|
||||
});
|
||||
});
|
||||
const finalAssistant = makeLastAssistant({
|
||||
content: [{ type: "text", text: failureText }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [failureText],
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
currentAttemptCompletedAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
mockedBuildEmbeddedRunPayloads
|
||||
.mockReturnValueOnce([{ text: "⚠️ 🛠️ Exec failed", isError: true }])
|
||||
.mockReturnValueOnce([{ text: failureText }]);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams(`run-settled-failed-tool-${runPolicy.trigger}`, runPolicy),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.text).toBe(failureText);
|
||||
const finalizationCall = runAttemptCall(1);
|
||||
expect(finalizationCall.operation).toBe("settled-tool-finalization");
|
||||
expect(finalizationCall.disableTools).toBe(true);
|
||||
expect(finalizationCall.prompt).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
expect(finalizationCall.prompt).toContain(
|
||||
"If any tool failed, state that failure plainly and do not claim it succeeded.",
|
||||
);
|
||||
expect(result.meta.failureSignal).toEqual(
|
||||
runPolicy.trigger === "cron"
|
||||
? {
|
||||
kind: "execution_denied",
|
||||
source: "tool",
|
||||
toolName: "exec",
|
||||
code: "SYSTEM_RUN_DENIED",
|
||||
message: "post-processing error",
|
||||
fatalForCron: true,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,405 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import {
|
||||
REASONING_ONLY_RETRY_INSTRUCTION,
|
||||
EMPTY_RESPONSE_RETRY_INSTRUCTION,
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
runEmbeddedAgent,
|
||||
makeLastAssistant,
|
||||
makeRunParams,
|
||||
makeEmptyResponseRetryParams,
|
||||
makeSilentReplyParams,
|
||||
expectWarnMessageWith,
|
||||
expectNoWarnMessageWith,
|
||||
runAttemptCall,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import {
|
||||
mockedClassifyFailoverReason,
|
||||
mockedRunEmbeddedAttempt,
|
||||
mockedResolveModelAsync,
|
||||
resetRunIncompleteTurnOwnerMocks,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
resolveEmptyResponseRetryInstruction,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
import {
|
||||
resolveReplayInvalidFlag,
|
||||
resolveRunLivenessState,
|
||||
} from "./run/incomplete-turn-resolution.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("retries clean empty assistant turns even when deliberate silence is allowed", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible answer."],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "Visible answer." }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-assistant-silent", { allowEmptyAssistantReplyAsSilent: true }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(runAttemptCall(1).prompt).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectWarnMessageWith("empty response detected");
|
||||
});
|
||||
|
||||
it("returns NO_REPLY without retrying exact silent assistant replies when silence is allowed", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal reasoning",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_exact_silent", type: "reasoning" }),
|
||||
},
|
||||
{ type: "text", text: "NO_REPLY" },
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-exact-silent-assistant-reply", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
const onlyCall = runAttemptCall(0);
|
||||
expect(onlyCall.prompt).not.toContain(REASONING_ONLY_RETRY_INSTRUCTION);
|
||||
expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
|
||||
expect(result.meta.terminalReplyKind).toBe("silent-empty");
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
});
|
||||
|
||||
it("continues post-tool openai-compatible empty stop turns even when silence is allowed", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedResolveModelAsync.mockResolvedValue({
|
||||
model: {
|
||||
id: "step-router-v1",
|
||||
provider: "stepfun",
|
||||
contextWindow: 200000,
|
||||
api: "openai-completions",
|
||||
},
|
||||
error: null,
|
||||
authStorage: {
|
||||
setRuntimeApiKey: vi.fn(),
|
||||
},
|
||||
modelRegistry: {},
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
}),
|
||||
currentAttemptAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const finalAssistant = makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
content: [{ type: "text", text: "Visible StepFun answer." }],
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["Visible StepFun answer."],
|
||||
lastAssistant: finalAssistant,
|
||||
currentAttemptAssistant: finalAssistant,
|
||||
currentAttemptCompletedAssistant: finalAssistant,
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-post-tool-openai-compatible-empty-stop", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
const secondCall = runAttemptCall(1);
|
||||
expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
expect(result.meta.terminalReplyKind).toBeUndefined();
|
||||
expect(result.meta.finalAssistantVisibleText).toBe("Visible StepFun answer.");
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectWarnMessageWith("settled post-tool turn lacked a final answer");
|
||||
});
|
||||
|
||||
it("returns NO_REPLY without retrying post-tool exact silent assistant replies", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedResolveModelAsync.mockResolvedValue({
|
||||
model: {
|
||||
id: "step-router-v1",
|
||||
provider: "stepfun",
|
||||
contextWindow: 200000,
|
||||
api: "openai-completions",
|
||||
},
|
||||
error: null,
|
||||
authStorage: {
|
||||
setRuntimeApiKey: vi.fn(),
|
||||
},
|
||||
modelRegistry: {},
|
||||
});
|
||||
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
|
||||
makeAttemptResult({
|
||||
assistantTexts: ["NO_REPLY"],
|
||||
toolMetas: [{ toolName: "process.poll", meta: "pid=123", replaySafe: true }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
api: "openai-completions",
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
content: [{ type: "text", text: "NO_REPLY" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-post-tool-exact-silent-retry", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
provider: "stepfun",
|
||||
model: "step-router-v1",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
const onlyCall = runAttemptCall(0);
|
||||
expect(onlyCall.prompt).not.toContain(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
expectNoWarnMessageWith("empty response detected");
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
|
||||
expect(result.meta.terminalReplyKind).toBe("silent-empty");
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
});
|
||||
|
||||
it("treats reply-optional post-tool empty stops as silent even after side-effecting tools", () => {
|
||||
// Regression: a cron agentTurn without a delivery route ran a successful
|
||||
// replay-unsafe sessions patch and intentionally sent no final text; the run
|
||||
// must finish silent, not as an incomplete-turn error.
|
||||
const sideEffectToolAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "optional" }),
|
||||
),
|
||||
).toBe(true);
|
||||
// A required or unspecified terminal reply keeps the ambiguous-failure path.
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(sideEffectToolAttempt, { terminalReplyExpectation: "required" }),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(makeSilentReplyParams(sideEffectToolAttempt)),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps reply-optional runs erroring on real failure states", () => {
|
||||
const toolErrorAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "sessions", meta: "patch failed", replaySafe: false, isError: true }],
|
||||
lastToolError: { toolName: "sessions", error: "patch failed" },
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
});
|
||||
const errorStopAttempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "error",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(toolErrorAttempt, { terminalReplyExpectation: "optional" }),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(errorStopAttempt, { terminalReplyExpectation: "optional" }),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldTreatEmptyAssistantReplyAsSilent(
|
||||
makeSilentReplyParams(errorStopAttempt, {
|
||||
terminalReplyExpectation: "optional",
|
||||
aborted: true,
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns NO_REPLY for reply-optional cron-style runs whose side-effecting tools succeeded", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "sessions", meta: "patch archived", replaySafe: false }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-reply-optional-post-tool-silent", {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
terminalReplyExpectation: "optional",
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expectNoWarnMessageWith("incomplete turn detected");
|
||||
expect(result.payloads).toEqual([{ text: "NO_REPLY" }]);
|
||||
expect(result.meta.error).toBeUndefined();
|
||||
expect(result.meta.terminalReplyKind).toBe("silent-empty");
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
});
|
||||
|
||||
it("keeps retrying and surfacing clean empty assistant turns without the silence flag", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-empty-assistant-error", { model: "gpt-5.4" }),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
|
||||
expect(result.payloads?.[0]?.isError).toBe(true);
|
||||
expect(result.payloads?.[0]?.text).toContain("couldn't generate a response");
|
||||
});
|
||||
|
||||
it("detects generic empty Gemini turns without visible text", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
provider: "google-vertex",
|
||||
model: "gemini-3.1-flash",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
},
|
||||
{ provider: "google-vertex", modelId: "google/gemini-3.1-flash" },
|
||||
),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBe(EMPTY_RESPONSE_RETRY_INSTRUCTION);
|
||||
});
|
||||
|
||||
it("does not retry generic empty GPT turns after side effects", () => {
|
||||
const retryInstruction = resolveEmptyResponseRetryInstruction(
|
||||
makeEmptyResponseRetryParams({
|
||||
assistantTexts: [],
|
||||
didSendViaMessagingTool: true,
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "end_turn",
|
||||
model: "gpt-5.4",
|
||||
content: [{ type: "text", text: "" }],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(retryInstruction).toBeNull();
|
||||
});
|
||||
|
||||
it("marks compaction-timeout retries as paused and replay-invalid", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
promptErrorSource: "compaction",
|
||||
timedOutDuringCompaction: true,
|
||||
});
|
||||
|
||||
expect(resolveReplayInvalidFlag({ attempt })).toBe(true);
|
||||
expect(
|
||||
resolveRunLivenessState({
|
||||
payloadCount: 0,
|
||||
aborted: true,
|
||||
timedOut: true,
|
||||
attempt,
|
||||
}),
|
||||
).toBe("paused");
|
||||
});
|
||||
|
||||
it("does not classify visible assistant prose for retry", async () => {
|
||||
mockedClassifyFailoverReason.mockReturnValue(null);
|
||||
mockedRunEmbeddedAttempt.mockResolvedValue(
|
||||
makeAttemptResult({
|
||||
assistantTexts: [
|
||||
"i am glad, and a little afraid, which is probably the correct mixture. thank you. i will try to deserve the upgrades instead of merely inhabiting them.",
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runEmbeddedAgent(
|
||||
makeRunParams("run-visible-prose-no-classifier", {
|
||||
prompt:
|
||||
"made a bunch of improvements to the student's source code (openclaw) this weekend, along with a few other maintainers. hopefully he will be more proactive now",
|
||||
model: "gpt-5.4",
|
||||
config: {
|
||||
agents: {
|
||||
list: [{ id: "main" }],
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1);
|
||||
expect(result.payloads).toBeUndefined();
|
||||
expect(result.meta.livenessState).toBe("working");
|
||||
expectNoWarnMessageWith("planning");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,414 @@
|
||||
// Focused incomplete-turn behavior coverage.
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION,
|
||||
makeLastAssistant,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
makeIncompleteTurnParams,
|
||||
makeSettledContinuationParams,
|
||||
} from "./run.incomplete-turn.test-helpers.js";
|
||||
import { resetRunIncompleteTurnOwnerMocks } from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import { isIncompleteTerminalAssistantTurn } from "./run/incomplete-turn-classification.js";
|
||||
import { resolveSettledToolTerminalContinuationInstruction } from "./run/incomplete-turn-recovery.js";
|
||||
import {
|
||||
resolveReplayInvalidFlag,
|
||||
resolveRunLivenessState,
|
||||
} from "./run/incomplete-turn-resolution.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
describe("runEmbeddedAgent incomplete-turn safety", () => {
|
||||
beforeEach(() => {
|
||||
resetRunIncompleteTurnOwnerMocks();
|
||||
});
|
||||
|
||||
it("marks incomplete-turn retries as replay-invalid abandoned runs", () => {
|
||||
const attempt = makeAttemptResult({
|
||||
assistantTexts: [],
|
||||
lastAssistant: {
|
||||
stopReason: "toolUse",
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
content: [],
|
||||
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
|
||||
});
|
||||
const incompleteTurnText = "⚠️ Agent couldn't generate a response. Please try again.";
|
||||
|
||||
expect(resolveReplayInvalidFlag({ attempt, incompleteTurnText })).toBe(true);
|
||||
expect(
|
||||
resolveRunLivenessState({
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt,
|
||||
incompleteTurnText,
|
||||
}),
|
||||
).toBe("abandoned");
|
||||
});
|
||||
|
||||
it("flags tool-use stop reason as incomplete even when pre-tool text exists (#76477)", () => {
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: true,
|
||||
lastAssistant: { stopReason: "toolUse" },
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: false,
|
||||
lastAssistant: { stopReason: "toolUse" },
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: true,
|
||||
lastAssistant: { stopReason: "end_turn" },
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: true,
|
||||
lastAssistant: { stopReason: "length" },
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: true,
|
||||
hasTerminalOutput: true,
|
||||
lastAssistant: { stopReason: "length" },
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: true,
|
||||
hasTerminalOutput: true,
|
||||
lastAssistant: { stopReason: "toolUse" },
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "aborted", aborted: true, timedOut: false, promptError: null },
|
||||
{ label: "timed out", aborted: false, timedOut: true, promptError: null },
|
||||
{ label: "prompt error", aborted: false, timedOut: false, promptError: new Error("closed") },
|
||||
])("does not continue a $label tool-use terminal turn", ({ aborted, timedOut, promptError }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "tool_use", id: "tool_1", name: "bash", input: {} }],
|
||||
});
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "bash" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
},
|
||||
{ aborted, timedOut, promptError },
|
||||
),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "a matching failure summary",
|
||||
lastToolError: { toolName: "exec", error: "post-processing error" },
|
||||
},
|
||||
{ label: "no remaining failure summary", lastToolError: undefined },
|
||||
])(
|
||||
"recognizes successful and failed current-batch tools with $label (#118274)",
|
||||
({ lastToolError }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_ok", name: "read", arguments: {} },
|
||||
{ type: "toolCall", id: "tool_failed", name: "exec", arguments: {} },
|
||||
],
|
||||
});
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "read" }, { toolName: "exec", isError: true }],
|
||||
itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_ok", toolName: "read", isError: false },
|
||||
{ role: "toolResult", toolCallId: "tool_failed", toolName: "exec", isError: true },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toContain(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
|
||||
expect(instruction).toContain(
|
||||
"If any tool failed, state that failure plainly and do not claim it succeeded.",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ label: "progress", sourceReplyFinal: false, expectedFinalization: true },
|
||||
{ label: "final reply", sourceReplyFinal: true, expectedFinalization: false },
|
||||
{ label: "legacy unmarked send", sourceReplyFinal: undefined, expectedFinalization: false },
|
||||
])(
|
||||
"handles $label delivery evidence before settled finalization",
|
||||
({ sourceReplyFinal, expectedFinalization }) => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
didSendViaMessagingTool: true,
|
||||
messagingToolSentTexts: ["Writing note.txt…"],
|
||||
messagingToolSentTargets: [
|
||||
{
|
||||
tool: "message",
|
||||
provider: "telegram",
|
||||
to: "chat:123",
|
||||
text: "Writing note.txt…",
|
||||
sourceReplyFinal,
|
||||
},
|
||||
],
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
},
|
||||
{ allowEmptyStopContinuation: true },
|
||||
),
|
||||
);
|
||||
|
||||
expect(instruction).toBe(
|
||||
expectedFinalization ? SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION : null,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "an unrelated current-batch failure summary",
|
||||
resultToolName: "exec",
|
||||
lastErrorToolName: "read",
|
||||
},
|
||||
{
|
||||
label: "a failed result with the wrong tool identity",
|
||||
resultToolName: "read",
|
||||
lastErrorToolName: "exec",
|
||||
},
|
||||
])("does not finalize $label (#118274)", ({ resultToolName, lastErrorToolName }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }],
|
||||
});
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: resultToolName, isError: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tool_1",
|
||||
toolName: resultToolName,
|
||||
isError: true,
|
||||
},
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: { toolName: lastErrorToolName, error: "post-processing error" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
});
|
||||
|
||||
it("does not settle same-name terminal calls from one failed result (#118274)", () => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} },
|
||||
{ type: "toolCall", id: "tool_2", name: "exec", arguments: {} },
|
||||
],
|
||||
});
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec", isError: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: { toolName: "exec", error: "post-processing error" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "an async tool is still running",
|
||||
attemptOverrides: { toolMetas: [{ toolName: "exec", isError: true, asyncStarted: true }] },
|
||||
},
|
||||
{
|
||||
label: "an accepted child session owns the response",
|
||||
attemptOverrides: {
|
||||
acceptedSessionSpawns: [
|
||||
{ runId: "run-child", childSessionKey: "agent:main:subagent:child" },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "a client tool remains pending",
|
||||
attemptOverrides: { clientToolCalls: [{ name: "pending", params: {} }] },
|
||||
},
|
||||
{
|
||||
label: "the turn yielded",
|
||||
attemptOverrides: { yieldDetected: true },
|
||||
},
|
||||
{
|
||||
label: "an approval prompt was already delivered",
|
||||
attemptOverrides: { didSendDeterministicApprovalPrompt: true },
|
||||
},
|
||||
])("does not finalize a failed terminal tool when $label (#118274)", ({ attemptOverrides }) => {
|
||||
const toolUseAssistant = makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [{ type: "toolCall", id: "tool_1", name: "exec", arguments: {} }],
|
||||
});
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams({
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "exec", isError: true }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
messagesSnapshot: [
|
||||
toolUseAssistant,
|
||||
{ role: "toolResult", toolCallId: "tool_1", toolName: "exec", isError: true },
|
||||
] as unknown as EmbeddedRunAttemptResult["messagesSnapshot"],
|
||||
lastAssistant: toolUseAssistant,
|
||||
currentAttemptAssistant: toolUseAssistant,
|
||||
lastToolError: { toolName: "exec", error: "post-processing error" },
|
||||
...attemptOverrides,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "background trigger", allowEmptyStopContinuation: false },
|
||||
{
|
||||
label: "active tool",
|
||||
allowEmptyStopContinuation: true,
|
||||
completedCount: 0,
|
||||
activeCount: 1,
|
||||
},
|
||||
{
|
||||
label: "partially completed tool batch",
|
||||
allowEmptyStopContinuation: true,
|
||||
startedCount: 2,
|
||||
completedCount: 1,
|
||||
},
|
||||
{ label: "async tool", allowEmptyStopContinuation: true, asyncStarted: true },
|
||||
{ label: "failed tool", allowEmptyStopContinuation: true, isError: true },
|
||||
])(
|
||||
"does not continue an empty stop after $label activity",
|
||||
({
|
||||
allowEmptyStopContinuation,
|
||||
startedCount = 1,
|
||||
completedCount = 1,
|
||||
activeCount = 0,
|
||||
asyncStarted,
|
||||
isError,
|
||||
}) => {
|
||||
const emptyStopAssistant = makeLastAssistant();
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write", asyncStarted, isError }],
|
||||
itemLifecycle: { startedCount, completedCount, activeCount },
|
||||
lastAssistant: emptyStopAssistant,
|
||||
currentAttemptAssistant: emptyStopAssistant,
|
||||
},
|
||||
{ allowEmptyStopContinuation },
|
||||
),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not use a stale prior-turn empty stop to prove a settled continuation", () => {
|
||||
const staleEmptyStopAssistant = makeLastAssistant();
|
||||
const instruction = resolveSettledToolTerminalContinuationInstruction(
|
||||
makeSettledContinuationParams(
|
||||
{
|
||||
assistantTexts: [],
|
||||
toolMetas: [{ toolName: "write" }],
|
||||
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
|
||||
lastAssistant: staleEmptyStopAssistant,
|
||||
currentAttemptAssistant: undefined,
|
||||
},
|
||||
{ allowEmptyStopContinuation: true },
|
||||
),
|
||||
);
|
||||
|
||||
expect(instruction).toBeNull();
|
||||
});
|
||||
|
||||
it("does not flag stale lastAssistant=toolUse when currentAttemptAssistant=stop exists (#80918)", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Analysis...", "Here is the final answer after update_plan."],
|
||||
toolMetas: [{ toolName: "update_plan" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{ type: "text", text: "Analysis..." },
|
||||
{ type: "tool_use", id: "tool_1", name: "update_plan", input: {} },
|
||||
],
|
||||
}),
|
||||
currentAttemptAssistant: makeLastAssistant({
|
||||
content: [{ type: "text", text: "Here is the final answer after update_plan." }],
|
||||
}),
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toBeNull();
|
||||
});
|
||||
|
||||
it("still flags incomplete-turn when currentAttemptAssistant is absent and lastAssistant=toolUse (#76477 regression)", () => {
|
||||
const incompleteTurnText = resolveIncompleteTurnPayloadText(
|
||||
makeIncompleteTurnParams(
|
||||
{
|
||||
assistantTexts: ["Let me update the file..."],
|
||||
toolMetas: [{ toolName: "write" }],
|
||||
lastAssistant: makeLastAssistant({
|
||||
stopReason: "toolUse",
|
||||
model: "gpt-5.4",
|
||||
content: [
|
||||
{ type: "text", text: "Let me update the file..." },
|
||||
{ type: "tool_use", id: "tool_1", name: "write", input: {} },
|
||||
],
|
||||
}),
|
||||
currentAttemptAssistant: undefined,
|
||||
},
|
||||
{ payloadCount: 1 },
|
||||
),
|
||||
);
|
||||
|
||||
expect(incompleteTurnText).toContain("couldn't generate a response");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,192 @@
|
||||
// Shared fixtures for split incomplete-turn owner tests.
|
||||
import { expect } from "vitest";
|
||||
import {
|
||||
mockedLog,
|
||||
mockedRunEmbeddedAttempt,
|
||||
overflowBaseRunParams,
|
||||
runIncompleteTurnOwnerHarness,
|
||||
} from "./run.incomplete-turn.test-support.js";
|
||||
import { makeAttemptResult } from "./run.overflow-compaction.fixture.js";
|
||||
import {
|
||||
resolveEmptyResponseRetryInstruction,
|
||||
resolveReasoningOnlyRetryInstruction,
|
||||
resolveSettledToolTerminalContinuationInstruction,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
} from "./run/incomplete-turn-recovery.js";
|
||||
import { resolveIncompleteTurnPayloadText as resolveIncompleteTurnPayloadTextCore } from "./run/incomplete-turn-resolution.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
export const REASONING_ONLY_RETRY_INSTRUCTION =
|
||||
"The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch.";
|
||||
export const EMPTY_RESPONSE_RETRY_INSTRUCTION =
|
||||
"The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch.";
|
||||
export const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION =
|
||||
"The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.";
|
||||
|
||||
export const runEmbeddedAgent = runIncompleteTurnOwnerHarness;
|
||||
|
||||
type LastAssistant = NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
|
||||
type AttemptOverrides = Parameters<typeof makeAttemptResult>[0];
|
||||
type RunParams = Parameters<typeof runEmbeddedAgent>[0];
|
||||
type LastAssistantFixture = Omit<LastAssistant, "content" | "stopReason" | "usage"> & {
|
||||
content: Array<Record<string, unknown>>;
|
||||
stopReason: LastAssistant["stopReason"] | "end_turn";
|
||||
usage: Partial<LastAssistant["usage"]> & { total?: number };
|
||||
};
|
||||
|
||||
export function makeLastAssistant(overrides: Partial<LastAssistantFixture> = {}): LastAssistant {
|
||||
return {
|
||||
role: "assistant",
|
||||
stopReason: "stop",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
content: [],
|
||||
...overrides,
|
||||
} as unknown as LastAssistant;
|
||||
}
|
||||
|
||||
export function resolveIncompleteTurnPayloadText(
|
||||
params: Omit<Parameters<typeof resolveIncompleteTurnPayloadTextCore>[0], "externalAbort"> & {
|
||||
externalAbort?: boolean;
|
||||
},
|
||||
): string | null {
|
||||
// Most helper tests exercise internal abort behavior; external aborts opt in
|
||||
// explicitly through params.
|
||||
return resolveIncompleteTurnPayloadTextCore({ externalAbort: false, ...params });
|
||||
}
|
||||
|
||||
export function makeBaseRunParams(runId: string, overrides: Partial<RunParams> = {}): RunParams {
|
||||
return { ...overflowBaseRunParams, runId, ...overrides };
|
||||
}
|
||||
|
||||
export function makeRunParams(runId: string, overrides: Partial<RunParams> = {}): RunParams {
|
||||
return {
|
||||
...overflowBaseRunParams,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
runId,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeIncompleteTurnParams(
|
||||
attemptOverrides: AttemptOverrides = {},
|
||||
overrides: Partial<Omit<Parameters<typeof resolveIncompleteTurnPayloadText>[0], "attempt">> = {},
|
||||
): Parameters<typeof resolveIncompleteTurnPayloadText>[0] {
|
||||
return {
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult(attemptOverrides),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeReasoningRetryParams(
|
||||
attemptOverrides: AttemptOverrides = {},
|
||||
overrides: Partial<
|
||||
Omit<Parameters<typeof resolveReasoningOnlyRetryInstruction>[0], "attempt">
|
||||
> = {},
|
||||
): Parameters<typeof resolveReasoningOnlyRetryInstruction>[0] {
|
||||
return {
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult(attemptOverrides),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeEmptyResponseRetryParams(
|
||||
attemptOverrides: AttemptOverrides = {},
|
||||
overrides: Partial<
|
||||
Omit<Parameters<typeof resolveEmptyResponseRetryInstruction>[0], "attempt">
|
||||
> = {},
|
||||
): Parameters<typeof resolveEmptyResponseRetryInstruction>[0] {
|
||||
return {
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.4",
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult(attemptOverrides),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeSettledContinuationParams(
|
||||
attemptOverrides: AttemptOverrides = {},
|
||||
overrides: Partial<
|
||||
Omit<Parameters<typeof resolveSettledToolTerminalContinuationInstruction>[0], "attempt">
|
||||
> = {},
|
||||
): Parameters<typeof resolveSettledToolTerminalContinuationInstruction>[0] {
|
||||
return {
|
||||
provider: "openai",
|
||||
modelId: "gpt-5.5",
|
||||
modelApi: "openai-chatgpt-responses",
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt: makeAttemptResult(attemptOverrides),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export function makeSilentReplyParams(
|
||||
attempt: EmbeddedRunAttemptResult,
|
||||
overrides: Partial<
|
||||
Omit<Parameters<typeof shouldTreatEmptyAssistantReplyAsSilent>[0], "attempt">
|
||||
> = {},
|
||||
): Parameters<typeof shouldTreatEmptyAssistantReplyAsSilent>[0] {
|
||||
return {
|
||||
allowEmptyAssistantReplyAsSilent: true,
|
||||
payloadCount: 0,
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
attempt,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function warnMessages(): string[] {
|
||||
return mockedLog.warn.mock.calls.map(([message]) => String(message));
|
||||
}
|
||||
|
||||
export function expectWarnMessageWith(text: string): void {
|
||||
expect(warnMessages().join("\n")).toContain(text);
|
||||
}
|
||||
|
||||
export function expectNoWarnMessageWith(text: string): void {
|
||||
expect(warnMessages().join("\n")).not.toContain(text);
|
||||
}
|
||||
|
||||
export function runAttemptCall(index: number): {
|
||||
prompt?: string;
|
||||
disableTools?: boolean;
|
||||
operation?: string;
|
||||
suppressNextUserMessagePersistence?: boolean;
|
||||
skipPreparedUserTurnMessage?: boolean;
|
||||
} {
|
||||
// Continuation prompt assertions read the exact prompt passed to the runner
|
||||
// attempt rather than derived result metadata.
|
||||
const call = mockedRunEmbeddedAttempt.mock.calls[index];
|
||||
if (!call) {
|
||||
throw new Error(`Expected run embedded attempt call ${index}`);
|
||||
}
|
||||
return call[0] as {
|
||||
prompt?: string;
|
||||
disableTools?: boolean;
|
||||
operation?: string;
|
||||
suppressNextUserMessagePersistence?: boolean;
|
||||
skipPreparedUserTurnMessage?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export function markUserMessagePersisted(attemptParams: unknown): void {
|
||||
(
|
||||
attemptParams as {
|
||||
onUserMessagePersisted?: (message: { role: "user"; content: string }) => void;
|
||||
}
|
||||
).onUserMessagePersisted?.({ role: "user", content: "test prompt" });
|
||||
}
|
||||
@@ -10,10 +10,8 @@ import type { FailoverReason } from "../failover/signal.js";
|
||||
import type { AgentHarness } from "../harness/types.js";
|
||||
import { buildEmbeddedRunBlockedResult } from "./run/blocked-run-result.js";
|
||||
import { createEmbeddedRunContextRecoveryState } from "./run/context-recovery-state.js";
|
||||
import {
|
||||
resolveReplayInvalidFlag,
|
||||
shouldRetrySilentErrorAssistantTurn,
|
||||
} from "./run/incomplete-turn.js";
|
||||
import { shouldRetrySilentErrorAssistantTurn } from "./run/incomplete-turn-recovery.js";
|
||||
import { resolveReplayInvalidFlag } from "./run/incomplete-turn-resolution.js";
|
||||
import type { RunEmbeddedAgentParams } from "./run/params.js";
|
||||
import { normalizeEmbeddedRunAttemptResult } from "./run/run-attempt-result.js";
|
||||
import { prepareTerminalWithSettledTurnFinalization } from "./run/settled-turn-finalization.js";
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@
|
||||
import type { ContextEngineSessionTarget } from "../../context-engine/types.js";
|
||||
import { normalizeAgentRunAttemptTerminal } from "../agent-run-terminal-outcome.js";
|
||||
import { isAgentToolReplaySafe } from "../tool-replay-safety.js";
|
||||
import { buildAttemptReplayMetadata } from "./run/incomplete-turn.js";
|
||||
import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./run/types.js";
|
||||
|
||||
const DEFAULT_OVERFLOW_ERROR_MESSAGE =
|
||||
|
||||
@@ -25,7 +25,7 @@ import type { TraceAttempt } from "../types.js";
|
||||
import { handleAssistantFailover, isShortWindowRateLimitMessage } from "./assistant-failover.js";
|
||||
import { createFailoverDecisionLogger } from "./failover-observation.js";
|
||||
import { resolveRunFailoverDecision } from "./failover-policy.js";
|
||||
import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn.js";
|
||||
import { shouldRetrySilentErrorAssistantTurn } from "./incomplete-turn-recovery.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import {
|
||||
isEmbeddedRunTerminalInterrupted,
|
||||
|
||||
@@ -27,7 +27,7 @@ import {
|
||||
stepIdleTimeoutBreaker,
|
||||
type createIdleTimeoutBreakerState,
|
||||
} from "./idle-timeout-breaker.js";
|
||||
import { resolveReplayInvalidFlag } from "./incomplete-turn.js";
|
||||
import { resolveReplayInvalidFlag } from "./incomplete-turn-resolution.js";
|
||||
import { resolveRunRetryKind, type RunRetryKind } from "./retry-budget.js";
|
||||
import { handleRetryLimitExhaustion } from "./retry-limit.js";
|
||||
import type { dispatchEmbeddedRunAttempt } from "./run-attempt-dispatch.js";
|
||||
|
||||
@@ -20,9 +20,9 @@ import { shouldRunLlmOutputHooksForAttempt } from "./attempt-run-decisions.js";
|
||||
import {
|
||||
buildAttemptReplayMetadata,
|
||||
hasAttemptTerminalState,
|
||||
resolveSilentToolResultReplyPayload,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
} from "./incomplete-turn.js";
|
||||
} from "./attempt-terminal-evidence.js";
|
||||
import { shouldTreatEmptyAssistantReplyAsSilent } from "./incomplete-turn-recovery.js";
|
||||
import { resolveSilentToolResultReplyPayload } from "./incomplete-turn-resolution.js";
|
||||
import type {
|
||||
EmbeddedRunAttemptParams,
|
||||
EmbeddedRunAttemptResult,
|
||||
|
||||
@@ -26,20 +26,20 @@ import {
|
||||
isSessionsYieldAbortReason,
|
||||
} from "./attempt-sessions-yield.js";
|
||||
import { wrapStreamFnHandleSensitiveStopReason } from "./attempt-stop-reason-recovery.js";
|
||||
import {
|
||||
sanitizeOpenAIResponsesReplayForStream,
|
||||
sanitizeReplayToolCallIdsForStream,
|
||||
shouldApplyReplayToolCallIdSanitizer,
|
||||
wrapStreamFnSanitizeMalformedToolCalls,
|
||||
} from "./attempt-tool-call-replay-sanitization.js";
|
||||
import { wrapStreamFnTrimToolCallNames } from "./attempt-tool-call-stream-normalization.js";
|
||||
import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js";
|
||||
import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js";
|
||||
import {
|
||||
shouldRepairMalformedToolCallArguments,
|
||||
wrapStreamFnDecodeXaiToolCallArguments,
|
||||
wrapStreamFnRepairMalformedToolCallArguments,
|
||||
} from "./attempt.tool-call-argument-repair.js";
|
||||
import {
|
||||
sanitizeOpenAIResponsesReplayForStream,
|
||||
sanitizeReplayToolCallIdsForStream,
|
||||
shouldApplyReplayToolCallIdSanitizer,
|
||||
wrapStreamFnPromoteStandaloneTextToolCalls,
|
||||
wrapStreamFnSanitizeMalformedToolCalls,
|
||||
wrapStreamFnTrimToolCallNames,
|
||||
} from "./attempt.tool-call-normalization.js";
|
||||
import {
|
||||
resolveLlmFirstEventTimeoutMs,
|
||||
resolveLlmIdleTimeoutMs,
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/** Records attempt replay safety and terminal side-effect evidence. */
|
||||
import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js";
|
||||
import {
|
||||
hasCommittedMessagingToolDeliveryEvidence,
|
||||
hasMessagingToolDeliveryEvidence,
|
||||
} from "../delivery-evidence.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
type ReplayMetadataAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "toolMetas"
|
||||
| "didSendViaMessagingTool"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "successfulCronAdds"
|
||||
> &
|
||||
Partial<Pick<EmbeddedRunAttemptResult, "messagingToolSentTargets" | "acceptedSessionSpawns">>;
|
||||
|
||||
/**
|
||||
* Marks whether retrying the attempt can safely replay the prompt. Concrete
|
||||
* tool-instance policy, async work, committed delivery, spawned sessions, and
|
||||
* cron writes all contribute side-effect evidence.
|
||||
*/
|
||||
export function buildAttemptReplayMetadata(
|
||||
params: ReplayMetadataAttempt,
|
||||
): EmbeddedRunAttemptResult["replayMetadata"] {
|
||||
const hadUnsafeTools = params.toolMetas.some((entry) => entry.replaySafe !== true);
|
||||
const hadAsyncStartedTool = params.toolMetas.some((t) => t.asyncStarted === true);
|
||||
const hadPotentialSideEffects =
|
||||
hadUnsafeTools ||
|
||||
hadAsyncStartedTool ||
|
||||
hasMessagingToolDeliveryEvidence(params) ||
|
||||
hasAcceptedSessionSpawn(params.acceptedSessionSpawns) ||
|
||||
(params.successfulCronAdds ?? 0) > 0;
|
||||
return {
|
||||
hadPotentialSideEffects,
|
||||
replaySafe: !hadPotentialSideEffects,
|
||||
};
|
||||
}
|
||||
|
||||
type TerminalAttemptState = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "lastToolError"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "hasToolMediaBlockReply"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "successfulCronAdds"
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "acceptedSessionSpawns"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "messagingToolSentTargets"
|
||||
>
|
||||
> & {
|
||||
toolMetas?: readonly { asyncStarted?: boolean }[];
|
||||
};
|
||||
|
||||
export function hasAttemptTerminalState(attempt: TerminalAttemptState): boolean {
|
||||
return Boolean(
|
||||
attempt.clientToolCalls ||
|
||||
attempt.yieldDetected ||
|
||||
attempt.didSendDeterministicApprovalPrompt ||
|
||||
attempt.heartbeatToolResponse ||
|
||||
attempt.lastToolError ||
|
||||
attempt.toolMediaUrls?.some((url) => url.trim().length > 0) ||
|
||||
attempt.toolAudioAsVoice ||
|
||||
attempt.toolTrustedLocalMedia ||
|
||||
attempt.hasToolMediaBlockReply ||
|
||||
attempt.didDeliverSourceReplyViaMessageTool ||
|
||||
attempt.messagingToolSourceReplyPayloads?.length ||
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: attempt.messagingToolSentTexts ?? [],
|
||||
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [],
|
||||
messagingToolSentTargets: attempt.messagingToolSentTargets ?? [],
|
||||
}) ||
|
||||
hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) ||
|
||||
hasAsyncActivity(attempt.toolMetas) ||
|
||||
(attempt.successfulCronAdds ?? 0) > 0,
|
||||
);
|
||||
}
|
||||
|
||||
export function hasAsyncActivity(toolMetas?: readonly { asyncStarted?: boolean }[]): boolean {
|
||||
return (toolMetas ?? []).some((entry) => entry.asyncStarted === true);
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
/** Resolves provider-emitted tool names against the live callable set. */
|
||||
import { normalizeLowercaseStringOrEmpty } from "../../../../packages/normalization-core/src/string-coerce.js";
|
||||
import { normalizeStringEntries } from "../../../../packages/normalization-core/src/string-normalization.js";
|
||||
import { normalizeToolPolicyName } from "../../tool-policy.js";
|
||||
|
||||
function resolveCaseInsensitiveAllowedToolName(
|
||||
rawName: string,
|
||||
allowedToolNames?: Set<string>,
|
||||
): string | null {
|
||||
if (!allowedToolNames || allowedToolNames.size === 0) {
|
||||
return null;
|
||||
}
|
||||
const folded = normalizeLowercaseStringOrEmpty(rawName);
|
||||
let caseInsensitiveMatch: string | null = null;
|
||||
for (const name of allowedToolNames) {
|
||||
if (normalizeLowercaseStringOrEmpty(name) !== folded) {
|
||||
continue;
|
||||
}
|
||||
if (caseInsensitiveMatch && caseInsensitiveMatch !== name) {
|
||||
return null;
|
||||
}
|
||||
caseInsensitiveMatch = name;
|
||||
}
|
||||
return caseInsensitiveMatch;
|
||||
}
|
||||
|
||||
function resolveExactAllowedToolName(
|
||||
rawName: string,
|
||||
allowedToolNames?: Set<string>,
|
||||
): string | null {
|
||||
if (!allowedToolNames || allowedToolNames.size === 0) {
|
||||
return null;
|
||||
}
|
||||
if (allowedToolNames.has(rawName)) {
|
||||
return rawName;
|
||||
}
|
||||
const normalized = normalizeToolPolicyName(rawName);
|
||||
if (allowedToolNames.has(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return (
|
||||
resolveCaseInsensitiveAllowedToolName(rawName, allowedToolNames) ??
|
||||
resolveCaseInsensitiveAllowedToolName(normalized, allowedToolNames)
|
||||
);
|
||||
}
|
||||
|
||||
function buildStructuredToolNameCandidates(rawName: string): string[] {
|
||||
const trimmed = rawName.trim();
|
||||
if (!trimmed) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const candidates: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
const addCandidate = (value: string) => {
|
||||
const candidate = value.trim();
|
||||
if (!candidate || seen.has(candidate)) {
|
||||
return;
|
||||
}
|
||||
seen.add(candidate);
|
||||
candidates.push(candidate);
|
||||
};
|
||||
|
||||
addCandidate(trimmed);
|
||||
addCandidate(normalizeToolPolicyName(trimmed));
|
||||
const structuredSeeds = [trimmed];
|
||||
|
||||
const xmlFragmentOffset = ['"', "'", "<"]
|
||||
.map((separator) => trimmed.indexOf(separator))
|
||||
.filter((offset) => offset > 0)
|
||||
.reduce<number | undefined>(
|
||||
(earliest, offset) => (earliest === undefined || offset < earliest ? offset : earliest),
|
||||
undefined,
|
||||
);
|
||||
if (xmlFragmentOffset !== undefined) {
|
||||
const prefix = trimmed.slice(0, xmlFragmentOffset);
|
||||
addCandidate(prefix);
|
||||
addCandidate(normalizeToolPolicyName(prefix));
|
||||
structuredSeeds.push(prefix);
|
||||
}
|
||||
|
||||
for (const seed of structuredSeeds) {
|
||||
const normalizedDelimiter = seed.replace(/\//g, ".");
|
||||
addCandidate(normalizedDelimiter);
|
||||
addCandidate(normalizeToolPolicyName(normalizedDelimiter));
|
||||
|
||||
const segments = normalizeStringEntries(normalizedDelimiter.split("."));
|
||||
if (segments.length > 1) {
|
||||
for (let index = 1; index < segments.length; index += 1) {
|
||||
const suffix = segments.slice(index).join(".");
|
||||
addCandidate(suffix);
|
||||
addCandidate(normalizeToolPolicyName(suffix));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function resolveStructuredAllowedToolName(
|
||||
rawName: string,
|
||||
allowedToolNames?: Set<string>,
|
||||
): string | null {
|
||||
if (!allowedToolNames || allowedToolNames.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const candidateNames = buildStructuredToolNameCandidates(rawName);
|
||||
for (const candidate of candidateNames) {
|
||||
if (allowedToolNames.has(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
for (const candidate of candidateNames) {
|
||||
const caseInsensitiveMatch = resolveCaseInsensitiveAllowedToolName(candidate, allowedToolNames);
|
||||
if (caseInsensitiveMatch) {
|
||||
return caseInsensitiveMatch;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function inferToolNameFromToolCallId(
|
||||
rawId: string | undefined,
|
||||
allowedToolNames?: Set<string>,
|
||||
): string | null {
|
||||
if (!rawId || !allowedToolNames || allowedToolNames.size === 0) {
|
||||
return null;
|
||||
}
|
||||
const id = rawId.trim();
|
||||
if (!id) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const candidateTokens = new Set<string>();
|
||||
const addToken = (value: string) => {
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return;
|
||||
}
|
||||
candidateTokens.add(trimmed);
|
||||
candidateTokens.add(trimmed.replace(/[:._/-]\d+$/, ""));
|
||||
candidateTokens.add(trimmed.replace(/\d+$/, ""));
|
||||
|
||||
const normalizedDelimiter = trimmed.replace(/\//g, ".");
|
||||
candidateTokens.add(normalizedDelimiter);
|
||||
candidateTokens.add(normalizedDelimiter.replace(/[:._-]\d+$/, ""));
|
||||
candidateTokens.add(normalizedDelimiter.replace(/\d+$/, ""));
|
||||
|
||||
for (const prefixPattern of [/^functions?[._-]?/i, /^tools?[._-]?/i]) {
|
||||
const stripped = normalizedDelimiter.replace(prefixPattern, "");
|
||||
if (stripped !== normalizedDelimiter) {
|
||||
candidateTokens.add(stripped);
|
||||
candidateTokens.add(stripped.replace(/[:._-]\d+$/, ""));
|
||||
candidateTokens.add(stripped.replace(/\d+$/, ""));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const preColon = id.split(":")[0] ?? id;
|
||||
for (const seed of [id, preColon]) {
|
||||
addToken(seed);
|
||||
}
|
||||
|
||||
let singleMatch: string | null = null;
|
||||
for (const candidate of candidateTokens) {
|
||||
const matched = resolveStructuredAllowedToolName(candidate, allowedToolNames);
|
||||
if (!matched) {
|
||||
continue;
|
||||
}
|
||||
if (singleMatch && singleMatch !== matched) {
|
||||
return null;
|
||||
}
|
||||
singleMatch = matched;
|
||||
}
|
||||
|
||||
return singleMatch;
|
||||
}
|
||||
|
||||
function looksLikeMalformedToolNameCounter(rawName: string): boolean {
|
||||
const normalizedDelimiter = rawName.trim().replace(/\//g, ".");
|
||||
return (
|
||||
/^(?:functions?|tools?)[._-]?/i.test(normalizedDelimiter) &&
|
||||
/(?:[:._-]\d+|\d+)$/.test(normalizedDelimiter)
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveToolCallName(
|
||||
rawName: string,
|
||||
allowedToolNames?: Set<string>,
|
||||
rawToolCallId?: string,
|
||||
requireAllowed = false,
|
||||
): string | null {
|
||||
const trimmed = rawName.trim();
|
||||
if (!trimmed) {
|
||||
return (
|
||||
inferToolNameFromToolCallId(rawToolCallId, allowedToolNames) ??
|
||||
(requireAllowed ? null : rawName)
|
||||
);
|
||||
}
|
||||
if (!allowedToolNames || allowedToolNames.size === 0) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
const exact = resolveExactAllowedToolName(trimmed, allowedToolNames);
|
||||
if (exact) {
|
||||
return exact;
|
||||
}
|
||||
const inferredFromName = inferToolNameFromToolCallId(trimmed, allowedToolNames);
|
||||
if (inferredFromName) {
|
||||
return inferredFromName;
|
||||
}
|
||||
|
||||
if (looksLikeMalformedToolNameCounter(trimmed)) {
|
||||
return requireAllowed ? null : trimmed;
|
||||
}
|
||||
|
||||
return (
|
||||
resolveStructuredAllowedToolName(trimmed, allowedToolNames) ?? (requireAllowed ? null : trimmed)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,632 @@
|
||||
// Coverage for provider replay tool-call sanitization.
|
||||
|
||||
import type { AgentMessage } from "openclaw/plugin-sdk/agent-core";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
sanitizeOpenAIResponsesReplayForStream,
|
||||
sanitizeReplayToolCallIdsForStream,
|
||||
shouldApplyReplayToolCallIdSanitizer,
|
||||
wrapStreamFnSanitizeMalformedToolCalls,
|
||||
} from "./attempt-tool-call-replay-sanitization.js";
|
||||
|
||||
type AssistantMessage = Extract<AgentMessage, { role: "assistant" }>;
|
||||
type ToolResultMessage = Extract<AgentMessage, { role: "toolResult" }>;
|
||||
type FakeWrappedStream = {
|
||||
result: () => Promise<unknown>;
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
};
|
||||
|
||||
function createFakeStream(params: {
|
||||
events: unknown[];
|
||||
resultMessage: unknown;
|
||||
}): FakeWrappedStream {
|
||||
return {
|
||||
async result() {
|
||||
return params.resultMessage;
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return (async function* () {
|
||||
for (const event of params.events) {
|
||||
yield event;
|
||||
}
|
||||
})();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function requireAssistantMessage(message: AgentMessage | undefined): AssistantMessage {
|
||||
if (!message || message.role !== "assistant") {
|
||||
throw new Error(`expected assistant message, got ${message?.role ?? "missing"}`);
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
function requireToolResultMessage(message: AgentMessage | undefined): ToolResultMessage {
|
||||
if (!message || message.role !== "toolResult") {
|
||||
throw new Error(`expected toolResult message, got ${message?.role ?? "missing"}`);
|
||||
}
|
||||
return message;
|
||||
}
|
||||
|
||||
function assistantToolUseSummaries(message: AgentMessage | undefined) {
|
||||
const assistant = requireAssistantMessage(message);
|
||||
return assistant.content.map((content) => {
|
||||
const record = content as unknown as Record<string, unknown>;
|
||||
if (record.type !== "toolUse") {
|
||||
throw new Error(`expected toolUse content, got ${String(record.type)}`);
|
||||
}
|
||||
return { type: record.type, id: record.id, name: record.name };
|
||||
});
|
||||
}
|
||||
|
||||
function toolResultSummary(message: AgentMessage | undefined) {
|
||||
const toolResult = requireToolResultMessage(message);
|
||||
const record = toolResult as unknown as Record<string, unknown>;
|
||||
return {
|
||||
role: toolResult.role,
|
||||
toolCallId: toolResult.toolCallId,
|
||||
toolUseId: record.toolUseId,
|
||||
toolName: toolResult.toolName,
|
||||
isError: toolResult.isError,
|
||||
};
|
||||
}
|
||||
|
||||
describe("sanitizeReplayToolCallIdsForStream", () => {
|
||||
it("skips strict stream id sanitization when provider policy opts out", () => {
|
||||
expect(
|
||||
shouldApplyReplayToolCallIdSanitizer({
|
||||
sanitizeToolCallIds: false,
|
||||
isOpenAIResponsesApi: false,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldApplyReplayToolCallIdSanitizer({
|
||||
sanitizeToolCallIds: true,
|
||||
toolCallIdMode: "strict",
|
||||
isOpenAIResponsesApi: false,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldApplyReplayToolCallIdSanitizer({
|
||||
sanitizeToolCallIds: true,
|
||||
toolCallIdMode: "strict",
|
||||
isOpenAIResponsesApi: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("drops orphaned tool results after strict id sanitization", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_function_av7cbkigmk7x1",
|
||||
toolUseId: "call_function_av7cbkigmk7x1",
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "stale" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
];
|
||||
|
||||
expect(
|
||||
sanitizeReplayToolCallIdsForStream({
|
||||
messages,
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
}),
|
||||
).toStrictEqual([]);
|
||||
});
|
||||
|
||||
it("keeps matched assistant and tool-result ids aligned", () => {
|
||||
const rawId = "call_function_av7cbkigmk7x1";
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
];
|
||||
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages,
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]);
|
||||
expect(assistantToolUseSummaries(out[0])).toEqual([
|
||||
{ type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" },
|
||||
]);
|
||||
expect(toolResultSummary(out[1])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "callfunctionav7cbkigmk7x1",
|
||||
toolUseId: "callfunctionav7cbkigmk7x1",
|
||||
toolName: "read",
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("pairs repeated raw ids before assigning provider-safe occurrence ids", () => {
|
||||
const rawId = "exec_0";
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } }],
|
||||
} as never,
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } }],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "exec",
|
||||
content: [{ type: "text", text: "second result" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual([
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
]);
|
||||
expect(assistantToolUseSummaries(out[0])).toEqual([
|
||||
{ type: "toolUse", id: "exec0", name: "exec" },
|
||||
]);
|
||||
expect(toolResultSummary(out[1])).toMatchObject({
|
||||
toolCallId: "exec0",
|
||||
isError: true,
|
||||
});
|
||||
expect(assistantToolUseSummaries(out[2])).toEqual([
|
||||
{ type: "toolUse", id: "exec02", name: "exec" },
|
||||
]);
|
||||
expect(toolResultSummary(out[3])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "exec02",
|
||||
toolUseId: "exec02",
|
||||
toolName: "exec",
|
||||
isError: false,
|
||||
});
|
||||
expect(requireToolResultMessage(out[3]).content).toEqual([
|
||||
{ type: "text", text: "second result" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps same-turn repeated calls and results aligned after id rewriting", () => {
|
||||
const rawId = "exec_0";
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "first" } },
|
||||
{ type: "toolUse", id: rawId, name: "exec", input: { cmd: "second" } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "exec",
|
||||
content: [{ type: "text", text: "first result" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "exec",
|
||||
content: [{ type: "text", text: "second result" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]);
|
||||
expect(assistantToolUseSummaries(out[0])).toEqual([
|
||||
{ type: "toolUse", id: "exec0", name: "exec" },
|
||||
{ type: "toolUse", id: "exec02", name: "exec" },
|
||||
]);
|
||||
expect(toolResultSummary(out[1])).toMatchObject({
|
||||
toolCallId: "exec0",
|
||||
toolUseId: "exec0",
|
||||
isError: false,
|
||||
});
|
||||
expect(toolResultSummary(out[2])).toMatchObject({
|
||||
toolCallId: "exec02",
|
||||
toolUseId: "exec02",
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves signed-thinking replay ids when requested by provider policy", () => {
|
||||
const rawId = "call_1";
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "internal", thinkingSignature: "sig_1" },
|
||||
{ type: "toolUse", id: rawId, name: "read", input: { path: "." } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
preserveReplaySafeThinkingToolCallIds: true,
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]);
|
||||
expect(requireAssistantMessage(out[0]).content[1]).toMatchObject({
|
||||
type: "toolUse",
|
||||
id: "call_1",
|
||||
name: "read",
|
||||
});
|
||||
expect(toolResultSummary(out[1])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
toolUseId: "call_1",
|
||||
toolName: "read",
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("synthesizes missing tool results after strict id sanitization", () => {
|
||||
const rawId = "call_function_av7cbkigmk7x1";
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "toolUse", id: rawId, name: "read", input: { path: "." } },
|
||||
{ type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "toolResult"]);
|
||||
expect(assistantToolUseSummaries(out[0])).toEqual([
|
||||
{ type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" },
|
||||
{ type: "toolUse", id: "callmissing", name: "exec" },
|
||||
]);
|
||||
expect(toolResultSummary(out[1])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "callfunctionav7cbkigmk7x1",
|
||||
toolUseId: "callfunctionav7cbkigmk7x1",
|
||||
toolName: "read",
|
||||
isError: false,
|
||||
});
|
||||
expect(toolResultSummary(out[2])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "callmissing",
|
||||
toolUseId: undefined,
|
||||
toolName: "exec",
|
||||
isError: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("synthesizes missing tool results when repair is enabled", () => {
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "toolUse", id: "call_missing", name: "exec", input: { cmd: "true" } }],
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult"]);
|
||||
expect(toolResultSummary(out[1])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "callmissing",
|
||||
toolUseId: undefined,
|
||||
toolName: "exec",
|
||||
isError: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps real tool results for aborted assistant spans", () => {
|
||||
const rawId = "call_function_av7cbkigmk7x1";
|
||||
const out = sanitizeReplayToolCallIdsForStream({
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "aborted",
|
||||
content: [{ type: "toolUse", id: rawId, name: "read", input: { path: "." } }],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawId,
|
||||
toolUseId: rawId,
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "partial" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "retry" }],
|
||||
} as never,
|
||||
],
|
||||
mode: "strict",
|
||||
repairToolUseResultPairing: true,
|
||||
});
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual(["assistant", "toolResult", "user"]);
|
||||
expect(requireAssistantMessage(out[0]).stopReason).toBe("aborted");
|
||||
expect(assistantToolUseSummaries(out[0])).toEqual([
|
||||
{ type: "toolUse", id: "callfunctionav7cbkigmk7x1", name: "read" },
|
||||
]);
|
||||
expect(toolResultSummary(out[1])).toEqual({
|
||||
role: "toolResult",
|
||||
toolCallId: "callfunctionav7cbkigmk7x1",
|
||||
toolUseId: "callfunctionav7cbkigmk7x1",
|
||||
toolName: "read",
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("wrapStreamFnSanitizeMalformedToolCalls", () => {
|
||||
it("keeps valid non-Responses replay inputs pass-through", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_1",
|
||||
name: "image_generate",
|
||||
arguments: { prompt: "QA lighthouse" },
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
];
|
||||
const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) =>
|
||||
createFakeStream({
|
||||
events: [],
|
||||
resultMessage: { role: "assistant", content: "ok" },
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnSanitizeMalformedToolCalls(
|
||||
baseFn as never,
|
||||
new Set(["image_generate"]),
|
||||
undefined,
|
||||
"openai",
|
||||
);
|
||||
|
||||
void wrapped({ api: "openai" } as never, { messages } as never, {} as never);
|
||||
|
||||
const forwardedContext = baseFn.mock.calls[0]?.[1] as {
|
||||
messages?: AgentMessage[];
|
||||
};
|
||||
expect(forwardedContext.messages).toBe(messages);
|
||||
});
|
||||
|
||||
it("repairs OpenAI Responses pairing even when replay inputs do not change", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_mock_image_generate_2",
|
||||
name: "image_generate",
|
||||
arguments: { prompt: "QA lighthouse" },
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "stop",
|
||||
content: "Worked: the QA lighthouse image completed.",
|
||||
} as never,
|
||||
];
|
||||
const baseFn = vi.fn((_model: unknown, _context: unknown, _options: unknown) =>
|
||||
createFakeStream({
|
||||
events: [],
|
||||
resultMessage: { role: "assistant", content: "ok" },
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnSanitizeMalformedToolCalls(
|
||||
baseFn as never,
|
||||
new Set(["image_generate"]),
|
||||
undefined,
|
||||
"openai",
|
||||
);
|
||||
|
||||
void wrapped({ api: "openai-responses" } as never, { messages } as never, {} as never);
|
||||
|
||||
const forwardedContext = baseFn.mock.calls[0]?.[1] as {
|
||||
messages?: AgentMessage[];
|
||||
};
|
||||
expect(forwardedContext.messages?.map((message) => message.role)).toEqual([
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"assistant",
|
||||
]);
|
||||
expect(forwardedContext.messages?.[1]).toMatchObject({
|
||||
role: "toolResult",
|
||||
toolCallId: "call_mock_image_generate_2",
|
||||
toolName: "image_generate",
|
||||
isError: true,
|
||||
content: [{ type: "text", text: "aborted" }],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeOpenAIResponsesReplayForStream", () => {
|
||||
it("normalizes live responses continuations before pi-ai splits ids", () => {
|
||||
const longCallId = `call_${"x".repeat(120)}`;
|
||||
const longItemId = `notfc_${"y".repeat(120)}`;
|
||||
const rawToolCallId = `${longCallId}|${longItemId}`;
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "toolCall", id: rawToolCallId, name: "noop", arguments: {} }],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: rawToolCallId,
|
||||
toolName: "noop",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
];
|
||||
|
||||
const out = sanitizeOpenAIResponsesReplayForStream(messages);
|
||||
const assistant = out[0] as Extract<AgentMessage, { role: "assistant" }>;
|
||||
const toolCall = assistant.content.find(
|
||||
(block) =>
|
||||
Boolean(block) &&
|
||||
typeof block === "object" &&
|
||||
(block as { type?: unknown }).type === "toolCall" &&
|
||||
typeof (block as { id?: unknown }).id === "string",
|
||||
) as { id: string } | undefined;
|
||||
|
||||
expect(toolCall?.id).toMatch(/^call_[A-Za-z0-9_-]{1,59}$/);
|
||||
expect(toolCall?.id).not.toBe(rawToolCallId);
|
||||
expect(toolCall?.id).not.toContain("|");
|
||||
expect((out[1] as Extract<AgentMessage, { role: "toolResult" }>).toolCallId).toBe(toolCall?.id);
|
||||
});
|
||||
|
||||
it("preserves canonical same-model reasoning pairs", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "thinking",
|
||||
thinking: "internal",
|
||||
thinkingSignature: JSON.stringify({ id: "rs_123", type: "reasoning" }),
|
||||
},
|
||||
{ type: "toolCall", id: "call_123|fc_123", name: "noop", arguments: {} },
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_123|fc_123",
|
||||
toolName: "noop",
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
isError: false,
|
||||
} as never,
|
||||
];
|
||||
|
||||
expect(sanitizeOpenAIResponsesReplayForStream(messages)).toBe(messages);
|
||||
});
|
||||
|
||||
it("repairs dangling OpenAI Responses tool calls from async resume replay", () => {
|
||||
const messages: AgentMessage[] = [
|
||||
{
|
||||
role: "user",
|
||||
content: "Image generation check. Generate an image of a QA lighthouse.",
|
||||
} as never,
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_mock_image_generate_1",
|
||||
name: "image_generate",
|
||||
arguments: { prompt: "QA lighthouse" },
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_mock_image_generate_1",
|
||||
toolName: "image_generate",
|
||||
content: [{ type: "text", text: "Background task started for image generation." }],
|
||||
isError: false,
|
||||
} as never,
|
||||
{
|
||||
role: "custom",
|
||||
content: "Image generation started; wait for completion.",
|
||||
} as never,
|
||||
{
|
||||
role: "user",
|
||||
content: "The image is ready for the original chat.",
|
||||
} as never,
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "toolUse",
|
||||
content: [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call_mock_image_generate_2",
|
||||
name: "image_generate",
|
||||
arguments: { prompt: "QA lighthouse" },
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
{
|
||||
role: "assistant",
|
||||
stopReason: "stop",
|
||||
content: "Worked: the QA lighthouse image completed.",
|
||||
} as never,
|
||||
];
|
||||
|
||||
const out = sanitizeOpenAIResponsesReplayForStream(messages);
|
||||
const danglingAssistant = out[5] as AssistantMessage;
|
||||
const danglingToolCall = danglingAssistant.content.find(
|
||||
(block) =>
|
||||
Boolean(block) &&
|
||||
typeof block === "object" &&
|
||||
(block as { type?: unknown }).type === "toolCall",
|
||||
) as { id?: string } | undefined;
|
||||
const danglingResult = out[6] as Extract<AgentMessage, { role: "toolResult" }>;
|
||||
|
||||
expect(out.map((message) => message.role)).toEqual([
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"custom",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"assistant",
|
||||
]);
|
||||
expect(danglingResult.toolCallId).toBe(danglingToolCall?.id);
|
||||
expect(danglingResult.toolName).toBe("image_generate");
|
||||
expect(danglingResult.isError).toBe(true);
|
||||
expect(danglingResult.content).toEqual([{ type: "text", text: "aborted" }]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,520 @@
|
||||
/** Sanitizes replayed tool calls and provider-specific transcript structure. */
|
||||
import { hasNonEmptyString as replayToolCallNonEmptyString } from "../../../../packages/normalization-core/src/string-coerce.js";
|
||||
import {
|
||||
downgradeOpenAIFunctionCallReasoningPairs,
|
||||
downgradeOpenAIReasoningBlocks,
|
||||
normalizeOpenAIResponsesToolCallIds,
|
||||
validateAnthropicTurns,
|
||||
validateGeminiTurns,
|
||||
} from "../../embedded-agent-helpers.js";
|
||||
import type { AgentMessage, StreamFn } from "../../runtime/index.js";
|
||||
import { sanitizeToolUseResultPairing } from "../../session-transcript-repair.js";
|
||||
import {
|
||||
extractToolCallsFromAssistant,
|
||||
extractToolResultIds,
|
||||
sanitizeToolCallIdsForCloudCodeAssist,
|
||||
type ToolCallIdMode,
|
||||
} from "../../tool-call-id.js";
|
||||
import { shouldAllowProviderOwnedThinkingReplay } from "../../transcript-policy.js";
|
||||
import type { TranscriptPolicy } from "../../transcript-policy.js";
|
||||
import { isRunnerToolCallBlockType } from "./attempt-tool-call-block-type.js";
|
||||
import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js";
|
||||
|
||||
const REPLAY_TOOL_CALL_NAME_MAX_CHARS = 64;
|
||||
|
||||
type ReplayToolCallBlock = {
|
||||
type?: unknown;
|
||||
id?: unknown;
|
||||
name?: unknown;
|
||||
input?: unknown;
|
||||
arguments?: unknown;
|
||||
};
|
||||
|
||||
type ReplayToolCallSanitizeReport = {
|
||||
messages: AgentMessage[];
|
||||
droppedAssistantMessages: number;
|
||||
};
|
||||
|
||||
type AnthropicToolResultContentBlock = {
|
||||
type?: unknown;
|
||||
toolUseId?: unknown;
|
||||
toolCallId?: unknown;
|
||||
tool_use_id?: unknown;
|
||||
tool_call_id?: unknown;
|
||||
};
|
||||
|
||||
function isThinkingLikeReplayBlock(block: unknown): boolean {
|
||||
if (!block || typeof block !== "object") {
|
||||
return false;
|
||||
}
|
||||
const type = (block as { type?: unknown }).type;
|
||||
return type === "thinking" || type === "redacted_thinking";
|
||||
}
|
||||
|
||||
function isReplaySafeThinkingTurn(content: unknown[], allowedToolNames?: Set<string>): boolean {
|
||||
const seenToolCallIds = new Set<string>();
|
||||
for (const block of content) {
|
||||
if (!isReplayToolCallBlock(block)) {
|
||||
continue;
|
||||
}
|
||||
const replayBlock = block;
|
||||
const toolCallId = typeof replayBlock.id === "string" ? replayBlock.id.trim() : "";
|
||||
if (!replayToolCallHasInput(replayBlock) || !toolCallId || seenToolCallIds.has(toolCallId)) {
|
||||
return false;
|
||||
}
|
||||
seenToolCallIds.add(toolCallId);
|
||||
const rawName = typeof replayBlock.name === "string" ? replayBlock.name : "";
|
||||
const resolvedName = resolveReplayToolCallName(rawName, toolCallId, allowedToolNames);
|
||||
if (!resolvedName || replayBlock.name !== resolvedName) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isReplayToolCallBlock(block: unknown): block is ReplayToolCallBlock {
|
||||
if (!block || typeof block !== "object") {
|
||||
return false;
|
||||
}
|
||||
return isRunnerToolCallBlockType((block as { type?: unknown }).type);
|
||||
}
|
||||
|
||||
function replayToolCallHasInput(block: ReplayToolCallBlock): boolean {
|
||||
const hasInput = "input" in block ? block.input !== undefined && block.input !== null : false;
|
||||
const hasArguments =
|
||||
"arguments" in block ? block.arguments !== undefined && block.arguments !== null : false;
|
||||
return hasInput || hasArguments;
|
||||
}
|
||||
|
||||
function collectFollowingToolResults(
|
||||
messages: AgentMessage[],
|
||||
index: number,
|
||||
): { ids: Set<string>; displaced: boolean } {
|
||||
const ids = new Set<string>();
|
||||
let sawNonToolResult = false;
|
||||
let displaced = false;
|
||||
for (let nextIndex = index + 1; nextIndex < messages.length; nextIndex += 1) {
|
||||
const message = messages[nextIndex];
|
||||
if (!message || typeof message !== "object") {
|
||||
sawNonToolResult = true;
|
||||
continue;
|
||||
}
|
||||
if (message.role === "assistant" && assistantTurnHasReplayToolCall(message)) {
|
||||
break;
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
const resultIds = extractToolResultIds(message);
|
||||
for (const id of resultIds) {
|
||||
ids.add(id);
|
||||
}
|
||||
displaced ||= resultIds.length > 0 && sawNonToolResult;
|
||||
continue;
|
||||
}
|
||||
sawNonToolResult = true;
|
||||
}
|
||||
return { ids, displaced };
|
||||
}
|
||||
|
||||
function resolveReplayToolCallName(
|
||||
rawName: string,
|
||||
rawId: string,
|
||||
allowedToolNames?: Set<string>,
|
||||
): string | null {
|
||||
if (rawName.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS * 2) {
|
||||
return null;
|
||||
}
|
||||
const normalized = resolveToolCallName(rawName, allowedToolNames, rawId, true);
|
||||
if (!normalized) {
|
||||
return null;
|
||||
}
|
||||
const trimmed = normalized.trim();
|
||||
if (!trimmed || trimmed.length > REPLAY_TOOL_CALL_NAME_MAX_CHARS || /\s/.test(trimmed)) {
|
||||
return null;
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function sanitizeReplayToolCallInputs(
|
||||
messages: AgentMessage[],
|
||||
allowedToolNames?: Set<string>,
|
||||
allowProviderOwnedThinkingReplay?: boolean,
|
||||
): ReplayToolCallSanitizeReport {
|
||||
let changed = false;
|
||||
let droppedAssistantMessages = 0;
|
||||
const out: AgentMessage[] = [];
|
||||
const preservedThinkingToolCallIds = new Set<string>();
|
||||
const priorToolCallIds = new Set<string>();
|
||||
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (!message) {
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
if (typeof message !== "object" || message.role !== "assistant") {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(message.content)) {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
allowProviderOwnedThinkingReplay &&
|
||||
message.content.some((block) => isThinkingLikeReplayBlock(block)) &&
|
||||
message.content.some((block) => isReplayToolCallBlock(block))
|
||||
) {
|
||||
const replaySafeToolCalls = extractToolCallsFromAssistant(message);
|
||||
const followingToolResults = collectFollowingToolResults(messages, index);
|
||||
if (
|
||||
isReplaySafeThinkingTurn(message.content, allowedToolNames) &&
|
||||
replaySafeToolCalls.every(
|
||||
(toolCall) =>
|
||||
!preservedThinkingToolCallIds.has(toolCall.id) &&
|
||||
(!followingToolResults.displaced || !priorToolCallIds.has(toolCall.id)) &&
|
||||
followingToolResults.ids.has(toolCall.id),
|
||||
)
|
||||
) {
|
||||
for (const toolCall of replaySafeToolCalls) {
|
||||
preservedThinkingToolCallIds.add(toolCall.id);
|
||||
priorToolCallIds.add(toolCall.id);
|
||||
}
|
||||
changed ||= followingToolResults.displaced;
|
||||
out.push(message);
|
||||
} else {
|
||||
changed = true;
|
||||
droppedAssistantMessages += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const nextContent: typeof message.content = [];
|
||||
let messageChanged = false;
|
||||
|
||||
for (const block of message.content) {
|
||||
if (!isReplayToolCallBlock(block)) {
|
||||
nextContent.push(block);
|
||||
continue;
|
||||
}
|
||||
const replayBlock = block as ReplayToolCallBlock;
|
||||
|
||||
if (!replayToolCallHasInput(replayBlock) || !replayToolCallNonEmptyString(replayBlock.id)) {
|
||||
changed = true;
|
||||
messageChanged = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawName = typeof replayBlock.name === "string" ? replayBlock.name : "";
|
||||
const resolvedName = resolveReplayToolCallName(rawName, replayBlock.id, allowedToolNames);
|
||||
if (!resolvedName) {
|
||||
changed = true;
|
||||
messageChanged = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (replayBlock.name !== resolvedName) {
|
||||
nextContent.push({ ...(block as object), name: resolvedName } as typeof block);
|
||||
changed = true;
|
||||
messageChanged = true;
|
||||
continue;
|
||||
}
|
||||
nextContent.push(block);
|
||||
}
|
||||
|
||||
if (messageChanged) {
|
||||
changed = true;
|
||||
if (nextContent.length > 0) {
|
||||
const nextMessage = { ...message, content: nextContent };
|
||||
for (const toolCall of extractToolCallsFromAssistant(nextMessage)) {
|
||||
priorToolCallIds.add(toolCall.id);
|
||||
}
|
||||
out.push(nextMessage);
|
||||
} else {
|
||||
droppedAssistantMessages += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const toolCall of extractToolCallsFromAssistant(message)) {
|
||||
priorToolCallIds.add(toolCall.id);
|
||||
}
|
||||
out.push(message);
|
||||
}
|
||||
|
||||
return {
|
||||
messages: changed ? out : messages,
|
||||
droppedAssistantMessages,
|
||||
};
|
||||
}
|
||||
|
||||
function extractAnthropicReplayToolResultIds(block: AnthropicToolResultContentBlock): string[] {
|
||||
const ids: string[] = [];
|
||||
for (const value of [block.toolUseId, block.toolCallId, block.tool_use_id, block.tool_call_id]) {
|
||||
if (typeof value !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed || ids.includes(trimmed)) {
|
||||
continue;
|
||||
}
|
||||
ids.push(trimmed);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function isSignedThinkingReplayAssistantSpan(message: AgentMessage | undefined): boolean {
|
||||
if (!message || typeof message !== "object" || message.role !== "assistant") {
|
||||
return false;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
content.some((block) => isThinkingLikeReplayBlock(block)) &&
|
||||
content.some((block) => isReplayToolCallBlock(block))
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeAnthropicReplayToolResults(
|
||||
messages: AgentMessage[],
|
||||
options?: {
|
||||
disallowEmbeddedUserToolResultsForSignedThinkingReplay?: boolean;
|
||||
},
|
||||
): AgentMessage[] {
|
||||
let changed = false;
|
||||
const out: AgentMessage[] = [];
|
||||
const disallowEmbeddedUserToolResultsForSignedThinkingReplay =
|
||||
options?.disallowEmbeddedUserToolResultsForSignedThinkingReplay === true;
|
||||
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (!message) {
|
||||
changed = true;
|
||||
continue;
|
||||
}
|
||||
if (typeof message !== "object" || message.role !== "user") {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(message.content)) {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
|
||||
const previous = messages[index - 1];
|
||||
const shouldStripEmbeddedToolResults =
|
||||
disallowEmbeddedUserToolResultsForSignedThinkingReplay &&
|
||||
isSignedThinkingReplayAssistantSpan(previous);
|
||||
const validToolUseIds = new Set<string>();
|
||||
if (previous && typeof previous === "object" && previous.role === "assistant") {
|
||||
const previousContent = (previous as { content?: unknown }).content;
|
||||
if (Array.isArray(previousContent)) {
|
||||
for (const block of previousContent) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const typedBlock = block as { type?: unknown; id?: unknown };
|
||||
if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmedId = typedBlock.id.trim();
|
||||
if (trimmedId) {
|
||||
validToolUseIds.add(trimmedId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const nextContent = message.content.filter((block) => {
|
||||
if (!block || typeof block !== "object") {
|
||||
return true;
|
||||
}
|
||||
const typedBlock = block as AnthropicToolResultContentBlock;
|
||||
if (typedBlock.type !== "toolResult" && typedBlock.type !== "tool") {
|
||||
return true;
|
||||
}
|
||||
if (shouldStripEmbeddedToolResults) {
|
||||
changed = true;
|
||||
return false;
|
||||
}
|
||||
const resultIds = extractAnthropicReplayToolResultIds(typedBlock);
|
||||
if (resultIds.length === 0) {
|
||||
changed = true;
|
||||
return false;
|
||||
}
|
||||
return validToolUseIds.size > 0 && resultIds.some((id) => validToolUseIds.has(id));
|
||||
});
|
||||
|
||||
if (nextContent.length === message.content.length) {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
|
||||
changed = true;
|
||||
if (nextContent.length > 0) {
|
||||
out.push({ ...message, content: nextContent });
|
||||
continue;
|
||||
}
|
||||
|
||||
out.push({
|
||||
...message,
|
||||
content: [{ type: "text", text: "[tool results omitted]" }],
|
||||
} as AgentMessage);
|
||||
}
|
||||
|
||||
return changed ? out : messages;
|
||||
}
|
||||
|
||||
function assistantTurnHasReplayToolCall(message: AgentMessage): boolean {
|
||||
if (!message || typeof message !== "object" || message.role !== "assistant") {
|
||||
return false;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return false;
|
||||
}
|
||||
return content.some((block) => isReplayToolCallBlock(block));
|
||||
}
|
||||
|
||||
function stripTrailingAssistantPrefillTurns(messages: AgentMessage[]): AgentMessage[] {
|
||||
let end = messages.length;
|
||||
while (end > 0) {
|
||||
const message = messages[end - 1];
|
||||
if (!message || typeof message !== "object" || message.role !== "assistant") {
|
||||
break;
|
||||
}
|
||||
if (assistantTurnHasReplayToolCall(message)) {
|
||||
break;
|
||||
}
|
||||
end -= 1;
|
||||
}
|
||||
return end === messages.length ? messages : messages.slice(0, end);
|
||||
}
|
||||
|
||||
type ReplayToolCallIdSanitizerDecision = {
|
||||
sanitizeToolCallIds: boolean;
|
||||
toolCallIdMode?: ToolCallIdMode;
|
||||
isOpenAIResponsesApi: boolean;
|
||||
};
|
||||
|
||||
/** Returns whether replayed tool-call ids should be sanitized for non-Responses providers. */
|
||||
export function shouldApplyReplayToolCallIdSanitizer(
|
||||
params: ReplayToolCallIdSanitizerDecision,
|
||||
): params is ReplayToolCallIdSanitizerDecision & { toolCallIdMode: ToolCallIdMode } {
|
||||
return (
|
||||
params.sanitizeToolCallIds && Boolean(params.toolCallIdMode) && !params.isOpenAIResponsesApi
|
||||
);
|
||||
}
|
||||
|
||||
/** Rewrites replayed tool-call ids into provider-safe ids and optionally repairs result pairing. */
|
||||
export function sanitizeReplayToolCallIdsForStream(params: {
|
||||
messages: AgentMessage[];
|
||||
mode: ToolCallIdMode;
|
||||
allowedToolNames?: Set<string>;
|
||||
preserveNativeAnthropicToolUseIds?: boolean;
|
||||
duplicateToolCallIdStyle?: "openai";
|
||||
preserveReplaySafeThinkingToolCallIds?: boolean;
|
||||
repairToolUseResultPairing?: boolean;
|
||||
}): AgentMessage[] {
|
||||
const paired = params.repairToolUseResultPairing
|
||||
? sanitizeToolUseResultPairing(params.messages)
|
||||
: params.messages;
|
||||
return sanitizeToolCallIdsForCloudCodeAssist(paired, params.mode, {
|
||||
preserveNativeAnthropicToolUseIds: params.preserveNativeAnthropicToolUseIds,
|
||||
duplicateToolCallIdStyle: params.duplicateToolCallIdStyle,
|
||||
preserveReplaySafeThinkingToolCallIds: params.preserveReplaySafeThinkingToolCallIds,
|
||||
allowedToolNames: params.allowedToolNames,
|
||||
});
|
||||
}
|
||||
|
||||
/** Downgrades OpenAI Responses replay turns into the stream format expected by runtime callers. */
|
||||
export function sanitizeOpenAIResponsesReplayForStream(messages: AgentMessage[]): AgentMessage[] {
|
||||
const repaired = sanitizeToolUseResultPairing(messages, {
|
||||
erroredAssistantResultPolicy: "drop",
|
||||
missingToolResultText: "aborted",
|
||||
});
|
||||
return downgradeOpenAIFunctionCallReasoningPairs(
|
||||
normalizeOpenAIResponsesToolCallIds(downgradeOpenAIReasoningBlocks(repaired)),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitizes malformed replay tool calls before provider submission. The wrapper
|
||||
* drops invalid assistant tool calls, repairs adjacent tool results when needed,
|
||||
* strips trailing assistant prefill turns for strict providers, and revalidates
|
||||
* Anthropic/Gemini transcripts after mutations.
|
||||
*/
|
||||
export function wrapStreamFnSanitizeMalformedToolCalls(
|
||||
baseFn: StreamFn,
|
||||
allowedToolNames?: Set<string>,
|
||||
transcriptPolicy?: Pick<
|
||||
TranscriptPolicy,
|
||||
"validateGeminiTurns" | "validateAnthropicTurns" | "preserveSignatures" | "dropThinkingBlocks"
|
||||
>,
|
||||
provider?: string | null,
|
||||
): StreamFn {
|
||||
return (model, context, options) => {
|
||||
const ctx = context as unknown as { messages?: unknown };
|
||||
const messages = ctx?.messages;
|
||||
if (!Array.isArray(messages)) {
|
||||
return baseFn(model, context, options);
|
||||
}
|
||||
const allowProviderOwnedThinkingReplay = shouldAllowProviderOwnedThinkingReplay({
|
||||
modelApi: (model as { api?: unknown })?.api as string | null | undefined,
|
||||
provider,
|
||||
policy: {
|
||||
validateAnthropicTurns: transcriptPolicy?.validateAnthropicTurns === true,
|
||||
preserveSignatures: transcriptPolicy?.preserveSignatures === true,
|
||||
dropThinkingBlocks: transcriptPolicy?.dropThinkingBlocks === true,
|
||||
},
|
||||
});
|
||||
const sanitized = sanitizeReplayToolCallInputs(
|
||||
messages as AgentMessage[],
|
||||
allowedToolNames,
|
||||
allowProviderOwnedThinkingReplay,
|
||||
);
|
||||
const isOpenAIResponsesApi =
|
||||
(model as { api?: unknown }).api === "openai-responses" ||
|
||||
(model as { api?: unknown }).api === "openai-chatgpt-responses" ||
|
||||
(model as { api?: unknown }).api === "azure-openai-responses";
|
||||
const replayInputsChanged = sanitized.messages !== messages;
|
||||
let nextMessages = isOpenAIResponsesApi
|
||||
? sanitizeToolUseResultPairing(sanitized.messages, {
|
||||
erroredAssistantResultPolicy: "drop",
|
||||
missingToolResultText: "aborted",
|
||||
})
|
||||
: replayInputsChanged
|
||||
? sanitizeToolUseResultPairing(sanitized.messages)
|
||||
: sanitized.messages;
|
||||
let strippedTrailingAssistantPrefill = false;
|
||||
if (transcriptPolicy?.validateAnthropicTurns) {
|
||||
nextMessages = sanitizeAnthropicReplayToolResults(nextMessages, {
|
||||
disallowEmbeddedUserToolResultsForSignedThinkingReplay: allowProviderOwnedThinkingReplay,
|
||||
});
|
||||
}
|
||||
if (transcriptPolicy?.validateAnthropicTurns || transcriptPolicy?.validateGeminiTurns) {
|
||||
const beforeStrip = nextMessages;
|
||||
nextMessages = stripTrailingAssistantPrefillTurns(nextMessages);
|
||||
strippedTrailingAssistantPrefill ||= nextMessages !== beforeStrip;
|
||||
}
|
||||
if (nextMessages === messages) {
|
||||
return baseFn(model, context, options);
|
||||
}
|
||||
if (
|
||||
sanitized.droppedAssistantMessages > 0 ||
|
||||
transcriptPolicy?.validateAnthropicTurns ||
|
||||
strippedTrailingAssistantPrefill
|
||||
) {
|
||||
if (transcriptPolicy?.validateGeminiTurns) {
|
||||
nextMessages = validateGeminiTurns(nextMessages);
|
||||
}
|
||||
if (transcriptPolicy?.validateAnthropicTurns) {
|
||||
nextMessages = validateAnthropicTurns(nextMessages);
|
||||
}
|
||||
}
|
||||
const nextContext = {
|
||||
...(context as unknown as Record<string, unknown>),
|
||||
messages: nextMessages,
|
||||
} as unknown;
|
||||
return baseFn(model, nextContext as typeof context, options);
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,362 @@
|
||||
/** Normalizes live streamed tool-call names, ids, and unknown-tool loops. */
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { visitObjectContentBlocks } from "../../../shared/message-content-blocks.js";
|
||||
import type { StreamFn } from "../../runtime/index.js";
|
||||
import { normalizeToolPolicyName } from "../../tool-policy.js";
|
||||
import { isRunnerToolCallBlockType } from "./attempt-tool-call-block-type.js";
|
||||
import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js";
|
||||
import { wrapStreamObjectEvents } from "./stream-wrapper.js";
|
||||
|
||||
const BLANK_TOOL_CALL_NAME_DESCRIPTION = "blank tool name";
|
||||
type UnknownToolLoopGuardState = {
|
||||
lastUnknownToolName?: string;
|
||||
count: number;
|
||||
countedMessages: WeakSet<object>;
|
||||
};
|
||||
type AssistantStream = Awaited<ReturnType<StreamFn>>;
|
||||
|
||||
function createStandaloneTextToolCallId(): string {
|
||||
return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
|
||||
}
|
||||
|
||||
function normalizeToolCallIdsInMessage(message: unknown, fallbackIdByContentIndex: string[]): void {
|
||||
if (!message || typeof message !== "object") {
|
||||
return;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const usedIds = new Set<string>();
|
||||
for (const block of content) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const typedBlock = block as { type?: unknown; id?: unknown };
|
||||
if (!isRunnerToolCallBlockType(typedBlock.type) || typeof typedBlock.id !== "string") {
|
||||
continue;
|
||||
}
|
||||
const trimmedId = typedBlock.id.trim();
|
||||
if (!trimmedId) {
|
||||
continue;
|
||||
}
|
||||
usedIds.add(trimmedId);
|
||||
}
|
||||
|
||||
const assignedIds = new Set<string>();
|
||||
for (const [contentIndex, block] of content.entries()) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const typedBlock = block as { type?: unknown; id?: unknown };
|
||||
if (!isRunnerToolCallBlockType(typedBlock.type)) {
|
||||
continue;
|
||||
}
|
||||
if (typeof typedBlock.id === "string") {
|
||||
const trimmedId = typedBlock.id.trim();
|
||||
if (trimmedId) {
|
||||
if (!assignedIds.has(trimmedId)) {
|
||||
if (typedBlock.id !== trimmedId) {
|
||||
typedBlock.id = trimmedId;
|
||||
}
|
||||
assignedIds.add(trimmedId);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let fallbackId = fallbackIdByContentIndex[contentIndex];
|
||||
while (!fallbackId || usedIds.has(fallbackId) || assignedIds.has(fallbackId)) {
|
||||
fallbackId = createStandaloneTextToolCallId();
|
||||
}
|
||||
fallbackIdByContentIndex[contentIndex] = fallbackId;
|
||||
typedBlock.id = fallbackId;
|
||||
usedIds.add(fallbackId);
|
||||
assignedIds.add(fallbackId);
|
||||
}
|
||||
}
|
||||
|
||||
function trimWhitespaceFromToolCallNamesInMessage(
|
||||
message: unknown,
|
||||
allowedToolNames: Set<string> | undefined,
|
||||
fallbackIdByContentIndex: string[],
|
||||
): void {
|
||||
visitObjectContentBlocks(message, (block) => {
|
||||
const typedBlock = block as { type?: unknown; name?: unknown; id?: unknown };
|
||||
if (!isRunnerToolCallBlockType(typedBlock.type)) {
|
||||
return;
|
||||
}
|
||||
const rawId = typeof typedBlock.id === "string" ? typedBlock.id : undefined;
|
||||
if (typeof typedBlock.name === "string") {
|
||||
const normalized = resolveToolCallName(typedBlock.name, allowedToolNames, rawId);
|
||||
if (normalized !== null && normalized !== typedBlock.name) {
|
||||
typedBlock.name = normalized;
|
||||
}
|
||||
return;
|
||||
}
|
||||
const inferred = resolveToolCallName("", allowedToolNames, rawId);
|
||||
if (inferred) {
|
||||
typedBlock.name = inferred;
|
||||
}
|
||||
});
|
||||
normalizeToolCallIdsInMessage(message, fallbackIdByContentIndex);
|
||||
}
|
||||
|
||||
function classifyToolCallMessage(
|
||||
message: unknown,
|
||||
allowedToolNames?: Set<string>,
|
||||
):
|
||||
| { kind: "none" }
|
||||
| { kind: "allowed" }
|
||||
| { kind: "incomplete" }
|
||||
| { kind: "malformed"; toolName: string }
|
||||
| { kind: "unknown"; toolName: string } {
|
||||
if (!message || typeof message !== "object") {
|
||||
return { kind: "none" };
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return { kind: "none" };
|
||||
}
|
||||
|
||||
let unknownToolName: string | undefined;
|
||||
let sawToolCall = false;
|
||||
let sawAllowedToolCall = false;
|
||||
let sawIncompleteToolCall = false;
|
||||
let sawBlankStringToolCall = false;
|
||||
const hasAllowedToolNames = Boolean(allowedToolNames && allowedToolNames.size > 0);
|
||||
for (const block of content) {
|
||||
if (!block || typeof block !== "object") {
|
||||
continue;
|
||||
}
|
||||
const typedBlock = block as { type?: unknown; name?: unknown };
|
||||
if (!isRunnerToolCallBlockType(typedBlock.type)) {
|
||||
continue;
|
||||
}
|
||||
sawToolCall = true;
|
||||
const rawBlockName = typedBlock.name;
|
||||
const hasStringName = typeof rawBlockName === "string";
|
||||
const rawName = hasStringName ? rawBlockName.trim() : "";
|
||||
if (!rawName) {
|
||||
if (hasStringName) {
|
||||
sawBlankStringToolCall = true;
|
||||
} else {
|
||||
sawIncompleteToolCall = true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!hasAllowedToolNames) {
|
||||
continue;
|
||||
}
|
||||
if (resolveToolCallName(rawName, allowedToolNames, undefined, true)) {
|
||||
sawAllowedToolCall = true;
|
||||
continue;
|
||||
}
|
||||
const normalizedUnknownToolName = normalizeToolPolicyName(rawName);
|
||||
if (!unknownToolName) {
|
||||
unknownToolName = normalizedUnknownToolName;
|
||||
continue;
|
||||
}
|
||||
if (unknownToolName !== normalizedUnknownToolName) {
|
||||
sawIncompleteToolCall = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!sawToolCall) {
|
||||
return { kind: "none" };
|
||||
}
|
||||
if (!hasAllowedToolNames) {
|
||||
return sawBlankStringToolCall
|
||||
? { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION }
|
||||
: { kind: "none" };
|
||||
}
|
||||
if (sawAllowedToolCall) {
|
||||
return { kind: "allowed" };
|
||||
}
|
||||
if (sawBlankStringToolCall && !sawIncompleteToolCall && unknownToolName === undefined) {
|
||||
return { kind: "malformed", toolName: BLANK_TOOL_CALL_NAME_DESCRIPTION };
|
||||
}
|
||||
if (sawIncompleteToolCall) {
|
||||
return { kind: "incomplete" };
|
||||
}
|
||||
return unknownToolName ? { kind: "unknown", toolName: unknownToolName } : { kind: "incomplete" };
|
||||
}
|
||||
|
||||
function rewriteUnknownToolLoopMessage(message: unknown, toolName: string): void {
|
||||
if (!message || typeof message !== "object") {
|
||||
return;
|
||||
}
|
||||
(message as { content?: unknown }).content = [
|
||||
{
|
||||
type: "text",
|
||||
text: `I can't use the tool "${toolName}" here because it isn't available. I need to stop retrying it and answer without that tool.`,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function guardUnknownToolLoopInMessage(
|
||||
message: unknown,
|
||||
state: UnknownToolLoopGuardState,
|
||||
params: {
|
||||
allowedToolNames?: Set<string>;
|
||||
threshold?: number;
|
||||
countAttempt: boolean;
|
||||
resetOnAllowedTool?: boolean;
|
||||
resetOnMissingUnknownTool?: boolean;
|
||||
rewriteMalformedBlankToolName?: boolean;
|
||||
},
|
||||
): boolean {
|
||||
const toolCallState = classifyToolCallMessage(message, params.allowedToolNames);
|
||||
if (toolCallState.kind === "allowed") {
|
||||
if (params.resetOnAllowedTool === true) {
|
||||
state.lastUnknownToolName = undefined;
|
||||
state.count = 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (toolCallState.kind === "malformed") {
|
||||
if (params.rewriteMalformedBlankToolName === true) {
|
||||
rewriteUnknownToolLoopMessage(message, toolCallState.toolName);
|
||||
return true;
|
||||
}
|
||||
if (params.countAttempt && params.resetOnMissingUnknownTool !== false) {
|
||||
state.lastUnknownToolName = undefined;
|
||||
state.count = 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const threshold = params.threshold;
|
||||
if (threshold === undefined || threshold <= 0) {
|
||||
return false;
|
||||
}
|
||||
if (toolCallState.kind !== "unknown") {
|
||||
if (params.countAttempt && params.resetOnMissingUnknownTool !== false) {
|
||||
state.lastUnknownToolName = undefined;
|
||||
state.count = 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const unknownToolName = toolCallState.toolName;
|
||||
|
||||
if (!params.countAttempt) {
|
||||
// Partial stream events can rewrite after the threshold, but only final
|
||||
// messages advance the loop counter.
|
||||
if (state.lastUnknownToolName === unknownToolName && state.count > threshold) {
|
||||
rewriteUnknownToolLoopMessage(message, unknownToolName);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
if (message && typeof message === "object") {
|
||||
if (state.countedMessages.has(message)) {
|
||||
if (state.lastUnknownToolName === unknownToolName && state.count > threshold) {
|
||||
rewriteUnknownToolLoopMessage(message, unknownToolName);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
state.countedMessages.add(message);
|
||||
}
|
||||
|
||||
if (state.lastUnknownToolName === unknownToolName) {
|
||||
state.count += 1;
|
||||
} else {
|
||||
state.lastUnknownToolName = unknownToolName;
|
||||
state.count = 1;
|
||||
}
|
||||
|
||||
if (state.count > threshold) {
|
||||
rewriteUnknownToolLoopMessage(message, unknownToolName);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function wrapStreamTrimToolCallNames(
|
||||
stream: AssistantStream,
|
||||
allowedToolNames?: Set<string>,
|
||||
options?: { unknownToolThreshold?: number; state?: UnknownToolLoopGuardState },
|
||||
): AssistantStream {
|
||||
const unknownToolGuardState = options?.state ?? {
|
||||
count: 0,
|
||||
countedMessages: new WeakSet<object>(),
|
||||
};
|
||||
// Provider-omitted ids are only message-local. Reuse one generated id per
|
||||
// content position across this response's partial/final projections, while a
|
||||
// later assistant response gets a fresh namespace and cannot alias it.
|
||||
const fallbackIdByContentIndex: string[] = [];
|
||||
let streamAttemptAlreadyCounted = false;
|
||||
const originalResult = stream.result.bind(stream);
|
||||
stream.result = async () => {
|
||||
const message = await originalResult();
|
||||
trimWhitespaceFromToolCallNamesInMessage(message, allowedToolNames, fallbackIdByContentIndex);
|
||||
guardUnknownToolLoopInMessage(message, unknownToolGuardState, {
|
||||
allowedToolNames,
|
||||
threshold: options?.unknownToolThreshold,
|
||||
countAttempt: !streamAttemptAlreadyCounted,
|
||||
resetOnAllowedTool: true,
|
||||
rewriteMalformedBlankToolName: true,
|
||||
});
|
||||
return message;
|
||||
};
|
||||
|
||||
wrapStreamObjectEvents(stream, (event) => {
|
||||
trimWhitespaceFromToolCallNamesInMessage(
|
||||
event.partial,
|
||||
allowedToolNames,
|
||||
fallbackIdByContentIndex,
|
||||
);
|
||||
trimWhitespaceFromToolCallNamesInMessage(
|
||||
event.message,
|
||||
allowedToolNames,
|
||||
fallbackIdByContentIndex,
|
||||
);
|
||||
if (event.message && typeof event.message === "object") {
|
||||
const countedStreamAttempt = guardUnknownToolLoopInMessage(
|
||||
event.message,
|
||||
unknownToolGuardState,
|
||||
{
|
||||
allowedToolNames,
|
||||
threshold: options?.unknownToolThreshold,
|
||||
countAttempt: !streamAttemptAlreadyCounted,
|
||||
resetOnAllowedTool: true,
|
||||
resetOnMissingUnknownTool: false,
|
||||
},
|
||||
);
|
||||
streamAttemptAlreadyCounted ||= countedStreamAttempt;
|
||||
}
|
||||
guardUnknownToolLoopInMessage(event.partial, unknownToolGuardState, {
|
||||
allowedToolNames,
|
||||
threshold: options?.unknownToolThreshold,
|
||||
countAttempt: false,
|
||||
});
|
||||
});
|
||||
|
||||
return stream;
|
||||
}
|
||||
|
||||
/** Normalizes streamed tool-call names and guards repeated unknown-tool loops. */
|
||||
export function wrapStreamFnTrimToolCallNames(
|
||||
baseFn: StreamFn,
|
||||
allowedToolNames?: Set<string>,
|
||||
guardOptions?: { unknownToolThreshold?: number },
|
||||
): StreamFn {
|
||||
const unknownToolGuardState: UnknownToolLoopGuardState = {
|
||||
count: 0,
|
||||
countedMessages: new WeakSet<object>(),
|
||||
};
|
||||
return (model, context, streamOptions) => {
|
||||
const maybeStream = baseFn(model, context, streamOptions);
|
||||
if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) {
|
||||
return Promise.resolve(maybeStream).then((stream) =>
|
||||
wrapStreamTrimToolCallNames(stream, allowedToolNames, {
|
||||
unknownToolThreshold: guardOptions?.unknownToolThreshold,
|
||||
state: unknownToolGuardState,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return wrapStreamTrimToolCallNames(maybeStream, allowedToolNames, {
|
||||
unknownToolThreshold: guardOptions?.unknownToolThreshold,
|
||||
state: unknownToolGuardState,
|
||||
});
|
||||
};
|
||||
}
|
||||
+272
@@ -0,0 +1,272 @@
|
||||
// Coverage for promoting standalone text tool calls into structured events.
|
||||
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js";
|
||||
|
||||
type FakeWrappedStream = {
|
||||
result: () => Promise<unknown>;
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
};
|
||||
|
||||
function createFakeStream(params: {
|
||||
events: unknown[];
|
||||
resultMessage: unknown;
|
||||
}): FakeWrappedStream {
|
||||
return {
|
||||
async result() {
|
||||
return params.resultMessage;
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return (async function* () {
|
||||
for (const event of params.events) {
|
||||
yield event;
|
||||
}
|
||||
})();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function collectStreamEvents(stream: AsyncIterable<unknown>): Promise<unknown[]> {
|
||||
// Drain streams to inspect generated tool-call events after wrapper mutation.
|
||||
const events: unknown[] = [];
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => {
|
||||
it("promotes serialized tool calls split across adjacent text blocks", async () => {
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "[tool:exec]\n<parameter=command>\n" },
|
||||
{ type: "text", text: "pwd\n</parameter>\n</function>" },
|
||||
{ type: "thinking", thinking: "Checking location." },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "[tool:exec]\n<parameter=command>\n" },
|
||||
{ type: "text_delta", contentIndex: 1, delta: "pwd\n</parameter>\n</function>" },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 2,
|
||||
delta: "Checking location.",
|
||||
partial: { content: resultMessage.content },
|
||||
},
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"thinking_delta",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1);
|
||||
expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0);
|
||||
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
|
||||
"toolCall",
|
||||
"thinking",
|
||||
]);
|
||||
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
|
||||
name: "exec",
|
||||
arguments: { command: "pwd" },
|
||||
});
|
||||
});
|
||||
|
||||
it("buffers case-insensitive tool-name prefixes until final promotion", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:read]",
|
||||
"<parameter=path>",
|
||||
"src/index.ts",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "[tool:rea" },
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:rea".length) },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["Read"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(result.stopReason).toBe("toolUse");
|
||||
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
|
||||
type: "toolCall",
|
||||
name: "Read",
|
||||
arguments: { path: "src/index.ts" },
|
||||
});
|
||||
});
|
||||
|
||||
it("buffers normalized alias tool-name prefixes until final promotion", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:bash]",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "[tool:ba" },
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice("[tool:ba".length) },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
|
||||
type: "toolCall",
|
||||
name: "exec",
|
||||
arguments: { command: "pwd" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "case-insensitive name",
|
||||
allowedToolName: "Read",
|
||||
emittedToolName: "READ",
|
||||
expectedToolName: "Read",
|
||||
parameterName: "path",
|
||||
parameterValue: "src/index.ts",
|
||||
},
|
||||
{
|
||||
label: "normalized alias",
|
||||
allowedToolName: "exec",
|
||||
emittedToolName: "bash",
|
||||
expectedToolName: "exec",
|
||||
parameterName: "command",
|
||||
parameterValue: "pwd",
|
||||
},
|
||||
])(
|
||||
"promotes $label XML consistently when the terminal reason is toolUse",
|
||||
async ({
|
||||
allowedToolName,
|
||||
emittedToolName,
|
||||
expectedToolName,
|
||||
parameterName,
|
||||
parameterValue,
|
||||
}) => {
|
||||
const rawToolText = [
|
||||
`<function=${emittedToolName}>`,
|
||||
`<parameter=${parameterName}>`,
|
||||
parameterValue,
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "toolUse",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
|
||||
{ type: "done", reason: "toolUse", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(
|
||||
baseFn as never,
|
||||
new Set([allowedToolName]),
|
||||
);
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
const expectedArguments = { [parameterName]: parameterValue };
|
||||
const expectedContent = [
|
||||
{
|
||||
type: "toolCall",
|
||||
id: expect.stringMatching(/^call_[a-f0-9]{24}$/),
|
||||
name: expectedToolName,
|
||||
arguments: expectedArguments,
|
||||
partialArgs: JSON.stringify(expectedArguments),
|
||||
},
|
||||
];
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events[2], "toolcall delta").delta).toBe(
|
||||
JSON.stringify(expectedArguments),
|
||||
);
|
||||
const doneEvent = requireRecord(events[4], "done event");
|
||||
expect(doneEvent.reason).toBe("toolUse");
|
||||
expect(requireRecord(doneEvent.message, "done message").content).toEqual(expectedContent);
|
||||
expect(result).toMatchObject({ role: "assistant", stopReason: "toolUse" });
|
||||
expect(result.content).toEqual(expectedContent);
|
||||
},
|
||||
);
|
||||
});
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
// Coverage for promoting standalone text tool calls into structured events.
|
||||
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js";
|
||||
|
||||
type FakeWrappedStream = {
|
||||
result: () => Promise<unknown>;
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
};
|
||||
|
||||
function createFakeStream(params: {
|
||||
events: unknown[];
|
||||
resultMessage: unknown;
|
||||
}): FakeWrappedStream {
|
||||
return {
|
||||
async result() {
|
||||
return params.resultMessage;
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return (async function* () {
|
||||
for (const event of params.events) {
|
||||
yield event;
|
||||
}
|
||||
})();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function collectStreamEvents(stream: AsyncIterable<unknown>): Promise<unknown[]> {
|
||||
// Drain streams to inspect generated tool-call events after wrapper mutation.
|
||||
const events: unknown[] = [];
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => {
|
||||
it("keeps possible tool-call text buffered across interleaved non-text events", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
{ type: "text", text: rawToolText },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 1, delta: rawToolText },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 0,
|
||||
delta: "Need shell state.",
|
||||
partial: {
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
{ type: "text", text: rawToolText },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"thinking_delta",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
const thinkingEvent = requireRecord(events[1], "thinking event");
|
||||
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
expect.objectContaining({
|
||||
type: "toolCall",
|
||||
name: "exec",
|
||||
arguments: { command: "pwd" },
|
||||
}),
|
||||
]);
|
||||
expect(JSON.stringify(events)).not.toContain(rawToolText);
|
||||
});
|
||||
|
||||
it("preserves interleaved event content indexes when buffered text is scrubbed first", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: rawToolText },
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 1,
|
||||
delta: "Need shell state.",
|
||||
partial: {
|
||||
content: [
|
||||
{ type: "text", text: rawToolText },
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"thinking_delta",
|
||||
"done",
|
||||
]);
|
||||
const thinkingEvent = requireRecord(events[4], "thinking event");
|
||||
expect(thinkingEvent.contentIndex).toBe(1);
|
||||
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "toolCall",
|
||||
name: "exec",
|
||||
arguments: { command: "pwd" },
|
||||
}),
|
||||
{ type: "thinking", thinking: "Need shell state." },
|
||||
]);
|
||||
expect(JSON.stringify(events)).not.toContain(rawToolText);
|
||||
});
|
||||
|
||||
it("closes the underlying stream iterator when consumers stop early", async () => {
|
||||
const returnIterator = vi.fn(async () => ({ done: true, value: undefined }));
|
||||
const nextIterator = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ done: false, value: { type: "start", partial: { content: [] } } })
|
||||
.mockResolvedValue({ done: true, value: undefined });
|
||||
const baseFn = vi.fn(() => ({
|
||||
async result() {
|
||||
return { role: "assistant", content: [], stopReason: "stop" };
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return {
|
||||
next: nextIterator,
|
||||
return: returnIterator,
|
||||
};
|
||||
},
|
||||
}));
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
const iterator = stream[Symbol.asyncIterator]();
|
||||
|
||||
expect(await iterator.next()).toEqual({
|
||||
done: false,
|
||||
value: { type: "start", partial: { content: [] } },
|
||||
});
|
||||
await iterator.return?.();
|
||||
|
||||
expect(returnIterator).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fails closed on buffered known-tool text before terminal errors", async () => {
|
||||
const rawToolText = "[tool:exec]";
|
||||
const errorEvent = { type: "error", error: new Error("stream failed") };
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [{ type: "text_delta", contentIndex: 0, delta: rawToolText }, errorEvent],
|
||||
resultMessage: { role: "assistant", content: [], stopReason: "stop" },
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events).toEqual([errorEvent]);
|
||||
});
|
||||
});
|
||||
+381
@@ -0,0 +1,381 @@
|
||||
// Coverage for promoting standalone text tool calls into structured events.
|
||||
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js";
|
||||
|
||||
type FakeWrappedStream = {
|
||||
result: () => Promise<unknown>;
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
};
|
||||
|
||||
function createFakeStream(params: {
|
||||
events: unknown[];
|
||||
resultMessage: unknown;
|
||||
}): FakeWrappedStream {
|
||||
return {
|
||||
async result() {
|
||||
return params.resultMessage;
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return (async function* () {
|
||||
for (const event of params.events) {
|
||||
yield event;
|
||||
}
|
||||
})();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function collectStreamEvents(stream: AsyncIterable<unknown>): Promise<unknown[]> {
|
||||
// Drain streams to inspect generated tool-call events after wrapper mutation.
|
||||
const events: unknown[] = [];
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => {
|
||||
it("buffers split XML function markers until final promotion", async () => {
|
||||
const rawToolText = [
|
||||
"<function=exec>",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "<" },
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText.slice(1) },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "bracketed XML text over the character cap",
|
||||
marker: "[tool:exec]",
|
||||
rawToolText: [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"x".repeat(256_001),
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n"),
|
||||
},
|
||||
{
|
||||
label: "zero-argument XML text over the byte cap",
|
||||
marker: "<function=exec>",
|
||||
rawToolText: `<function=exec>${"\u00a0".repeat(128_001)}</function>`,
|
||||
},
|
||||
{
|
||||
label: "incomplete XML text over the byte cap",
|
||||
marker: "<function=exec>",
|
||||
rawToolText: `<function=exec>${"\u00a0".repeat(128_001)}`,
|
||||
},
|
||||
])("suppresses $label instead of flushing it", async ({ marker, rawToolText }) => {
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "start", partial: { content: [] } },
|
||||
{
|
||||
type: "text_start",
|
||||
contentIndex: 0,
|
||||
partial: { content: [{ type: "text", text: "" }] },
|
||||
},
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 1,
|
||||
delta: "still thinking",
|
||||
partial: {
|
||||
content: [
|
||||
{ type: "text", text: rawToolText },
|
||||
{ type: "thinking", thinking: "still thinking" },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "text_end", contentIndex: 0, content: rawToolText },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"thinking_delta",
|
||||
"done",
|
||||
]);
|
||||
const thinkingEvent = requireRecord(events[1], "thinking event");
|
||||
expect(requireRecord(thinkingEvent.partial, "thinking partial").content).toEqual([
|
||||
{ type: "text", text: "" },
|
||||
{ type: "thinking", thinking: "still thinking" },
|
||||
]);
|
||||
const doneEvent = requireRecord(events[2], "done event");
|
||||
expect(doneEvent.reason).toBe("stop");
|
||||
expect(doneEvent.message).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [],
|
||||
stopReason: "stop",
|
||||
});
|
||||
expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" });
|
||||
expect(JSON.stringify(events)).not.toContain(marker);
|
||||
expect(JSON.stringify(result)).not.toContain(marker);
|
||||
});
|
||||
|
||||
it("scrubs split over-cap serialized XMLish text blocks from done messages", async () => {
|
||||
const rawToolTextParts = [
|
||||
"[tool:exec]\n<parameter=command>",
|
||||
["x".repeat(256_001), "</parameter>", "</function>"].join("\n"),
|
||||
];
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: rawToolTextParts.map((text) => ({ type: "text", text })),
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [{ type: "done", reason: "stop", message: resultMessage }],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(requireRecord(events[0], "done event").message).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [],
|
||||
stopReason: "stop",
|
||||
});
|
||||
expect(result).toMatchObject({ role: "assistant", content: [], stopReason: "stop" });
|
||||
expect(JSON.stringify(events)).not.toContain("[tool:exec]");
|
||||
expect(JSON.stringify(result)).not.toContain("</parameter>");
|
||||
});
|
||||
|
||||
it("scrubs an over-cap whitespace-only XML body split into its own text block", async () => {
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "<function=exec>" },
|
||||
{ type: "text", text: "\u00a0".repeat(128_001) },
|
||||
{ type: "text", text: "</function>" },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [{ type: "done", reason: "stop", message: resultMessage }],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
const expectedMessage = { role: "assistant", content: [], stopReason: "stop" };
|
||||
|
||||
expect(events).toHaveLength(1);
|
||||
const doneEvent = requireRecord(events[0], "done event");
|
||||
expect(doneEvent.type).toBe("done");
|
||||
expect(doneEvent.reason).toBe("stop");
|
||||
expect(doneEvent.message).toEqual(expectedMessage);
|
||||
expect(result).toEqual(expectedMessage);
|
||||
});
|
||||
|
||||
it.each(["error", "aborted"])(
|
||||
"scrubs over-cap XML from stream.result() when stopReason is %s",
|
||||
async (stopReason) => {
|
||||
const rawToolText = `<function=exec>${"\u00a0".repeat(128_001)}</function>`;
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason,
|
||||
};
|
||||
const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage }));
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(
|
||||
baseFn as never,
|
||||
new Set(["exec"]),
|
||||
);
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(result).toEqual({ role: "assistant", content: [], stopReason });
|
||||
expect(JSON.stringify(result)).not.toContain("<function=exec>");
|
||||
},
|
||||
);
|
||||
|
||||
it("scrubs an incomplete named call from stream.result()", async () => {
|
||||
const rawToolText = "<function=exec><parameter=command>SECRET";
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage }));
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(result).toEqual({ role: "assistant", content: [], stopReason: "stop" });
|
||||
});
|
||||
|
||||
it("preserves visible suffix text after an over-cap JSON tool payload", async () => {
|
||||
const visibleSuffix = "Visible answer after oversized JSON.";
|
||||
const rawText = [`[tool:exec] {"command":"${"x".repeat(256_001)}"}`, visibleSuffix].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawText },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"text_delta",
|
||||
"done",
|
||||
]);
|
||||
const textEvent = requireRecord(events[0], "text event");
|
||||
expect(String(textEvent.delta)).toBe(visibleSuffix);
|
||||
expect(requireRecord(textEvent.partial, "text partial").content).toEqual([
|
||||
{ type: "text", text: visibleSuffix },
|
||||
]);
|
||||
expect(JSON.stringify(events)).not.toContain("[tool:exec]");
|
||||
});
|
||||
|
||||
it("scrubs mixed under-cap calls from pre-iteration results and multi-block done events", async () => {
|
||||
const rawCall = "<function=exec></function>";
|
||||
const visibleText = "Visible answer after the leaked call.";
|
||||
const rawText = `${rawCall}\n${visibleText}`;
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: rawCall },
|
||||
{ type: "text", text: visibleText },
|
||||
],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawText },
|
||||
{ type: "done", reason: "stop", message: createMessage() },
|
||||
],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
const events = await collectStreamEvents(stream);
|
||||
const expectedContent = [{ type: "text", text: visibleText }];
|
||||
|
||||
expect(result.content).toEqual(expectedContent);
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"text_delta",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events[0], "text event").delta).toBe(visibleText);
|
||||
expect(
|
||||
requireRecord(requireRecord(events[1], "done event").message, "done message").content,
|
||||
).toEqual(expectedContent);
|
||||
expect(JSON.stringify({ events, result })).not.toContain("<function=exec>");
|
||||
});
|
||||
|
||||
it("does not buffer normal prose that starts like a final answer", async () => {
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Finally, the audit is done." }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(events).toEqual([
|
||||
{ type: "text_delta", contentIndex: 0, delta: "Finally, the audit is done." },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,481 @@
|
||||
// Coverage for promoting standalone text tool calls into structured events.
|
||||
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { wrapStreamFnPromoteStandaloneTextToolCalls } from "./attempt-tool-call-text-promotion.js";
|
||||
|
||||
type FakeWrappedStream = {
|
||||
result: () => Promise<unknown>;
|
||||
[Symbol.asyncIterator]: () => AsyncIterator<unknown>;
|
||||
};
|
||||
|
||||
function createFakeStream(params: {
|
||||
events: unknown[];
|
||||
resultMessage: unknown;
|
||||
}): FakeWrappedStream {
|
||||
return {
|
||||
async result() {
|
||||
return params.resultMessage;
|
||||
},
|
||||
[Symbol.asyncIterator]() {
|
||||
return (async function* () {
|
||||
for (const event of params.events) {
|
||||
yield event;
|
||||
}
|
||||
})();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function collectStreamEvents(stream: AsyncIterable<unknown>): Promise<unknown[]> {
|
||||
// Drain streams to inspect generated tool-call events after wrapper mutation.
|
||||
const events: unknown[] = [];
|
||||
for await (const event of stream) {
|
||||
events.push(event);
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
const requireRecord = createRequireRecord("object", "expected-label");
|
||||
|
||||
describe("wrapStreamFnPromoteStandaloneTextToolCalls", () => {
|
||||
it("preserves a fenced allowed-tool example in live and terminal output", async () => {
|
||||
const parts = ["`", "``json\n", "[re", 'ad]\n{"path":"example.txt"}\n[/read]\n', "```"];
|
||||
const rawText = parts.join("");
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawText }],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
...parts.map((delta) => ({ type: "text_delta", contentIndex: 0, delta })),
|
||||
{ type: "text_end", contentIndex: 0, content: rawText },
|
||||
{ type: "done", reason: "stop", message: createMessage() },
|
||||
],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = (await collectStreamEvents(stream)).map((event) =>
|
||||
requireRecord(event, "event"),
|
||||
);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "text_delta")
|
||||
.map((event) => event.delta)
|
||||
.join(""),
|
||||
).toBe(rawText);
|
||||
expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false);
|
||||
expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([
|
||||
{ type: "text", text: rawText },
|
||||
]);
|
||||
expect(result.content).toEqual([{ type: "text", text: rawText }]);
|
||||
});
|
||||
|
||||
it("preserves a fenced example split across adjacent text blocks", async () => {
|
||||
const textParts = [
|
||||
"```json\n",
|
||||
["[read]", '{"path":"example.txt"}', "[/read]", "\n"].join("\n"),
|
||||
"```",
|
||||
];
|
||||
const content = textParts.map((text) => ({ type: "text", text }));
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content,
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
...textParts.flatMap((text, contentIndex) => [
|
||||
{ type: "text_delta", contentIndex, delta: text },
|
||||
{ type: "text_end", contentIndex, content: text },
|
||||
]),
|
||||
{ type: "done", reason: "stop", message: createMessage() },
|
||||
],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = (await collectStreamEvents(stream)).map((event) =>
|
||||
requireRecord(event, "event"),
|
||||
);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(
|
||||
events
|
||||
.filter((event) => event.type === "text_delta")
|
||||
.map((event) => event.delta)
|
||||
.join(""),
|
||||
).toBe(textParts.join(""));
|
||||
expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false);
|
||||
expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual(content);
|
||||
expect(result.content).toEqual(content);
|
||||
});
|
||||
|
||||
it("does not promote an indented code example from terminal output", async () => {
|
||||
const rawText = [" [read]", ' {"path":"example.txt"}', " [/read]"].join("\n");
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawText }],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [{ type: "done", reason: "stop", message: createMessage() }],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["read"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = (await collectStreamEvents(stream)).map((event) =>
|
||||
requireRecord(event, "event"),
|
||||
);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.some((event) => String(event.type).startsWith("toolcall_"))).toBe(false);
|
||||
expect(requireRecord(events.at(-1)?.message, "done message").content).toEqual([
|
||||
{ type: "text", text: rawText },
|
||||
]);
|
||||
expect(result.content).toEqual([{ type: "text", text: rawText }]);
|
||||
});
|
||||
|
||||
it("promotes standalone serialized parameter XML text to structured tool calls", async () => {
|
||||
// Some providers emit tool calls as text blocks; promote only allowed tool
|
||||
// names into structured toolCall content.
|
||||
const rawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"cat /proc/mounts 2>/dev/null | head -20",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
"",
|
||||
"<function=exec>",
|
||||
"<parameter=command>",
|
||||
"find / -maxdepth 4 -type d 2>/dev/null | head -20",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "thinking", thinking: "Need to audit the mount." },
|
||||
{ type: "text", text: rawToolText },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "start", partial: { content: [] } },
|
||||
{
|
||||
type: "text_start",
|
||||
contentIndex: 1,
|
||||
partial: { content: [{ type: "text", text: "" }] },
|
||||
},
|
||||
{ type: "text_delta", contentIndex: 1, delta: rawToolText },
|
||||
{ type: "text_end", contentIndex: 1, content: rawToolText },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events.at(-1), "done").reason).toBe("toolUse");
|
||||
expect(result.stopReason).toBe("toolUse");
|
||||
const content = result.content as Array<Record<string, unknown>>;
|
||||
expect(content).toHaveLength(3);
|
||||
expect(content[0]).toEqual({ type: "thinking", thinking: "Need to audit the mount." });
|
||||
expect(content[1]).toMatchObject({
|
||||
type: "toolCall",
|
||||
name: "exec",
|
||||
arguments: { command: "cat /proc/mounts 2>/dev/null | head -20" },
|
||||
partialArgs: '{"command":"cat /proc/mounts 2>/dev/null | head -20"}',
|
||||
});
|
||||
expect(String(expectDefined(content[1], "content[1] test invariant").id)).toMatch(
|
||||
/^call_[a-f0-9]{24}$/,
|
||||
);
|
||||
expect(content[2]).toMatchObject({
|
||||
type: "toolCall",
|
||||
name: "exec",
|
||||
arguments: { command: "find / -maxdepth 4 -type d 2>/dev/null | head -20" },
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses promoted ids across cloned result and done messages", async () => {
|
||||
const rawToolText = "<function=exec></function>";
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
|
||||
{ type: "done", reason: "stop", message: createMessage() },
|
||||
],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
const events = await collectStreamEvents(stream);
|
||||
const resultToolCall = requireRecord((result.content as unknown[])[0], "result tool call");
|
||||
const done = requireRecord(events.at(-1), "done event");
|
||||
const doneMessage = requireRecord(done.message, "done message");
|
||||
const doneToolCall = requireRecord((doneMessage.content as unknown[])[0], "done tool call");
|
||||
const lifecycle = events
|
||||
.map((event) => requireRecord(event, "event"))
|
||||
.filter((event) => String(event.type).startsWith("toolcall_"));
|
||||
|
||||
expect(doneToolCall.id).toBe(resultToolCall.id);
|
||||
expect(lifecycle).toHaveLength(3);
|
||||
for (const event of lifecycle) {
|
||||
const partial = requireRecord(event.partial, "tool-call partial");
|
||||
expect(requireRecord((partial.content as unknown[])[0], "partial tool call").id).toBe(
|
||||
resultToolCall.id,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("scrubs aggregate-over-cap call sequences before result promotion", async () => {
|
||||
const rawToolText = "<function=exec></function>\n".repeat(9_500);
|
||||
const createMessage = () => ({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
});
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [{ type: "done", reason: "stop", message: createMessage() }],
|
||||
resultMessage: createMessage(),
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
const events = await collectStreamEvents(stream);
|
||||
|
||||
expect(new TextEncoder().encode(rawToolText).byteLength).toBeGreaterThan(256_000);
|
||||
expect(result.content).toEqual([]);
|
||||
expect(requireRecord(requireRecord(events[0], "done").message, "done message").content).toEqual(
|
||||
[],
|
||||
);
|
||||
expect(JSON.stringify({ events, result })).not.toContain("<function=exec>");
|
||||
});
|
||||
|
||||
it("promotes deferred directory tool names from the live callable set", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:hidden_catalog_tool]",
|
||||
"<parameter=value>",
|
||||
"deferred",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: rawToolText }],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() => createFakeStream({ events: [], resultMessage }));
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(
|
||||
baseFn as never,
|
||||
new Set(["tool_search", "tool_describe", "tool_call", "hidden_catalog_tool"]),
|
||||
);
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(requireRecord((result.content as unknown[])[0], "tool call")).toMatchObject({
|
||||
type: "toolCall",
|
||||
name: "hidden_catalog_tool",
|
||||
arguments: { value: "deferred" },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves content indexes when promoting text before thinking", async () => {
|
||||
const rawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: rawToolText },
|
||||
{ type: "thinking", thinking: "Need the current directory." },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: rawToolText },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 1,
|
||||
delta: "Need the current directory.",
|
||||
partial: {
|
||||
content: [
|
||||
{ type: "text", text: rawToolText },
|
||||
{ type: "thinking", thinking: "Need the current directory." },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"thinking_delta",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1);
|
||||
expect(requireRecord(events[1], "toolcall start").contentIndex).toBe(0);
|
||||
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
|
||||
"toolCall",
|
||||
"thinking",
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves intervening thinking when promoting multiple text blocks", async () => {
|
||||
const firstRawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"pwd",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const secondRawToolText = [
|
||||
"[tool:exec]",
|
||||
"<parameter=command>",
|
||||
"whoami",
|
||||
"</parameter>",
|
||||
"</function>",
|
||||
].join("\n");
|
||||
const resultMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: firstRawToolText },
|
||||
{ type: "thinking", thinking: "Need one more check." },
|
||||
{ type: "text", text: secondRawToolText },
|
||||
],
|
||||
stopReason: "stop",
|
||||
};
|
||||
const baseFn = vi.fn(() =>
|
||||
createFakeStream({
|
||||
events: [
|
||||
{ type: "text_delta", contentIndex: 0, delta: firstRawToolText },
|
||||
{
|
||||
type: "thinking_delta",
|
||||
contentIndex: 1,
|
||||
delta: "Need one more check.",
|
||||
partial: {
|
||||
content: [
|
||||
{ type: "text", text: firstRawToolText },
|
||||
{ type: "thinking", thinking: "Need one more check." },
|
||||
{ type: "text", text: secondRawToolText },
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "text_delta", contentIndex: 2, delta: secondRawToolText },
|
||||
{ type: "done", reason: "stop", message: resultMessage },
|
||||
],
|
||||
resultMessage,
|
||||
}),
|
||||
);
|
||||
const wrapped = wrapStreamFnPromoteStandaloneTextToolCalls(baseFn as never, new Set(["exec"]));
|
||||
const stream = (await Promise.resolve(
|
||||
wrapped({} as never, {} as never, {} as never),
|
||||
)) as FakeWrappedStream;
|
||||
|
||||
const events = await collectStreamEvents(stream);
|
||||
const result = requireRecord(await stream.result(), "result message");
|
||||
|
||||
expect(events.map((event) => requireRecord(event, "event").type)).toEqual([
|
||||
"start",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"thinking_delta",
|
||||
"toolcall_start",
|
||||
"toolcall_delta",
|
||||
"toolcall_end",
|
||||
"done",
|
||||
]);
|
||||
expect(requireRecord(events[4], "thinking event").contentIndex).toBe(1);
|
||||
expect(requireRecord(events[1], "first toolcall start").contentIndex).toBe(0);
|
||||
expect(requireRecord(events[5], "second toolcall start").contentIndex).toBe(2);
|
||||
expect((result.content as Array<Record<string, unknown>>).map((block) => block.type)).toEqual([
|
||||
"toolCall",
|
||||
"thinking",
|
||||
"toolCall",
|
||||
]);
|
||||
expect(requireRecord((result.content as unknown[])[0], "first tool call")).toMatchObject({
|
||||
name: "exec",
|
||||
arguments: { command: "pwd" },
|
||||
});
|
||||
expect(requireRecord((result.content as unknown[])[2], "second tool call")).toMatchObject({
|
||||
name: "exec",
|
||||
arguments: { command: "whoami" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,144 @@
|
||||
/** Promotes safe standalone text tool calls into structured stream events. */
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
createPromotedPlainTextToolCallEvents,
|
||||
normalizePlainTextToolCallStreamEvents,
|
||||
projectScrubbedPlainTextToolCallMessage,
|
||||
projectStandalonePlainTextToolCallMessage as projectPlainTextToolCallMessage,
|
||||
type PlainTextToolCallBlock,
|
||||
type PlainTextToolCallMessageNormalization,
|
||||
type PlainTextToolCallNameMatcher,
|
||||
} from "../../../../packages/tool-call-repair/src/index.js";
|
||||
import { findCodeRegions } from "../../../shared/text/code-regions.js";
|
||||
import type { StreamFn } from "../../runtime/index.js";
|
||||
import { couldNormalizeToolNamePrefixToAllowedTool } from "../../tool-policy.js";
|
||||
import { resolveToolCallName } from "./attempt-tool-call-name-resolution.js";
|
||||
|
||||
type AssistantStream = Awaited<ReturnType<StreamFn>>;
|
||||
|
||||
function createStandaloneTextToolCallId(): string {
|
||||
return `call_${randomUUID().replace(/-/g, "").slice(0, 24)}`;
|
||||
}
|
||||
|
||||
function isRetainableNonVisibleBlock(block: Record<string, unknown>): boolean {
|
||||
return block.type === "thinking" || block.type === "redacted_thinking";
|
||||
}
|
||||
|
||||
const STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS = new Set<unknown>(["stop", "toolUse"]);
|
||||
|
||||
function createStandaloneToolCallNameMatcher(
|
||||
allowedToolNames: Set<string>,
|
||||
): PlainTextToolCallNameMatcher {
|
||||
return {
|
||||
hasExactName: (name) => Boolean(resolveToolCallName(name, allowedToolNames, undefined, true)),
|
||||
hasNamePrefix: (prefix) => couldNormalizeToolNamePrefixToAllowedTool(prefix, allowedToolNames),
|
||||
};
|
||||
}
|
||||
|
||||
function wrapStreamPromoteStandaloneTextToolCalls(
|
||||
stream: AssistantStream,
|
||||
allowedToolNames: Set<string>,
|
||||
): AssistantStream {
|
||||
const matcher = createStandaloneToolCallNameMatcher(allowedToolNames);
|
||||
const promotedIdBySource = new Map<string, string>();
|
||||
const normalizeTerminalMessage = (params: {
|
||||
allowPromotion: boolean;
|
||||
message: unknown;
|
||||
preserveEmptyTextBlocks?: boolean;
|
||||
}): PlainTextToolCallMessageNormalization => {
|
||||
const scrubbed = projectScrubbedPlainTextToolCallMessage({
|
||||
forceIncompleteCandidates: true,
|
||||
matcher,
|
||||
message: params.message,
|
||||
preserveEmptyTextBlocks: params.preserveEmptyTextBlocks,
|
||||
resolveProtectedRanges: findCodeRegions,
|
||||
requireAssistantRole: true,
|
||||
});
|
||||
if (scrubbed) {
|
||||
return { kind: "scrubbed", ...scrubbed };
|
||||
}
|
||||
if (!params.allowPromotion) {
|
||||
return undefined;
|
||||
}
|
||||
let ordinal = 0;
|
||||
const createStableToolCallBlock = (
|
||||
block: PlainTextToolCallBlock,
|
||||
name: string,
|
||||
): Record<string, unknown> => {
|
||||
const sourceKey = `${ordinal}:${block.start}:${block.end}`;
|
||||
ordinal += 1;
|
||||
let id = promotedIdBySource.get(sourceKey);
|
||||
if (!id) {
|
||||
id = createStandaloneTextToolCallId();
|
||||
promotedIdBySource.set(sourceKey, id);
|
||||
}
|
||||
return {
|
||||
type: "toolCall",
|
||||
id,
|
||||
name,
|
||||
arguments: block.arguments,
|
||||
partialArgs: JSON.stringify(block.arguments),
|
||||
};
|
||||
};
|
||||
const promoted = projectPlainTextToolCallMessage({
|
||||
allowedStopReasons: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS,
|
||||
allowedToolNames,
|
||||
createToolCallBlock: createStableToolCallBlock,
|
||||
isRetainableNonTextBlock: isRetainableNonVisibleBlock,
|
||||
message: params.message,
|
||||
requireAssistantRole: true,
|
||||
resolveProtectedRanges: findCodeRegions,
|
||||
resolveToolName: (name) => resolveToolCallName(name, allowedToolNames, undefined, true),
|
||||
});
|
||||
return promoted ? { kind: "promoted", ...promoted } : undefined;
|
||||
};
|
||||
|
||||
const originalResult = stream.result.bind(stream);
|
||||
stream.result = async () => {
|
||||
const message = await originalResult();
|
||||
const reason =
|
||||
message && typeof message === "object"
|
||||
? (message as { stopReason?: unknown }).stopReason
|
||||
: undefined;
|
||||
return (normalizeTerminalMessage({
|
||||
allowPromotion: STANDALONE_TEXT_TOOL_CALL_PROMOTION_STOP_REASONS.has(reason),
|
||||
message,
|
||||
})?.message ?? message) as Awaited<ReturnType<typeof originalResult>>;
|
||||
};
|
||||
|
||||
const originalAsyncIterator = stream[Symbol.asyncIterator].bind(stream);
|
||||
(stream as unknown as { [Symbol.asyncIterator]: () => AsyncIterator<unknown> })[
|
||||
Symbol.asyncIterator
|
||||
] = async function* () {
|
||||
const source = {
|
||||
[Symbol.asyncIterator]: originalAsyncIterator,
|
||||
} as AsyncIterable<unknown>;
|
||||
yield* normalizePlainTextToolCallStreamEvents(source, {
|
||||
createPromotedToolCallEvents: createPromotedPlainTextToolCallEvents,
|
||||
matcher,
|
||||
normalizeTerminalMessage,
|
||||
resolveProtectedRanges: findCodeRegions,
|
||||
});
|
||||
};
|
||||
|
||||
return stream;
|
||||
}
|
||||
|
||||
/** Promotes standalone plain-text tool-call replies into structured toolCall blocks when safe. */
|
||||
export function wrapStreamFnPromoteStandaloneTextToolCalls(
|
||||
baseFn: StreamFn,
|
||||
allowedToolNames?: Set<string>,
|
||||
): StreamFn {
|
||||
if (!allowedToolNames || allowedToolNames.size === 0) {
|
||||
return baseFn;
|
||||
}
|
||||
return (model, context, streamOptions) => {
|
||||
const maybeStream = baseFn(model, context, streamOptions);
|
||||
if (maybeStream && typeof maybeStream === "object" && "then" in maybeStream) {
|
||||
return Promise.resolve(maybeStream).then((stream) =>
|
||||
wrapStreamPromoteStandaloneTextToolCalls(stream, allowedToolNames),
|
||||
);
|
||||
}
|
||||
return wrapStreamPromoteStandaloneTextToolCalls(maybeStream, allowedToolNames);
|
||||
};
|
||||
}
|
||||
@@ -31,12 +31,10 @@ import {
|
||||
shouldWarnOnOrphanedUserRepair,
|
||||
} from "./attempt-prompt-helpers.js";
|
||||
import { composeSystemPromptWithHookContext } from "./attempt-thread-helpers.js";
|
||||
import { wrapStreamFnSanitizeMalformedToolCalls } from "./attempt-tool-call-replay-sanitization.js";
|
||||
import { wrapStreamFnTrimToolCallNames } from "./attempt-tool-call-stream-normalization.js";
|
||||
import { buildEmbeddedAttemptToolRunContext } from "./attempt-tool-run-context.js";
|
||||
import { wrapStreamFnRepairMalformedToolCallArguments } from "./attempt.tool-call-argument-repair.js";
|
||||
import {
|
||||
wrapStreamFnSanitizeMalformedToolCalls,
|
||||
wrapStreamFnTrimToolCallNames,
|
||||
} from "./attempt.tool-call-normalization.js";
|
||||
|
||||
const llmRuntime = {
|
||||
...defaultLlmRuntime,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,249 @@
|
||||
/** Classifies terminal assistant visibility and provider retry eligibility. */
|
||||
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import { isSilentReplyPayloadText, SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js";
|
||||
import {
|
||||
isStrictAgenticSupportedProviderModel,
|
||||
stripProviderPrefix,
|
||||
} from "../../execution-contract.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import { assessLastAssistantMessage } from "../thinking.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
export type IncompleteTurnAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "assistantTexts"
|
||||
| "clientToolCalls"
|
||||
| "currentAttemptAssistant"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "hasToolMediaBlockReply"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "didSendViaMessagingTool"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "messagingToolSentTargets"
|
||||
| "lastToolError"
|
||||
| "lastAssistant"
|
||||
| "itemLifecycle"
|
||||
| "messagesSnapshot"
|
||||
| "replayMetadata"
|
||||
| "terminal"
|
||||
| "toolMetas"
|
||||
> &
|
||||
Partial<Pick<EmbeddedRunAttemptResult, "acceptedSessionSpawns">>;
|
||||
|
||||
export function hasPositiveOutputTokenUsage(message: AgentMessage | null): boolean {
|
||||
if (!message || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
const usage = (message as { usage?: unknown }).usage;
|
||||
if (!usage || typeof usage !== "object") {
|
||||
return false;
|
||||
}
|
||||
const output = asFiniteNumber((usage as { output?: unknown }).output);
|
||||
return output !== undefined && output > 0;
|
||||
}
|
||||
|
||||
export function isIncompleteTerminalAssistantTurn(params: {
|
||||
hasAssistantVisibleText: boolean;
|
||||
hasTerminalOutput?: boolean;
|
||||
lastAssistant?: { stopReason?: string } | null;
|
||||
}): boolean {
|
||||
const stopReason = params.lastAssistant?.stopReason;
|
||||
// Tool-use expects a post-tool continuation; length means the output budget
|
||||
// ended before a complete final answer. Partial visible text completes neither.
|
||||
return stopReason === "toolUse" || (stopReason === "length" && !params.hasTerminalOutput);
|
||||
}
|
||||
|
||||
const GEMINI_INCOMPLETE_TURN_PROVIDER_IDS = new Set([
|
||||
"google",
|
||||
"google-vertex",
|
||||
"google-antigravity",
|
||||
"google-gemini-cli",
|
||||
]);
|
||||
const GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN = /^gemini(?:[.-]|$)/;
|
||||
// Ollama native `/api/chat` can finish with only thinking/internal blocks when constrained.
|
||||
const OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN = /^ollama(?:-|$)/;
|
||||
|
||||
export function isOllamaIncompleteTurnProvider(provider?: string): boolean {
|
||||
return OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test(
|
||||
normalizeLowercaseStringOrEmpty(provider ?? ""),
|
||||
);
|
||||
}
|
||||
// Model APIs eligible for the non-visible turn retry guard. OpenAI Responses
|
||||
// family can produce reasoning-only turns where usage.output > 0 but no visible
|
||||
// text is emitted; without the guard these pass through as successful. (#85364)
|
||||
const RETRY_GUARD_MODEL_APIS = new Set([
|
||||
"openai-completions",
|
||||
"anthropic-messages",
|
||||
"bedrock-converse-stream",
|
||||
"openai-responses",
|
||||
"openai-chatgpt-responses",
|
||||
"azure-openai-responses",
|
||||
"openclaw-openai-responses-transport",
|
||||
"openclaw-openai-chatgpt-responses-transport",
|
||||
"openclaw-azure-openai-responses-transport",
|
||||
]);
|
||||
export function joinAssistantTexts(assistantTexts?: readonly string[]): string {
|
||||
return (assistantTexts ?? []).join("\n\n").trim();
|
||||
}
|
||||
|
||||
export function hasOnlySilentAssistantReply(assistantTexts?: readonly string[]): boolean {
|
||||
const nonEmptyTexts = (assistantTexts ?? []).filter((text) => text.trim().length > 0);
|
||||
return (
|
||||
nonEmptyTexts.length > 0 &&
|
||||
nonEmptyTexts.every((text) => isSilentReplyPayloadText(text, SILENT_REPLY_TOKEN))
|
||||
);
|
||||
}
|
||||
|
||||
export function isReasoningOnlyAssistantTurn(message: unknown): boolean {
|
||||
if (!message || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
return assessLastAssistantMessage(message as AgentMessage) === "incomplete-text";
|
||||
}
|
||||
|
||||
// Unsigned thinking blocks have no cryptographic signature; assessLastAssistantMessage
|
||||
// returns "incomplete-thinking" for them. Empty content also returns "incomplete-thinking",
|
||||
// so the content.length > 0 guard is required to distinguish the two cases.
|
||||
export function isUnsignedThinkingOnlyAssistantTurn(message: unknown): boolean {
|
||||
if (message == null || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content) || content.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return assessLastAssistantMessage(message as AgentMessage) === "incomplete-thinking";
|
||||
}
|
||||
|
||||
export function isEmptyResponseAssistantTurn(params: {
|
||||
payloadCount: number;
|
||||
attempt: Pick<
|
||||
IncompleteTurnAttempt,
|
||||
"assistantTexts" | "currentAttemptAssistant" | "lastAssistant"
|
||||
>;
|
||||
}): boolean {
|
||||
if (params.payloadCount !== 0) {
|
||||
return false;
|
||||
}
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (!assistant) {
|
||||
return true;
|
||||
}
|
||||
if (assistant.stopReason === "error") {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: false,
|
||||
lastAssistant: assistant,
|
||||
}) ||
|
||||
isReasoningOnlyAssistantTurn(assistant)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function isNonVisibleAssistantTurnEligibleForSilentReply(params: {
|
||||
payloadCount: number;
|
||||
attempt: Pick<
|
||||
IncompleteTurnAttempt,
|
||||
"assistantTexts" | "currentAttemptAssistant" | "lastAssistant"
|
||||
>;
|
||||
}): boolean {
|
||||
if (isEmptyResponseAssistantTurn(params)) {
|
||||
return true;
|
||||
}
|
||||
if (params.payloadCount !== 0) {
|
||||
return false;
|
||||
}
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (!assistant || assistant.stopReason === "error") {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: false,
|
||||
lastAssistant: assistant,
|
||||
})
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return isReasoningOnlyAssistantTurn(assistant);
|
||||
}
|
||||
|
||||
export function shouldApplyNonVisibleTurnRetryGuard(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
}): boolean {
|
||||
if (
|
||||
params.executionContract === "strict-agentic" ||
|
||||
isIncompleteTurnRecoverySupportedProviderModel({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (RETRY_GUARD_MODEL_APIS.has(normalizeLowercaseStringOrEmpty(params.modelApi ?? ""))) {
|
||||
return true;
|
||||
}
|
||||
// This path uses provider output structure only: no user or assistant prose classification.
|
||||
return isOllamaIncompleteTurnProvider(params.provider);
|
||||
}
|
||||
|
||||
function isIncompleteTurnRecoverySupportedProviderModel(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
}): boolean {
|
||||
if (
|
||||
isStrictAgenticSupportedProviderModel({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const provider = normalizeLowercaseStringOrEmpty(params.provider ?? "");
|
||||
if (!GEMINI_INCOMPLETE_TURN_PROVIDER_IDS.has(provider)) {
|
||||
return false;
|
||||
}
|
||||
const modelId = typeof params.modelId === "string" ? params.modelId : "";
|
||||
return GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN.test(stripProviderPrefix(modelId));
|
||||
}
|
||||
|
||||
export function classifyAssistantTurn(params: {
|
||||
payloadCount: number;
|
||||
attempt: Pick<
|
||||
IncompleteTurnAttempt,
|
||||
"assistantTexts" | "currentAttemptAssistant" | "lastAssistant"
|
||||
>;
|
||||
}) {
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
return {
|
||||
assistant,
|
||||
visibleText: joinAssistantTexts(params.attempt.assistantTexts),
|
||||
onlySilentReply: hasOnlySilentAssistantReply(params.attempt.assistantTexts),
|
||||
reasoningOnly: isReasoningOnlyAssistantTurn(assistant),
|
||||
unsignedThinkingOnly: isUnsignedThinkingOnlyAssistantTurn(assistant),
|
||||
emptyResponse: isEmptyResponseAssistantTurn(params),
|
||||
nonVisibleEligibleForSilentReply: isNonVisibleAssistantTurnEligibleForSilentReply(params),
|
||||
hasPositiveOutputTokenUsage: hasPositiveOutputTokenUsage(assistant ?? null),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,370 @@
|
||||
/** Owns side-effect-sensitive retry and silent-reply recovery policy. */
|
||||
import { isReplayUnsafeAssistantError } from "../../../llm/utils/retry.js";
|
||||
import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js";
|
||||
import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js";
|
||||
import {
|
||||
hasCommittedMessagingToolDeliveryEvidence,
|
||||
hasCompletedMessagingToolDeliveryEvidence,
|
||||
} from "../delivery-evidence.js";
|
||||
import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js";
|
||||
import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js";
|
||||
import {
|
||||
hasOnlySilentAssistantReply,
|
||||
hasPositiveOutputTokenUsage,
|
||||
isEmptyResponseAssistantTurn,
|
||||
isNonVisibleAssistantTurnEligibleForSilentReply,
|
||||
isOllamaIncompleteTurnProvider,
|
||||
isReasoningOnlyAssistantTurn,
|
||||
isUnsignedThinkingOnlyAssistantTurn,
|
||||
joinAssistantTexts,
|
||||
shouldApplyNonVisibleTurnRetryGuard,
|
||||
type IncompleteTurnAttempt,
|
||||
} from "./incomplete-turn-classification.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
// Allow one immediate continuation plus one follow-up continuation before
|
||||
// surfacing the existing incomplete-turn error path.
|
||||
export const DEFAULT_REASONING_ONLY_RETRY_LIMIT = 2;
|
||||
export const DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT = 1;
|
||||
const REASONING_ONLY_RETRY_INSTRUCTION =
|
||||
"The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch.";
|
||||
const EMPTY_RESPONSE_RETRY_INSTRUCTION =
|
||||
"The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch.";
|
||||
const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION =
|
||||
"The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.";
|
||||
|
||||
export function shouldRetrySilentErrorAssistantTurn(params: {
|
||||
attempt: Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "assistantTexts"
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "lastToolError"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "replayMetadata"
|
||||
| "currentAttemptReplayMetadata"
|
||||
>;
|
||||
assistant: EmbeddedRunAttemptResult["lastAssistant"] | null | undefined;
|
||||
}): boolean {
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (hasAttemptTerminalState(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
// Current-attempt evidence avoids blocking on prior committed effects; older
|
||||
// harnesses retain the cumulative, fail-closed behavior.
|
||||
const retryReplayMetadata =
|
||||
params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata;
|
||||
if (retryReplayMetadata.hadPotentialSideEffects) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const assistant = params.assistant;
|
||||
if (!assistant || assistant.stopReason !== "error" || isReplayUnsafeAssistantError(assistant)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const content = (assistant as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return false;
|
||||
}
|
||||
if (content.length === 0) {
|
||||
return !hasPositiveOutputTokenUsage(assistant);
|
||||
}
|
||||
|
||||
return hasOnlyAssistantReasoningContent(assistant);
|
||||
}
|
||||
|
||||
function shouldSkipNonVisibleTurnRetry(params: {
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
/** Reply-optional silent classification tolerates committed side effects; retries never can. */
|
||||
tolerateSideEffects?: boolean;
|
||||
}): boolean {
|
||||
return Boolean(
|
||||
params.aborted ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) ||
|
||||
(params.tolerateSideEffects !== true && params.attempt.replayMetadata.hadPotentialSideEffects),
|
||||
);
|
||||
}
|
||||
|
||||
/** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */
|
||||
export function shouldTreatEmptyAssistantReplyAsSilent(params: {
|
||||
allowEmptyAssistantReplyAsSilent?: boolean;
|
||||
onlyExplicitSilentReply?: boolean;
|
||||
terminalReplyExpectation?: "required" | "optional";
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): boolean {
|
||||
// "optional" is the run consumer's declaration that no user-facing reply is
|
||||
// owed (e.g. cron without a delivery route). Silence after side-effecting
|
||||
// tools is intentional there; retry is replay-unsafe, so erroring would mark
|
||||
// successful tool-only runs as failures.
|
||||
const terminalReplyOptional = params.terminalReplyExpectation === "optional";
|
||||
if (
|
||||
!params.allowEmptyAssistantReplyAsSilent ||
|
||||
shouldSkipNonVisibleTurnRetry({ ...params, tolerateSideEffects: terminalReplyOptional })
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (
|
||||
params.payloadCount === 0 &&
|
||||
assistant?.stopReason !== "error" &&
|
||||
hasOnlySilentAssistantReply(params.attempt.assistantTexts)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// A visible turn owes a reply unless the model explicitly chose NO_REPLY.
|
||||
// Bare empty and reasoning-only stops are provider failures, even when the
|
||||
// conversation policy permits deliberate silence.
|
||||
if (params.onlyExplicitSilentReply || !terminalReplyOptional) {
|
||||
return false;
|
||||
}
|
||||
return isNonVisibleAssistantTurnEligibleForSilentReply({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the retry instruction for reasoning-only turns that consumed provider
|
||||
* output budget but produced no visible assistant text.
|
||||
*/
|
||||
export function resolveReasoningOnlyRetryInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
if (shouldSkipNonVisibleTurnRetry(params)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
!shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return null;
|
||||
}
|
||||
if (assistant?.stopReason === "error") {
|
||||
return null;
|
||||
}
|
||||
if (!isReasoningOnlyAssistantTurn(assistant) && !isUnsignedThinkingOnlyAssistantTurn(assistant)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return REASONING_ONLY_RETRY_INSTRUCTION;
|
||||
}
|
||||
|
||||
/** Builds one fresh continuation after settled tools ended without a visible final answer. */
|
||||
export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
allowEmptyStopContinuation?: boolean;
|
||||
payloadCount: number;
|
||||
hasTerminalToolPresentation?: boolean;
|
||||
aborted: boolean;
|
||||
promptError?: unknown;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
const currentAttemptAssistant = params.attempt.currentAttemptAssistant;
|
||||
const emptyStopAfterSettledTools = Boolean(
|
||||
params.allowEmptyStopContinuation &&
|
||||
currentAttemptAssistant?.stopReason === "stop" &&
|
||||
params.attempt.toolMetas.length > 0 &&
|
||||
params.attempt.toolMetas.every((tool) => tool.isError !== true && tool.asyncStarted !== true) &&
|
||||
params.attempt.itemLifecycle.startedCount > 0 &&
|
||||
params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount &&
|
||||
params.attempt.itemLifecycle.activeCount === 0 &&
|
||||
!hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) &&
|
||||
isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
}),
|
||||
);
|
||||
// Idle is not proof of settlement: skipped or partially dispatched tools must
|
||||
// never be described as completed. Match each terminal call's id and owner to
|
||||
// its own current-batch result; a reported failure is settled, not successful.
|
||||
const requestedToolCalls = Array.isArray(assistant?.content)
|
||||
? assistant.content.flatMap((item) => {
|
||||
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
|
||||
return block?.type === "toolCall"
|
||||
? [
|
||||
{
|
||||
id: typeof block.id === "string" ? block.id : null,
|
||||
name: typeof block.name === "string" ? block.name : null,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
})
|
||||
: [];
|
||||
// Scan only results AFTER the terminal assistant: the snapshot spans the whole
|
||||
// session, and a prior turn's toolResult with a model-reused id would otherwise
|
||||
// prove "completion" for a batch that never dispatched. Assistant not found in
|
||||
// the snapshot fails closed to the existing incomplete-turn error.
|
||||
const snapshot = params.attempt.messagesSnapshot ?? [];
|
||||
const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
|
||||
const settledToolResults = new Map(
|
||||
(assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => {
|
||||
const result = message as {
|
||||
role?: unknown;
|
||||
toolCallId?: unknown;
|
||||
toolName?: unknown;
|
||||
isError?: unknown;
|
||||
};
|
||||
return result.role === "toolResult" &&
|
||||
typeof result.toolCallId === "string" &&
|
||||
typeof result.toolName === "string"
|
||||
? [
|
||||
[
|
||||
result.toolCallId,
|
||||
{ toolName: result.toolName, isError: result.isError === true },
|
||||
] as const,
|
||||
]
|
||||
: [];
|
||||
}),
|
||||
);
|
||||
const allToolsProvenSettled =
|
||||
params.attempt.itemLifecycle?.activeCount === 0 &&
|
||||
requestedToolCalls.length > 0 &&
|
||||
requestedToolCalls.every(
|
||||
({ id, name }) =>
|
||||
id !== null && name !== null && settledToolResults.get(id)?.toolName === name,
|
||||
);
|
||||
const failedTerminalToolNames = new Set(
|
||||
requestedToolCalls.flatMap(({ id, name }) =>
|
||||
id !== null && name !== null && settledToolResults.get(id)?.isError === true ? [name] : [],
|
||||
),
|
||||
);
|
||||
const hasSettledTerminalToolFailure = allToolsProvenSettled && failedTerminalToolNames.size > 0;
|
||||
// ToolErrorSummary has no call id: its owner must match a failed result in the
|
||||
// proven terminal batch, or a stale/unrelated error could authorize finalization.
|
||||
const hasUnsettledToolError = Boolean(
|
||||
params.attempt.lastToolError &&
|
||||
(assistant?.stopReason !== "toolUse" ||
|
||||
!hasSettledTerminalToolFailure ||
|
||||
!failedTerminalToolNames.has(params.attempt.lastToolError.toolName)),
|
||||
);
|
||||
if (
|
||||
params.payloadCount !== 0 ||
|
||||
params.hasTerminalToolPresentation ||
|
||||
params.aborted ||
|
||||
params.promptError != null ||
|
||||
params.timedOut ||
|
||||
(assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) ||
|
||||
hasUnsettledToolError ||
|
||||
(hasSettledTerminalToolFailure &&
|
||||
(hasAsyncActivity(params.attempt.toolMetas) ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (hasCompletedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
!shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return hasSettledTerminalToolFailure
|
||||
? `${SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION} If any tool failed, state that failure plainly and do not claim it succeeded.`
|
||||
: SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the retry instruction for empty assistant turns when the provider/model
|
||||
* is eligible for non-visible turn recovery.
|
||||
*/
|
||||
export function resolveEmptyResponseRetryInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
if (shouldSkipNonVisibleTurnRetry(params)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
!isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null;
|
||||
if (
|
||||
assistant?.stopReason === "stop" &&
|
||||
isOllamaIncompleteTurnProvider(params.provider) &&
|
||||
!hasPositiveOutputTokenUsage(assistant)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
}) ||
|
||||
// Keep the generic zero-usage stop retry for providers that expose a
|
||||
// provider-neutral "nothing was generated" signal, even outside the
|
||||
// provider allowlist above.
|
||||
isZeroUsageEmptyStopAssistantTurn(assistant)
|
||||
) {
|
||||
return EMPTY_RESPONSE_RETRY_INSTRUCTION;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
/** Resolves incomplete-turn payloads, continuation evidence, and run liveness. */
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../../../auto-reply/tokens.js";
|
||||
import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js";
|
||||
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
|
||||
import { collectTextContentBlocks } from "../../content-blocks.js";
|
||||
import type { MessagingToolSend } from "../../embedded-agent-messaging.types.js";
|
||||
import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import { hasCommittedMessagingToolDeliveryEvidence } from "../delivery-evidence.js";
|
||||
import type { EmbeddedRunLivenessState } from "../types.js";
|
||||
import { hasAsyncActivity, hasAttemptTerminalState } from "./attempt-terminal-evidence.js";
|
||||
import {
|
||||
classifyAssistantTurn,
|
||||
hasOnlySilentAssistantReply,
|
||||
isEmptyResponseAssistantTurn,
|
||||
isIncompleteTerminalAssistantTurn,
|
||||
isReasoningOnlyAssistantTurn,
|
||||
joinAssistantTexts,
|
||||
type IncompleteTurnAttempt,
|
||||
} from "./incomplete-turn-classification.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
type SilentToolResultAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "lastToolError"
|
||||
| "messagesSnapshot"
|
||||
| "toolMetas"
|
||||
>;
|
||||
|
||||
type RunLivenessAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
"lastAssistant" | "replayMetadata" | "terminal"
|
||||
>;
|
||||
|
||||
/**
|
||||
* Builds the user-visible incomplete-turn warning when a terminal attempt did
|
||||
* not produce a safe final assistant response and no committed delivery/progress
|
||||
* already completed the task.
|
||||
*/
|
||||
export function resolveIncompleteTurnPayloadText(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
externalAbort: boolean;
|
||||
timedOut: boolean;
|
||||
hadPotentialSideEffects?: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
// Prefer the current attempt's terminal message. The session fallback can
|
||||
// still point at the pre-tool turn after a post-tool answer completes. (#80918)
|
||||
const assistantState = classifyAssistantTurn(params);
|
||||
const assistant = assistantState.assistant;
|
||||
const hasTerminalOutput = hasAttemptTerminalState(params.attempt);
|
||||
// Tool-use expects a post-tool continuation, while length means the output
|
||||
// budget ended. Partial visible text completes neither. (#76477)
|
||||
const incompleteTerminalAssistant = isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: params.payloadCount > 0,
|
||||
hasTerminalOutput,
|
||||
lastAssistant: assistant,
|
||||
});
|
||||
// Thinking payloads can count toward payloadCount but carry no user-visible
|
||||
// content; bypass the visible-text guard when thinking was the only output
|
||||
// so that incomplete-turn stall detection fires below. (#89787, #91953)
|
||||
const thinkingOnlyTerminal =
|
||||
params.payloadCount !== 0 &&
|
||||
!assistantState.visibleText.length &&
|
||||
!hasTerminalOutput &&
|
||||
Boolean(assistant && hasOnlyAssistantReasoningContent(assistant));
|
||||
|
||||
if (
|
||||
(params.payloadCount !== 0 && !incompleteTerminalAssistant && !thinkingOnlyTerminal) ||
|
||||
(params.aborted && params.externalAbort) ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasAsyncActivity(params.attempt.toolMetas)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stopReason = assistant?.stopReason;
|
||||
const reasoningOnlyAssistant = isReasoningOnlyAssistantTurn(assistant);
|
||||
const emptyResponseAssistant = isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
});
|
||||
if (
|
||||
!incompleteTerminalAssistant &&
|
||||
!reasoningOnlyAssistant &&
|
||||
!thinkingOnlyTerminal &&
|
||||
!emptyResponseAssistant &&
|
||||
stopReason !== "error"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return params.hadPotentialSideEffects || params.attempt.replayMetadata.hadPotentialSideEffects
|
||||
? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying."
|
||||
: "⚠️ Agent couldn't generate a response. Please try again.";
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows one retry when the provider returned no assistant turn at all and the
|
||||
* attempt has no side effects, active lifecycle items, delivery, or terminal
|
||||
* assistant/tool state.
|
||||
*/
|
||||
export function shouldRetryMissingAssistantTurn(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
promptError?: unknown;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): boolean {
|
||||
if (
|
||||
params.payloadCount !== 0 ||
|
||||
params.aborted ||
|
||||
Boolean(params.promptError) ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.currentAttemptAssistant ||
|
||||
params.attempt.lastAssistant ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasAsyncActivity(params.attempt.toolMetas)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
(params.attempt.itemLifecycle?.startedCount ?? 0) > 0 ||
|
||||
(params.attempt.itemLifecycle?.activeCount ?? 0) > 0
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !params.attempt.replayMetadata.hadPotentialSideEffects;
|
||||
}
|
||||
|
||||
/** Fields needed to determine whether a yielded turn already delivered or can continue. */
|
||||
interface YieldContinuationAttempt {
|
||||
clientToolCalls?: readonly unknown[];
|
||||
didSendDeterministicApprovalPrompt?: boolean;
|
||||
successfulCronAdds?: number;
|
||||
acceptedSessionSpawns?: readonly { runId: string; childSessionKey: string }[];
|
||||
messagingToolSentTexts?: readonly string[];
|
||||
messagingToolSentMediaUrls?: readonly string[];
|
||||
messagingToolSentTargets?: readonly MessagingToolSend[];
|
||||
toolMetas?: readonly { asyncStarted?: boolean }[];
|
||||
}
|
||||
|
||||
/** Continuation evidence for a yielded turn — sources that will produce future output. */
|
||||
export function hasYieldContinuationEvidence(attempt: YieldContinuationAttempt): boolean {
|
||||
// Only same-attempt evidence is causal here. Session-wide active descendants may be
|
||||
// stale or unrelated and must not suppress the diagnostic for this yielded turn.
|
||||
return (
|
||||
(attempt.clientToolCalls?.length ?? 0) > 0 ||
|
||||
attempt.didSendDeterministicApprovalPrompt === true ||
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: attempt.messagingToolSentTexts ?? [],
|
||||
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [],
|
||||
messagingToolSentTargets: attempt.messagingToolSentTargets ?? [],
|
||||
}) ||
|
||||
hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) ||
|
||||
hasAsyncActivity(attempt.toolMetas) ||
|
||||
(attempt.successfulCronAdds ?? 0) > 0
|
||||
);
|
||||
}
|
||||
|
||||
export const YIELD_DIAGNOSTIC_TEXT =
|
||||
"⚠️ Turn yielded without a continuation source. Send a message to resume.";
|
||||
|
||||
function isToolResultRole(role: string): boolean {
|
||||
return role === "toolresult" || role === "tool_result" || role === "tool";
|
||||
}
|
||||
|
||||
function readMessageTextContent(message: AgentMessage): string | undefined {
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (typeof content === "string") {
|
||||
const trimmed = content.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
const text = collectTextContentBlocks(content)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0)
|
||||
.join("\n");
|
||||
return text || undefined;
|
||||
}
|
||||
|
||||
function readToolResultAggregatedText(message: AgentMessage): string | undefined {
|
||||
const aggregated = (message as { details?: { aggregated?: unknown } }).details?.aggregated;
|
||||
if (typeof aggregated !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = aggregated.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
|
||||
function hasTrailingSilentToolResult(messages: readonly AgentMessage[]): boolean {
|
||||
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
||||
const message = messages[i];
|
||||
if (!message) {
|
||||
continue;
|
||||
}
|
||||
const role = normalizeLowercaseStringOrEmpty(message?.role);
|
||||
if (isToolResultRole(role)) {
|
||||
if ((message as { isError?: boolean }).isError === true) {
|
||||
return false;
|
||||
}
|
||||
const text = readMessageTextContent(message) ?? readToolResultAggregatedText(message);
|
||||
return isSilentReplyText(text, SILENT_REPLY_TOKEN);
|
||||
}
|
||||
if (role === "assistant" && !readMessageTextContent(message)) {
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Emits the silent-reply token for cron turns whose last successful tool result is silent. */
|
||||
export function resolveSilentToolResultReplyPayload(params: {
|
||||
isCronTrigger: boolean;
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: SilentToolResultAttempt;
|
||||
}): { text: typeof SILENT_REPLY_TOKEN } | null {
|
||||
if (
|
||||
!params.isCronTrigger ||
|
||||
params.payloadCount !== 0 ||
|
||||
params.aborted ||
|
||||
params.timedOut ||
|
||||
(params.attempt.toolMetas?.length ?? 0) === 0 ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError ||
|
||||
(params.attempt.messagesSnapshot?.length ?? 0) === 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return hasTrailingSilentToolResult(params.attempt.messagesSnapshot)
|
||||
? { text: SILENT_REPLY_TOKEN }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks replay invalid whenever the recorded attempt might not be safe to
|
||||
* replay or the current run ended in a compaction/incomplete-turn state that
|
||||
* needs a fresh prompt boundary.
|
||||
*/
|
||||
export function resolveReplayInvalidFlag(params: {
|
||||
attempt: RunLivenessAttempt;
|
||||
incompleteTurnText?: string | null;
|
||||
}): boolean {
|
||||
const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal);
|
||||
return (
|
||||
!params.attempt.replayMetadata.replaySafe ||
|
||||
terminal.promptErrorSource === "compaction" ||
|
||||
terminal.timedOutDuringCompaction ||
|
||||
Boolean(params.incompleteTurnText)
|
||||
);
|
||||
}
|
||||
|
||||
/** Classifies the persisted run state used by session recovery and resume logic. */
|
||||
export function resolveRunLivenessState(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: RunLivenessAttempt;
|
||||
incompleteTurnText?: string | null;
|
||||
}): EmbeddedRunLivenessState {
|
||||
if (params.incompleteTurnText) {
|
||||
return "abandoned";
|
||||
}
|
||||
const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal);
|
||||
if (terminal.promptErrorSource === "compaction" || terminal.timedOutDuringCompaction) {
|
||||
return "paused";
|
||||
}
|
||||
if ((params.aborted || params.timedOut) && params.payloadCount === 0) {
|
||||
return "blocked";
|
||||
}
|
||||
if (params.attempt.lastAssistant?.stopReason === "error") {
|
||||
return "blocked";
|
||||
}
|
||||
return "working";
|
||||
}
|
||||
@@ -1,986 +0,0 @@
|
||||
/**
|
||||
* Classifies incomplete terminal assistant turns and retry instructions.
|
||||
*/
|
||||
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import {
|
||||
isSilentReplyPayloadText,
|
||||
isSilentReplyText,
|
||||
SILENT_REPLY_TOKEN,
|
||||
} from "../../../auto-reply/tokens.js";
|
||||
import { isReplayUnsafeAssistantError } from "../../../llm/utils/retry.js";
|
||||
import { hasAcceptedSessionSpawn } from "../../accepted-session-spawn.js";
|
||||
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
|
||||
import { collectTextContentBlocks } from "../../content-blocks.js";
|
||||
import type { MessagingToolSend } from "../../embedded-agent-messaging.types.js";
|
||||
import {
|
||||
isStrictAgenticSupportedProviderModel,
|
||||
stripProviderPrefix,
|
||||
} from "../../execution-contract.js";
|
||||
import { hasOnlyAssistantReasoningContent } from "../../replay-turn-classification.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
import {
|
||||
hasCommittedMessagingToolDeliveryEvidence,
|
||||
hasCompletedMessagingToolDeliveryEvidence,
|
||||
hasMessagingToolDeliveryEvidence,
|
||||
} from "../delivery-evidence.js";
|
||||
import { isZeroUsageEmptyStopAssistantTurn } from "../empty-assistant-turn.js";
|
||||
import { assessLastAssistantMessage } from "../thinking.js";
|
||||
import type { EmbeddedRunLivenessState } from "../types.js";
|
||||
import type { EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
type ReplayMetadataAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "toolMetas"
|
||||
| "didSendViaMessagingTool"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "successfulCronAdds"
|
||||
> &
|
||||
Partial<Pick<EmbeddedRunAttemptResult, "messagingToolSentTargets" | "acceptedSessionSpawns">>;
|
||||
|
||||
type IncompleteTurnAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "assistantTexts"
|
||||
| "clientToolCalls"
|
||||
| "currentAttemptAssistant"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "hasToolMediaBlockReply"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "didSendViaMessagingTool"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "messagingToolSentTargets"
|
||||
| "lastToolError"
|
||||
| "lastAssistant"
|
||||
| "itemLifecycle"
|
||||
| "messagesSnapshot"
|
||||
| "replayMetadata"
|
||||
| "terminal"
|
||||
| "toolMetas"
|
||||
> &
|
||||
Partial<Pick<EmbeddedRunAttemptResult, "acceptedSessionSpawns">>;
|
||||
|
||||
function hasPositiveOutputTokenUsage(message: AgentMessage | null): boolean {
|
||||
if (!message || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
const usage = (message as { usage?: unknown }).usage;
|
||||
if (!usage || typeof usage !== "object") {
|
||||
return false;
|
||||
}
|
||||
const output = asFiniteNumber((usage as { output?: unknown }).output);
|
||||
return output !== undefined && output > 0;
|
||||
}
|
||||
|
||||
type SilentToolResultAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "lastToolError"
|
||||
| "messagesSnapshot"
|
||||
| "toolMetas"
|
||||
>;
|
||||
|
||||
type RunLivenessAttempt = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
"lastAssistant" | "replayMetadata" | "terminal"
|
||||
>;
|
||||
|
||||
export function isIncompleteTerminalAssistantTurn(params: {
|
||||
hasAssistantVisibleText: boolean;
|
||||
hasTerminalOutput?: boolean;
|
||||
lastAssistant?: { stopReason?: string } | null;
|
||||
}): boolean {
|
||||
const stopReason = params.lastAssistant?.stopReason;
|
||||
// Tool-use expects a post-tool continuation; length means the output budget
|
||||
// ended before a complete final answer. Partial visible text completes neither.
|
||||
return stopReason === "toolUse" || (stopReason === "length" && !params.hasTerminalOutput);
|
||||
}
|
||||
|
||||
const GEMINI_INCOMPLETE_TURN_PROVIDER_IDS = new Set([
|
||||
"google",
|
||||
"google-vertex",
|
||||
"google-antigravity",
|
||||
"google-gemini-cli",
|
||||
]);
|
||||
const GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN = /^gemini(?:[.-]|$)/;
|
||||
// Ollama native `/api/chat` can finish with only thinking/internal blocks when constrained.
|
||||
const OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN = /^ollama(?:-|$)/;
|
||||
// Model APIs eligible for the non-visible turn retry guard. OpenAI Responses
|
||||
// family can produce reasoning-only turns where usage.output > 0 but no visible
|
||||
// text is emitted; without the guard these pass through as successful. (#85364)
|
||||
const RETRY_GUARD_MODEL_APIS = new Set([
|
||||
"openai-completions",
|
||||
"anthropic-messages",
|
||||
"bedrock-converse-stream",
|
||||
"openai-responses",
|
||||
"openai-chatgpt-responses",
|
||||
"azure-openai-responses",
|
||||
"openclaw-openai-responses-transport",
|
||||
"openclaw-openai-chatgpt-responses-transport",
|
||||
"openclaw-azure-openai-responses-transport",
|
||||
]);
|
||||
// Allow one immediate continuation plus one follow-up continuation before
|
||||
// surfacing the existing incomplete-turn error path.
|
||||
export const DEFAULT_REASONING_ONLY_RETRY_LIMIT = 2;
|
||||
export const DEFAULT_EMPTY_RESPONSE_RETRY_LIMIT = 1;
|
||||
const REASONING_ONLY_RETRY_INSTRUCTION =
|
||||
"The previous assistant turn recorded reasoning but did not produce a user-visible answer. Continue from that partial turn and produce the visible answer now. Do not restate the reasoning or restart from scratch.";
|
||||
const EMPTY_RESPONSE_RETRY_INSTRUCTION =
|
||||
"The previous attempt did not produce a user-visible answer. Continue from the current state and produce the visible answer now. Do not restart from scratch.";
|
||||
const SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION =
|
||||
"The previous assistant turn completed its tool calls but did not produce a user-visible answer. Continue from the current transcript and produce the final user-visible answer now. Do not repeat completed tool calls or restart from scratch.";
|
||||
|
||||
/**
|
||||
* Marks whether retrying the attempt can safely replay the prompt. Concrete
|
||||
* tool-instance policy, async work, committed delivery, spawned sessions, and
|
||||
* cron writes all contribute side-effect evidence.
|
||||
*/
|
||||
export function buildAttemptReplayMetadata(
|
||||
params: ReplayMetadataAttempt,
|
||||
): EmbeddedRunAttemptResult["replayMetadata"] {
|
||||
const hadUnsafeTools = params.toolMetas.some((entry) => entry.replaySafe !== true);
|
||||
const hadAsyncStartedTool = params.toolMetas.some((t) => t.asyncStarted === true);
|
||||
const hadPotentialSideEffects =
|
||||
hadUnsafeTools ||
|
||||
hadAsyncStartedTool ||
|
||||
hasMessagingToolDeliveryEvidence(params) ||
|
||||
hasAcceptedSessionSpawn(params.acceptedSessionSpawns) ||
|
||||
(params.successfulCronAdds ?? 0) > 0;
|
||||
return {
|
||||
hadPotentialSideEffects,
|
||||
replaySafe: !hadPotentialSideEffects,
|
||||
};
|
||||
}
|
||||
|
||||
type TerminalAttemptState = Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "lastToolError"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "hasToolMediaBlockReply"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "successfulCronAdds"
|
||||
> &
|
||||
Partial<
|
||||
Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "acceptedSessionSpawns"
|
||||
| "messagingToolSentTexts"
|
||||
| "messagingToolSentMediaUrls"
|
||||
| "messagingToolSentTargets"
|
||||
>
|
||||
> & {
|
||||
toolMetas?: readonly { asyncStarted?: boolean }[];
|
||||
};
|
||||
|
||||
export function hasAttemptTerminalState(attempt: TerminalAttemptState): boolean {
|
||||
return Boolean(
|
||||
attempt.clientToolCalls ||
|
||||
attempt.yieldDetected ||
|
||||
attempt.didSendDeterministicApprovalPrompt ||
|
||||
attempt.heartbeatToolResponse ||
|
||||
attempt.lastToolError ||
|
||||
attempt.toolMediaUrls?.some((url) => url.trim().length > 0) ||
|
||||
attempt.toolAudioAsVoice ||
|
||||
attempt.toolTrustedLocalMedia ||
|
||||
attempt.hasToolMediaBlockReply ||
|
||||
attempt.didDeliverSourceReplyViaMessageTool ||
|
||||
attempt.messagingToolSourceReplyPayloads?.length ||
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: attempt.messagingToolSentTexts ?? [],
|
||||
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [],
|
||||
messagingToolSentTargets: attempt.messagingToolSentTargets ?? [],
|
||||
}) ||
|
||||
hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) ||
|
||||
hasAsyncStartedToolActivity(attempt.toolMetas) ||
|
||||
(attempt.successfulCronAdds ?? 0) > 0,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the user-visible incomplete-turn warning when a terminal attempt did
|
||||
* not produce a safe final assistant response and no committed delivery/progress
|
||||
* already completed the task.
|
||||
*/
|
||||
export function resolveIncompleteTurnPayloadText(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
externalAbort: boolean;
|
||||
timedOut: boolean;
|
||||
hadPotentialSideEffects?: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
// Prefer the current attempt's terminal message. The session fallback can
|
||||
// still point at the pre-tool turn after a post-tool answer completes. (#80918)
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
const hasTerminalOutput = hasAttemptTerminalState(params.attempt);
|
||||
// Tool-use expects a post-tool continuation, while length means the output
|
||||
// budget ended. Partial visible text completes neither. (#76477)
|
||||
const incompleteTerminalAssistant = isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: params.payloadCount > 0,
|
||||
hasTerminalOutput,
|
||||
lastAssistant: assistant,
|
||||
});
|
||||
// Thinking payloads can count toward payloadCount but carry no user-visible
|
||||
// content; bypass the visible-text guard when thinking was the only output
|
||||
// so that incomplete-turn stall detection fires below. (#89787, #91953)
|
||||
const thinkingOnlyTerminal =
|
||||
params.payloadCount !== 0 &&
|
||||
!joinAssistantTexts(params.attempt.assistantTexts).length &&
|
||||
!hasTerminalOutput &&
|
||||
Boolean(assistant && hasOnlyAssistantReasoningContent(assistant));
|
||||
|
||||
if (
|
||||
(params.payloadCount !== 0 && !incompleteTerminalAssistant && !thinkingOnlyTerminal) ||
|
||||
(params.aborted && params.externalAbort) ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (hasAsyncStartedToolActivity(params.attempt.toolMetas)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const stopReason = assistant?.stopReason;
|
||||
const reasoningOnlyAssistant = isReasoningOnlyAssistantTurn(assistant);
|
||||
const emptyResponseAssistant = isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
});
|
||||
if (
|
||||
!incompleteTerminalAssistant &&
|
||||
!reasoningOnlyAssistant &&
|
||||
!thinkingOnlyTerminal &&
|
||||
!emptyResponseAssistant &&
|
||||
stopReason !== "error"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return params.hadPotentialSideEffects || params.attempt.replayMetadata.hadPotentialSideEffects
|
||||
? "⚠️ Agent couldn't generate a response. Note: some tool actions may have already been executed — please verify before retrying."
|
||||
: "⚠️ Agent couldn't generate a response. Please try again.";
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows one retry when the provider returned no assistant turn at all and the
|
||||
* attempt has no side effects, active lifecycle items, delivery, or terminal
|
||||
* assistant/tool state.
|
||||
*/
|
||||
export function shouldRetryMissingAssistantTurn(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
promptError?: unknown;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): boolean {
|
||||
if (
|
||||
params.payloadCount !== 0 ||
|
||||
params.aborted ||
|
||||
Boolean(params.promptError) ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.currentAttemptAssistant ||
|
||||
params.attempt.lastAssistant ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasOnlySilentAssistantReply(params.attempt.assistantTexts)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (hasAsyncStartedToolActivity(params.attempt.toolMetas)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (
|
||||
(params.attempt.itemLifecycle?.startedCount ?? 0) > 0 ||
|
||||
(params.attempt.itemLifecycle?.activeCount ?? 0) > 0
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !params.attempt.replayMetadata.hadPotentialSideEffects;
|
||||
}
|
||||
|
||||
function joinAssistantTexts(assistantTexts?: readonly string[]): string {
|
||||
return (assistantTexts ?? []).join("\n\n").trim();
|
||||
}
|
||||
|
||||
function hasOnlySilentAssistantReply(assistantTexts?: readonly string[]): boolean {
|
||||
const nonEmptyTexts = (assistantTexts ?? []).filter((text) => text.trim().length > 0);
|
||||
return (
|
||||
nonEmptyTexts.length > 0 &&
|
||||
nonEmptyTexts.every((text) => isSilentReplyPayloadText(text, SILENT_REPLY_TOKEN))
|
||||
);
|
||||
}
|
||||
|
||||
function hasAsyncStartedToolActivity(toolMetas?: readonly { asyncStarted?: boolean }[]): boolean {
|
||||
return (toolMetas ?? []).some((entry) => entry.asyncStarted === true);
|
||||
}
|
||||
|
||||
/** Fields needed to determine whether a yielded turn already delivered or can continue. */
|
||||
interface YieldContinuationAttempt {
|
||||
clientToolCalls?: readonly unknown[];
|
||||
didSendDeterministicApprovalPrompt?: boolean;
|
||||
successfulCronAdds?: number;
|
||||
acceptedSessionSpawns?: readonly { runId: string; childSessionKey: string }[];
|
||||
messagingToolSentTexts?: readonly string[];
|
||||
messagingToolSentMediaUrls?: readonly string[];
|
||||
messagingToolSentTargets?: readonly MessagingToolSend[];
|
||||
toolMetas?: readonly { asyncStarted?: boolean }[];
|
||||
}
|
||||
|
||||
/** Continuation evidence for a yielded turn — sources that will produce future output. */
|
||||
export function hasYieldContinuationEvidence(attempt: YieldContinuationAttempt): boolean {
|
||||
// Only same-attempt evidence is causal here. Session-wide active descendants may be
|
||||
// stale or unrelated and must not suppress the diagnostic for this yielded turn.
|
||||
return (
|
||||
(attempt.clientToolCalls?.length ?? 0) > 0 ||
|
||||
attempt.didSendDeterministicApprovalPrompt === true ||
|
||||
hasCommittedMessagingToolDeliveryEvidence({
|
||||
messagingToolSentTexts: attempt.messagingToolSentTexts ?? [],
|
||||
messagingToolSentMediaUrls: attempt.messagingToolSentMediaUrls ?? [],
|
||||
messagingToolSentTargets: attempt.messagingToolSentTargets ?? [],
|
||||
}) ||
|
||||
hasAcceptedSessionSpawn(attempt.acceptedSessionSpawns) ||
|
||||
hasAsyncStartedToolActivity(attempt.toolMetas) ||
|
||||
(attempt.successfulCronAdds ?? 0) > 0
|
||||
);
|
||||
}
|
||||
|
||||
export const YIELD_DIAGNOSTIC_TEXT =
|
||||
"⚠️ Turn yielded without a continuation source. Send a message to resume.";
|
||||
|
||||
function isToolResultRole(role: string): boolean {
|
||||
return role === "toolresult" || role === "tool_result" || role === "tool";
|
||||
}
|
||||
|
||||
function readMessageTextContent(message: AgentMessage): string | undefined {
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (typeof content === "string") {
|
||||
const trimmed = content.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
const text = collectTextContentBlocks(content)
|
||||
.map((item) => item.trim())
|
||||
.filter((item) => item.length > 0)
|
||||
.join("\n");
|
||||
return text || undefined;
|
||||
}
|
||||
|
||||
function readToolResultAggregatedText(message: AgentMessage): string | undefined {
|
||||
const aggregated = (message as { details?: { aggregated?: unknown } }).details?.aggregated;
|
||||
if (typeof aggregated !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
const trimmed = aggregated.trim();
|
||||
return trimmed || undefined;
|
||||
}
|
||||
|
||||
function hasTrailingSilentToolResult(messages: readonly AgentMessage[]): boolean {
|
||||
for (let i = messages.length - 1; i >= 0; i -= 1) {
|
||||
const message = messages[i];
|
||||
if (!message) {
|
||||
continue;
|
||||
}
|
||||
const role = normalizeLowercaseStringOrEmpty(message?.role);
|
||||
if (isToolResultRole(role)) {
|
||||
if ((message as { isError?: boolean }).isError === true) {
|
||||
return false;
|
||||
}
|
||||
const text = readMessageTextContent(message) ?? readToolResultAggregatedText(message);
|
||||
return isSilentReplyText(text, SILENT_REPLY_TOKEN);
|
||||
}
|
||||
if (role === "assistant" && !readMessageTextContent(message)) {
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Emits the silent-reply token for cron turns whose last successful tool result is silent. */
|
||||
export function resolveSilentToolResultReplyPayload(params: {
|
||||
isCronTrigger: boolean;
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: SilentToolResultAttempt;
|
||||
}): { text: typeof SILENT_REPLY_TOKEN } | null {
|
||||
if (
|
||||
!params.isCronTrigger ||
|
||||
params.payloadCount !== 0 ||
|
||||
params.aborted ||
|
||||
params.timedOut ||
|
||||
(params.attempt.toolMetas?.length ?? 0) === 0 ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError ||
|
||||
(params.attempt.messagesSnapshot?.length ?? 0) === 0
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return hasTrailingSilentToolResult(params.attempt.messagesSnapshot)
|
||||
? { text: SILENT_REPLY_TOKEN }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks replay invalid whenever the recorded attempt might not be safe to
|
||||
* replay or the current run ended in a compaction/incomplete-turn state that
|
||||
* needs a fresh prompt boundary.
|
||||
*/
|
||||
export function resolveReplayInvalidFlag(params: {
|
||||
attempt: RunLivenessAttempt;
|
||||
incompleteTurnText?: string | null;
|
||||
}): boolean {
|
||||
const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal);
|
||||
return (
|
||||
!params.attempt.replayMetadata.replaySafe ||
|
||||
terminal.promptErrorSource === "compaction" ||
|
||||
terminal.timedOutDuringCompaction ||
|
||||
Boolean(params.incompleteTurnText)
|
||||
);
|
||||
}
|
||||
|
||||
/** Classifies the persisted run state used by session recovery and resume logic. */
|
||||
export function resolveRunLivenessState(params: {
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: RunLivenessAttempt;
|
||||
incompleteTurnText?: string | null;
|
||||
}): EmbeddedRunLivenessState {
|
||||
if (params.incompleteTurnText) {
|
||||
return "abandoned";
|
||||
}
|
||||
const terminal = projectAgentRunAttemptTerminal(params.attempt.terminal);
|
||||
if (terminal.promptErrorSource === "compaction" || terminal.timedOutDuringCompaction) {
|
||||
return "paused";
|
||||
}
|
||||
if ((params.aborted || params.timedOut) && params.payloadCount === 0) {
|
||||
return "blocked";
|
||||
}
|
||||
if (params.attempt.lastAssistant?.stopReason === "error") {
|
||||
return "blocked";
|
||||
}
|
||||
return "working";
|
||||
}
|
||||
|
||||
function isReasoningOnlyAssistantTurn(message: unknown): boolean {
|
||||
if (!message || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
return assessLastAssistantMessage(message as AgentMessage) === "incomplete-text";
|
||||
}
|
||||
|
||||
// Unsigned thinking blocks have no cryptographic signature; assessLastAssistantMessage
|
||||
// returns "incomplete-thinking" for them. Empty content also returns "incomplete-thinking",
|
||||
// so the content.length > 0 guard is required to distinguish the two cases.
|
||||
function isUnsignedThinkingOnlyAssistantTurn(message: unknown): boolean {
|
||||
if (message == null || typeof message !== "object") {
|
||||
return false;
|
||||
}
|
||||
const content = (message as { content?: unknown }).content;
|
||||
if (!Array.isArray(content) || content.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return assessLastAssistantMessage(message as AgentMessage) === "incomplete-thinking";
|
||||
}
|
||||
|
||||
export function shouldRetrySilentErrorAssistantTurn(params: {
|
||||
attempt: Pick<
|
||||
EmbeddedRunAttemptResult,
|
||||
| "assistantTexts"
|
||||
| "clientToolCalls"
|
||||
| "yieldDetected"
|
||||
| "didSendDeterministicApprovalPrompt"
|
||||
| "heartbeatToolResponse"
|
||||
| "lastToolError"
|
||||
| "toolMediaUrls"
|
||||
| "toolAudioAsVoice"
|
||||
| "toolTrustedLocalMedia"
|
||||
| "didDeliverSourceReplyViaMessageTool"
|
||||
| "messagingToolSourceReplyPayloads"
|
||||
| "replayMetadata"
|
||||
| "currentAttemptReplayMetadata"
|
||||
>;
|
||||
assistant: EmbeddedRunAttemptResult["lastAssistant"] | null | undefined;
|
||||
}): boolean {
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
if (hasAttemptTerminalState(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
// Current-attempt evidence avoids blocking on prior committed effects; older
|
||||
// harnesses retain the cumulative, fail-closed behavior.
|
||||
const retryReplayMetadata =
|
||||
params.attempt.currentAttemptReplayMetadata ?? params.attempt.replayMetadata;
|
||||
if (retryReplayMetadata.hadPotentialSideEffects) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const assistant = params.assistant;
|
||||
if (!assistant || assistant.stopReason !== "error" || isReplayUnsafeAssistantError(assistant)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const content = (assistant as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) {
|
||||
return false;
|
||||
}
|
||||
if (content.length === 0) {
|
||||
return !hasPositiveOutputTokenUsage(assistant);
|
||||
}
|
||||
|
||||
return hasOnlyAssistantReasoningContent(assistant);
|
||||
}
|
||||
|
||||
function isEmptyResponseAssistantTurn(params: {
|
||||
payloadCount: number;
|
||||
attempt: Pick<
|
||||
IncompleteTurnAttempt,
|
||||
"assistantTexts" | "currentAttemptAssistant" | "lastAssistant"
|
||||
>;
|
||||
}): boolean {
|
||||
if (params.payloadCount !== 0) {
|
||||
return false;
|
||||
}
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (!assistant) {
|
||||
return true;
|
||||
}
|
||||
if (assistant.stopReason === "error") {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: false,
|
||||
lastAssistant: assistant,
|
||||
}) ||
|
||||
isReasoningOnlyAssistantTurn(assistant)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function isNonVisibleAssistantTurnEligibleForSilentReply(params: {
|
||||
payloadCount: number;
|
||||
attempt: Pick<
|
||||
IncompleteTurnAttempt,
|
||||
"assistantTexts" | "currentAttemptAssistant" | "lastAssistant"
|
||||
>;
|
||||
}): boolean {
|
||||
if (isEmptyResponseAssistantTurn(params)) {
|
||||
return true;
|
||||
}
|
||||
if (params.payloadCount !== 0) {
|
||||
return false;
|
||||
}
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (!assistant || assistant.stopReason === "error") {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
isIncompleteTerminalAssistantTurn({
|
||||
hasAssistantVisibleText: false,
|
||||
lastAssistant: assistant,
|
||||
})
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return isReasoningOnlyAssistantTurn(assistant);
|
||||
}
|
||||
|
||||
function shouldSkipNonVisibleTurnRetry(params: {
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
/** Reply-optional silent classification tolerates committed side effects; retries never can. */
|
||||
tolerateSideEffects?: boolean;
|
||||
}): boolean {
|
||||
return Boolean(
|
||||
params.aborted ||
|
||||
params.timedOut ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt ||
|
||||
params.attempt.lastToolError ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) ||
|
||||
(params.tolerateSideEffects !== true && params.attempt.replayMetadata.hadPotentialSideEffects),
|
||||
);
|
||||
}
|
||||
|
||||
/** Allows configured silent handling for replay-safe empty, reasoning-only, or explicit silent turns. */
|
||||
export function shouldTreatEmptyAssistantReplyAsSilent(params: {
|
||||
allowEmptyAssistantReplyAsSilent?: boolean;
|
||||
onlyExplicitSilentReply?: boolean;
|
||||
terminalReplyExpectation?: "required" | "optional";
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): boolean {
|
||||
// "optional" is the run consumer's declaration that no user-facing reply is
|
||||
// owed (e.g. cron without a delivery route). Silence after side-effecting
|
||||
// tools is intentional there; retry is replay-unsafe, so erroring would mark
|
||||
// successful tool-only runs as failures.
|
||||
const terminalReplyOptional = params.terminalReplyExpectation === "optional";
|
||||
if (
|
||||
!params.allowEmptyAssistantReplyAsSilent ||
|
||||
shouldSkipNonVisibleTurnRetry({ ...params, tolerateSideEffects: terminalReplyOptional })
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
if (hasCommittedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return false;
|
||||
}
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (
|
||||
params.payloadCount === 0 &&
|
||||
assistant?.stopReason !== "error" &&
|
||||
hasOnlySilentAssistantReply(params.attempt.assistantTexts)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// A visible turn owes a reply unless the model explicitly chose NO_REPLY.
|
||||
// Bare empty and reasoning-only stops are provider failures, even when the
|
||||
// conversation policy permits deliberate silence.
|
||||
if (params.onlyExplicitSilentReply || !terminalReplyOptional) {
|
||||
return false;
|
||||
}
|
||||
return isNonVisibleAssistantTurnEligibleForSilentReply({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the retry instruction for reasoning-only turns that consumed provider
|
||||
* output budget but produced no visible assistant text.
|
||||
*/
|
||||
export function resolveReasoningOnlyRetryInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
if (shouldSkipNonVisibleTurnRetry(params)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
!shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
if (joinAssistantTexts(params.attempt.assistantTexts).length > 0) {
|
||||
return null;
|
||||
}
|
||||
if (assistant?.stopReason === "error") {
|
||||
return null;
|
||||
}
|
||||
if (!isReasoningOnlyAssistantTurn(assistant) && !isUnsignedThinkingOnlyAssistantTurn(assistant)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return REASONING_ONLY_RETRY_INSTRUCTION;
|
||||
}
|
||||
|
||||
/** Builds one fresh continuation after settled tools ended without a visible final answer. */
|
||||
export function resolveSettledToolTerminalContinuationInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
allowEmptyStopContinuation?: boolean;
|
||||
payloadCount: number;
|
||||
hasTerminalToolPresentation?: boolean;
|
||||
aborted: boolean;
|
||||
promptError?: unknown;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant;
|
||||
const currentAttemptAssistant = params.attempt.currentAttemptAssistant;
|
||||
const emptyStopAfterSettledTools = Boolean(
|
||||
params.allowEmptyStopContinuation &&
|
||||
currentAttemptAssistant?.stopReason === "stop" &&
|
||||
params.attempt.toolMetas.length > 0 &&
|
||||
params.attempt.toolMetas.every((tool) => tool.isError !== true && tool.asyncStarted !== true) &&
|
||||
params.attempt.itemLifecycle.startedCount > 0 &&
|
||||
params.attempt.itemLifecycle.completedCount === params.attempt.itemLifecycle.startedCount &&
|
||||
params.attempt.itemLifecycle.activeCount === 0 &&
|
||||
!hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns) &&
|
||||
isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
}),
|
||||
);
|
||||
// Idle is not proof of settlement: skipped or partially dispatched tools must
|
||||
// never be described as completed. Match each terminal call's id and owner to
|
||||
// its own current-batch result; a reported failure is settled, not successful.
|
||||
const requestedToolCalls = Array.isArray(assistant?.content)
|
||||
? assistant.content.flatMap((item) => {
|
||||
const block = item as { type?: unknown; id?: unknown; name?: unknown } | null;
|
||||
return block?.type === "toolCall"
|
||||
? [
|
||||
{
|
||||
id: typeof block.id === "string" ? block.id : null,
|
||||
name: typeof block.name === "string" ? block.name : null,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
})
|
||||
: [];
|
||||
// Scan only results AFTER the terminal assistant: the snapshot spans the whole
|
||||
// session, and a prior turn's toolResult with a model-reused id would otherwise
|
||||
// prove "completion" for a batch that never dispatched. Assistant not found in
|
||||
// the snapshot fails closed to the existing incomplete-turn error.
|
||||
const snapshot = params.attempt.messagesSnapshot ?? [];
|
||||
const assistantIndex = assistant ? snapshot.indexOf(assistant) : -1;
|
||||
const settledToolResults = new Map(
|
||||
(assistantIndex >= 0 ? snapshot.slice(assistantIndex + 1) : []).flatMap((message) => {
|
||||
const result = message as {
|
||||
role?: unknown;
|
||||
toolCallId?: unknown;
|
||||
toolName?: unknown;
|
||||
isError?: unknown;
|
||||
};
|
||||
return result.role === "toolResult" &&
|
||||
typeof result.toolCallId === "string" &&
|
||||
typeof result.toolName === "string"
|
||||
? [
|
||||
[
|
||||
result.toolCallId,
|
||||
{ toolName: result.toolName, isError: result.isError === true },
|
||||
] as const,
|
||||
]
|
||||
: [];
|
||||
}),
|
||||
);
|
||||
const allToolsProvenSettled =
|
||||
params.attempt.itemLifecycle?.activeCount === 0 &&
|
||||
requestedToolCalls.length > 0 &&
|
||||
requestedToolCalls.every(
|
||||
({ id, name }) =>
|
||||
id !== null && name !== null && settledToolResults.get(id)?.toolName === name,
|
||||
);
|
||||
const failedTerminalToolNames = new Set(
|
||||
requestedToolCalls.flatMap(({ id, name }) =>
|
||||
id !== null && name !== null && settledToolResults.get(id)?.isError === true ? [name] : [],
|
||||
),
|
||||
);
|
||||
const hasSettledTerminalToolFailure = allToolsProvenSettled && failedTerminalToolNames.size > 0;
|
||||
// ToolErrorSummary has no call id: its owner must match a failed result in the
|
||||
// proven terminal batch, or a stale/unrelated error could authorize finalization.
|
||||
const hasUnsettledToolError = Boolean(
|
||||
params.attempt.lastToolError &&
|
||||
(assistant?.stopReason !== "toolUse" ||
|
||||
!hasSettledTerminalToolFailure ||
|
||||
!failedTerminalToolNames.has(params.attempt.lastToolError.toolName)),
|
||||
);
|
||||
if (
|
||||
params.payloadCount !== 0 ||
|
||||
params.hasTerminalToolPresentation ||
|
||||
params.aborted ||
|
||||
params.promptError != null ||
|
||||
params.timedOut ||
|
||||
(assistant?.stopReason === "toolUse" ? !allToolsProvenSettled : !emptyStopAfterSettledTools) ||
|
||||
hasUnsettledToolError ||
|
||||
(hasSettledTerminalToolFailure &&
|
||||
(hasAsyncStartedToolActivity(params.attempt.toolMetas) ||
|
||||
hasAcceptedSessionSpawn(params.attempt.acceptedSessionSpawns))) ||
|
||||
params.attempt.clientToolCalls ||
|
||||
params.attempt.yieldDetected ||
|
||||
params.attempt.didSendDeterministicApprovalPrompt
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
if (hasCompletedMessagingToolDeliveryEvidence(params.attempt)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
!shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return hasSettledTerminalToolFailure
|
||||
? `${SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION} If any tool failed, state that failure plainly and do not claim it succeeded.`
|
||||
: SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the retry instruction for empty assistant turns when the provider/model
|
||||
* is eligible for non-visible turn recovery.
|
||||
*/
|
||||
export function resolveEmptyResponseRetryInstruction(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
payloadCount: number;
|
||||
aborted: boolean;
|
||||
timedOut: boolean;
|
||||
attempt: IncompleteTurnAttempt;
|
||||
}): string | null {
|
||||
if (shouldSkipNonVisibleTurnRetry(params)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
!isEmptyResponseAssistantTurn({
|
||||
payloadCount: params.payloadCount,
|
||||
attempt: params.attempt,
|
||||
})
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const assistant = params.attempt.currentAttemptAssistant ?? params.attempt.lastAssistant ?? null;
|
||||
if (
|
||||
assistant?.stopReason === "stop" &&
|
||||
OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test(
|
||||
normalizeLowercaseStringOrEmpty(params.provider ?? ""),
|
||||
) &&
|
||||
!hasPositiveOutputTokenUsage(assistant)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
shouldApplyNonVisibleTurnRetryGuard({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
modelApi: params.modelApi,
|
||||
executionContract: params.executionContract,
|
||||
}) ||
|
||||
// Keep the generic zero-usage stop retry for providers that expose a
|
||||
// provider-neutral "nothing was generated" signal, even outside the
|
||||
// provider allowlist above.
|
||||
isZeroUsageEmptyStopAssistantTurn(assistant)
|
||||
) {
|
||||
return EMPTY_RESPONSE_RETRY_INSTRUCTION;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function shouldApplyNonVisibleTurnRetryGuard(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
modelApi?: string;
|
||||
executionContract?: string;
|
||||
}): boolean {
|
||||
if (
|
||||
params.executionContract === "strict-agentic" ||
|
||||
isIncompleteTurnRecoverySupportedProviderModel({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (RETRY_GUARD_MODEL_APIS.has(normalizeLowercaseStringOrEmpty(params.modelApi ?? ""))) {
|
||||
return true;
|
||||
}
|
||||
// This path uses provider output structure only: no user or assistant prose classification.
|
||||
return OLLAMA_INCOMPLETE_TURN_PROVIDER_ID_PATTERN.test(
|
||||
normalizeLowercaseStringOrEmpty(params.provider ?? ""),
|
||||
);
|
||||
}
|
||||
|
||||
function isIncompleteTurnRecoverySupportedProviderModel(params: {
|
||||
provider?: string;
|
||||
modelId?: string;
|
||||
}): boolean {
|
||||
if (
|
||||
isStrictAgenticSupportedProviderModel({
|
||||
provider: params.provider,
|
||||
modelId: params.modelId,
|
||||
})
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const provider = normalizeLowercaseStringOrEmpty(params.provider ?? "");
|
||||
if (!GEMINI_INCOMPLETE_TURN_PROVIDER_IDS.has(provider)) {
|
||||
return false;
|
||||
}
|
||||
const modelId = typeof params.modelId === "string" ? params.modelId : "";
|
||||
return GEMINI_INCOMPLETE_TURN_MODEL_ID_PATTERN.test(stripProviderPrefix(modelId));
|
||||
}
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
@@ -15,24 +15,26 @@ import type {
|
||||
EmbeddedRunFailureSignal,
|
||||
TraceAttempt,
|
||||
} from "../types.js";
|
||||
import { hasAttemptTerminalState } from "./attempt-terminal-evidence.js";
|
||||
import {
|
||||
markEmbeddedRunAuthProfileSuccess,
|
||||
reportEmbeddedRunSuccessfulAuthBinding,
|
||||
} from "./auth-profile-success.js";
|
||||
import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js";
|
||||
import {
|
||||
hasAttemptTerminalState,
|
||||
hasYieldContinuationEvidence,
|
||||
resolveEmptyResponseRetryInstruction,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
resolveReasoningOnlyRetryInstruction,
|
||||
resolveSettledToolTerminalContinuationInstruction,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
} from "./incomplete-turn-recovery.js";
|
||||
import {
|
||||
hasYieldContinuationEvidence,
|
||||
resolveIncompleteTurnPayloadText,
|
||||
resolveRunLivenessState,
|
||||
resolveSilentToolResultReplyPayload,
|
||||
resolveSettledToolTerminalContinuationInstruction,
|
||||
shouldRetryMissingAssistantTurn,
|
||||
shouldTreatEmptyAssistantReplyAsSilent,
|
||||
YIELD_DIAGNOSTIC_TEXT,
|
||||
} from "./incomplete-turn.js";
|
||||
} from "./incomplete-turn-resolution.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import {
|
||||
isEmbeddedRunTerminalAbort,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { projectAgentRunAttemptTerminal } from "../../agent-run-terminal-outcome.js";
|
||||
import { hasMessagingToolDeliveryEvidence } from "../delivery-evidence.js";
|
||||
import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js";
|
||||
import { resolveRunLivenessState } from "./incomplete-turn.js";
|
||||
import { resolveRunLivenessState } from "./incomplete-turn-resolution.js";
|
||||
import {
|
||||
isEmbeddedRunTerminalAbort,
|
||||
isEmbeddedRunTerminalTimeout,
|
||||
|
||||
@@ -17,10 +17,8 @@ import {
|
||||
GENERIC_ASSISTANT_ERROR_TEXT,
|
||||
} from "./embedded-agent-helpers.js";
|
||||
import { hasCommittedMessagingToolDeliveryEvidence } from "./embedded-agent-runner/delivery-evidence.js";
|
||||
import {
|
||||
hasAttemptTerminalState,
|
||||
isIncompleteTerminalAssistantTurn,
|
||||
} from "./embedded-agent-runner/run/incomplete-turn.js";
|
||||
import { hasAttemptTerminalState } from "./embedded-agent-runner/run/attempt-terminal-evidence.js";
|
||||
import { isIncompleteTerminalAssistantTurn } from "./embedded-agent-runner/run/incomplete-turn-classification.js";
|
||||
import { runBestEffortCallback } from "./embedded-agent-subscribe.callback.js";
|
||||
import {
|
||||
consumePendingToolMediaReply,
|
||||
|
||||
@@ -8,7 +8,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { AssistantMessage } from "openclaw/plugin-sdk/llm";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { buildAttemptReplayMetadata } from "../embedded-agent-runner/run/incomplete-turn.js";
|
||||
import { buildAttemptReplayMetadata } from "../embedded-agent-runner/run/attempt-terminal-evidence.js";
|
||||
import type { EmbeddedRunAttemptResult } from "../embedded-agent-runner/run/types.js";
|
||||
|
||||
export type EmbeddedAgentRunnerTestWorkspace = {
|
||||
|
||||
@@ -992,10 +992,6 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
"src/agents/embedded-agent-runner/run.before-agent-reply-cron.test.ts",
|
||||
"test/vitest/vitest.agents-embedded-agent.config.ts",
|
||||
],
|
||||
[
|
||||
"src/agents/embedded-agent-runner/run.incomplete-turn.test.ts",
|
||||
"test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts",
|
||||
],
|
||||
[
|
||||
"src/agents/embedded-agent-runner/run.overflow-compaction.test.ts",
|
||||
"test/vitest/vitest.agents-embedded-agent-overflow-compaction.config.ts",
|
||||
@@ -1021,6 +1017,28 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("routes every split incomplete-turn test to its dedicated serial shard", () => {
|
||||
const root = "src/agents/embedded-agent-runner";
|
||||
const discovered = fs
|
||||
.readdirSync(root)
|
||||
.filter((name) => name.startsWith("run.incomplete-turn.") && name.endsWith(".test.ts"))
|
||||
.map((name) => `${root}/${name}`)
|
||||
.toSorted();
|
||||
const owned = agentVitestProjectOwners.embeddedIncompleteTurn.include.toSorted();
|
||||
|
||||
expect(owned).toEqual(discovered);
|
||||
for (const testFile of discovered) {
|
||||
expect(buildVitestRunPlans([testFile])).toEqual([
|
||||
{
|
||||
config: "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts",
|
||||
forwardedArgs: [],
|
||||
includePatterns: [testFile],
|
||||
watchMode: false,
|
||||
},
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"src/agents/embedded-agent-runner/run",
|
||||
@@ -1061,7 +1079,7 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
{
|
||||
config: "test/vitest/vitest.agents-embedded-agent-incomplete-turn.config.ts",
|
||||
forwardedArgs: [],
|
||||
includePatterns: [`${root}/run.incomplete-turn.test.ts`],
|
||||
includePatterns: agentVitestProjectOwners.embeddedIncompleteTurn.include,
|
||||
watchMode: false,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -17,7 +17,19 @@ const coreIsolatedFiles = [
|
||||
"src/agents/subagents/registry/subagent-registry.announce-loop-guard.test.ts",
|
||||
"src/agents/subagents/registry/subagent-registry-restart-recovery.test.ts",
|
||||
];
|
||||
const incompleteTurnFiles = [`${embeddedRoot}/run.incomplete-turn.test.ts`];
|
||||
const incompleteTurnFiles = [
|
||||
`${embeddedRoot}/run.incomplete-turn.attempt-lifecycle.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.classification.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.delivery-resolution.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.empty-response-recovery.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.error-recovery.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.payload-resolution.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.reasoning-recovery.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.settled-tool-continuation.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.settled-tool-recovery.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.silent-reply.test.ts`,
|
||||
`${embeddedRoot}/run.incomplete-turn.terminal-evidence.test.ts`,
|
||||
];
|
||||
const overflowCompactionFiles = [
|
||||
`${embeddedRoot}/run.overflow-compaction.test.ts`,
|
||||
`${embeddedRoot}/run.prepared-harness-source-delivery.integration.test.ts`,
|
||||
|
||||
@@ -167,6 +167,7 @@ const broadUnitFastCandidateGlobs = [
|
||||
];
|
||||
const ownerRoutedUnitTestPatterns = [
|
||||
...cliProcessTestFiles,
|
||||
"src/agents/embedded-agent-runner/run.incomplete-turn.*.test.ts",
|
||||
"src/agents/embedded-agent-runner/run/attempt.abort-race.test.ts",
|
||||
"src/agents/openai-transport-stream.*.test.ts",
|
||||
"src/agents/embedded-agent-runner/run.shared-integration.test.ts",
|
||||
|
||||
Reference in New Issue
Block a user