fix(qa): preserve nested tool success evidence (#122189)

Record successful logical target tools at the attempt owner so Code Mode exec receipts retain the nested tool evidence needed by release qualification.

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Vincent Koc
2026-08-12 07:19:56 +08:00
committed by GitHub
parent 9c9ab5836d
commit b3cb8f7ab2
12 changed files with 242 additions and 37 deletions
@@ -141,6 +141,21 @@ describe("model-switch tool continuity terminal evidence", () => {
);
});
it("accepts a logical read appended after the physical Code Mode exec", async () => {
const { result } = await runToolContinuity(["exec", "read"]);
expect(result.status).toBe("pass");
expect(result.modelSwitchEvidence).toMatchObject({
alternate: { runId: "run-2", successfulToolNames: ["exec", "read"] },
});
});
it("rejects a bare successful Code Mode exec without logical read evidence", async () => {
await expect(runToolContinuity(["exec"])).rejects.toThrow(
"alternate-model run did not return exact owned successful read evidence",
);
});
it("does not let a successful prior-run read satisfy the alternate run", async () => {
await expect(runToolContinuity([])).rejects.toThrow(
"alternate-model run did not return exact owned successful read evidence",
@@ -442,6 +442,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
{ toolName: "cron" },
{ toolName: "sessions_spawn" },
],
successfulNestedToolNames: ["read"],
acceptedSessionSpawns,
successfulCronAdds: 1,
itemLifecycle: { startedCount: 3, completedCount: 3, activeCount: 0 },
@@ -484,6 +485,9 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
codeModeEngaged: true,
assistantTurns: 2,
bridgeCalls: { search: 1, describe: 2, call: 3 },
terminalReceipt: {
successfulToolNames: ["read"],
},
});
const secondCall = runAttemptCall(1);
expect(secondCall.prompt).toBe(SETTLED_TOOL_TERMINAL_CONTINUATION_INSTRUCTION);
@@ -4,8 +4,8 @@
import type { ContextEngineSessionTarget } from "../../context-engine/types.js";
import { normalizeAgentRunAttemptTerminal } from "../agent-run-terminal-outcome.js";
import { isAgentToolReplaySafe } from "../tool-replay-safety.js";
import type { EmbeddedRunAttemptWithReceiptEvidence } from "./run/attempt-result.js";
import { buildAttemptReplayMetadata } from "./run/attempt-terminal-evidence.js";
import type { EmbeddedRunAttemptResult } from "./run/types.js";
const DEFAULT_OVERFLOW_ERROR_MESSAGE =
"request_too_large: Request size exceeds model context window";
@@ -38,7 +38,7 @@ export function makeCompactionSuccess(params: {
};
}
type AttemptResultOverrides = Partial<EmbeddedRunAttemptResult> &
type AttemptResultOverrides = Partial<EmbeddedRunAttemptWithReceiptEvidence> &
Parameters<typeof normalizeAgentRunAttemptTerminal>[0];
function resolveFixtureTerminal(overrides: AttemptResultOverrides) {
@@ -47,7 +47,7 @@ function resolveFixtureTerminal(overrides: AttemptResultOverrides) {
export function makeAttemptResult(
overrides: AttemptResultOverrides = {},
): EmbeddedRunAttemptResult {
): EmbeddedRunAttemptWithReceiptEvidence {
const toolMetas = (overrides.toolMetas ?? []).map((entry) =>
Object.assign({}, entry, {
replaySafe: entry.replaySafe ?? isAgentToolReplaySafe({ name: entry.toolName }),
@@ -1,4 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { createTestAdmittedRunContext } from "../../admitted-run-context.test-support.js";
import { createUsageAccumulator } from "../usage-accumulator.js";
const mocks = vi.hoisted(() => ({
clearActiveEmbeddedRun: vi.fn(),
@@ -15,9 +17,14 @@ const mocks = vi.hoisted(() => ({
vi.mock("../logger.js", () => ({
log: { debug: mocks.logDebug, error: mocks.logError, warn: mocks.logWarn },
}));
vi.mock("../../subagents/registry/subagent-registry.js", () => ({
settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns,
}));
vi.mock("../../subagents/registry/subagent-registry.js", async (importOriginal) => {
const actual =
await importOriginal<typeof import("../../subagents/registry/subagent-registry.js")>();
return {
...actual,
settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns,
};
});
vi.mock("../runs.js", () => ({ clearActiveEmbeddedRun: mocks.clearActiveEmbeddedRun }));
vi.mock("./attempt-prompt-phase.js", () => ({
runEmbeddedAttemptPromptPhase: mocks.runPrompt,
@@ -38,6 +45,8 @@ vi.mock("./attempt-stream-settle.js", () => ({
import { SESSIONS_YIELD_ABORT_REASON } from "./attempt-sessions-yield.js";
import { runEmbeddedAttemptSettledPhase } from "./attempt-settle.js";
import { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js";
import { prepareEmbeddedRunTerminal } from "./terminal-preparation.js";
type SettledInput = Parameters<typeof runEmbeddedAttemptSettledPhase>[0];
@@ -46,14 +55,68 @@ function createFixture() {
const queueHandle = { kind: "embedded", runId: "run-1" };
const unsubscribe = vi.fn(() => order.push("unsubscribe"));
const waitForPendingEvents = vi.fn(async () => undefined);
const subscription = { unsubscribe, waitForPendingEvents };
const subscription = {
assistantTexts: [],
didSendDeterministicApprovalPrompt: vi.fn(() => false),
didSendViaMessagingTool: vi.fn(() => false),
getAcceptedSessionSpawns: vi.fn(() => []),
getAssistantTurnCount: vi.fn(() => 1),
getCompactionCount: vi.fn(() => 0),
getCurrentAttemptAssistant: vi.fn(() => undefined),
getHeartbeatToolResponse: vi.fn(() => undefined),
getItemLifecycle: vi.fn(() => ({ startedCount: 0, completedCount: 0, activeCount: 0 })),
getLastAssistantTextMessageIndex: vi.fn(() => undefined),
getLastAssistantUsage: vi.fn(() => undefined),
getLastCompactionTokensAfter: vi.fn(() => undefined),
getLastToolError: vi.fn(() => undefined),
getLatestMcpAppChannelView: vi.fn(() => undefined),
getLatestMcpConnectAction: vi.fn(() => undefined),
getMessagingToolSentMediaUrls: vi.fn(() => []),
getMessagingToolSentTargets: vi.fn(() => []),
getMessagingToolSentTexts: vi.fn(() => []),
getMessagingToolSourceReplyPayloads: vi.fn(() => []),
getPendingToolMediaReply: vi.fn(() => undefined),
getReplayState: vi.fn(() => ({ replayInvalid: false, hadPotentialSideEffects: false })),
getSuccessfulCronAdds: vi.fn(() => []),
getUsageTotals: vi.fn(() => ({ input: 1, output: 2, total: 3 })),
getVisibleBlockReplyCount: vi.fn(() => 0),
hasToolMediaBlockReply: vi.fn(() => false),
isCompactionInFlight: vi.fn(() => false),
setTerminalLifecycleMeta: vi.fn(),
toolMetas: [{ toolName: "exec", isError: false }],
unsubscribe,
waitForCompactionRetry: vi.fn(async () => undefined),
waitForPendingEvents,
};
const detachBackend = vi.fn(() => order.push("detach-backend"));
const clearTimers = vi.fn(() => order.push("clear-timers"));
const getBeforeAgentFinalizeRevisionReason = vi.fn(() => "revision");
const getBeforeAgentFinalizeRevisionEntryId = vi.fn(() => undefined);
const promptActiveSession = vi.fn(async () => undefined);
const messages = [
{
role: "assistant",
content: [{ type: "text", text: "done" }],
api: "openai-responses",
provider: "openai",
model: "model",
usage: {
input: 1,
output: 2,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 3,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: 100,
},
];
const activeSession = {
agent: { state: { messages: [] } },
agent: { state: { messages } },
isCompacting: false,
isStreaming: false,
messages,
sessionId: "active-session",
getActiveToolNames: vi.fn(() => ["read"]),
};
@@ -61,9 +124,9 @@ function createFixture() {
kind: "session-manager",
buildSessionContext: vi.fn(() => ({ messages: [] })),
};
const hookRunner = { kind: "hook-runner" };
const cacheTrace = { kind: "cache-trace" };
const trajectoryRecorder = { kind: "trajectory" };
const hookRunner = { hasHooks: vi.fn(() => false) };
const cacheTrace = { recordStage: vi.fn() };
const trajectoryRecorder = { recordEvent: vi.fn(), flush: vi.fn(async () => undefined) };
const toolResultPromptProjectionState = { kind: "tool-result-projection" };
const sessionPromptState = { toolResults: toolResultPromptProjectionState };
const sessionRuntimeState = {
@@ -144,11 +207,19 @@ function createFixture() {
};
const input = {
attempt: {
admittedRunContext: createTestAdmittedRunContext("run-1"),
config: {},
model: { api: "openai-responses" },
modelId: "model",
promptCacheKey: undefined,
provider: "openai",
replyOperation: { detachBackend },
runId: "run-1",
sessionFile: "/tmp/session.jsonl",
sessionId: "session-1",
sessionKey: "agent:main",
trigger: "user",
workspaceDir: "/workspace",
},
agentDir: "/agent",
isRawModelRun: false,
@@ -156,7 +227,7 @@ function createFixture() {
runAbortController: new AbortController(),
prepared: {
bootstrap: {
bootstrapPromptWarning: undefined,
bootstrapPromptWarning: {},
shouldRecordCompletedBootstrapTurn: false,
},
bundleTools: {
@@ -168,7 +239,7 @@ function createFixture() {
runtimeInfo: { model: { id: "model" } },
systemPromptReport: { chars: 13 },
},
toolBase: { toolSearchTargetTranscriptProjections: new Map() },
toolBase: { toolSearchTargetTranscriptProjections: [] },
toolCatalog: {
effectiveTools: [{ name: "read" }],
emptyExplicitToolAllowlistError: undefined,
@@ -176,7 +247,7 @@ function createFixture() {
},
},
sessionLock: {
withOwnedTranscriptWrite: vi.fn(),
withOwnedTranscriptWrite: vi.fn(async (operation: () => unknown) => await operation()),
},
setup: {
effectiveFsWorkspaceOnly: false,
@@ -327,6 +398,78 @@ describe("runEmbeddedAttemptSettledPhase", () => {
);
});
it("carries a successful hidden target through settlement into the terminal receipt", async () => {
const fixture = createFixture();
fixture.input.prepared.toolBase.toolSearchTargetTranscriptProjections.push(
{
parentToolCallId: "outer-exec",
toolCallId: "tool_search_code:outer-exec:read:1",
toolName: "read",
input: { path: "qa/scenarios/index.yaml" },
result: {
content: [{ type: "text", text: "QA scenario pack mission" }],
details: {},
},
isError: false,
},
{
parentToolCallId: "outer-exec",
toolCallId: "tool_search_code:outer-exec:write:2",
toolName: "write",
input: { path: "qa/scenarios/index.yaml", content: "invalid" },
result: {
content: [{ type: "text", text: "write failed" }],
details: {},
},
isError: true,
},
);
const actualStreamSettle = await vi.importActual<typeof import("./attempt-stream-settle.js")>(
"./attempt-stream-settle.js",
);
const actualAttemptResult =
await vi.importActual<typeof import("./attempt-result.js")>("./attempt-result.js");
mocks.settleStream.mockImplementationOnce(actualStreamSettle.settleEmbeddedAttemptStream);
mocks.completeResult.mockImplementationOnce(actualAttemptResult.completeEmbeddedAttemptResult);
const attempt = await runEmbeddedAttemptSettledPhase(fixture.input);
const prepared = prepareEmbeddedRunTerminal({
runParams: {
admittedRunContext: createTestAdmittedRunContext("run-1"),
sessionId: "session-1",
runId: "run-1",
workspaceDir: "/workspace",
prompt: "read the QA scenario index",
trigger: "user",
timeoutMs: 60_000,
},
attempt,
currentAttemptCompletedAssistant: attempt.currentAttemptCompletedAssistant,
provider: "openai",
model: "model",
activeErrorContext: { provider: "openai", model: "model" },
authProfileStore: { version: 1, profiles: {} },
sessionIdUsed: attempt.sessionIdUsed,
sessionFileUsed: attempt.sessionFileUsed,
outerContextTokenMeta: {},
usageAccumulator: createUsageAccumulator(),
contextRecoveryState: createEmbeddedRunContextRecoveryState(),
resolvedToolResultFormat: "markdown",
terminalState: {
outcome: { reason: "completed", status: "ok", stopReason: "stop" },
signalOwnedInterruption: false,
},
});
expect(
(
prepared.agentMeta as {
terminalReceipt?: { successfulToolNames?: string[] };
}
).terminalReceipt?.successfulToolNames,
).toEqual(["exec", "read"]);
});
it("preserves a prompt failure while still completing stream cleanup", async () => {
const fixture = createFixture();
const failure = new Error("prompt failed");
@@ -292,6 +292,7 @@ describe("embedded attempt phase lifecycle state", () => {
expect(result.lastAssistant).toBe(modelAssistant);
expect(result.currentAttemptAssistant).toBe(modelAssistant);
expect(result.currentAttemptCompletedAssistant).toEqual(modelAssistant);
expect(result.successfulNestedToolNames).toEqual([]);
expect(result.messagesSnapshot).toHaveLength(5);
expect(result.messagesSnapshot.at(-2)).toMatchObject({
role: "assistant",
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import { completeEmbeddedAttemptResult, createMcpAttemptCarryover } from "./attempt-result.js";
function completeResult(params?: {
successfulNestedToolNames?: string[];
latestMcpAppChannelView?: { viewId: string };
clientToolCallSlots?: Array<{
toolCallId: string;
@@ -59,6 +60,7 @@ function completeResult(params?: {
terminal: { kind: "ok" },
sessionIdUsed: "session-1",
messagesSnapshot: [],
successfulNestedToolNames: params?.successfulNestedToolNames,
yieldDetected: false,
didDeliverSourceReplyViaMessageTool: false,
diagnosticTrace: { traceId: "trace-1", spanId: "span-1" },
@@ -156,6 +158,13 @@ describe("attempt result projection", () => {
]);
});
it("projects successful nested tool names from settled attempt state", () => {
expect(
completeResult({ successfulNestedToolNames: ["read", "memory_search"] })
.successfulNestedToolNames,
).toEqual(["read", "memory_search"]);
});
it("projects pending media and voice fields", () => {
expect(completeResult().toolMediaUrls).toBeUndefined();
expect(completeResult({ pendingToolMediaReply: { mediaUrls: [" "] } }).toolMediaUrls).toEqual([
@@ -49,6 +49,10 @@ export function createMcpAttemptCarryover() {
};
}
export type EmbeddedRunAttemptWithReceiptEvidence = EmbeddedRunAttemptResult & {
successfulNestedToolNames?: string[];
};
export type EmbeddedAttemptClientToolCallSlot = {
toolCallId: string;
name: string;
@@ -57,7 +61,7 @@ export type EmbeddedAttemptClientToolCallSlot = {
};
type EmbeddedAttemptResultState = Pick<
EmbeddedRunAttemptResult,
EmbeddedRunAttemptWithReceiptEvidence,
| "terminal"
| "preflightRecovery"
| "sessionIdUsed"
@@ -69,6 +73,7 @@ type EmbeddedAttemptResultState = Pick<
| "lastAssistant"
| "currentAttemptAssistant"
| "currentAttemptCompletedAssistant"
| "successfulNestedToolNames"
| "attemptUsage"
| "promptCache"
| "contextBudgetStatus"
@@ -158,7 +163,7 @@ function hasVisiblePendingToolMediaReply(
/** Runs output hooks, classifies terminal effects, and returns the finalized attempt result. */
export function completeEmbeddedAttemptResult(
input: CompleteEmbeddedAttemptResultInput,
): EmbeddedRunAttemptResult {
): EmbeddedRunAttemptWithReceiptEvidence {
const { attempt, state, subscription } = input;
const terminal = projectAgentRunAttemptTerminal(state.terminal);
const {
@@ -389,7 +394,7 @@ export function completeEmbeddedAttemptResult(
terminal: state.terminal,
},
});
const result: EmbeddedRunAttemptResult = {
const result: EmbeddedRunAttemptWithReceiptEvidence = {
...state,
replayMetadata,
currentAttemptReplayMetadata,
@@ -403,6 +408,7 @@ export function completeEmbeddedAttemptResult(
latestMcpConnectAction: getLatestMcpConnectAction(),
lastAssistantTextMessageIndex: getLastAssistantTextMessageIndex(),
toolMetas: toolMetasNormalized,
successfulNestedToolNames: state.successfulNestedToolNames,
acceptedSessionSpawns,
lastToolError,
didSendViaMessagingTool: didSendViaMessagingTool(),
@@ -23,7 +23,10 @@ import type {
import { completeEmbeddedAttemptAfterTurn } from "./attempt-finalize.js";
import type { prepareEmbeddedAttemptHistory } from "./attempt-history.js";
import { runEmbeddedAttemptPromptPhase } from "./attempt-prompt-phase.js";
import { completeEmbeddedAttemptResult } from "./attempt-result.js";
import {
completeEmbeddedAttemptResult,
type EmbeddedRunAttemptWithReceiptEvidence,
} from "./attempt-result.js";
import type { prepareEmbeddedAttemptStream } from "./attempt-stream-prepare.js";
import { settleEmbeddedAttemptStream } from "./attempt-stream-settle.js";
import type { installEmbeddedAttemptStreamGuards } from "./attempt-stream.js";
@@ -107,7 +110,7 @@ export async function runEmbeddedAttemptSettledPhase(
getRepairedRejectedThinkingReplay: () => boolean;
preparedStreamRuntime: PreparedStreamRuntime;
},
): Promise<EmbeddedRunAttemptResult> {
): Promise<EmbeddedRunAttemptWithReceiptEvidence> {
const { attempt, state } = input;
const { bootstrap, bundleTools, sessionRuntime, systemPrompt, toolBase, toolCatalog } =
input.prepared;
@@ -173,6 +176,7 @@ export async function runEmbeddedAttemptSettledPhase(
let lastAssistant: AssistantMessage | undefined;
let currentAttemptAssistant: EmbeddedRunAttemptResult["currentAttemptAssistant"];
let currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"];
let successfulNestedToolNames: EmbeddedRunAttemptWithReceiptEvidence["successfulNestedToolNames"];
let attemptUsage: NormalizedUsage | undefined;
let cacheBreak: PromptCacheBreak | null = null;
let contextBudgetStatus: EmbeddedRunAttemptResult["contextBudgetStatus"];
@@ -447,6 +451,7 @@ export async function runEmbeddedAttemptSettledPhase(
lastAssistant = settledStream.lastAssistant;
currentAttemptAssistant = settledStream.currentAttemptAssistant;
currentAttemptCompletedAssistant = settledStream.currentAttemptCompletedAssistant;
successfulNestedToolNames = settledStream.successfulNestedToolNames;
attemptUsage = settledStream.attemptUsage;
cacheBreak = settledStream.cacheBreak;
sessionRuntimeState.promptCache = settledStream.promptCache;
@@ -530,6 +535,7 @@ export async function runEmbeddedAttemptSettledPhase(
lastAssistant,
currentAttemptAssistant,
currentAttemptCompletedAssistant,
successfulNestedToolNames,
attemptUsage,
promptCache: sessionRuntimeState.promptCache,
contextBudgetStatus,
@@ -92,6 +92,7 @@ type StreamSettleResult = {
lastAssistant: EmbeddedRunAttemptResult["lastAssistant"];
currentAttemptAssistant: EmbeddedRunAttemptResult["currentAttemptAssistant"];
currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"];
successfulNestedToolNames: string[];
attemptUsage: EmbeddedRunAttemptResult["attemptUsage"];
cacheBreak: PromptCacheBreak | null;
lastCallUsage: NormalizedUsage | undefined;
@@ -426,6 +427,15 @@ export async function settleEmbeddedAttemptStream(input: {
lastAssistant,
currentAttemptAssistant,
currentAttemptCompletedAssistant,
successfulNestedToolNames: [
...new Set(
input.toolSearchTargetTranscriptProjections
// Receipt evidence admits only projections explicitly recorded as successful.
.filter((projection) => Object.is(projection.isError, false))
.map((projection) => projection.toolName.trim())
.filter(Boolean),
),
],
attemptUsage,
cacheBreak,
lastCallUsage,
@@ -9,6 +9,7 @@ import {
mergeAttemptRunStatsIntoAccumulator,
mergeUsageIntoAccumulator,
} from "../usage-accumulator.js";
import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js";
import { runEmbeddedSettledTurnFinalizationWithBackend } from "./backend.js";
import { withEmbeddedRunLaneProgressHeartbeat } from "./lane-runtime.js";
import {
@@ -20,7 +21,7 @@ import {
copyAttemptDeliveryState,
resolveSettledTurnFinalizationRequest,
} from "./terminal-resolution.js";
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js";
import type { EmbeddedRunAttemptParams } from "./types.js";
type TerminalPreparationInput = Parameters<typeof prepareEmbeddedRunTerminal>[0];
type TerminalPreparationBase = Omit<
@@ -35,9 +36,9 @@ type TerminalPreparationBase = Omit<
export async function prepareTerminalWithSettledTurnFinalization(input: {
initial: {
attempt: EmbeddedRunAttemptResult;
attemptAssistant: EmbeddedRunAttemptResult["lastAssistant"];
currentAttemptCompletedAssistant: EmbeddedRunAttemptResult["currentAttemptCompletedAssistant"];
attempt: EmbeddedRunAttemptWithReceiptEvidence;
attemptAssistant: EmbeddedRunAttemptWithReceiptEvidence["lastAssistant"];
currentAttemptCompletedAssistant: EmbeddedRunAttemptWithReceiptEvidence["currentAttemptCompletedAssistant"];
sessionIdUsed: string;
sessionFileUsed?: string;
terminalState: EmbeddedRunTerminalState;
@@ -181,12 +182,12 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
async function runPreparedSettledTurnFinalization(input: {
attempt: EmbeddedRunAttemptParams;
settledAttempt: EmbeddedRunAttemptResult;
settledAttempt: EmbeddedRunAttemptWithReceiptEvidence;
harness: AgentHarness;
prompt: string;
noteLaneTaskProgress: () => void;
}): Promise<
| { outcome: "answered"; attempt: EmbeddedRunAttemptResult }
| { outcome: "answered"; attempt: EmbeddedRunAttemptWithReceiptEvidence }
| {
outcome: "empty";
result: AgentHarnessSettledTurnFinalizationResult;
@@ -222,13 +223,13 @@ async function runPreparedSettledTurnFinalization(input: {
function buildSettledTurnFinalizationAttemptResult(input: {
result: AgentHarnessSettledTurnFinalizationResult;
settledAttempt: EmbeddedRunAttemptResult;
settledAttempt: EmbeddedRunAttemptWithReceiptEvidence;
prompt: string;
agentHarnessId?: string;
}): EmbeddedRunAttemptResult {
}): EmbeddedRunAttemptWithReceiptEvidence {
const { result, settledAttempt } = input;
const text = resolveSettledTurnFinalizationText(result);
// Finalization replaces terminal ownership, not facts from already-settled tools.
// Finalization replaces terminal ownership, not host-private facts from settled tools.
// Keep those facts while replay, abort, and lifecycle state remain finalizer-local.
return {
terminal: { kind: "ok" },
@@ -249,6 +250,7 @@ function buildSettledTurnFinalizationAttemptResult(input: {
currentAttemptAssistant: result.assistant,
currentAttemptCompletedAssistant: result.assistant,
toolMetas: settledAttempt.toolMetas,
successfulNestedToolNames: settledAttempt.successfulNestedToolNames,
hasToolMediaBlockReply: false,
cloudCodeAssistFormatError: false,
attemptUsage: result.usage,
@@ -2,8 +2,8 @@ import type { AssistantMessage } from "openclaw/plugin-sdk/llm";
import { describe, expect, it, vi } from "vitest";
import { createTestAdmittedRunContext } from "../../admitted-run-context.test-support.js";
import { createUsageAccumulator } from "../usage-accumulator.js";
import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js";
import { createEmbeddedRunContextRecoveryState } from "./context-recovery-state.js";
import type { EmbeddedRunAttemptResult } from "./types.js";
vi.mock("./payloads.js", () => ({
buildEmbeddedRunPayloads: () => [],
@@ -43,8 +43,8 @@ function assistantMessage(stopReason: AssistantMessage["stopReason"] = "stop"):
}
function attemptResult(
overrides: Partial<EmbeddedRunAttemptResult> = {},
): EmbeddedRunAttemptResult {
overrides: Partial<EmbeddedRunAttemptWithReceiptEvidence> = {},
): EmbeddedRunAttemptWithReceiptEvidence {
const assistant = assistantMessage("error");
return {
terminal: { kind: "ok" },
@@ -114,7 +114,7 @@ describe("prepareEmbeddedRunTerminal", () => {
describe("prepareEmbeddedRunTerminal run stats", () => {
type StatsInput = {
attempt?: Partial<EmbeddedRunAttemptResult> & {
attempt?: Partial<EmbeddedRunAttemptWithReceiptEvidence> & {
terminalTurnId?: string;
};
assistantTurns?: number;
@@ -257,12 +257,13 @@ describe("prepareEmbeddedRunTerminal run stats", () => {
attempt: {
terminalTurnId: "turn-7",
toolMetas: [
{ toolName: "started" },
{ toolName: "exec", isError: false },
{ toolName: "unknown" },
{ toolName: "write", isError: true },
{ toolName: "read", isError: false },
{ toolName: "read", isError: false },
{ toolName: "exec", isError: false },
],
successfulNestedToolNames: ["read", "zeta", "alpha", "Zeta", " exec ", "alpha", " "],
},
});
@@ -278,7 +279,7 @@ describe("prepareEmbeddedRunTerminal run stats", () => {
model: "cost-model-rerouted",
responseModel: "cost-model-rerouted",
},
successfulToolNames: ["read"],
successfulToolNames: ["exec", "read", "Zeta", "alpha", "zeta"],
rerouted: true,
});
expect(
@@ -9,6 +9,7 @@ import type { NormalizedUsage, UsageLike } from "../../usage.js";
import { resolveEmbeddedRunFailureSignal } from "../failure-signal.js";
import type { EmbeddedAgentMeta, EmbeddedAgentRunResult } from "../types.js";
import type { UsageAccumulator } from "../usage-accumulator.js";
import type { EmbeddedRunAttemptWithReceiptEvidence } from "./attempt-result.js";
import type { EmbeddedRunContextRecoveryState } from "./context-recovery-state.js";
import {
buildUsageAgentMetaFields,
@@ -25,11 +26,10 @@ import {
type EmbeddedRunTerminalState,
} from "./terminal-outcome.js";
import { mergeAttemptToolMediaPayloads } from "./tool-media-payloads.js";
import type { EmbeddedRunAttemptResult } from "./types.js";
export function prepareEmbeddedRunTerminal(input: {
runParams: RunEmbeddedAgentParams;
attempt: EmbeddedRunAttemptResult;
attempt: EmbeddedRunAttemptWithReceiptEvidence;
currentAttemptCompletedAssistant?: AssistantMessage;
provider: string;
providerOwner?: PreparedProviderFailoverOwner;
@@ -143,6 +143,14 @@ export function prepareEmbeddedRunTerminal(input: {
.filter(Boolean),
),
];
const missingNestedToolNames = [
...new Set(
(attempt.successfulNestedToolNames ?? []).map((name) => name.trim()).filter(Boolean),
),
]
.filter((name) => !successfulToolNames.includes(name))
.toSorted();
successfulToolNames.push(...missingNestedToolNames);
Object.assign(agentMeta, {
terminalReceipt: {
runId: runParams.runId,