From 4e51e7addaf2d876d63dce486cbc3c6602be04c1 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 25 Jul 2026 08:44:45 +0800 Subject: [PATCH] fix(qa): capture trajectory-only runtime tools (#113452) --- extensions/qa-lab/src/runtime-parity.test.ts | 314 ++++++++++++++++++- extensions/qa-lab/src/runtime-parity.ts | 300 +++++++++++++++++- 2 files changed, 591 insertions(+), 23 deletions(-) diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index a3970399b594..f9a319f9f36e 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -2,13 +2,14 @@ import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import path from "node:path"; -import { - formatSqliteSessionFileMarker, - resolveStorePath, - upsertSessionEntry, -} from "openclaw/plugin-sdk/session-store-runtime"; +import { resolveStorePath, upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { + appendSqliteTrajectoryRuntimeEvents, + formatSqliteSessionFileMarker, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { stableHash } from "./parity-shared.js"; import { captureRuntimeParityCell, isRuntimeParityResultPass, @@ -28,11 +29,18 @@ afterEach(async () => { }); async function seedRuntimeParityTranscript(params: { + heartbeatIsolatedBaseSessionKey?: string; messages: Array>; sessionId: string; sessionKey: string; + tempRoot?: string; + trajectoryEvents?: Array<{ + data?: Record; + type: string; + }>; + updatedAt?: number; }) { - const tempRoot = await tempDirs.makeTempDir("openclaw-qa-runtime-parity-"); + const tempRoot = params.tempRoot ?? (await tempDirs.makeTempDir("openclaw-qa-runtime-parity-")); const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(tempRoot, "state") }; const storePath = resolveStorePath(undefined, { agentId: "qa", env }); await upsertSessionEntry({ @@ -47,7 +55,10 @@ async function seedRuntimeParityTranscript(params: { sessionId: params.sessionId, storePath, }), - updatedAt: 100, + updatedAt: params.updatedAt ?? 100, + ...(params.heartbeatIsolatedBaseSessionKey + ? { heartbeatIsolatedBaseSessionKey: params.heartbeatIsolatedBaseSessionKey } + : {}), }, }); for (const [index, message] of params.messages.entries()) { @@ -61,6 +72,24 @@ async function seedRuntimeParityTranscript(params: { message: message as never, }); } + if (params.trajectoryEvents?.length) { + appendSqliteTrajectoryRuntimeEvents( + { agentId: "qa", env, sessionId: params.sessionId, storePath }, + params.trajectoryEvents.map((event, index) => ({ + traceSchema: "openclaw-trajectory", + schemaVersion: 1, + traceId: params.sessionId, + source: "runtime", + type: event.type, + ts: new Date(index + 1).toISOString(), + seq: index + 1, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + runId: "run-1", + data: event.data, + })), + ); + } return tempRoot; } @@ -68,12 +97,17 @@ async function captureRuntimeParityWithMockRequests(params: { messages?: Array>; requests: Array>; scenarioResult?: Parameters[0]["scenarioResult"]; + trajectoryEvents?: Array<{ + data?: Record; + type: string; + }>; }) { const parentPrompt = "Delegate one bounded QA task to a subagent."; const tempRoot = await seedRuntimeParityTranscript({ sessionId: "mock-runtime-parity", sessionKey: "agent:qa:mock-runtime-parity", messages: params.messages ?? [{ role: "user", content: parentPrompt }], + trajectoryEvents: params.trajectoryEvents, }); const requests = params.requests.map((request) => ({ prompt: parentPrompt, @@ -202,6 +236,272 @@ describe("runtime parity", () => { expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); }); + it("captures native tool execution from the canonical SQLite trajectory", async () => { + const cell = await captureRuntimeParityWithMockRequests({ + messages: [], + requests: [], + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "search-1", + name: "web_search", + arguments: { query: "OpenClaw runtime parity fixed query" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "search-1", + name: "web_search", + status: "completed", + isError: false, + result: { + status: "completed", + query: "OpenClaw runtime parity fixed query", + }, + }, + }, + ], + }); + + expect(cell.toolCalls).toEqual([expect.objectContaining({ tool: "web_search" })]); + expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); + expect(cell.providerPlanToolCalls).toEqual([]); + }); + + it("merges trajectory-only calls without duplicating transcript calls", async () => { + const tempRoot = await seedRuntimeParityTranscript({ + sessionId: "mixed-runtime-tools", + sessionKey: "agent:qa:mixed-runtime-tools", + messages: [ + { role: "user", content: "Read the file, run the command, then search." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "read-1", + name: "read", + arguments: { path: "README.md" }, + }, + { + type: "toolCall", + id: "exec-1", + name: "exec", + arguments: { command: "pwd" }, + }, + ], + }, + { + role: "toolResult", + toolCallId: "read-1", + toolName: "read", + content: [{ type: "text", text: "README contents" }], + }, + ], + trajectoryEvents: [ + { + type: "tool.result", + data: { + toolCallId: "exec-1", + name: "exec", + success: true, + contentItems: [{ type: "text", text: "/workspace" }], + }, + }, + { + type: "tool.call", + data: { + toolCallId: "search-1", + name: "web_search", + arguments: { query: "OpenClaw runtime parity fixed query" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "search-1", + name: "web_search", + status: "completed", + result: { status: "completed" }, + }, + }, + ], + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.toolCalls.map((toolCall) => toolCall.tool)).toEqual(["read", "exec", "web_search"]); + expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); + expect(cell.toolCalls[1]?.errorClass).toBeUndefined(); + expect(cell.toolCalls[1]?.argsHash).toBe(stableHash({ command: "pwd" })); + }); + + it("keeps distinct same-tool calls with identical arguments", async () => { + const tempRoot = await seedRuntimeParityTranscript({ + sessionId: "distinct-web-searches", + sessionKey: "agent:qa:distinct-web-searches", + messages: [ + { role: "user", content: "Search for both QA markers." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + name: "web_search", + arguments: { query: "same marker" }, + }, + ], + }, + { + role: "toolResult", + toolName: "web_search", + content: [{ type: "text", text: "result A" }], + }, + ], + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "search-b", + name: "web_search", + arguments: { query: "same marker" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "search-b", + name: "web_search", + status: "completed", + result: { status: "completed", query: "same marker" }, + }, + }, + ], + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.toolCalls.map((toolCall) => toolCall.tool)).toEqual(["web_search", "web_search"]); + expect(cell.toolCalls[0]?.argsHash).toBe(cell.toolCalls[1]?.argsHash); + }); + + it("skips newer trajectory-only heartbeat sessions", async () => { + const now = Date.now(); + const tempRoot = await seedRuntimeParityTranscript({ + sessionId: "web-search-session", + sessionKey: "agent:qa:web-search-session", + messages: [], + updatedAt: now - 1_000, + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "search-1", + name: "web_search", + arguments: { query: "release marker" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "search-1", + name: "web_search", + status: "completed", + result: { status: "completed" }, + }, + }, + ], + }); + await seedRuntimeParityTranscript({ + tempRoot, + sessionId: "heartbeat-session", + sessionKey: "agent:qa:main:heartbeat", + heartbeatIsolatedBaseSessionKey: "agent:qa:main", + messages: [], + updatedAt: now, + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "heartbeat-1", + name: "web_search", + arguments: { query: "heartbeat background search" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "heartbeat-1", + name: "web_search", + success: true, + result: { status: "completed" }, + }, + }, + ], + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.toolCalls.map((toolCall) => toolCall.tool)).toEqual(["web_search"]); + expect(cell.toolCalls[0]?.argsHash).toBe(stableHash({ query: "release marker" })); + }); + + it("keeps an explicitly identified orphan result separate", async () => { + const tempRoot = await seedRuntimeParityTranscript({ + sessionId: "orphan-trajectory-result", + sessionKey: "agent:qa:orphan-trajectory-result", + messages: [], + trajectoryEvents: [ + { + type: "tool.call", + data: { + toolCallId: "read-pending", + name: "read", + arguments: { path: "README.md" }, + }, + }, + { + type: "tool.result", + data: { + toolCallId: "read-orphan", + name: "read", + success: true, + contentItems: [{ type: "text", text: "orphan result" }], + }, + }, + ], + }); + + const cell = await captureRuntimeParityCell({ + runtime: "codex", + gateway: { tempRoot }, + scenarioResult: { status: "pass" }, + wallClockMs: 10, + }); + + expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([ + "tool-result-missing", + undefined, + ]); + }); + it("keeps a retry pass diagnostic from failing the captured cell", async () => { const cell = await captureRuntimeParityCell({ runtime: "openclaw", diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index c0439d2c3ace..e869076e5c80 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -1,7 +1,12 @@ import { listSessionEntries, loadTranscriptEventsSync, + resolveStorePath, } from "openclaw/plugin-sdk/session-store-runtime"; +import { + loadSqliteTrajectoryRuntimeEvents, + type SqliteTrajectoryRuntimeEventForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; // Qa Lab plugin module implements runtime parity behavior. import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { @@ -134,6 +139,7 @@ type RuntimeParitySessionEntry = { sessionId?: string; sessionFile?: string; updatedAt?: number; + heartbeatIsolatedBaseSessionKey?: string; spawnedBy?: string; parentSessionKey?: string; spawnDepth?: number; @@ -158,10 +164,21 @@ type RuntimeParityMockRequestSnapshot = { toolOutput?: string; }; -type RuntimeParityPendingToolCall = RuntimeParityToolCall & { +type RuntimeParityObservedToolCall = RuntimeParityToolCall & { + callId?: string; + hasArguments?: boolean; + hasResult?: boolean; +}; + +type RuntimeParityPendingToolCall = RuntimeParityObservedToolCall & { _resolved: boolean; }; +type RuntimeParityCaptureSources = { + transcriptBytes: string; + trajectoryToolCalls: RuntimeParityObservedToolCall[]; +}; + const DEFAULT_AGENT_ID = "qa"; const HEARTBEAT_RESPONSE_TOOL_NAME = "heartbeat_respond"; const HEARTBEAT_TRANSCRIPT_PROMPT = "[OpenClaw heartbeat poll]"; @@ -406,7 +423,9 @@ function classifyToolResultError(params: { return undefined; } -function finalizeToolCallOrder(ordered: RuntimeParityPendingToolCall[]): RuntimeParityToolCall[] { +function finalizeToolCallOrder( + ordered: RuntimeParityPendingToolCall[], +): RuntimeParityObservedToolCall[] { return ordered.map(({ _resolved, ...toolCall }) => _resolved ? toolCall @@ -417,7 +436,9 @@ function finalizeToolCallOrder(ordered: RuntimeParityPendingToolCall[]): Runtime ); } -function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): RuntimeParityToolCall[] { +function resolveToolCallOrder( + records: RuntimeParityTranscriptRecord[], +): RuntimeParityObservedToolCall[] { const ordered: RuntimeParityPendingToolCall[] = []; const byId = new Map(); const unresolvedByTool = new Map(); @@ -473,6 +494,9 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime tool: call.tool, argsHash: parity.stableHash(call.args), resultHash: parity.stableHash(null), + callId: call.id, + hasArguments: true, + hasResult: false, _resolved: false, }) - 1; if (call.id) { @@ -484,7 +508,7 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime if (record.role === "user" || record.role === "tool" || record.role === "toolResult") { for (const result of extractToolResults(record.message)) { const pendingIndex = matchPendingIndex(result); - const nextValue: RuntimeParityToolCall = { + const nextValue: RuntimeParityObservedToolCall = { tool: result.tool ?? (pendingIndex !== undefined ? ordered[pendingIndex]?.tool : undefined) ?? @@ -494,6 +518,10 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime ? (ordered[pendingIndex]?.argsHash ?? parity.stableHash(null)) : parity.stableHash(null), resultHash: parity.stableHash(result.result), + callId: pendingIndex !== undefined ? ordered[pendingIndex]?.callId : result.id, + hasArguments: + pendingIndex !== undefined ? ordered[pendingIndex]?.hasArguments === true : false, + hasResult: true, ...(result.errorClass ? { errorClass: result.errorClass } : {}), }; if (pendingIndex === undefined || !ordered[pendingIndex]) { @@ -580,6 +608,220 @@ function resolveToolCallOrderFromMockRequests( return finalizeToolCallOrder(ordered); } +function trajectoryToolCallId(data: Record) { + return ( + normalizeToolCallId(data.toolCallId) ?? + normalizeToolCallId(data.itemId) ?? + normalizeToolCallId(data.id) + ); +} + +function trajectoryToolResultValue(data: Record) { + if (Object.hasOwn(data, "result")) { + return data.result; + } + if (Object.hasOwn(data, "output")) { + return data.output; + } + if (Object.hasOwn(data, "contentItems")) { + return data.contentItems; + } + return { + ...(Object.hasOwn(data, "status") ? { status: data.status } : {}), + ...(Object.hasOwn(data, "success") ? { success: data.success } : {}), + }; +} + +function isTrajectoryToolResultError(data: Record) { + if (data.isError === true || data.success === false) { + return true; + } + const status = readNonEmptyString(data.status)?.toLowerCase(); + return ( + status === "blocked" || + status === "cancelled" || + status === "declined" || + status === "error" || + status === "failed" + ); +} + +function resolveTrajectoryToolCallOrder( + events: readonly SqliteTrajectoryRuntimeEventForTest[], +): RuntimeParityObservedToolCall[] { + const ordered: Array<{ + call: RuntimeParityObservedToolCall; + resolved: boolean; + }> = []; + + const matchPendingIndex = (data: Record, tool?: string) => { + const id = trajectoryToolCallId(data); + if (id) { + const idMatch = ordered.findIndex((pending) => pending.call.callId === id); + if (idMatch >= 0) { + return idMatch; + } + return undefined; + } + const toolMatch = ordered.findIndex( + (pending) => !pending.resolved && (!tool || pending.call.tool === tool), + ); + if (toolMatch >= 0) { + return toolMatch; + } + return undefined; + }; + + for (const event of events) { + const data = event.data ?? {}; + if (event.type === "tool.call") { + const tool = readNonEmptyString(data.name) ?? "unknown"; + ordered.push({ + call: { + tool, + argsHash: parity.stableHash(data.arguments ?? null), + resultHash: parity.stableHash(null), + callId: trajectoryToolCallId(data), + hasArguments: true, + hasResult: false, + }, + resolved: false, + }); + continue; + } + if (event.type !== "tool.result") { + continue; + } + const tool = readNonEmptyString(data.name); + const pendingIndex = matchPendingIndex(data, tool); + const pendingCall = pendingIndex !== undefined ? ordered[pendingIndex]?.call : undefined; + const nextValue: RuntimeParityObservedToolCall = { + tool: tool ?? pendingCall?.tool ?? "unknown", + argsHash: pendingCall?.argsHash ?? parity.stableHash(null), + resultHash: parity.stableHash(trajectoryToolResultValue(data)), + callId: trajectoryToolCallId(data) ?? pendingCall?.callId, + hasArguments: pendingCall?.hasArguments === true, + hasResult: true, + ...(isTrajectoryToolResultError(data) ? { errorClass: "tool-result-error" } : {}), + }; + if (pendingIndex === undefined) { + ordered.push({ + call: nextValue, + resolved: true, + }); + continue; + } + ordered[pendingIndex] = { + ...ordered[pendingIndex], + call: nextValue, + resolved: true, + }; + } + + for (const pending of ordered) { + if (!pending.resolved) { + pending.call.errorClass ??= TOOL_RESULT_MISSING_ERROR_CLASS; + } + } + return ordered.map((pending) => pending.call); +} + +function mergeRuntimeParityToolCalls(params: { + transcriptToolCalls: RuntimeParityObservedToolCall[]; + trajectoryToolCalls: RuntimeParityObservedToolCall[]; +}): RuntimeParityObservedToolCall[] { + if (params.trajectoryToolCalls.length === 0) { + return params.transcriptToolCalls; + } + if (params.transcriptToolCalls.length === 0) { + return params.trajectoryToolCalls; + } + + const transcript = params.transcriptToolCalls; + const trajectory = params.trajectoryToolCalls; + const callsMatch = ( + transcriptCall: RuntimeParityObservedToolCall, + trajectoryCall: RuntimeParityObservedToolCall, + ) => { + if (transcriptCall.callId && trajectoryCall.callId) { + return transcriptCall.callId === trajectoryCall.callId; + } + if (transcriptCall.callId || trajectoryCall.callId) { + return false; + } + return ( + transcriptCall.tool === trajectoryCall.tool && + transcriptCall.argsHash === trajectoryCall.argsHash + ); + }; + const mergeMatchingCalls = ( + transcriptCall: RuntimeParityObservedToolCall, + trajectoryCall: RuntimeParityObservedToolCall, + ): RuntimeParityObservedToolCall => { + const invocation = transcriptCall.hasArguments ? transcriptCall : trajectoryCall; + const completion = transcriptCall.hasResult ? transcriptCall : trajectoryCall; + return { + tool: invocation.tool, + argsHash: invocation.argsHash, + resultHash: completion.resultHash, + callId: transcriptCall.callId ?? trajectoryCall.callId, + hasArguments: invocation.hasArguments, + hasResult: completion.hasResult, + ...(completion.errorClass ? { errorClass: completion.errorClass } : {}), + }; + }; + const sharedSuffixLengths = Array.from({ length: transcript.length + 1 }, () => + Array(trajectory.length + 1).fill(0), + ); + for (let transcriptIndex = transcript.length - 1; transcriptIndex >= 0; transcriptIndex -= 1) { + for (let trajectoryIndex = trajectory.length - 1; trajectoryIndex >= 0; trajectoryIndex -= 1) { + sharedSuffixLengths[transcriptIndex]![trajectoryIndex] = callsMatch( + transcript[transcriptIndex]!, + trajectory[trajectoryIndex]!, + ) + ? 1 + sharedSuffixLengths[transcriptIndex + 1]![trajectoryIndex + 1]! + : Math.max( + sharedSuffixLengths[transcriptIndex + 1]![trajectoryIndex]!, + sharedSuffixLengths[transcriptIndex]![trajectoryIndex + 1]!, + ); + } + } + + const merged: RuntimeParityObservedToolCall[] = []; + let transcriptIndex = 0; + let trajectoryIndex = 0; + while (transcriptIndex < transcript.length && trajectoryIndex < trajectory.length) { + const transcriptCall = transcript[transcriptIndex]!; + const trajectoryCall = trajectory[trajectoryIndex]!; + if (callsMatch(transcriptCall, trajectoryCall)) { + merged.push(mergeMatchingCalls(transcriptCall, trajectoryCall)); + transcriptIndex += 1; + trajectoryIndex += 1; + continue; + } + if ( + sharedSuffixLengths[transcriptIndex + 1]![trajectoryIndex]! >= + sharedSuffixLengths[transcriptIndex]![trajectoryIndex + 1]! + ) { + merged.push(transcriptCall); + transcriptIndex += 1; + continue; + } + merged.push(trajectoryCall); + trajectoryIndex += 1; + } + return [...merged, ...transcript.slice(transcriptIndex), ...trajectory.slice(trajectoryIndex)]; +} + +function removeRuntimeParityToolCallIdentity( + toolCalls: RuntimeParityObservedToolCall[], +): RuntimeParityToolCall[] { + return toolCalls.map( + ({ callId: _callId, hasArguments: _hasArguments, hasResult: _hasResult, ...toolCall }) => + toolCall, + ); +} + function classifyScenarioError(details: string | undefined): string | undefined { const normalized = normalizeTextForParity(details ?? "").toLowerCase(); if (!normalized) { @@ -969,7 +1211,8 @@ function readRuntimeParitySessionEntries(params: { .map(({ entry, sessionKey }) => ({ entry: entry as RuntimeParitySessionEntry, sessionKey, - })); + })) + .filter(({ entry }) => !readNonEmptyString(entry.heartbeatIsolatedBaseSessionKey)); const rootEntries = entries.filter(({ entry }) => isRuntimeParityRootSession(entry)); const candidates = rootEntries.length > 0 ? rootEntries : entries; return candidates.toSorted( @@ -980,38 +1223,57 @@ function readRuntimeParitySessionEntries(params: { } } -async function loadRuntimeParityTranscripts(params: { +async function loadRuntimeParityCaptureSources(params: { gateway: QaGatewayLike; agentId: string; -}): Promise { +}): Promise { const stateDir = `${params.gateway.tempRoot}/state`; + const env = runtimeParitySessionEnv(stateDir); + const storePath = resolveStorePath(undefined, { agentId: params.agentId, env }); const sessionEntries = readRuntimeParitySessionEntries({ stateDir, agentId: params.agentId, }); - const transcripts: string[] = []; for (const { entry, sessionKey } of sessionEntries) { const sessionId = readNonEmptyString(entry.sessionId); if (!sessionId) { continue; } + let transcriptBytes = ""; try { const events = loadTranscriptEventsSync({ agentId: params.agentId, - env: runtimeParitySessionEnv(stateDir), + env, sessionId, sessionKey, }); - const transcript = events.map((event) => JSON.stringify(event)).join("\n"); - if (transcript.trim().length > 0 && !isHeartbeatOnlyRuntimeTranscript(transcript)) { - transcripts.push(transcript.trimEnd()); - break; + transcriptBytes = events + .map((event) => JSON.stringify(event)) + .join("\n") + .trimEnd(); + if (transcriptBytes && isHeartbeatOnlyRuntimeTranscript(transcriptBytes)) { + continue; } } catch { // Ignore missing transcript files so failed cells still render. } + let trajectoryToolCalls: RuntimeParityObservedToolCall[] = []; + try { + const trajectoryEvents = await loadSqliteTrajectoryRuntimeEvents({ + agentId: params.agentId, + env, + sessionId, + storePath, + }); + trajectoryToolCalls = resolveTrajectoryToolCallOrder(trajectoryEvents); + } catch { + // Transcript evidence remains authoritative when diagnostics are unavailable. + } + if (transcriptBytes || trajectoryToolCalls.length > 0) { + return { transcriptBytes, trajectoryToolCalls }; + } } - return transcripts.join("\n"); + return { transcriptBytes: "", trajectoryToolCalls: [] }; } async function loadRuntimeParityMockToolCalls( @@ -1063,12 +1325,18 @@ export async function captureRuntimeParityCell( params: RuntimeParityCaptureParams, ): Promise { const agentId = params.agentId ?? DEFAULT_AGENT_ID; - const transcriptBytes = await loadRuntimeParityTranscripts({ + const { transcriptBytes, trajectoryToolCalls } = await loadRuntimeParityCaptureSources({ gateway: params.gateway, agentId, }); const transcriptRecords = buildTranscriptRecords(transcriptBytes); const transcriptToolCalls = resolveToolCallOrder(transcriptRecords); + const runtimeToolCalls = removeRuntimeParityToolCallIdentity( + mergeRuntimeParityToolCalls({ + transcriptToolCalls, + trajectoryToolCalls, + }), + ); const parentPrompts = transcriptRecords .filter((record) => record.role === "user") .map((record) => extractAssistantText(record.message)) @@ -1096,7 +1364,7 @@ export async function captureRuntimeParityCell( runtime: params.runtime, transcriptBytes, toolCalls: resolveRuntimeParityToolCalls({ - transcriptToolCalls, + transcriptToolCalls: runtimeToolCalls, terminalImageResultProven, }), ...(mockToolCalls ? { providerPlanToolCalls: mockToolCalls } : {}),