From e23ba67142fe54a98e124520ae1917d03d18b87e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 14 Jul 2026 07:00:49 -0400 Subject: [PATCH] fix(qa): reconcile duplicate parity capture rows --- extensions/qa-lab/src/parity-shared.test.ts | 21 ++++++++ extensions/qa-lab/src/parity-shared.ts | 50 ++++++++++++++++++++ extensions/qa-lab/src/runtime-parity.test.ts | 7 ++- extensions/qa-lab/src/runtime-parity.ts | 39 ++++++++------- 4 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 extensions/qa-lab/src/parity-shared.test.ts diff --git a/extensions/qa-lab/src/parity-shared.test.ts b/extensions/qa-lab/src/parity-shared.test.ts new file mode 100644 index 000000000000..0ffaafa72e46 --- /dev/null +++ b/extensions/qa-lab/src/parity-shared.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { compareCapturedToolCallShape } from "./parity-shared.js"; + +const call = { tool: "image_generate", argsHash: "same-args" }; + +describe("compareCapturedToolCallShape", () => { + it("accepts exact repeated executions", () => { + expect(compareCapturedToolCallShape([call, call], [call, call])).toBeUndefined(); + }); + + it("accepts a duplicated process-global capture row", () => { + expect(compareCapturedToolCallShape([call, call], [call])).toBeUndefined(); + expect(compareCapturedToolCallShape([call, call, call], [call, call])).toBeUndefined(); + }); + + it("preserves canonical execution count", () => { + expect(compareCapturedToolCallShape([call], [call, call])).toBe( + "tool call count differs (1 vs 2)", + ); + }); +}); diff --git a/extensions/qa-lab/src/parity-shared.ts b/extensions/qa-lab/src/parity-shared.ts index 8558bc46867e..c5bd0cc4a77a 100644 --- a/extensions/qa-lab/src/parity-shared.ts +++ b/extensions/qa-lab/src/parity-shared.ts @@ -48,3 +48,53 @@ export function compareToolCallShape( } return undefined; } + +export function distinctToolCallShapes(toolCalls: readonly ParityToolCallShape[]) { + return toolCalls.filter( + (toolCall, index) => + toolCalls.findIndex( + (candidate) => candidate.tool === toolCall.tool && candidate.argsHash === toolCall.argsHash, + ) === index, + ); +} + +export function compareCapturedToolCallShape( + left: readonly ParityToolCallShape[], + right: readonly ParityToolCallShape[], +) { + const exactMatch = compareToolCallShape(left, right); + if (exactMatch === undefined) { + return undefined; + } + // Process-global captures can repeat planned rows. The canonical transcript + // must remain an ordered subsequence; unknown shapes still fail comparison. + let rightIndex = 0; + for (const leftCall of left) { + const expected = right[rightIndex]; + if (expected?.tool === leftCall.tool && expected.argsHash === leftCall.argsHash) { + rightIndex += 1; + continue; + } + const knownShape = right.some( + (candidate) => candidate.tool === leftCall.tool && candidate.argsHash === leftCall.argsHash, + ); + if (!knownShape) { + return exactMatch; + } + } + return rightIndex === right.length ? undefined : exactMatch; +} + +export function hasSingleDistinctLeftToolCallShape( + left: readonly ParityToolCallShape[], + right: readonly ParityToolCallShape[], +) { + const distinctLeft = distinctToolCallShapes(left); + return ( + distinctLeft.length <= 1 && + right.length <= 1 && + (distinctLeft.length === 0 || + right.length === 0 || + compareToolCallShape(distinctLeft, right) === undefined) + ); +} diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index adbbb7f1eeb3..3e5af061c9ab 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -330,9 +330,12 @@ describe("runtime parity", () => { expect(isRuntimeParityResultPass(result)).toBe(false); }); - it("prefers transcript tool results when mock debug rows are incomplete", async () => { + it("prefers transcript tool results when mock debug rows repeat an incomplete call", async () => { const cell = await captureRuntimeParityWithMockRequests({ - requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }], + requests: [ + { plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }, + { plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }, + ], messages: [ { role: "user", content: "Delegate one bounded QA task to a subagent." }, { diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 8ad829342abb..6ebb98a92862 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -14,7 +14,7 @@ import { scanGatewayLogSentinels, type GatewayLogSentinelFinding, } from "./gateway-log-sentinel.js"; -import { compareToolCallShape, stableHash } from "./parity-shared.js"; +import * as parity from "./parity-shared.js"; export type RuntimeId = "openclaw" | "codex"; @@ -469,8 +469,8 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime const index = ordered.push({ tool: call.tool, - argsHash: stableHash(call.args), - resultHash: stableHash(null), + argsHash: parity.stableHash(call.args), + resultHash: parity.stableHash(null), _resolved: false, }) - 1; if (call.id) { @@ -489,9 +489,9 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime "unknown", argsHash: pendingIndex !== undefined - ? (ordered[pendingIndex]?.argsHash ?? stableHash(null)) - : stableHash(null), - resultHash: stableHash(result.result), + ? (ordered[pendingIndex]?.argsHash ?? parity.stableHash(null)) + : parity.stableHash(null), + resultHash: parity.stableHash(result.result), ...(result.errorClass ? { errorClass: result.errorClass } : {}), }; if (pendingIndex === undefined || !ordered[pendingIndex]) { @@ -541,9 +541,9 @@ function resolveToolCallOrderFromMockRequests( tool: pendingIndex !== undefined ? (ordered[pendingIndex]?.tool ?? "unknown") : "unknown", argsHash: pendingIndex !== undefined - ? (ordered[pendingIndex]?.argsHash ?? stableHash(null)) - : stableHash(null), - resultHash: stableHash(parsedOutput ?? rawToolOutput), + ? (ordered[pendingIndex]?.argsHash ?? parity.stableHash(null)) + : parity.stableHash(null), + resultHash: parity.stableHash(parsedOutput ?? rawToolOutput), ...(classifyToolResultError({ rawOutput: rawToolOutput, parsedOutput, @@ -568,8 +568,8 @@ function resolveToolCallOrderFromMockRequests( } ordered.push({ tool: plannedToolName, - argsHash: stableHash(request.plannedToolArgs ?? null), - resultHash: stableHash(null), + argsHash: parity.stableHash(request.plannedToolArgs ?? null), + resultHash: parity.stableHash(null), _resolved: false, }); enqueueUnresolved(ordered.length - 1); @@ -788,7 +788,7 @@ function hasProvenTerminalImageResult(scenarioResult: QaSuiteScenarioLike) { ); } -const PROVEN_TERMINAL_IMAGE_RESULT_HASH = stableHash({ kind: "media", status: "success" }); +const PROVEN_TERMINAL_IMAGE_RESULT_HASH = parity.stableHash({ kind: "media", status: "success" }); function resolveRuntimeParityToolCalls(params: { mockToolCalls: RuntimeParityToolCall[] | null; @@ -801,19 +801,18 @@ function resolveRuntimeParityToolCalls(params: { const transcriptImageCalls = params.transcriptToolCalls.filter( (toolCall) => toolCall.tool === "image_generate", ); - const imageCaptureIsUnambiguous = - mockImageCalls.length <= 1 && - transcriptImageCalls.length <= 1 && - (mockImageCalls.length === 0 || - transcriptImageCalls.length === 0 || - compareToolCallShape(mockImageCalls, transcriptImageCalls) === undefined); + const imageCaptureIsUnambiguous = parity.hasSingleDistinctLeftToolCallShape( + mockImageCalls, + transcriptImageCalls, + ); let selected: RuntimeParityToolCall[]; if (!params.mockToolCalls) { selected = params.transcriptToolCalls; } else if ( hasMissingToolResult(params.mockToolCalls) && !hasMissingToolResult(params.transcriptToolCalls) && - compareToolCallShape(params.mockToolCalls, params.transcriptToolCalls) === undefined + parity.compareCapturedToolCallShape(params.mockToolCalls, params.transcriptToolCalls) === + undefined ) { selected = params.transcriptToolCalls; } else { @@ -917,7 +916,7 @@ function classifyRuntimeParityCells(params: { }; } - const toolCallShapeDetails = compareToolCallShape( + const toolCallShapeDetails = parity.compareToolCallShape( params.openclaw.toolCalls, params.codex.toolCalls, );