From c0c12da9f25bd91fe91d54cf5d1ec999b6eebcdf Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 25 Jul 2026 04:55:54 +0800 Subject: [PATCH] fix(qa): separate provider plans from runtime calls (#113399) * fix(qa): separate provider plans from runtime calls * fix(claws): keep profile limit outside env ratchet * refactor(qa): remove obsolete capture comparators * chore(release): leave notes to release workflow --- config/env-var-count-budget.txt | 2 +- extensions/qa-lab/src/parity-shared.test.ts | 28 ---- extensions/qa-lab/src/parity-shared.ts | 52 ------- extensions/qa-lab/src/runtime-parity.test.ts | 136 ++++++++++++++++--- extensions/qa-lab/src/runtime-parity.ts | 30 +--- src/claws/openclaw-profile.ts | 6 +- 6 files changed, 123 insertions(+), 131 deletions(-) delete mode 100644 extensions/qa-lab/src/parity-shared.test.ts diff --git a/config/env-var-count-budget.txt b/config/env-var-count-budget.txt index 84d268f02b52..fa5ca8837afa 100644 --- a/config/env-var-count-budget.txt +++ b/config/env-var-count-budget.txt @@ -1,3 +1,3 @@ # Distinct OPENCLAW_* names in production source under src, packages, and extensions. # Ratchet: lower this number when cleanup removes names; never raise it. -522 +521 diff --git a/extensions/qa-lab/src/parity-shared.test.ts b/extensions/qa-lab/src/parity-shared.test.ts deleted file mode 100644 index 6a39978dc5e4..000000000000 --- a/extensions/qa-lab/src/parity-shared.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { compareCapturedToolCallShape } from "./parity-shared.js"; - -const call = { tool: "image_generate", argsHash: "same-args" }; - -describe("compareCapturedToolCallShape", () => { - it("accepts exact repeated executions", () => { - expect(compareCapturedToolCallShape([call, call], [call, call])).toBeUndefined(); - }); - - it("accepts a duplicated process-global capture row", () => { - expect(compareCapturedToolCallShape([call, call], [call])).toBeUndefined(); - expect(compareCapturedToolCallShape([call, call, call], [call, call])).toBeUndefined(); - }); - - it("preserves execution count for non-image tools", () => { - const readCall = { tool: "read", argsHash: "same-args" }; - expect(compareCapturedToolCallShape([readCall, readCall], [readCall])).toBe( - "tool call count differs (2 vs 1)", - ); - }); - - it("preserves canonical execution count", () => { - expect(compareCapturedToolCallShape([call], [call, call])).toBe( - "tool call count differs (1 vs 2)", - ); - }); -}); diff --git a/extensions/qa-lab/src/parity-shared.ts b/extensions/qa-lab/src/parity-shared.ts index ff6c181130a3..8558bc46867e 100644 --- a/extensions/qa-lab/src/parity-shared.ts +++ b/extensions/qa-lab/src/parity-shared.ts @@ -48,55 +48,3 @@ export function compareToolCallShape( } return undefined; } - -function distinctToolCallShapes(toolCalls: readonly ParityToolCallShape[]) { - return toolCalls.filter( - (toolCall, index) => - toolCalls.findIndex( - (candidate) => candidate.tool === toolCall.tool && candidate.argsHash === toolCall.argsHash, - ) === index, - ); -} - -export function compareCapturedToolCallShape( - left: readonly ParityToolCallShape[], - right: readonly ParityToolCallShape[], -) { - const exactMatch = compareToolCallShape(left, right); - if (exactMatch === undefined) { - return undefined; - } - // Process-global captures can repeat planned rows. The canonical transcript - // must remain an ordered subsequence; unknown shapes still fail comparison. - let rightIndex = 0; - for (const leftCall of left) { - const expected = right[rightIndex]; - if (expected?.tool === leftCall.tool && expected.argsHash === leftCall.argsHash) { - rightIndex += 1; - continue; - } - const duplicatedImageShape = - leftCall.tool === "image_generate" && - right.some( - (candidate) => candidate.tool === leftCall.tool && candidate.argsHash === leftCall.argsHash, - ); - if (!duplicatedImageShape) { - return exactMatch; - } - } - return rightIndex === right.length ? undefined : exactMatch; -} - -export function hasSingleDistinctLeftToolCallShape( - left: readonly ParityToolCallShape[], - right: readonly ParityToolCallShape[], -) { - const distinctLeft = distinctToolCallShapes(left); - return ( - distinctLeft.length <= 1 && - right.length <= 1 && - (distinctLeft.length === 0 || - right.length === 0 || - compareToolCallShape(distinctLeft, right) === undefined) - ); -} diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index 86a5b5889dbe..a3970399b594 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -234,19 +234,20 @@ describe("runtime parity", () => { expect(cell.runtimeErrorClass).toBe("timeout"); }); - it("marks planned mock tool calls without outputs as missing tool results", async () => { + it("keeps planned mock calls diagnostic instead of promoting them to runtime calls", async () => { const cell = await captureRuntimeParityWithMockRequests({ requests: [{ plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } }], }); - expect(cell.toolCalls).toHaveLength(1); - expect(cell.toolCalls[0]).toMatchObject({ + expect(cell.toolCalls).toEqual([]); + expect(cell.providerPlanToolCalls).toHaveLength(1); + expect(cell.providerPlanToolCalls?.[0]).toMatchObject({ tool: "read_file", errorClass: "tool-result-missing", }); }); - it("keeps resolved mock tool calls eligible for no-drift parity", async () => { + it("records resolved mock calls as provider-plan evidence", async () => { const cell = await captureRuntimeParityWithMockRequests({ requests: [ { plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } }, @@ -254,8 +255,9 @@ describe("runtime parity", () => { ], }); - expect(cell.toolCalls).toHaveLength(1); - expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); + expect(cell.toolCalls).toEqual([]); + expect(cell.providerPlanToolCalls).toHaveLength(1); + expect(cell.providerPlanToolCalls?.[0]?.errorClass).toBeUndefined(); const result = await runRuntimeParityScenario({ scenarioId: "resolved-tool", @@ -299,7 +301,7 @@ describe("runtime parity", () => { ).toEqual({ expectation: "assistant-message-required" }); }); - it("classifies planned-only matching tool calls as failure-mode", async () => { + it("does not classify planned-only provider evidence as a runtime failure", async () => { const cell = await captureRuntimeParityWithMockRequests({ requests: [{ plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } }], }); @@ -312,10 +314,8 @@ describe("runtime parity", () => { }), }); - expect(result).toMatchObject({ - drift: "failure-mode", - driftDetails: "at least one runtime planned a tool call without a tool result", - }); + expect(result.drift).toBe("none"); + expect(isRuntimeParityResultPass(result)).toBe(true); }); it("treats matching controlled tool errors as equivalent results", async () => { @@ -398,6 +398,20 @@ describe("runtime parity", () => { it("accepts a fresh scenario MEDIA result for terminal image tools", async () => { const cell = await captureRuntimeParityWithMockRequests({ requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }], + messages: [ + { role: "user", content: "Generate the QA image." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "image-call", + name: "image_generate", + arguments: { prompt: "same" }, + }, + ], + }, + ], scenarioResult: { status: "pass", steps: [ @@ -412,9 +426,58 @@ describe("runtime parity", () => { expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); }); + it("keeps multiple image provider plans from invalidating one proven runtime call", async () => { + const cell = await captureRuntimeParityWithMockRequests({ + requests: [ + { plannedToolName: "image_generate", plannedToolArgs: { prompt: "first" } }, + { plannedToolName: "image_generate", plannedToolArgs: { prompt: "second" } }, + ], + messages: [ + { role: "user", content: "Generate the QA image." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "image-call", + name: "image_generate", + arguments: { prompt: "runtime" }, + }, + ], + }, + ], + scenarioResult: { + status: "pass", + steps: [ + { + status: "pass", + details: "QA-CAPABILITY-1234\nimage_generate=true\nMEDIA:/tmp/qa-image.png", + }, + ], + }, + }); + + expect(cell.toolCalls[0]?.errorClass).toBeUndefined(); + expect(cell.providerPlanToolCalls).toHaveLength(2); + }); + it("requires call-linked passed step evidence for terminal image results", async () => { const proven = await captureRuntimeParityWithMockRequests({ - requests: [{ plannedToolName: "image_generate" }], + requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }], + messages: [ + { role: "user", content: "Generate the QA image." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "image-call", + name: "image_generate", + arguments: { prompt: "same" }, + }, + ], + }, + ], scenarioResult: { status: "pass", steps: [ @@ -426,14 +489,42 @@ describe("runtime parity", () => { }, }); const unrelated = await captureRuntimeParityWithMockRequests({ - requests: [{ plannedToolName: "image_generate" }], + requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }], + messages: [ + { role: "user", content: "Generate the QA image." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "image-call", + name: "image_generate", + arguments: { prompt: "same" }, + }, + ], + }, + ], scenarioResult: { status: "pass", steps: [{ status: "pass", details: "MEDIA:/tmp/unrelated-screenshot.png" }], }, }); const failed = await captureRuntimeParityWithMockRequests({ - requests: [{ plannedToolName: "image_generate" }], + requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }], + messages: [ + { role: "user", content: "Generate the QA image." }, + { + role: "assistant", + content: [ + { + type: "toolCall", + id: "image-call", + name: "image_generate", + arguments: { prompt: "same" }, + }, + ], + }, + ], scenarioResult: { status: "pass", steps: [ @@ -450,7 +541,7 @@ describe("runtime parity", () => { expect(failed.toolCalls[0]?.errorClass).toBe("tool-result-missing"); }); - it("preserves a missing image result when MEDIA may belong to another call", async () => { + it("preserves incomplete image provider plans as diagnostic evidence", async () => { const cell = await captureRuntimeParityWithMockRequests({ requests: [ { plannedToolName: "image_generate", plannedToolArgs: { prompt: "first" } }, @@ -468,7 +559,8 @@ describe("runtime parity", () => { }, }); - expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([ + expect(cell.toolCalls).toEqual([]); + expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.errorClass)).toEqual([ undefined, "tool-result-missing", ]); @@ -514,8 +606,9 @@ describe("runtime parity", () => { }, }); - expect(cell.toolCalls).toEqual([ - expect.objectContaining({ errorClass: "tool-result-missing" }), + expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([ + undefined, + "tool-result-missing", ]); }); @@ -559,12 +652,13 @@ describe("runtime parity", () => { ], }); - expect(cell.toolCalls).toHaveLength(2); - expect(cell.toolCalls.map((toolCall) => toolCall.tool)).toEqual([ + expect(cell.toolCalls).toEqual([]); + expect(cell.providerPlanToolCalls).toHaveLength(2); + expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.tool)).toEqual([ "sessions_spawn", "sessions_spawn", ]); - expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([ + expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.errorClass)).toEqual([ undefined, "tool-result-missing", ]); diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index 9269dc56fe3e..c0439d2c3ace 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -42,6 +42,7 @@ export type RuntimeParityCell = { runtime: RuntimeId; transcriptBytes: string; toolCalls: RuntimeParityToolCall[]; + providerPlanToolCalls?: RuntimeParityToolCall[]; finalText: string; usage: RuntimeParityUsage; wallClockMs: number; @@ -792,35 +793,12 @@ function hasProvenTerminalImageResult(scenarioResult: QaSuiteScenarioLike) { const PROVEN_TERMINAL_IMAGE_RESULT_HASH = parity.stableHash({ kind: "media", status: "success" }); function resolveRuntimeParityToolCalls(params: { - mockToolCalls: RuntimeParityToolCall[] | null; transcriptToolCalls: RuntimeParityToolCall[]; terminalImageResultProven?: boolean; }): RuntimeParityToolCall[] { - const mockImageCalls = (params.mockToolCalls ?? []).filter( - (toolCall) => toolCall.tool === "image_generate", - ); - const transcriptImageCalls = params.transcriptToolCalls.filter( - (toolCall) => toolCall.tool === "image_generate", - ); - const imageCaptureIsUnambiguous = parity.hasSingleDistinctLeftToolCallShape( - mockImageCalls, - transcriptImageCalls, - ); - let selected: RuntimeParityToolCall[]; - if (!params.mockToolCalls) { - selected = params.transcriptToolCalls; - } else if ( - hasMissingToolResult(params.mockToolCalls) && - !hasMissingToolResult(params.transcriptToolCalls) && - parity.compareCapturedToolCallShape(params.mockToolCalls, params.transcriptToolCalls) === - undefined - ) { - selected = params.transcriptToolCalls; - } else { - selected = params.mockToolCalls; - } + let selected = params.transcriptToolCalls; const imageCalls = selected.filter((toolCall) => toolCall.tool === "image_generate"); - if (params.terminalImageResultProven && imageCaptureIsUnambiguous && imageCalls.length === 1) { + if (params.terminalImageResultProven && imageCalls.length === 1) { selected = selected.map((toolCall) => { if ( toolCall.tool !== "image_generate" || @@ -1118,10 +1096,10 @@ export async function captureRuntimeParityCell( runtime: params.runtime, transcriptBytes, toolCalls: resolveRuntimeParityToolCalls({ - mockToolCalls, transcriptToolCalls, terminalImageResultProven, }), + ...(mockToolCalls ? { providerPlanToolCalls: mockToolCalls } : {}), finalText: extractFinalAssistantText(transcriptRecords), usage: aggregateUsage(transcriptRecords), wallClockMs: params.wallClockMs, diff --git a/src/claws/openclaw-profile.ts b/src/claws/openclaw-profile.ts index 532a4c0d1478..ae1bd6fcf3e3 100644 --- a/src/claws/openclaw-profile.ts +++ b/src/claws/openclaw-profile.ts @@ -5,7 +5,7 @@ import { isSafeClawRelativePath } from "./schema-portability.js"; import { parseClawOpenClawProfile } from "./schema.js"; import type { ClawDiagnostic, ClawManifest, ClawOpenClawProfile } from "./types.js"; -const MAX_OPENCLAW_PROFILE_BYTES = 256 * 1024; +const MAX_PROFILE_BYTES = 256 * 1024; function diagnostic(code: string, message: string, path = "$"): ClawDiagnostic { return { level: "error", code, phase: "parse", path, message }; @@ -75,7 +75,7 @@ async function readProfileFile(packageRoot: string, path: string): Promise