fix(qa): separate provider plans from runtime calls (#113399)

* fix(qa): separate provider plans from runtime calls

* fix(claws): keep profile limit outside env ratchet

* refactor(qa): remove obsolete capture comparators

* chore(release): leave notes to release workflow

config/env-var-count-budget.txt

@@ -1,3 +1,3 @@
 # Distinct OPENCLAW_* names in production source under src, packages, and extensions.
 # Ratchet: lower this number when cleanup removes names; never raise it.
-522
+521

extensions/qa-lab/src/parity-shared.test.ts

@@ -1,28 +0,0 @@
-import { describe, expect, it } from "vitest";
-import { compareCapturedToolCallShape } from "./parity-shared.js";
-
-const call = { tool: "image_generate", argsHash: "same-args" };
-
-describe("compareCapturedToolCallShape", () => {
-  it("accepts exact repeated executions", () => {
-    expect(compareCapturedToolCallShape([call, call], [call, call])).toBeUndefined();
-  });
-
-  it("accepts a duplicated process-global capture row", () => {
-    expect(compareCapturedToolCallShape([call, call], [call])).toBeUndefined();
-    expect(compareCapturedToolCallShape([call, call, call], [call, call])).toBeUndefined();
-  });
-
-  it("preserves execution count for non-image tools", () => {
-    const readCall = { tool: "read", argsHash: "same-args" };
-    expect(compareCapturedToolCallShape([readCall, readCall], [readCall])).toBe(
-      "tool call count differs (2 vs 1)",
-    );
-  });
-
-  it("preserves canonical execution count", () => {
-    expect(compareCapturedToolCallShape([call], [call, call])).toBe(
-      "tool call count differs (1 vs 2)",
-    );
-  });
-});

extensions/qa-lab/src/parity-shared.ts

@@ -48,55 +48,3 @@ export function compareToolCallShape(
   }
   return undefined;
 }
-
-function distinctToolCallShapes(toolCalls: readonly ParityToolCallShape[]) {
-  return toolCalls.filter(
-    (toolCall, index) =>
-      toolCalls.findIndex(
-        (candidate) => candidate.tool === toolCall.tool && candidate.argsHash === toolCall.argsHash,
-      ) === index,
-  );
-}
-
-export function compareCapturedToolCallShape(
-  left: readonly ParityToolCallShape[],
-  right: readonly ParityToolCallShape[],
-) {
-  const exactMatch = compareToolCallShape(left, right);
-  if (exactMatch === undefined) {
-    return undefined;
-  }
-  // Process-global captures can repeat planned rows. The canonical transcript
-  // must remain an ordered subsequence; unknown shapes still fail comparison.
-  let rightIndex = 0;
-  for (const leftCall of left) {
-    const expected = right[rightIndex];
-    if (expected?.tool === leftCall.tool && expected.argsHash === leftCall.argsHash) {
-      rightIndex += 1;
-      continue;
-    }
-    const duplicatedImageShape =
-      leftCall.tool === "image_generate" &&
-      right.some(
-        (candidate) => candidate.tool === leftCall.tool && candidate.argsHash === leftCall.argsHash,
-      );
-    if (!duplicatedImageShape) {
-      return exactMatch;
-    }
-  }
-  return rightIndex === right.length ? undefined : exactMatch;
-}
-
-export function hasSingleDistinctLeftToolCallShape(
-  left: readonly ParityToolCallShape[],
-  right: readonly ParityToolCallShape[],
-) {
-  const distinctLeft = distinctToolCallShapes(left);
-  return (
-    distinctLeft.length <= 1 &&
-    right.length <= 1 &&
-    (distinctLeft.length === 0 ||
-      right.length === 0 ||
-      compareToolCallShape(distinctLeft, right) === undefined)
-  );
-}

extensions/qa-lab/src/runtime-parity.test.ts

@@ -234,19 +234,20 @@ describe("runtime parity", () => {
     expect(cell.runtimeErrorClass).toBe("timeout");
   });
 
-  it("marks planned mock tool calls without outputs as missing tool results", async () => {
+  it("keeps planned mock calls diagnostic instead of promoting them to runtime calls", async () => {
     const cell = await captureRuntimeParityWithMockRequests({
       requests: [{ plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } }],
     });
 
-    expect(cell.toolCalls).toHaveLength(1);
-    expect(cell.toolCalls[0]).toMatchObject({
+    expect(cell.toolCalls).toEqual([]);
+    expect(cell.providerPlanToolCalls).toHaveLength(1);
+    expect(cell.providerPlanToolCalls?.[0]).toMatchObject({
       tool: "read_file",
       errorClass: "tool-result-missing",
     });
   });
 
-  it("keeps resolved mock tool calls eligible for no-drift parity", async () => {
+  it("records resolved mock calls as provider-plan evidence", async () => {
     const cell = await captureRuntimeParityWithMockRequests({
       requests: [
         { plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } },
@@ -254,8 +255,9 @@ describe("runtime parity", () => {
       ],
     });
 
-    expect(cell.toolCalls).toHaveLength(1);
-    expect(cell.toolCalls[0]?.errorClass).toBeUndefined();
+    expect(cell.toolCalls).toEqual([]);
+    expect(cell.providerPlanToolCalls).toHaveLength(1);
+    expect(cell.providerPlanToolCalls?.[0]?.errorClass).toBeUndefined();
 
     const result = await runRuntimeParityScenario({
       scenarioId: "resolved-tool",
@@ -299,7 +301,7 @@ describe("runtime parity", () => {
     ).toEqual({ expectation: "assistant-message-required" });
   });
 
-  it("classifies planned-only matching tool calls as failure-mode", async () => {
+  it("does not classify planned-only provider evidence as a runtime failure", async () => {
     const cell = await captureRuntimeParityWithMockRequests({
       requests: [{ plannedToolName: "read_file", plannedToolArgs: { path: "README.md" } }],
     });
@@ -312,10 +314,8 @@ describe("runtime parity", () => {
       }),
     });
 
-    expect(result).toMatchObject({
-      drift: "failure-mode",
-      driftDetails: "at least one runtime planned a tool call without a tool result",
-    });
+    expect(result.drift).toBe("none");
+    expect(isRuntimeParityResultPass(result)).toBe(true);
   });
 
   it("treats matching controlled tool errors as equivalent results", async () => {
@@ -398,6 +398,20 @@ describe("runtime parity", () => {
   it("accepts a fresh scenario MEDIA result for terminal image tools", async () => {
     const cell = await captureRuntimeParityWithMockRequests({
       requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }],
+      messages: [
+        { role: "user", content: "Generate the QA image." },
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "toolCall",
+              id: "image-call",
+              name: "image_generate",
+              arguments: { prompt: "same" },
+            },
+          ],
+        },
+      ],
       scenarioResult: {
         status: "pass",
         steps: [
@@ -412,9 +426,58 @@ describe("runtime parity", () => {
     expect(cell.toolCalls[0]?.errorClass).toBeUndefined();
   });
 
+  it("keeps multiple image provider plans from invalidating one proven runtime call", async () => {
+    const cell = await captureRuntimeParityWithMockRequests({
+      requests: [
+        { plannedToolName: "image_generate", plannedToolArgs: { prompt: "first" } },
+        { plannedToolName: "image_generate", plannedToolArgs: { prompt: "second" } },
+      ],
+      messages: [
+        { role: "user", content: "Generate the QA image." },
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "toolCall",
+              id: "image-call",
+              name: "image_generate",
+              arguments: { prompt: "runtime" },
+            },
+          ],
+        },
+      ],
+      scenarioResult: {
+        status: "pass",
+        steps: [
+          {
+            status: "pass",
+            details: "QA-CAPABILITY-1234\nimage_generate=true\nMEDIA:/tmp/qa-image.png",
+          },
+        ],
+      },
+    });
+
+    expect(cell.toolCalls[0]?.errorClass).toBeUndefined();
+    expect(cell.providerPlanToolCalls).toHaveLength(2);
+  });
+
   it("requires call-linked passed step evidence for terminal image results", async () => {
     const proven = await captureRuntimeParityWithMockRequests({
-      requests: [{ plannedToolName: "image_generate" }],
+      requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }],
+      messages: [
+        { role: "user", content: "Generate the QA image." },
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "toolCall",
+              id: "image-call",
+              name: "image_generate",
+              arguments: { prompt: "same" },
+            },
+          ],
+        },
+      ],
       scenarioResult: {
         status: "pass",
         steps: [
@@ -426,14 +489,42 @@ describe("runtime parity", () => {
       },
     });
     const unrelated = await captureRuntimeParityWithMockRequests({
-      requests: [{ plannedToolName: "image_generate" }],
+      requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }],
+      messages: [
+        { role: "user", content: "Generate the QA image." },
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "toolCall",
+              id: "image-call",
+              name: "image_generate",
+              arguments: { prompt: "same" },
+            },
+          ],
+        },
+      ],
       scenarioResult: {
         status: "pass",
         steps: [{ status: "pass", details: "MEDIA:/tmp/unrelated-screenshot.png" }],
       },
     });
     const failed = await captureRuntimeParityWithMockRequests({
-      requests: [{ plannedToolName: "image_generate" }],
+      requests: [{ plannedToolName: "image_generate", plannedToolArgs: { prompt: "same" } }],
+      messages: [
+        { role: "user", content: "Generate the QA image." },
+        {
+          role: "assistant",
+          content: [
+            {
+              type: "toolCall",
+              id: "image-call",
+              name: "image_generate",
+              arguments: { prompt: "same" },
+            },
+          ],
+        },
+      ],
       scenarioResult: {
         status: "pass",
         steps: [
@@ -450,7 +541,7 @@ describe("runtime parity", () => {
     expect(failed.toolCalls[0]?.errorClass).toBe("tool-result-missing");
   });
 
-  it("preserves a missing image result when MEDIA may belong to another call", async () => {
+  it("preserves incomplete image provider plans as diagnostic evidence", async () => {
     const cell = await captureRuntimeParityWithMockRequests({
       requests: [
         { plannedToolName: "image_generate", plannedToolArgs: { prompt: "first" } },
@@ -468,7 +559,8 @@ describe("runtime parity", () => {
       },
     });
 
-    expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([
+    expect(cell.toolCalls).toEqual([]);
+    expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.errorClass)).toEqual([
       undefined,
       "tool-result-missing",
     ]);
@@ -514,8 +606,9 @@ describe("runtime parity", () => {
       },
     });
 
-    expect(cell.toolCalls).toEqual([
-      expect.objectContaining({ errorClass: "tool-result-missing" }),
+    expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([
+      undefined,
+      "tool-result-missing",
     ]);
   });
 
@@ -559,12 +652,13 @@ describe("runtime parity", () => {
       ],
     });
 
-    expect(cell.toolCalls).toHaveLength(2);
-    expect(cell.toolCalls.map((toolCall) => toolCall.tool)).toEqual([
+    expect(cell.toolCalls).toEqual([]);
+    expect(cell.providerPlanToolCalls).toHaveLength(2);
+    expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.tool)).toEqual([
       "sessions_spawn",
       "sessions_spawn",
     ]);
-    expect(cell.toolCalls.map((toolCall) => toolCall.errorClass)).toEqual([
+    expect(cell.providerPlanToolCalls?.map((toolCall) => toolCall.errorClass)).toEqual([
       undefined,
       "tool-result-missing",
     ]);

extensions/qa-lab/src/runtime-parity.ts

@@ -42,6 +42,7 @@ export type RuntimeParityCell = {
   runtime: RuntimeId;
   transcriptBytes: string;
   toolCalls: RuntimeParityToolCall[];
+  providerPlanToolCalls?: RuntimeParityToolCall[];
   finalText: string;
   usage: RuntimeParityUsage;
   wallClockMs: number;
@@ -792,35 +793,12 @@ function hasProvenTerminalImageResult(scenarioResult: QaSuiteScenarioLike) {
 const PROVEN_TERMINAL_IMAGE_RESULT_HASH = parity.stableHash({ kind: "media", status: "success" });
 
 function resolveRuntimeParityToolCalls(params: {
-  mockToolCalls: RuntimeParityToolCall[] | null;
   transcriptToolCalls: RuntimeParityToolCall[];
   terminalImageResultProven?: boolean;
 }): RuntimeParityToolCall[] {
-  const mockImageCalls = (params.mockToolCalls ?? []).filter(
-    (toolCall) => toolCall.tool === "image_generate",
-  );
-  const transcriptImageCalls = params.transcriptToolCalls.filter(
-    (toolCall) => toolCall.tool === "image_generate",
-  );
-  const imageCaptureIsUnambiguous = parity.hasSingleDistinctLeftToolCallShape(
-    mockImageCalls,
-    transcriptImageCalls,
-  );
-  let selected: RuntimeParityToolCall[];
-  if (!params.mockToolCalls) {
-    selected = params.transcriptToolCalls;
-  } else if (
-    hasMissingToolResult(params.mockToolCalls) &&
-    !hasMissingToolResult(params.transcriptToolCalls) &&
-    parity.compareCapturedToolCallShape(params.mockToolCalls, params.transcriptToolCalls) ===
-      undefined
-  ) {
-    selected = params.transcriptToolCalls;
-  } else {
-    selected = params.mockToolCalls;
-  }
+  let selected = params.transcriptToolCalls;
   const imageCalls = selected.filter((toolCall) => toolCall.tool === "image_generate");
-  if (params.terminalImageResultProven && imageCaptureIsUnambiguous && imageCalls.length === 1) {
+  if (params.terminalImageResultProven && imageCalls.length === 1) {
     selected = selected.map((toolCall) => {
       if (
         toolCall.tool !== "image_generate" ||
@@ -1118,10 +1096,10 @@ export async function captureRuntimeParityCell(
     runtime: params.runtime,
     transcriptBytes,
     toolCalls: resolveRuntimeParityToolCalls({
-      mockToolCalls,
       transcriptToolCalls,
       terminalImageResultProven,
     }),
+    ...(mockToolCalls ? { providerPlanToolCalls: mockToolCalls } : {}),
     finalText: extractFinalAssistantText(transcriptRecords),
     usage: aggregateUsage(transcriptRecords),
     wallClockMs: params.wallClockMs,

src/claws/openclaw-profile.ts

@@ -5,7 +5,7 @@ import { isSafeClawRelativePath } from "./schema-portability.js";
 import { parseClawOpenClawProfile } from "./schema.js";
 import type { ClawDiagnostic, ClawManifest, ClawOpenClawProfile } from "./types.js";
 
-const MAX_OPENCLAW_PROFILE_BYTES = 256 * 1024;
+const MAX_PROFILE_BYTES = 256 * 1024;
 
 function diagnostic(code: string, message: string, path = "$"): ClawDiagnostic {
   return { level: "error", code, phase: "parse", path, message };
@@ -75,7 +75,7 @@ async function readProfileFile(packageRoot: string, path: string): Promise<Buffe
   const packageFiles = await fsSafeRoot(packageRoot);
   const read = await packageFiles.read(path, {
     hardlinks: "reject",
-    maxBytes: MAX_OPENCLAW_PROFILE_BYTES,
+    maxBytes: MAX_PROFILE_BYTES,
     nonBlockingRead: true,
     symlinks: "reject",
   });
@@ -130,7 +130,7 @@ export async function readClawOpenClawProfile(params: {
           unsafe
             ? "The OpenClaw profile must be a regular, non-symlinked, non-hardlinked file."
             : tooLarge
-              ? `The OpenClaw profile exceeds ${MAX_OPENCLAW_PROFILE_BYTES} bytes.`
+              ? `The OpenClaw profile exceeds ${MAX_PROFILE_BYTES} bytes.`
               : `Could not read ${declaredPath}: ${(error as Error).message}`,
           "$.metadata.openclaw.config",
         ),