fix(qa): repair agentic runtime parity mocks (#105500)

* fix(qa): repair agentic runtime parity mocks

* fix(qa): scope terminal media parity evidence

* fix(qa): require one image call for media fallback

* fix(qa): bind terminal media proof to image step

* fix(qa): require image capture agreement

* fix(qa): allow skipped parity steps

extensions/qa-lab/src/runtime-parity.test.ts

@@ -221,11 +221,127 @@ describe("runtime parity", () => {
     });
 
     expect(resolved).toEqual([
-      {
+      expect.objectContaining({
         tool: "image_generate",
         argsHash: "same-args",
-        resultHash: "async-started",
-      },
+      }),
+    ]);
+    expect(resolved[0]?.errorClass).toBeUndefined();
+  });
+
+  it("accepts a fresh scenario MEDIA result for terminal image tools", () => {
+    const resolved = __testing.resolveRuntimeParityToolCalls({
+      mockToolCalls: [
+        {
+          tool: "image_generate",
+          argsHash: "same-args",
+          resultHash: "missing",
+          errorClass: "tool-result-missing",
+        },
+      ],
+      transcriptToolCalls: [],
+      terminalImageResultProven: true,
+    });
+    const codexResult = __testing.resolveRuntimeParityToolCalls({
+      mockToolCalls: [
+        {
+          tool: "image_generate",
+          argsHash: "same-args",
+          resultHash: "runtime-specific-path",
+        },
+      ],
+      transcriptToolCalls: [],
+      terminalImageResultProven: true,
+    });
+
+    expect(resolved).toEqual(codexResult);
+    expect(resolved[0]?.errorClass).toBeUndefined();
+  });
+
+  it("requires call-linked passed step evidence for terminal image results", () => {
+    expect(
+      __testing.hasProvenTerminalImageResult({
+        status: "pass",
+        steps: [
+          {
+            status: "pass",
+            details: "QA-CAPABILITY-1234\nimage_generate=true\nMEDIA:/tmp/qa-image.png",
+          },
+        ],
+      }),
+    ).toBe(true);
+    expect(
+      __testing.hasProvenTerminalImageResult({
+        status: "pass",
+        steps: [{ status: "pass", details: "MEDIA:/tmp/unrelated-screenshot.png" }],
+      }),
+    ).toBe(false);
+    expect(
+      __testing.hasProvenTerminalImageResult({
+        status: "pass",
+        steps: [
+          {
+            status: "fail",
+            details: "image_generate=true\nMEDIA:/tmp/failed-image.png",
+          },
+        ],
+      }),
+    ).toBe(false);
+  });
+
+  it("preserves a missing image result when MEDIA may belong to another call", () => {
+    const resolved = __testing.resolveRuntimeParityToolCalls({
+      mockToolCalls: [
+        {
+          tool: "image_generate",
+          argsHash: "first-args",
+          resultHash: "first-success",
+        },
+        {
+          tool: "image_generate",
+          argsHash: "second-args",
+          resultHash: "second-missing",
+          errorClass: "tool-result-missing",
+        },
+      ],
+      transcriptToolCalls: [],
+      terminalImageResultProven: true,
+    });
+
+    expect(resolved.map((toolCall) => toolCall.errorClass)).toEqual([
+      undefined,
+      "tool-result-missing",
+    ]);
+  });
+
+  it("preserves missing image results when capture sources disagree on call count", () => {
+    const resolved = __testing.resolveRuntimeParityToolCalls({
+      mockToolCalls: [
+        {
+          tool: "image_generate",
+          argsHash: "first-args",
+          resultHash: "mock-missing",
+          errorClass: "tool-result-missing",
+        },
+      ],
+      transcriptToolCalls: [
+        {
+          tool: "image_generate",
+          argsHash: "first-args",
+          resultHash: "first-success",
+        },
+        {
+          tool: "image_generate",
+          argsHash: "second-args",
+          resultHash: "second-missing",
+          errorClass: "tool-result-missing",
+        },
+      ],
+      terminalImageResultProven: true,
+    });
+
+    expect(resolved).toEqual([
+      expect.objectContaining({ errorClass: "tool-result-missing", resultHash: "mock-missing" }),
     ]);
   });
 

extensions/qa-lab/src/runtime-parity.ts

@@ -114,6 +114,7 @@ type QaGatewayLike = {
 type QaSuiteScenarioLike = {
   details?: string;
   status: "pass" | "fail";
+  steps?: Array<{ details?: string; status?: "pass" | "fail" | "skip" }>;
 };
 
 type RuntimeParityCaptureParams = {
@@ -761,21 +762,66 @@ function hasMissingToolResult(toolCalls: readonly RuntimeParityToolCall[]) {
   return toolCalls.some((toolCall) => toolCall.errorClass === TOOL_RESULT_MISSING_ERROR_CLASS);
 }
 
+function hasProvenTerminalImageResult(scenarioResult: QaSuiteScenarioLike) {
+  return (
+    scenarioResult.status === "pass" &&
+    (scenarioResult.steps ?? []).some(
+      (step) =>
+        step.status === "pass" &&
+        /(?:^|\n)image_generate=true\r?\nMEDIA:\S+/u.test(step.details ?? ""),
+    )
+  );
+}
+
+const PROVEN_TERMINAL_IMAGE_RESULT_HASH = stableHash({ kind: "media", status: "success" });
+
 function resolveRuntimeParityToolCalls(params: {
   mockToolCalls: RuntimeParityToolCall[] | null;
   transcriptToolCalls: RuntimeParityToolCall[];
+  terminalImageResultProven?: boolean;
 }): RuntimeParityToolCall[] {
+  const mockImageCalls = (params.mockToolCalls ?? []).filter(
+    (toolCall) => toolCall.tool === "image_generate",
+  );
+  const transcriptImageCalls = params.transcriptToolCalls.filter(
+    (toolCall) => toolCall.tool === "image_generate",
+  );
+  const imageCaptureIsUnambiguous =
+    mockImageCalls.length <= 1 &&
+    transcriptImageCalls.length <= 1 &&
+    (mockImageCalls.length === 0 ||
+      transcriptImageCalls.length === 0 ||
+      compareToolCallShape(mockImageCalls, transcriptImageCalls) === undefined);
+  let selected: RuntimeParityToolCall[];
   if (!params.mockToolCalls) {
-    return params.transcriptToolCalls;
-  }
-  if (
+    selected = params.transcriptToolCalls;
+  } else if (
     hasMissingToolResult(params.mockToolCalls) &&
     !hasMissingToolResult(params.transcriptToolCalls) &&
     compareToolCallShape(params.mockToolCalls, params.transcriptToolCalls) === undefined
   ) {
-    return params.transcriptToolCalls;
+    selected = params.transcriptToolCalls;
+  } else {
+    selected = params.mockToolCalls;
   }
-  return params.mockToolCalls;
+  const imageCalls = selected.filter((toolCall) => toolCall.tool === "image_generate");
+  if (params.terminalImageResultProven && imageCaptureIsUnambiguous && imageCalls.length === 1) {
+    selected = selected.map((toolCall) => {
+      if (
+        toolCall.tool !== "image_generate" ||
+        (toolCall.errorClass !== undefined &&
+          toolCall.errorClass !== TOOL_RESULT_MISSING_ERROR_CLASS)
+      ) {
+        return toolCall;
+      }
+      return {
+        ...toolCall,
+        resultHash: PROVEN_TERMINAL_IMAGE_RESULT_HASH,
+        errorClass: undefined,
+      };
+    });
+  }
+  return selected;
 }
 
 function filterMockRequestsForParentPrompt(
@@ -1069,10 +1115,15 @@ export async function captureRuntimeParityCell(
       ? classifyScenarioError(params.scenarioResult.details)
       : undefined;
   const sentinelErrorClass = summarizeSentinelErrorClass(sentinelFindings);
+  const terminalImageResultProven = hasProvenTerminalImageResult(params.scenarioResult);
   return {
     runtime: params.runtime,
     transcriptBytes,
-    toolCalls: resolveRuntimeParityToolCalls({ mockToolCalls, transcriptToolCalls }),
+    toolCalls: resolveRuntimeParityToolCalls({
+      mockToolCalls,
+      transcriptToolCalls,
+      terminalImageResultProven,
+    }),
     finalText: extractFinalAssistantText(transcriptRecords),
     usage: aggregateUsage(transcriptRecords),
     wallClockMs: params.wallClockMs,
@@ -1112,6 +1163,7 @@ export async function runRuntimeParityScenario(params: {
 export const testing = {
   classifyRuntimeParityCells,
   filterMockRequestsForParentPrompt,
+  hasProvenTerminalImageResult,
   resolveRuntimeParityToolCalls,
   resolveToolCallOrderFromMockRequests,
 };