diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts index 5aa842fd52ee..714e368768b0 100644 --- a/extensions/qa-lab/src/runtime-parity.test.ts +++ b/extensions/qa-lab/src/runtime-parity.test.ts @@ -221,11 +221,127 @@ describe("runtime parity", () => { }); expect(resolved).toEqual([ - { + expect.objectContaining({ tool: "image_generate", argsHash: "same-args", - resultHash: "async-started", - }, + }), + ]); + expect(resolved[0]?.errorClass).toBeUndefined(); + }); + + it("accepts a fresh scenario MEDIA result for terminal image tools", () => { + const resolved = __testing.resolveRuntimeParityToolCalls({ + mockToolCalls: [ + { + tool: "image_generate", + argsHash: "same-args", + resultHash: "missing", + errorClass: "tool-result-missing", + }, + ], + transcriptToolCalls: [], + terminalImageResultProven: true, + }); + const codexResult = __testing.resolveRuntimeParityToolCalls({ + mockToolCalls: [ + { + tool: "image_generate", + argsHash: "same-args", + resultHash: "runtime-specific-path", + }, + ], + transcriptToolCalls: [], + terminalImageResultProven: true, + }); + + expect(resolved).toEqual(codexResult); + expect(resolved[0]?.errorClass).toBeUndefined(); + }); + + it("requires call-linked passed step evidence for terminal image results", () => { + expect( + __testing.hasProvenTerminalImageResult({ + status: "pass", + steps: [ + { + status: "pass", + details: "QA-CAPABILITY-1234\nimage_generate=true\nMEDIA:/tmp/qa-image.png", + }, + ], + }), + ).toBe(true); + expect( + __testing.hasProvenTerminalImageResult({ + status: "pass", + steps: [{ status: "pass", details: "MEDIA:/tmp/unrelated-screenshot.png" }], + }), + ).toBe(false); + expect( + __testing.hasProvenTerminalImageResult({ + status: "pass", + steps: [ + { + status: "fail", + details: "image_generate=true\nMEDIA:/tmp/failed-image.png", + }, + ], + }), + ).toBe(false); + }); + + it("preserves a missing image result when MEDIA may belong to another call", () => { + const resolved = __testing.resolveRuntimeParityToolCalls({ + mockToolCalls: [ + { + tool: "image_generate", + argsHash: "first-args", + resultHash: "first-success", + }, + { + tool: "image_generate", + argsHash: "second-args", + resultHash: "second-missing", + errorClass: "tool-result-missing", + }, + ], + transcriptToolCalls: [], + terminalImageResultProven: true, + }); + + expect(resolved.map((toolCall) => toolCall.errorClass)).toEqual([ + undefined, + "tool-result-missing", + ]); + }); + + it("preserves missing image results when capture sources disagree on call count", () => { + const resolved = __testing.resolveRuntimeParityToolCalls({ + mockToolCalls: [ + { + tool: "image_generate", + argsHash: "first-args", + resultHash: "mock-missing", + errorClass: "tool-result-missing", + }, + ], + transcriptToolCalls: [ + { + tool: "image_generate", + argsHash: "first-args", + resultHash: "first-success", + }, + { + tool: "image_generate", + argsHash: "second-args", + resultHash: "second-missing", + errorClass: "tool-result-missing", + }, + ], + terminalImageResultProven: true, + }); + + expect(resolved).toEqual([ + expect.objectContaining({ errorClass: "tool-result-missing", resultHash: "mock-missing" }), ]); }); diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts index dff2d67cea32..ce811f637de8 100644 --- a/extensions/qa-lab/src/runtime-parity.ts +++ b/extensions/qa-lab/src/runtime-parity.ts @@ -114,6 +114,7 @@ type QaGatewayLike = { type QaSuiteScenarioLike = { details?: string; status: "pass" | "fail"; + steps?: Array<{ details?: string; status?: "pass" | "fail" | "skip" }>; }; type RuntimeParityCaptureParams = { @@ -761,21 +762,66 @@ function hasMissingToolResult(toolCalls: readonly RuntimeParityToolCall[]) { return toolCalls.some((toolCall) => toolCall.errorClass === TOOL_RESULT_MISSING_ERROR_CLASS); } +function hasProvenTerminalImageResult(scenarioResult: QaSuiteScenarioLike) { + return ( + scenarioResult.status === "pass" && + (scenarioResult.steps ?? []).some( + (step) => + step.status === "pass" && + /(?:^|\n)image_generate=true\r?\nMEDIA:\S+/u.test(step.details ?? ""), + ) + ); +} + +const PROVEN_TERMINAL_IMAGE_RESULT_HASH = stableHash({ kind: "media", status: "success" }); + function resolveRuntimeParityToolCalls(params: { mockToolCalls: RuntimeParityToolCall[] | null; transcriptToolCalls: RuntimeParityToolCall[]; + terminalImageResultProven?: boolean; }): RuntimeParityToolCall[] { + const mockImageCalls = (params.mockToolCalls ?? []).filter( + (toolCall) => toolCall.tool === "image_generate", + ); + const transcriptImageCalls = params.transcriptToolCalls.filter( + (toolCall) => toolCall.tool === "image_generate", + ); + const imageCaptureIsUnambiguous = + mockImageCalls.length <= 1 && + transcriptImageCalls.length <= 1 && + (mockImageCalls.length === 0 || + transcriptImageCalls.length === 0 || + compareToolCallShape(mockImageCalls, transcriptImageCalls) === undefined); + let selected: RuntimeParityToolCall[]; if (!params.mockToolCalls) { - return params.transcriptToolCalls; - } - if ( + selected = params.transcriptToolCalls; + } else if ( hasMissingToolResult(params.mockToolCalls) && !hasMissingToolResult(params.transcriptToolCalls) && compareToolCallShape(params.mockToolCalls, params.transcriptToolCalls) === undefined ) { - return params.transcriptToolCalls; + selected = params.transcriptToolCalls; + } else { + selected = params.mockToolCalls; } - return params.mockToolCalls; + const imageCalls = selected.filter((toolCall) => toolCall.tool === "image_generate"); + if (params.terminalImageResultProven && imageCaptureIsUnambiguous && imageCalls.length === 1) { + selected = selected.map((toolCall) => { + if ( + toolCall.tool !== "image_generate" || + (toolCall.errorClass !== undefined && + toolCall.errorClass !== TOOL_RESULT_MISSING_ERROR_CLASS) + ) { + return toolCall; + } + return { + ...toolCall, + resultHash: PROVEN_TERMINAL_IMAGE_RESULT_HASH, + errorClass: undefined, + }; + }); + } + return selected; } function filterMockRequestsForParentPrompt( @@ -1069,10 +1115,15 @@ export async function captureRuntimeParityCell( ? classifyScenarioError(params.scenarioResult.details) : undefined; const sentinelErrorClass = summarizeSentinelErrorClass(sentinelFindings); + const terminalImageResultProven = hasProvenTerminalImageResult(params.scenarioResult); return { runtime: params.runtime, transcriptBytes, - toolCalls: resolveRuntimeParityToolCalls({ mockToolCalls, transcriptToolCalls }), + toolCalls: resolveRuntimeParityToolCalls({ + mockToolCalls, + transcriptToolCalls, + terminalImageResultProven, + }), finalText: extractFinalAssistantText(transcriptRecords), usage: aggregateUsage(transcriptRecords), wallClockMs: params.wallClockMs, @@ -1112,6 +1163,7 @@ export async function runRuntimeParityScenario(params: { export const testing = { classifyRuntimeParityCells, filterMockRequestsForParentPrompt, + hasProvenTerminalImageResult, resolveRuntimeParityToolCalls, resolveToolCallOrderFromMockRequests, };