From b9e9f2dd0963ffce337e0f96da3a2d4d6e2ee9aa Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 20 Aug 2026 19:45:41 -0700 Subject: [PATCH] fix(qa): keep failure projection internal --- ...ol-search-gateway-request-evidence.test.ts | 120 ------------------ .../tool-search-gateway-request-evidence.ts | 8 +- .../qa-lab/src/tool-search-gateway.fixture.ts | 2 +- 3 files changed, 6 insertions(+), 124 deletions(-) delete mode 100644 extensions/qa-lab/src/tool-search-gateway-request-evidence.test.ts diff --git a/extensions/qa-lab/src/tool-search-gateway-request-evidence.test.ts b/extensions/qa-lab/src/tool-search-gateway-request-evidence.test.ts deleted file mode 100644 index 7490f5030f05..000000000000 --- a/extensions/qa-lab/src/tool-search-gateway-request-evidence.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - projectToolSearchGatewayLogFacts, - projectToolSearchProviderRequests, -} from "./tool-search-gateway-request-evidence.js"; - -describe("Tool Search provider request evidence", () => { - const targetTool = "fake_plugin_tool_17"; - - const absentStage = { - plannedToolName: null, - declaredToolCount: 0, - targetDeclared: false, - bridgeDeclared: false, - targetResultObserved: false, - }; - - it.each([ - { - label: "wrapped OpenAI", - tools: [ - { type: "function", function: { name: targetTool } }, - { type: "function", function: { name: "tool_search_code" } }, - ], - plannedToolName: targetTool, - targetResultObserved: false, - }, - { - label: "flat Anthropic", - tools: [ - { name: targetTool, input_schema: { type: "object" } }, - { name: "tool_search_code", input_schema: { type: "object" } }, - ], - plannedToolName: "tool_search_code", - targetResultObserved: true, - }, - ])("projects $label declarations into stage-only fields", (testCase) => { - expect( - projectToolSearchProviderRequests( - [ - { - body: { tools: testCase.tools }, - plannedToolName: testCase.plannedToolName, - toolOutput: testCase.targetResultObserved - ? `FAKE_PLUGIN_OK ${targetTool} private output` - : undefined, - }, - ], - targetTool, - ), - ).toEqual([ - { - plannedToolName: testCase.plannedToolName, - declaredToolCount: 2, - targetDeclared: true, - bridgeDeclared: true, - targetResultObserved: testCase.targetResultObserved, - }, - ]); - }); - - it("treats malformed snapshots as absent stage evidence", () => { - expect(projectToolSearchProviderRequests({ requests: [] }, targetTool)).toEqual([]); - expect( - projectToolSearchProviderRequests([null, [], { body: { tools: [null, 42] } }], targetTool), - ).toEqual([absentStage, absentStage, { ...absentStage, declaredToolCount: 2 }]); - }); - - it("keeps only the latest request window and never returns raw diagnostic text", () => { - const secrets = { - body: "raw-body-secret", - prompt: "raw-prompt-secret", - output: "raw-tool-output-secret", - planned: "raw-planned-tool-secret", - }; - const requests = Array.from({ length: 14 }, (_, index) => ({ - body: { tools: index === 2 ? [{ name: "tool_search_code" }] : [] }, - plannedToolName: index < 2 ? "tool_call" : index === 2 ? "tool_search" : secrets.planned, - raw: secrets.body, - prompt: secrets.prompt, - toolOutput: secrets.output, - })); - - const summary = projectToolSearchProviderRequests(requests, targetTool); - expect(summary).toHaveLength(12); - expect(summary[0]).toMatchObject({ - plannedToolName: "tool_search", - bridgeDeclared: true, - }); - expect(summary.at(-1)?.plannedToolName).toBe(""); - const serialized = JSON.stringify(summary); - expect(serialized).not.toContain(secrets.body); - expect(serialized).not.toContain(secrets.prompt); - expect(serialized).not.toContain(secrets.output); - expect(serialized).not.toContain(secrets.planned); - }); - - it("projects only allowlisted Gateway log facts", () => { - const prompt = "arbitrary prompt sentinel"; - const output = "arbitrary tool output sentinel"; - const facts = projectToolSearchGatewayLogFacts( - `${prompt}\ntool_search_code\n${targetTool}\n${output}`, - targetTool, - ); - - expect(facts).toEqual({ - captured: true, - mentions: { - tool_search_code: true, - tool_search: true, - tool_describe: false, - tool_call: false, - [targetTool]: true, - }, - }); - const serialized = JSON.stringify(facts); - expect(serialized).not.toContain(prompt); - expect(serialized).not.toContain(output); - }); -}); diff --git a/extensions/qa-lab/src/tool-search-gateway-request-evidence.ts b/extensions/qa-lab/src/tool-search-gateway-request-evidence.ts index 295efca71f37..ea5c950ada62 100644 --- a/extensions/qa-lab/src/tool-search-gateway-request-evidence.ts +++ b/extensions/qa-lab/src/tool-search-gateway-request-evidence.ts @@ -11,7 +11,7 @@ const SAFE_TOOL_SEARCH_STAGE_NAMES = new Set([ "tool_call", ]); -export function projectToolSearchGatewayLogFacts(logs: string, targetTool: string) { +function projectToolSearchGatewayLogFacts(logs: string, targetTool: string) { const safeTargets = [...SAFE_TOOL_SEARCH_STAGE_NAMES, targetTool].filter(Boolean); return { captured: logs.length > 0, @@ -19,7 +19,7 @@ export function projectToolSearchGatewayLogFacts(logs: string, targetTool: strin }; } -export function projectToolSearchProviderRequests(requests: unknown, targetTool: string) { +function projectToolSearchProviderRequests(requests: unknown, targetTool: string) { if (!Array.isArray(requests)) { return []; } @@ -101,7 +101,9 @@ export async function throwToolSearchGatewayRequestFailure(params: { params.cause instanceof Error ? params.cause.message : "", )?.[1]; const errorCode = - params.cause instanceof Error ? (params.cause as NodeJS.ErrnoException).code : undefined; + params.cause instanceof Error && "code" in params.cause && typeof params.cause.code === "string" + ? params.cause.code + : undefined; const safeFailure = httpStatus ? `HTTP ${httpStatus}` : errorCode === "ETIMEDOUT" || errorCode === "ETOOBIG" diff --git a/extensions/qa-lab/src/tool-search-gateway.fixture.ts b/extensions/qa-lab/src/tool-search-gateway.fixture.ts index 7fdafe0fc9e3..fe22656689fa 100644 --- a/extensions/qa-lab/src/tool-search-gateway.fixture.ts +++ b/extensions/qa-lab/src/tool-search-gateway.fixture.ts @@ -447,7 +447,7 @@ export async function runToolSearchGatewayLane(params: { }), }, { timeoutMs: liveTurnTimeoutMs(env, 30_000) }, - ).catch((cause) => + ).catch((cause: unknown) => throwToolSearchGatewayRequestFailure({ cause, fetchJson,