From 5a2bc73c28561346a4ce663aeca219a4943aaa08 Mon Sep 17 00:00:00 2001 From: Alix-007 Date: Thu, 16 Jul 2026 13:45:30 +0800 Subject: [PATCH] fix(qa-lab): keep suite HTTP probes within deadlines (#108064) * fix(qa-lab): bound suite HTTP probes * style(qa-lab): format gateway timeout proof * test(qa-lab): pin media deadline request count --- .../src/suite-runtime-agent-media.test.ts | 3 + .../qa-lab/src/suite-runtime-agent-media.ts | 14 +++-- .../qa-lab/src/suite-runtime-gateway.test.ts | 55 +++++++++++++++++++ .../qa-lab/src/suite-runtime-gateway.ts | 15 +++-- extensions/qa-lab/src/suite.test.ts | 30 ++++++++++ extensions/qa-lab/src/suite.ts | 10 +++- 6 files changed, 115 insertions(+), 12 deletions(-) diff --git a/extensions/qa-lab/src/suite-runtime-agent-media.test.ts b/extensions/qa-lab/src/suite-runtime-agent-media.test.ts index 106142725866..ddcbd822ca17 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-media.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-media.test.ts @@ -93,6 +93,9 @@ describe("qa suite runtime agent media helpers", () => { timeoutMs: 2_000, }), ).resolves.toBe("/tmp/generated.png"); + expect(fetchJsonMock).toHaveBeenCalledOnce(); + expect(fetchJsonMock).toHaveBeenCalledWith(expect.any(String), expect.any(Number)); + expect(fetchJsonMock.mock.calls[0]?.[1]).toBeLessThanOrEqual(2_000); }); it("falls back to generated image files under the gateway temp root", async () => { diff --git a/extensions/qa-lab/src/suite-runtime-agent-media.ts b/extensions/qa-lab/src/suite-runtime-agent-media.ts index 805cface931d..035217b8ade4 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-media.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-media.ts @@ -86,12 +86,13 @@ async function resolveGeneratedImagePath(params: { startedAtMs: number; timeoutMs: number; }) { - const startedAt = Date.now(); - while (Date.now() - startedAt < params.timeoutMs) { + const deadline = Date.now() + params.timeoutMs; + while (Date.now() < deadline) { if (params.env.mock) { try { const requests = await fetchJson>( `${params.env.mock.baseUrl}/debug/requests`, + Math.max(1, deadline - Date.now()), ); for (const request of requests.toReversed()) { if (!(request.allInputText ?? "").includes(params.promptSnippet)) { @@ -135,9 +136,12 @@ async function resolveGeneratedImagePath(params: { if (match) { return match; } - await new Promise((resolve) => { - setTimeout(resolve, 250); - }); + const remainingMs = deadline - Date.now(); + if (remainingMs > 0) { + await new Promise((resolve) => { + setTimeout(resolve, Math.min(250, remainingMs)); + }); + } } throw new Error(`timed out after ${params.timeoutMs}ms`); } diff --git a/extensions/qa-lab/src/suite-runtime-gateway.test.ts b/extensions/qa-lab/src/suite-runtime-gateway.test.ts index 15d8cb432785..42e58ef12fbe 100644 --- a/extensions/qa-lab/src/suite-runtime-gateway.test.ts +++ b/extensions/qa-lab/src/suite-runtime-gateway.test.ts @@ -9,6 +9,7 @@ import { patchConfig, restartGatewayWithConfigPatch, waitForConfigRestartSettle, + waitForGatewayHealthy, } from "./suite-runtime-gateway.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; @@ -122,9 +123,63 @@ describe("qa suite gateway helpers", () => { await expect(fetchJson("http://127.0.0.1:43123/config")).rejects.toThrow( "qa-lab-suite-fetch-json: JSON response exceeds 16777216 bytes", ); + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 15_000 }), + ); expect(release).toHaveBeenCalledTimes(1); }); + it("bounds stalled suite gateway JSON response bodies", async () => { + vi.useFakeTimers(); + const release = vi.fn(async () => {}); + fetchWithSsrFGuardMock.mockImplementation(async ({ timeoutMs }: { timeoutMs: number }) => { + let bodyController: ReadableStreamDefaultController | undefined; + const response = new Response( + new ReadableStream({ + start(controller) { + bodyController = controller; + controller.enqueue(new TextEncoder().encode('{"pending":')); + }, + }), + { headers: { "content-type": "application/json" } }, + ); + setTimeout(() => bodyController?.error(new Error("request timed out")), timeoutMs); + return { response, release }; + }); + + const request = fetchJson("http://127.0.0.1:43123/config", 1_000); + const rejection = expect(request).rejects.toThrow("request timed out"); + + await vi.advanceTimersByTimeAsync(1_000); + await rejection; + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 1_000 }), + ); + expect(release).toHaveBeenCalledTimes(1); + }); + + it("bounds a hung gateway health request by the remaining readiness deadline", async () => { + vi.useFakeTimers(); + fetchWithSsrFGuardMock.mockImplementation( + async ({ timeoutMs }: { timeoutMs: number }) => + await new Promise((_, reject) => { + setTimeout(() => reject(new Error("request timed out")), timeoutMs); + }), + ); + + const readiness = waitForGatewayHealthy( + { gateway: { baseUrl: "http://127.0.0.1:43123" } } as never, + 1_000, + ); + const rejection = expect(readiness).rejects.toThrow("timed out after 1000ms"); + + await vi.advanceTimersByTimeAsync(1_000); + await rejection; + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 1_000 }), + ); + }); + it("skips config mutations that would not change the snapshot", async () => { const config = { tools: { diff --git a/extensions/qa-lab/src/suite-runtime-gateway.ts b/extensions/qa-lab/src/suite-runtime-gateway.ts index 8f2d869fc29b..429b2190cfc3 100644 --- a/extensions/qa-lab/src/suite-runtime-gateway.ts +++ b/extensions/qa-lab/src/suite-runtime-gateway.ts @@ -16,10 +16,13 @@ type QaGatewayMutationEnv = Pick< "gateway" | "transport" | "providerMode" | "primaryModel" | "alternateModel" >; -async function fetchJson(url: string): Promise { +const QA_SUITE_FETCH_JSON_TIMEOUT_MS = 15_000; + +async function fetchJson(url: string, timeoutMs = QA_SUITE_FETCH_JSON_TIMEOUT_MS): Promise { const { response, release } = await fetchWithSsrFGuard({ url, policy: { allowPrivateNetwork: true }, + timeoutMs, auditContext: "qa-lab-suite-fetch-json", }); try { @@ -33,12 +36,13 @@ async function fetchJson(url: string): Promise { } async function waitForGatewayHealthy(env: Pick, timeoutMs = 45_000) { - const startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { try { const { response, release } = await fetchWithSsrFGuard({ url: `${env.gateway.baseUrl}/readyz`, policy: { allowPrivateNetwork: true }, + timeoutMs: Math.max(1, deadline - Date.now()), auditContext: "qa-lab-suite-wait-for-gateway-healthy", }); try { @@ -51,7 +55,10 @@ async function waitForGatewayHealthy(env: Pick, ti } catch { // retry } - await sleep(250); + const remainingMs = deadline - Date.now(); + if (remainingMs > 0) { + await sleep(Math.min(250, remainingMs)); + } } throw new QaSuiteInfraError("gateway_ready_timeout", `timed out after ${timeoutMs}ms`); } diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 1bb0103fe950..82369bb7c2f0 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -213,6 +213,36 @@ describe("qa suite", () => { expect(stop).toHaveBeenCalledTimes(1); }); + it("bounds a hung lab readiness request by the remaining startup deadline", async () => { + vi.useFakeTimers(); + const stop = vi.fn(async () => {}); + fetchWithSsrFGuardMock.mockImplementation( + async ({ timeoutMs }: { timeoutMs: number }) => + await new Promise((_, reject) => { + setTimeout(() => reject(new Error("request timed out")), timeoutMs); + }), + ); + + const readiness = qaSuiteProgressTesting.waitForQaLabReadyOrStopOwned({ + lab: { + listenUrl: "http://127.0.0.1:43123", + stop, + }, + ownsLab: true, + timeoutMs: 1_000, + }); + const rejection = expect(readiness).rejects.toThrow( + "timed out after 1000ms waiting for qa-lab ready", + ); + + await vi.advanceTimersByTimeAsync(1_000); + await rejection; + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: 1_000 }), + ); + expect(stop).toHaveBeenCalledTimes(1); + }); + it("leaves caller-owned labs running when readiness never becomes healthy", async () => { const stop = vi.fn(async () => {}); fetchWithSsrFGuardMock.mockResolvedValue({ diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 143186f14a8d..77336ba6f0a5 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -264,12 +264,13 @@ function formatQaSuiteRunStartProgress(params: { } async function waitForQaLabReady(baseUrl: string, timeoutMs = 10_000) { - const startedAt = Date.now(); - while (Date.now() - startedAt < timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { try { const { response, release } = await fetchWithSsrFGuard({ url: `${baseUrl}/readyz`, policy: { allowPrivateNetwork: true }, + timeoutMs: Math.max(1, deadline - Date.now()), auditContext: "qa-lab-suite-wait-for-lab-ready", }); try { @@ -282,7 +283,10 @@ async function waitForQaLabReady(baseUrl: string, timeoutMs = 10_000) { } catch { // retry } - await sleep(100); + const remainingMs = deadline - Date.now(); + if (remainingMs > 0) { + await sleep(Math.min(100, remainingMs)); + } } throw new Error(`timed out after ${timeoutMs}ms waiting for qa-lab ready`); }