From 8122e5cdccfb6e35e20cd13fb50bd60b02ab4c77 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sun, 23 Aug 2026 14:15:11 -0700 Subject: [PATCH] fix(qa): let scenario-owned timeouts settle before watchdog (#127289) * fix(qa): let scenario timeouts settle before watchdog * test(qa): satisfy promise executor lint * fix(qa): cap scenario lifecycle watchdog --------- Co-authored-by: Vincent Koc --- .../qa-lab/src/suite-runtime-flow.test.ts | 190 +++++++++++++----- extensions/qa-lab/src/suite-runtime-flow.ts | 8 +- 2 files changed, 149 insertions(+), 49 deletions(-) diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index 22faec7713b0..b4ab65de5a0b 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -1,5 +1,6 @@ // Qa Lab tests cover suite runtime flow plugin behavior. import { parseModelRef, resolveModelRefFromString } from "openclaw/plugin-sdk/agent-runtime"; +import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn()); @@ -388,45 +389,45 @@ describe("qa suite runtime flow", () => { }); it("bounds preparation and actions with one aborting scenario deadline", async () => { - let preparationSignal: AbortSignal | undefined; - let actionSignal: AbortSignal | undefined; - const prepareFlow = vi.fn(async (input: { signal?: AbortSignal }) => { - preparationSignal = input.signal; - await new Promise((resolve) => { - setTimeout(resolve, 10); + vi.useFakeTimers(); + try { + let preparationSignal: AbortSignal | undefined; + let actionSignal: AbortSignal | undefined; + const prepareFlow = vi.fn(async (input: { signal?: AbortSignal }) => { + preparationSignal = input.signal; + await new Promise((resolve) => { + setTimeout(resolve, 10); + }); + }); + const env = createQaSuiteRuntimeFlowTestEnv({ prepareFlow }); + const scenario = makeQaSuiteTestScenario("flow-deadline", { config: {} }); + if (scenario.execution.kind !== "flow") { + throw new Error("expected flow scenario"); + } + scenario.execution.timeoutMs = 30; + createQaScenarioRuntimeApi.mockImplementationOnce( + (params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({ + runScenario: params.deps.runScenario, + }), + ); + runScenarioFlow.mockImplementationOnce(async (params) => { + const api = params.api as { + runScenario: typeof runQaSuiteScenarioSteps; + signal?: AbortSignal; + }; + return await api.runScenario("Flow deadline", [ + { + name: "Never settles without abort", + run: async () => + await new Promise(() => { + actionSignal = api.signal; + api.signal?.addEventListener("abort", () => {}, { once: true }); + }), + }, + ]); }); - }); - const env = createQaSuiteRuntimeFlowTestEnv({ prepareFlow }); - const scenario = makeQaSuiteTestScenario("flow-deadline", { config: {} }); - if (scenario.execution.kind !== "flow") { - throw new Error("expected flow scenario"); - } - scenario.execution.timeoutMs = 30; - createQaScenarioRuntimeApi.mockImplementationOnce( - (params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({ - runScenario: params.deps.runScenario, - }), - ); - runScenarioFlow.mockImplementationOnce(async (params) => { - const api = params.api as { - runScenario: typeof runQaSuiteScenarioSteps; - signal?: AbortSignal; - }; - return await api.runScenario("Flow deadline", [ - { - name: "Never settles without abort", - run: async () => - await new Promise(() => { - actionSignal = api.signal; - api.signal?.addEventListener("abort", () => {}, { once: true }); - }), - }, - ]); - }); - let boundedTimer: ReturnType | undefined; - const result = await Promise.race([ - runQaSuiteScenarioDefinition({ + const pending = runQaSuiteScenarioDefinition({ env, scenario, runScenario: runQaSuiteScenarioSteps, @@ -435,16 +436,111 @@ describe("qa suite runtime flow", () => { liveTurnTimeoutMs: () => 60_000, resolveQaLiveTurnTimeoutMs: () => 60_000, constants: qaSuiteRuntimeFlowTestConstants, - }), - new Promise<"bounded-window-expired">((resolve) => { - boundedTimer = setTimeout(() => resolve("bounded-window-expired"), 250); - }), - ]); - clearTimeout(boundedTimer); + }); + await vi.advanceTimersByTimeAsync(5_029); + expect(actionSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + const result = await pending; - expect(result).not.toBe("bounded-window-expired"); - expect(result).toMatchObject({ status: "fail", details: expect.stringContaining("30ms") }); - expect(preparationSignal).toBe(actionSignal); - expect(actionSignal?.aborted).toBe(true); + expect(result).toMatchObject({ status: "fail", details: expect.stringContaining("30ms") }); + expect(preparationSignal).toBe(actionSignal); + expect(actionSignal?.aborted).toBe(true); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it("lets a scenario-owned timeout settle before the lifecycle watchdog", async () => { + vi.useFakeTimers(); + try { + const env = createQaSuiteRuntimeFlowTestEnv(); + const scenario = makeQaSuiteTestScenario("flow-owned-timeout", { config: {} }); + if (scenario.execution.kind !== "flow") { + throw new Error("expected flow scenario"); + } + scenario.execution.timeoutMs = 20; + createQaScenarioRuntimeApi.mockImplementationOnce( + (params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({ + runScenario: params.deps.runScenario, + }), + ); + runScenarioFlow.mockImplementationOnce(async (params) => { + const api = params.api as { runScenario: typeof runQaSuiteScenarioSteps }; + return await api.runScenario("Scenario-owned timeout", [ + { + name: "Complete the observation window", + run: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 30); + }); + }, + }, + ]); + }); + + const pending = runQaSuiteScenarioDefinition({ + env, + scenario, + runScenario: runQaSuiteScenarioSteps, + splitModelRef: (raw) => parseModelRef(raw, "openai"), + formatErrorMessage: (error) => String(error), + liveTurnTimeoutMs: () => 60_000, + resolveQaLiveTurnTimeoutMs: () => 60_000, + constants: qaSuiteRuntimeFlowTestConstants, + }); + await vi.advanceTimersByTimeAsync(30); + const result = await pending; + + expect(result.status).toBe("pass"); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + + it("caps and disposes the lifecycle watchdog without advancing the maximum timer", async () => { + vi.useFakeTimers(); + const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); + try { + const env = createQaSuiteRuntimeFlowTestEnv(); + const scenario = makeQaSuiteTestScenario("flow-capped-deadline", { config: {} }); + if (scenario.execution.kind !== "flow") { + throw new Error("expected flow scenario"); + } + scenario.execution.timeoutMs = MAX_TIMER_TIMEOUT_MS; + createQaScenarioRuntimeApi.mockImplementationOnce( + (params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({ + runScenario: params.deps.runScenario, + }), + ); + runScenarioFlow.mockImplementationOnce(async (params) => { + const api = params.api as { runScenario: typeof runQaSuiteScenarioSteps }; + return await api.runScenario("Capped deadline", [ + { name: "Settles immediately", run: async () => undefined }, + ]); + }); + + const result = await runQaSuiteScenarioDefinition({ + env, + scenario, + runScenario: runQaSuiteScenarioSteps, + splitModelRef: (raw) => parseModelRef(raw, "openai"), + formatErrorMessage: (error) => String(error), + liveTurnTimeoutMs: () => 60_000, + resolveQaLiveTurnTimeoutMs: () => 60_000, + constants: qaSuiteRuntimeFlowTestConstants, + }); + + expect(result.status).toBe("pass"); + expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS); + expect(vi.getTimerCount()).toBe(0); + } finally { + timeoutSpy.mockRestore(); + vi.clearAllTimers(); + vi.useRealTimers(); + } }); }); diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 72fbec607cdc..566075019a66 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -30,6 +30,7 @@ import * as suiteRuntimeAgent from "./suite-runtime-agent.js"; import * as suiteRuntimeGateway from "./suite-runtime-gateway.js"; import * as suiteRuntimeTransport from "./suite-runtime-transport.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; +import { resolveQaGatewayTimeoutWithGraceMs } from "./timer-timeouts.js"; import * as webRuntime from "./web-runtime.js"; type QaSuiteScenarioFlowEnv = { @@ -258,15 +259,18 @@ function createQaSuiteScenarioFlowApi( function createQaScenarioDeadline(timeoutMs?: number) { const controller = new AbortController(); let timer: ReturnType | undefined; + const deadlineTimeoutMs = resolveQaGatewayTimeoutWithGraceMs(timeoutMs); const deadline = - timeoutMs === undefined + deadlineTimeoutMs === undefined ? undefined : new Promise((_resolve, reject) => { const timeoutError = new Error(`QA scenario flow timed out after ${timeoutMs}ms`); + // Scenario-owned polls may consume the full declared timeout. Keep the outer + // lifecycle fence later so their terminal result and cleanup stay authoritative. timer = setTimeout(() => { controller.abort(timeoutError); reject(timeoutError); - }, timeoutMs); + }, deadlineTimeoutMs); }); return { signal: controller.signal,