ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-25 03:45:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): let scenario-owned timeouts settle before watchdog (#127289)

Browse Source 
* fix(qa): let scenario timeouts settle before watchdog

* test(qa): satisfy promise executor lint

* fix(qa): cap scenario lifecycle watchdog

---------

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
Dallin Romney
2026-08-23 14:15:11 -07:00
committed by GitHub
parent 901b62262e
commit 8122e5cdcc
2 changed files with 149 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/src/suite-runtime-flow.test.ts
+143 -47
View File
@@ -1,5 +1,6 @@
// Qa Lab tests cover suite runtime flow plugin behavior.
import { parseModelRef, resolveModelRefFromString } from "openclaw/plugin-sdk/agent-runtime";
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import { beforeEach, describe, expect, it, vi } from "vitest";
const createQaScenarioRuntimeApi = vi.hoisted(() => vi.fn());
@@ -388,45 +389,45 @@ describe("qa suite runtime flow", () => {
});
it("bounds preparation and actions with one aborting scenario deadline", async () => {
let preparationSignal: AbortSignal | undefined;
let actionSignal: AbortSignal | undefined;
const prepareFlow = vi.fn(async (input: { signal?: AbortSignal }) => {
preparationSignal = input.signal;
await new Promise<void>((resolve) => {
setTimeout(resolve, 10);
vi.useFakeTimers();
try {
let preparationSignal: AbortSignal | undefined;
let actionSignal: AbortSignal | undefined;
const prepareFlow = vi.fn(async (input: { signal?: AbortSignal }) => {
preparationSignal = input.signal;
await new Promise<void>((resolve) => {
setTimeout(resolve, 10);
});
});
const env = createQaSuiteRuntimeFlowTestEnv({ prepareFlow });
const scenario = makeQaSuiteTestScenario("flow-deadline", { config: {} });
if (scenario.execution.kind !== "flow") {
throw new Error("expected flow scenario");
}
scenario.execution.timeoutMs = 30;
createQaScenarioRuntimeApi.mockImplementationOnce(
(params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({
runScenario: params.deps.runScenario,
}),
);
runScenarioFlow.mockImplementationOnce(async (params) => {
const api = params.api as {
runScenario: typeof runQaSuiteScenarioSteps;
signal?: AbortSignal;
};
return await api.runScenario("Flow deadline", [
{
name: "Never settles without abort",
run: async () =>
await new Promise<void>(() => {
actionSignal = api.signal;
api.signal?.addEventListener("abort", () => {}, { once: true });
}),
},
]);
});
});
const env = createQaSuiteRuntimeFlowTestEnv({ prepareFlow });
const scenario = makeQaSuiteTestScenario("flow-deadline", { config: {} });
if (scenario.execution.kind !== "flow") {
throw new Error("expected flow scenario");
}
scenario.execution.timeoutMs = 30;
createQaScenarioRuntimeApi.mockImplementationOnce(
(params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({
runScenario: params.deps.runScenario,
}),
);
runScenarioFlow.mockImplementationOnce(async (params) => {
const api = params.api as {
runScenario: typeof runQaSuiteScenarioSteps;
signal?: AbortSignal;
};
return await api.runScenario("Flow deadline", [
{
name: "Never settles without abort",
run: async () =>
await new Promise<void>(() => {
actionSignal = api.signal;
api.signal?.addEventListener("abort", () => {}, { once: true });
}),
},
]);
});
let boundedTimer: ReturnType<typeof setTimeout> | undefined;
const result = await Promise.race([
runQaSuiteScenarioDefinition({
const pending = runQaSuiteScenarioDefinition({
env,
scenario,
runScenario: runQaSuiteScenarioSteps,
@@ -435,16 +436,111 @@ describe("qa suite runtime flow", () => {
liveTurnTimeoutMs: () => 60_000,
resolveQaLiveTurnTimeoutMs: () => 60_000,
constants: qaSuiteRuntimeFlowTestConstants,
}),
new Promise<"bounded-window-expired">((resolve) => {
boundedTimer = setTimeout(() => resolve("bounded-window-expired"), 250);
}),
]);
clearTimeout(boundedTimer);
});
await vi.advanceTimersByTimeAsync(5_029);
expect(actionSignal?.aborted).toBe(false);
await vi.advanceTimersByTimeAsync(1);
const result = await pending;
expect(result).not.toBe("bounded-window-expired");
expect(result).toMatchObject({ status: "fail", details: expect.stringContaining("30ms") });
expect(preparationSignal).toBe(actionSignal);
expect(actionSignal?.aborted).toBe(true);
expect(result).toMatchObject({ status: "fail", details: expect.stringContaining("30ms") });
expect(preparationSignal).toBe(actionSignal);
expect(actionSignal?.aborted).toBe(true);
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.clearAllTimers();
vi.useRealTimers();
}
});
it("lets a scenario-owned timeout settle before the lifecycle watchdog", async () => {
vi.useFakeTimers();
try {
const env = createQaSuiteRuntimeFlowTestEnv();
const scenario = makeQaSuiteTestScenario("flow-owned-timeout", { config: {} });
if (scenario.execution.kind !== "flow") {
throw new Error("expected flow scenario");
}
scenario.execution.timeoutMs = 20;
createQaScenarioRuntimeApi.mockImplementationOnce(
(params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({
runScenario: params.deps.runScenario,
}),
);
runScenarioFlow.mockImplementationOnce(async (params) => {
const api = params.api as { runScenario: typeof runQaSuiteScenarioSteps };
return await api.runScenario("Scenario-owned timeout", [
{
name: "Complete the observation window",
run: async () => {
await new Promise<void>((resolve) => {
setTimeout(resolve, 30);
});
},
},
]);
});
const pending = runQaSuiteScenarioDefinition({
env,
scenario,
runScenario: runQaSuiteScenarioSteps,
splitModelRef: (raw) => parseModelRef(raw, "openai"),
formatErrorMessage: (error) => String(error),
liveTurnTimeoutMs: () => 60_000,
resolveQaLiveTurnTimeoutMs: () => 60_000,
constants: qaSuiteRuntimeFlowTestConstants,
});
await vi.advanceTimersByTimeAsync(30);
const result = await pending;
expect(result.status).toBe("pass");
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.clearAllTimers();
vi.useRealTimers();
}
});
it("caps and disposes the lifecycle watchdog without advancing the maximum timer", async () => {
vi.useFakeTimers();
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
try {
const env = createQaSuiteRuntimeFlowTestEnv();
const scenario = makeQaSuiteTestScenario("flow-capped-deadline", { config: {} });
if (scenario.execution.kind !== "flow") {
throw new Error("expected flow scenario");
}
scenario.execution.timeoutMs = MAX_TIMER_TIMEOUT_MS;
createQaScenarioRuntimeApi.mockImplementationOnce(
(params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({
runScenario: params.deps.runScenario,
}),
);
runScenarioFlow.mockImplementationOnce(async (params) => {
const api = params.api as { runScenario: typeof runQaSuiteScenarioSteps };
return await api.runScenario("Capped deadline", [
{ name: "Settles immediately", run: async () => undefined },
]);
});
const result = await runQaSuiteScenarioDefinition({
env,
scenario,
runScenario: runQaSuiteScenarioSteps,
splitModelRef: (raw) => parseModelRef(raw, "openai"),
formatErrorMessage: (error) => String(error),
liveTurnTimeoutMs: () => 60_000,
resolveQaLiveTurnTimeoutMs: () => 60_000,
constants: qaSuiteRuntimeFlowTestConstants,
});
expect(result.status).toBe("pass");
expect(timeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS);
expect(vi.getTimerCount()).toBe(0);
} finally {
timeoutSpy.mockRestore();
vi.clearAllTimers();
vi.useRealTimers();
}
});
});
extensions/qa-lab/src/suite-runtime-flow.ts
+6 -2
View File
@@ -30,6 +30,7 @@ import * as suiteRuntimeAgent from "./suite-runtime-agent.js";
import * as suiteRuntimeGateway from "./suite-runtime-gateway.js";
import * as suiteRuntimeTransport from "./suite-runtime-transport.js";
import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js";
import { resolveQaGatewayTimeoutWithGraceMs } from "./timer-timeouts.js";
import * as webRuntime from "./web-runtime.js";
type QaSuiteScenarioFlowEnv = {
@@ -258,15 +259,18 @@ function createQaSuiteScenarioFlowApi(
function createQaScenarioDeadline(timeoutMs?: number) {
const controller = new AbortController();
let timer: ReturnType<typeof setTimeout> | undefined;
const deadlineTimeoutMs = resolveQaGatewayTimeoutWithGraceMs(timeoutMs);
const deadline =
timeoutMs === undefined
deadlineTimeoutMs === undefined
? undefined
: new Promise<never>((_resolve, reject) => {
const timeoutError = new Error(`QA scenario flow timed out after ${timeoutMs}ms`);
// Scenario-owned polls may consume the full declared timeout. Keep the outer
// lifecycle fence later so their terminal result and cleanup stay authoritative.
timer = setTimeout(() => {
controller.abort(timeoutError);
reject(timeoutError);
}, timeoutMs);
}, deadlineTimeoutMs);
});
return {
signal: controller.signal,