From 745b780025c9577a6c19a4787c37fbca3532c396 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Tue, 25 Aug 2026 10:23:09 -0700 Subject: [PATCH] test(qa): derive flow deadline ownership structurally --- .../matrix/matrix-scenario-flows.test.ts | 22 ++++---- extensions/qa-lab/src/scenario-catalog.ts | 12 ++++- .../qa-lab/src/suite-runtime-flow.test.ts | 51 +++++++++++++++++++ .../matrix-e2ee-thread-follow-up.yaml | 1 + ...rix-inbound-edit-no-duplicate-trigger.yaml | 1 + .../matrix-thread-nested-reply-shape.yaml | 1 + 6 files changed, 74 insertions(+), 14 deletions(-) diff --git a/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts b/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts index af44568fedcb..2ff71436e3d8 100644 --- a/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts @@ -24,12 +24,6 @@ const MATRIX_ISOLATED_ALLOWBOTS_ADMISSION_SCENARIOS = [ "matrix-allowbots-true-unmentioned-open-room", ] as const; -const MATRIX_PROVIDER_DEADLINE_SCENARIOS = new Set([ - "matrix-e2ee-thread-follow-up", - "matrix-inbound-edit-no-duplicate-trigger", - "matrix-thread-nested-reply-shape", -]); - function readModuleBinding( scenario: ReturnType["scenarios"][number], ) { @@ -99,9 +93,6 @@ describe("Matrix QA Lab scenario flows", () => { } expect(scenario.execution.channel, scenario.id).toBe("matrix"); expect(scenario.execution.retryCount, scenario.id).toBe(0); - if (!MATRIX_PROVIDER_DEADLINE_SCENARIOS.has(scenario.id)) { - expect(scenario.execution.timeoutMs, scenario.id).toBeGreaterThan(0); - } expect(scenario.execution.flow?.steps.at(-1)?.detailsExpr, scenario.id).toBe( "result.details ?? (result.artifacts ? JSON.stringify(result.artifacts, null, 2) : undefined)", ); @@ -126,11 +117,16 @@ describe("Matrix QA Lab scenario flows", () => { } }); - it("uses provider-owned deadlines for model-driven multi-phase Matrix flows", () => { - for (const scenarioId of MATRIX_PROVIDER_DEADLINE_SCENARIOS) { - const execution = requireFlowScenario(readQaScenarioById(scenarioId)).execution; - expect(execution.timeoutMs, scenarioId).toBeUndefined(); + it("keeps provider-owned deadlines outside the scenario lifecycle", () => { + let providerDeadlineScenarioCount = 0; + for (const scenario of scenarios) { + if (scenario.execution.kind !== "flow" || scenario.execution.deadlineOwner !== "provider") { + continue; + } + providerDeadlineScenarioCount += 1; + expect(scenario.execution.timeoutMs, scenario.id).toBeUndefined(); } + expect(providerDeadlineScenarioCount).toBeGreaterThan(0); }); it("applies the portable thread override through Matrix flow preparation", () => { diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 19828fe67d27..e38d2354a2bb 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -97,9 +97,19 @@ const qaFlowScenarioExecutionSchema = z suiteIsolation: z.literal("isolated").optional(), isolationReason: z.string().trim().min(1).optional(), transportPolicy: qaScenarioTransportPolicySchema.optional(), + deadlineOwner: z.literal("provider").optional(), config: qaScenarioConfigSchema.optional(), }) - .extend(qaScenarioModuleFlow.executionShape); + .extend(qaScenarioModuleFlow.executionShape) + .superRefine((execution, ctx) => { + if (execution.deadlineOwner === "provider" && execution.timeoutMs !== undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["timeoutMs"], + message: "provider-owned deadlines cannot declare a scenario timeout", + }); + } + }); const qaTestFileScenarioExecutionBaseSchema = z.object({ summary: z.string().trim().min(1).optional(), diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index b4ab65de5a0b..829f221aee5d 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -501,6 +501,57 @@ describe("qa suite runtime flow", () => { } }); + it("leaves the lifecycle deadline unset when the scenario delegates timeout ownership", async () => { + vi.useFakeTimers(); + const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); + try { + const env = createQaSuiteRuntimeFlowTestEnv(); + const scenario = makeQaSuiteTestScenario("provider-owned-timeout", { config: {} }); + if (scenario.execution.kind !== "flow") { + throw new Error("expected flow scenario"); + } + createQaScenarioRuntimeApi.mockImplementationOnce( + (params: { deps: { runScenario: typeof runQaSuiteScenarioSteps } }) => ({ + runScenario: params.deps.runScenario, + }), + ); + runScenarioFlow.mockImplementationOnce(async (params) => { + const api = params.api as { runScenario: typeof runQaSuiteScenarioSteps }; + return await api.runScenario("Provider-owned timeout", [ + { + name: "Complete under the provider deadline", + run: async () => { + await new Promise((resolve) => { + setTimeout(resolve, 10_000); + }); + }, + }, + ]); + }); + + const pending = runQaSuiteScenarioDefinition({ + env, + scenario, + runScenario: runQaSuiteScenarioSteps, + splitModelRef: (raw) => parseModelRef(raw, "openai"), + formatErrorMessage: (error) => String(error), + liveTurnTimeoutMs: () => 60_000, + resolveQaLiveTurnTimeoutMs: () => 60_000, + constants: qaSuiteRuntimeFlowTestConstants, + }); + expect(timeoutSpy).toHaveBeenCalledTimes(1); + expect(timeoutSpy).toHaveBeenLastCalledWith(expect.any(Function), 10_000); + await vi.advanceTimersByTimeAsync(10_000); + + expect(await pending).toMatchObject({ status: "pass" }); + expect(vi.getTimerCount()).toBe(0); + } finally { + timeoutSpy.mockRestore(); + vi.clearAllTimers(); + vi.useRealTimers(); + } + }); + it("caps and disposes the lifecycle watchdog without advancing the maximum timer", async () => { vi.useFakeTimers(); const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); diff --git a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml index 476a431fd4d6..6f5c3869386b 100644 --- a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml +++ b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml @@ -8,6 +8,7 @@ scenario: execution: kind: flow channel: matrix + deadlineOwner: provider retryCount: 0 config: matrixConfigOverrides: diff --git a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml index c84c16e7c6c0..b978e4e40129 100644 --- a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml +++ b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml @@ -8,6 +8,7 @@ scenario: execution: kind: flow channel: matrix + deadlineOwner: provider retryCount: 0 flow: diff --git a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml index 6d1e8a07f373..54592e9280a1 100644 --- a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml +++ b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml @@ -8,6 +8,7 @@ scenario: execution: kind: flow channel: matrix + deadlineOwner: provider retryCount: 0 flow: