fix(qa): use canonical scenario deadline grace

This commit is contained in:
Dallin Romney
2026-08-24 16:06:26 -07:00
parent 03cf96a326
commit 2fd3db332a
12 changed files with 11 additions and 36 deletions
@@ -24,20 +24,6 @@ const MATRIX_ISOLATED_ALLOWBOTS_ADMISSION_SCENARIOS = [
"matrix-allowbots-true-unmentioned-open-room",
] as const;
const MATRIX_NO_REPLY_SCENARIOS = [
"matrix-allowbots-default-block",
"matrix-allowbots-mentions-unmentioned-open-room-block",
"matrix-allowbots-room-override-blocks-account-true",
"matrix-allowbots-self-sender-ignored",
"matrix-attachment-only-ignored",
"matrix-inbound-edit-ignored",
"matrix-mention-metadata-spoof-block",
"matrix-mxid-prefixed-command-block",
"matrix-reaction-not-a-reply",
] as const;
const MATRIX_NO_REPLY_WINDOW_MS = 8_000;
const MATRIX_PROVIDER_DEADLINE_SCENARIOS = new Set([
"matrix-e2ee-thread-follow-up",
"matrix-inbound-edit-no-duplicate-trigger",
@@ -140,13 +126,6 @@ describe("Matrix QA Lab scenario flows", () => {
}
});
it("reserves flow cleanup time after the no-reply observation window", () => {
for (const scenarioId of MATRIX_NO_REPLY_SCENARIOS) {
const execution = requireFlowScenario(readQaScenarioById(scenarioId)).execution;
expect(execution.timeoutMs, scenarioId).toBeGreaterThan(MATRIX_NO_REPLY_WINDOW_MS);
}
});
it("uses provider-owned deadlines for model-driven multi-phase Matrix flows", () => {
for (const scenarioId of MATRIX_PROVIDER_DEADLINE_SCENARIOS) {
const execution = requireFlowScenario(readQaScenarioById(scenarioId)).execution;
@@ -17,8 +17,7 @@ describe("matrix scenario runtime shared", () => {
vi.stubEnv("OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS", "12000");
expect(resolveMatrixQaNoReplyWindowMs(30_000)).toBe(12_000);
expect(resolveMatrixQaNoReplyWindowMs(15_000)).toBe(10_000);
expect(resolveMatrixQaNoReplyWindowMs(5_000)).toBe(1);
expect(resolveMatrixQaNoReplyWindowMs(5_000)).toBe(5_000);
for (const value of ["1e3", "0x1000", "1.5", "nope"]) {
vi.stubEnv("OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS", value);
@@ -81,8 +81,6 @@ export type MatrixQaScenarioContext = {
const NO_REPLY_WINDOW_MS = 8_000;
const NO_REPLY_WINDOW_ENV = "OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS";
// The outer deadline starts before Matrix preparation and must also cover cleanup.
const NO_REPLY_DEADLINE_RESERVE_MS = 5_000;
const MATRIX_QA_PREVIEW_LIMIT = 200;
export function truncateMatrixQaPreview(value: string | null | undefined) {
@@ -96,8 +94,7 @@ export function resolveMatrixQaNoReplyWindowMs(timeoutMs: number) {
const parsed =
raw === undefined ? NO_REPLY_WINDOW_MS : /^\d+$/.test(raw) ? Number(raw) : Number.NaN;
const windowMs = Number.isSafeInteger(parsed) && parsed >= 1 ? parsed : NO_REPLY_WINDOW_MS;
const availableWindowMs = Math.max(1, timeoutMs - NO_REPLY_DEADLINE_RESERVE_MS);
return Math.min(windowMs, availableWindowMs);
return Math.min(windowMs, timeoutMs);
}
export {
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixConfigOverrides:
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixConfigOverrides:
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixConfigOverrides:
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixConfigOverrides:
@@ -10,7 +10,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixTopology:
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
flow:
@@ -8,7 +8,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
flow:
@@ -10,7 +10,7 @@ scenario:
execution:
kind: flow
channel: matrix
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixConfigOverrides:
@@ -10,7 +10,7 @@ scenario:
channel: matrix
suiteIsolation: isolated
isolationReason: Posts native reaction state against a fresh canary reply and cannot inherit a shared Matrix session.
timeoutMs: 15000
timeoutMs: 8000
retryCount: 0
config:
matrixRequireCanary: true