diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 914b9369be94..c1f2d4995f59 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -192,6 +192,22 @@ describe("qa scenario catalog", () => { expect(cronAuthorityFlow).not.toContain("waitForCronRunCompletion"); }); + it("keeps the audited parallel script allowlist exact", () => { + const expected = + "active-talk-agent-run-status agent-run-identity-inspection cached-health-snapshot-boundaries channel-health-monitor-lifecycle diagnostic-events-boundary gateway-loopback-lan-access gateway-rpc-account-health gateway-smoke gateway-ssh-tunnels gateway-stability-runtime gateway-support-export gateway-tls-pinning gateway-websocket-protocol-contracts logging-file-boundary mcp-gateway-connect-startup-retry mcp-plugin-tools-call otel-generation-config-watcher qa-otel-smoke remote-log-tailing tui-command-surfaces-pty tui-editor-input-pty tui-entrypoints-pty tui-gateway-boundary-pty tui-local-runtime-recovery-pty tui-local-shell-pty tui-pty-evidence-producer-contract tui-session-management-pty tui-streaming-tool-cards-pty tui-terminal-safety-pty voice-call-cli-rpc-agent-tool webchat-auto-tts".split( + " ", + ); + const marked = readQaScenarioPack().scenarios.filter( + (scenario) => + scenario.execution.kind === "script" && scenario.execution.parallelSafe === true, + ); + + expect(marked.map((scenario) => scenario.id).toSorted()).toEqual(expected); + const ssh = readQaScenarioById("gateway-ssh-tunnels"); + expect(ssh.execution).toMatchObject({ kind: "script", parallelSafe: true }); + expect(ssh.execution).not.toHaveProperty("allowBlockedEvidence"); + }); + it("rejects invalid provider metadata at the catalog boundary", () => { const scenario = structuredClone( requireFlowScenario(readQaScenarioById("subagent-completion-direct-fallback")), diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 24da9eb0e74b..e25f0081c9fb 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -111,6 +111,7 @@ const qaTestFileScenarioExecutionSchema = z.discriminatedUnion("kind", [ kind: z.literal("script"), allowBlockedEvidence: z.boolean().optional(), args: z.array(z.string()).optional(), + parallelSafe: z.boolean().optional(), timeoutMs: z.number().int().positive().optional(), }), ]); diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index f8fd5ff36aaa..880968c9761f 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -57,6 +57,14 @@ async function writeEvidence(pathLocal: string, writeFile = true) { return evidence; } +function createDeferred() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + function trackMaxActiveFlowRuns() { const run = runQaFlowSuite.getMockImplementation(); if (!run) { @@ -1966,6 +1974,213 @@ describe("qa suite runtime launcher", () => { ); }); + it("settles flow and native work, then runs serial scripts before a bounded parallel tail", async () => { + const repoRoot = await makeTempRepo("qa-suite-parallel-scripts-"); + const defaultFlowImplementation = runQaFlowSuite.getMockImplementation(); + const defaultTestFileImplementation = runQaTestFileScenarios.getMockImplementation(); + if (!defaultFlowImplementation || !defaultTestFileImplementation) { + throw new Error("expected default QA suite mock implementations"); + } + const flow = createDeferred(); + const native = createDeferred(); + const serial = createDeferred(); + const parallel = createDeferred(); + const started: string[] = []; + const parallelScriptIds: string[] = []; + let activeParallelScripts = 0; + let maxActiveParallelScripts = 0; + runQaFlowSuite.mockImplementationOnce(async (params) => { + started.push("flow"); + await flow.promise; + return await defaultFlowImplementation(params); + }); + runQaTestFileScenarios.mockImplementation(async (params) => { + const scenarioIds = params.scenarios.map((scenario: QaTestFileScenario) => scenario.id); + const kind = params.scenarios[0]?.execution.kind; + if (kind === "playwright") { + started.push("native"); + await native.promise; + } else if (scenarioIds.includes("docker-npm-onboard-channel-agent")) { + started.push("serial"); + await serial.promise; + } else { + parallelScriptIds.push(...scenarioIds); + activeParallelScripts += 1; + maxActiveParallelScripts = Math.max(maxActiveParallelScripts, activeParallelScripts); + try { + await parallel.promise; + } finally { + activeParallelScripts -= 1; + } + } + return await defaultTestFileImplementation(params); + }); + + const runPromise = runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/parallel-scripts", + concurrency: 8, + scenarioIds: [ + "dm-chat-baseline", + "control-ui-chat-flow-playwright", + "docker-npm-onboard-channel-agent", + "remote-log-tailing", + "gateway-smoke", + "logging-file-boundary", + "diagnostic-events-boundary", + ], + }); + await vi.waitFor(() => expect(started).toEqual(["flow", "native"])); + + flow.resolve(); + await Promise.resolve(); + expect(started).toEqual(["flow", "native"]); + + native.resolve(); + await vi.waitFor(() => expect(started).toContain("serial")); + expect(parallelScriptIds).toEqual([]); + + serial.resolve(); + await vi.waitFor(() => expect(parallelScriptIds).toHaveLength(3)); + expect(maxActiveParallelScripts).toBe(3); + expect(parallelScriptIds).not.toContain("diagnostic-events-boundary"); + + parallel.resolve(); + await runPromise; + expect(parallelScriptIds.slice(0, 3)).toEqual( + expect.arrayContaining(["remote-log-tailing", "gateway-smoke", "logging-file-boundary"]), + ); + expect(parallelScriptIds[3]).toBe("diagnostic-events-boundary"); + expect(maxActiveParallelScripts).toBe(3); + }); + + it("keeps selected evidence order and successful siblings when a parallel script rejects", async () => { + const repoRoot = await makeTempRepo("qa-suite-parallel-script-rejection-"); + const defaultTestFileImplementation = runQaTestFileScenarios.getMockImplementation(); + if (!defaultTestFileImplementation) { + throw new Error("expected default QA test-file mock implementation"); + } + const first = createDeferred(); + runQaTestFileScenarios.mockImplementation(async (params) => { + const scenario = params.scenarios[0] as QaTestFileScenario | undefined; + if (!scenario) { + throw new Error("expected one script scenario"); + } + if (scenario.id === "gateway-smoke") { + throw new Error("audited producer rejected"); + } + if (scenario.id === "remote-log-tailing") { + await first.promise; + } + const result = await defaultTestFileImplementation(params); + return { + ...result, + evidence: { + ...result.evidence, + entries: [ + { + test: { kind: "qa-scenario", id: scenario.id, title: scenario.title }, + coverage: [], + result: { status: "pass" as const }, + }, + ], + }, + }; + }); + + const runPromise = runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/parallel-script-rejection", + concurrency: 3, + scenarioIds: ["remote-log-tailing", "gateway-smoke", "logging-file-boundary"], + }); + await vi.waitFor(() => expect(runQaTestFileScenarios).toHaveBeenCalledTimes(3)); + first.resolve(); + const result = await runPromise; + + expect(result.executionKind).toBe("suite"); + if (result.executionKind !== "suite") { + throw new Error("expected unified suite result"); + } + const evidence = JSON.parse(await fs.readFile(result.result.evidencePath, "utf8")) as { + entries: Array<{ + result: { failure?: { reason?: string }; status: string }; + test: { id: string }; + }>; + }; + expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ + "remote-log-tailing", + "gateway-smoke", + "logging-file-boundary", + ]); + expect(evidence.entries[1]).toMatchObject({ + result: { + failure: { reason: "suite partition failed: audited producer rejected" }, + status: "fail", + }, + }); + }); + + it("serializes every fail-fast script and stops before post-failure work", async () => { + const repoRoot = await makeTempRepo("qa-suite-fail-fast-scripts-"); + const defaultTestFileImplementation = runQaTestFileScenarios.getMockImplementation(); + if (!defaultTestFileImplementation) { + throw new Error("expected default QA test-file mock implementation"); + } + const first = createDeferred(); + const started: string[] = []; + let active = 0; + let maxActive = 0; + runQaTestFileScenarios.mockImplementation(async (params) => { + const scenario = params.scenarios[0] as QaTestFileScenario | undefined; + if (!scenario) { + throw new Error("expected one script scenario"); + } + started.push(scenario.id); + active += 1; + maxActive = Math.max(maxActive, active); + try { + if (scenario.id === "remote-log-tailing") { + await first.promise; + } + const result = await defaultTestFileImplementation(params); + if (scenario.id !== "docker-npm-onboard-channel-agent") { + return result; + } + return { + ...result, + results: result.results.map((scenarioResult: QaTestFileScenarioRunResult) => + Object.assign({}, scenarioResult, { + status: "fail" as const, + failureMessage: "serial owner failed", + }), + ), + }; + } finally { + active -= 1; + } + }); + + const runPromise = runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/fail-fast-scripts", + concurrency: 8, + failFast: true, + scenarioIds: ["remote-log-tailing", "docker-npm-onboard-channel-agent", "gateway-smoke"], + }); + await vi.waitFor(() => expect(started).toEqual(["remote-log-tailing"])); + expect(maxActive).toBe(1); + + first.resolve(); + await runPromise; + expect(started).toEqual(["remote-log-tailing", "docker-npm-onboard-channel-agent"]); + expect(maxActive).toBe(1); + expect(runQaTestFileScenarios).toHaveBeenCalledTimes(2); + expect(runQaTestFileScenarios).toHaveBeenLastCalledWith( + expect.objectContaining({ failFast: true }), + ); + }); + it("keeps multiple isolated flow scenarios in separate serial partitions", async () => { const repoRoot = await makeTempRepo("qa-suite-serial-isolated-"); await runQaSuite({ diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index f1be725cd3e0..1fa4d311f988 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -92,6 +92,9 @@ type QaSuiteExecutionPlan = { const MAX_SHARED_FLOW_PARTITIONS = 4; const MAX_ISOLATED_FLOW_CONCURRENCY = 8; +// Three is the audited ceiling for concurrent Gateway and process-group lifecycles. +// Raising it risks cleanup overlap and shared port/listener contention. +const MAX_PARALLEL_SCRIPT_CONCURRENCY = 3; const ISOLATED_FLOW_WORKER_START_STAGGER_MS = 1_500; const QA_SUITE_INFRA_RETRY_LIMIT = 1; const QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES = new Set([ @@ -807,7 +810,8 @@ async function runUnifiedQaSuite(params: { const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = []; const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = []; const testFilePartitionTasks: QaUnifiedPartitionTask[] = []; - const scriptPartitionTasks: QaUnifiedPartitionTask[] = []; + const serialScriptPartitionTasks: QaUnifiedPartitionTask[] = []; + const parallelScriptPartitionTasks: QaUnifiedPartitionTask[] = []; const unavailableChannelCredentialDetails = new Map(); if (params.plan.channelGroups.length > 0) { const channelGroups = params.plan.channelGroups; @@ -1139,16 +1143,32 @@ async function runUnifiedQaSuite(params: { testFilePartitionTasks.push(createTestFilePartitionTask(concurrentTestFileScenariosByKind)); } } - const scriptScenarios = params.plan.testFileScenariosByKind.get("script"); + const scriptScenarios = params.plan.testFileScenariosByKind + .get("script") + ?.filter((scenario) => scenario.execution.kind === "script"); if (scriptScenarios?.length) { + const isParallelSafeScript = (scenario: QaTestFileScenario) => + scenario.execution.kind === "script" && scenario.execution.parallelSafe === true; if (failFast) { for (const scenario of scriptScenarios) { - scriptPartitionTasks.push(createTestFilePartitionTask(new Map([["script", [scenario]]]))); + serialScriptPartitionTasks.push( + createTestFilePartitionTask(new Map([["script", [scenario]]])), + ); } } else { - scriptPartitionTasks.push( - createTestFilePartitionTask(new Map([["script", scriptScenarios]])), - ); + const serialScenarios = scriptScenarios.filter((scenario) => !isParallelSafeScript(scenario)); + if (serialScenarios.length > 0) { + serialScriptPartitionTasks.push( + createTestFilePartitionTask(new Map([["script", serialScenarios]])), + ); + } + for (const scenario of scriptScenarios) { + if (isParallelSafeScript(scenario)) { + parallelScriptPartitionTasks.push( + createTestFilePartitionTask(new Map([["script", [scenario]]])), + ); + } + } } } const concurrentPartitionTasks = [ @@ -1299,13 +1319,24 @@ async function runUnifiedQaSuite(params: { : await runWeightedUnifiedPartitionTasks(retryingTasks, maxWeight); }; const concurrentPartitionResults = await runPartitionTasks(concurrentPartitionTasks, concurrency); - // Script scenarios may rebuild the checkout's shared dist tree. Wait until every - // flow Gateway has stopped so package postbuild cannot invalidate its loaded chunks. - const scriptPartitionResults = + // Unmarked scripts may rebuild shared checkout state. Run them exclusively + // after every flow and native partition settles, then start only audited peers. + const serialScriptPartitionResults = failFast && concurrentPartitionResults.some(partitionFailed) ? [] - : await runPartitionTasks(scriptPartitionTasks, 1); - const partitionResults = [...concurrentPartitionResults, ...scriptPartitionResults]; + : await runPartitionTasks(serialScriptPartitionTasks, 1); + const parallelScriptPartitionResults = + failFast && serialScriptPartitionResults.some(partitionFailed) + ? [] + : await runPartitionTasks( + parallelScriptPartitionTasks, + Math.min(concurrency, MAX_PARALLEL_SCRIPT_CONCURRENCY), + ); + const partitionResults = [ + ...concurrentPartitionResults, + ...serialScriptPartitionResults, + ...parallelScriptPartitionResults, + ]; for (const partitionResult of partitionResults) { for (const scenarioResult of partitionResult.scenarioResults) { const results = scenarioResultsById.get(scenarioResult.scenarioId) ?? []; @@ -1315,10 +1346,21 @@ async function runUnifiedQaSuite(params: { evidenceSummaries.push(...partitionResult.evidenceSummaries); } const finishedAt = new Date(); - const evidence = mergeQaEvidenceSummaries({ + const mergedEvidence = mergeQaEvidenceSummaries({ evidenceSummaries, generatedAt: finishedAt.toISOString(), }); + const scenarioOrder = new Map( + params.plan.scenarios.map((scenario, index) => [scenario.id, index]), + ); + const evidence = { + ...mergedEvidence, + entries: mergedEvidence.entries.toSorted( + (left, right) => + (scenarioOrder.get(left.test.id) ?? Number.MAX_SAFE_INTEGER) - + (scenarioOrder.get(right.test.id) ?? Number.MAX_SAFE_INTEGER), + ), + }; const channel = summarizeQaEvidenceChannel([evidence]); const scenarios = params.plan.scenarios.flatMap((scenario) => { const results = scenarioResultsById.get(scenario.id); diff --git a/qa/scenarios/cli/remote-log-tailing.yaml b/qa/scenarios/cli/remote-log-tailing.yaml index 7d7276229595..73a680e4e9db 100644 --- a/qa/scenarios/cli/remote-log-tailing.yaml +++ b/qa/scenarios/cli/remote-log-tailing.yaml @@ -23,6 +23,7 @@ scenario: - test/e2e/qa-lab/runtime/remote-log-tailing-runtime.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/remote-log-tailing-runtime.ts summary: Starts an authenticated Gateway and exercises logs.tail through RPC and the packaged CLI. args: diff --git a/qa/scenarios/index.yaml b/qa/scenarios/index.yaml index a2be4f193409..b8efa6e37dfa 100644 --- a/qa/scenarios/index.yaml +++ b/qa/scenarios/index.yaml @@ -34,6 +34,9 @@ title: OpenClaw QA Scenario Pack # - use `scenario.execution.suiteIsolation: isolated` for flow scenarios that # mutate gateway/runtime state in non-obvious ways; add `isolationReason` # so reviewers know why the suite scheduler must not share the worker +# - set `scenario.execution.parallelSafe: true` only for script producers that +# own their mutable state, ports, caches, processes, and credentials, and do +# not rebuild or mutate shared checkout output # - use `runtimePairLane` only for runtime-pair batch membership: `core`, # `extended`, or `soak`; provider/model/auth/channel eligibility stays in # independent execution constraints diff --git a/qa/scenarios/media/webchat-auto-tts.yaml b/qa/scenarios/media/webchat-auto-tts.yaml index e9cbb21c4f33..68c78064d4c1 100644 --- a/qa/scenarios/media/webchat-auto-tts.yaml +++ b/qa/scenarios/media/webchat-auto-tts.yaml @@ -26,6 +26,7 @@ scenario: - test/e2e/qa-lab/runtime/media-talk-gateway.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/media-talk-gateway.ts summary: Starts a real Gateway and mock model/TTS providers, sends a WebChat turn, and verifies the history attachment plus scoped media-ticket route. args: diff --git a/qa/scenarios/observability/cached-health-snapshot-boundaries.yaml b/qa/scenarios/observability/cached-health-snapshot-boundaries.yaml index f64211ed79bb..8dd790e5a586 100644 --- a/qa/scenarios/observability/cached-health-snapshot-boundaries.yaml +++ b/qa/scenarios/observability/cached-health-snapshot-boundaries.yaml @@ -25,6 +25,7 @@ scenario: - test/e2e/qa-lab/runtime/cached-health-snapshot-boundaries.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/cached-health-snapshot-boundaries.ts summary: Exercises the production health handler cache boundary and crosses a real authenticated Gateway plugin-tool invocation. args: diff --git a/qa/scenarios/observability/channel-health-monitor-lifecycle.yaml b/qa/scenarios/observability/channel-health-monitor-lifecycle.yaml index 4c8f853be7ee..a7fd025638bc 100644 --- a/qa/scenarios/observability/channel-health-monitor-lifecycle.yaml +++ b/qa/scenarios/observability/channel-health-monitor-lifecycle.yaml @@ -24,6 +24,7 @@ scenario: - test/e2e/qa-lab/runtime/channel-health-monitor-lifecycle-runtime.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/channel-health-monitor-lifecycle-runtime.ts summary: Runs the production channel health monitor with controlled real-time account snapshots and captures lifecycle operations and logs. args: diff --git a/qa/scenarios/observability/gateway-rpc-account-health.yaml b/qa/scenarios/observability/gateway-rpc-account-health.yaml index bef68aedd252..79d06faf9065 100644 --- a/qa/scenarios/observability/gateway-rpc-account-health.yaml +++ b/qa/scenarios/observability/gateway-rpc-account-health.yaml @@ -23,6 +23,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-rpc-account-health.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-rpc-account-health.ts summary: Starts a real QA bus and CLI Gateway child, performs authenticated health and status RPCs, and patches one account through config CAS and hot reload. args: diff --git a/qa/scenarios/observability/gateway-stability-runtime.yaml b/qa/scenarios/observability/gateway-stability-runtime.yaml index 8c0ba79ee6e5..c2fe94c675f0 100644 --- a/qa/scenarios/observability/gateway-stability-runtime.yaml +++ b/qa/scenarios/observability/gateway-stability-runtime.yaml @@ -29,6 +29,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-stability-runtime.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-stability-runtime.ts summary: Exercise live stability RPC, bounded retention, persisted bundle filtering, and support export through the real child CLI. args: diff --git a/qa/scenarios/observability/gateway-support-export.yaml b/qa/scenarios/observability/gateway-support-export.yaml index 5f66e2b511ee..161fc487aeb1 100644 --- a/qa/scenarios/observability/gateway-support-export.yaml +++ b/qa/scenarios/observability/gateway-support-export.yaml @@ -29,6 +29,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-support-export-runtime.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-support-export-runtime.ts summary: Run the real child CLI against a live isolated Gateway and inspect the resulting support zip privacy and artifact-safety contract. args: diff --git a/qa/scenarios/observability/otel-generation-config-watcher.yaml b/qa/scenarios/observability/otel-generation-config-watcher.yaml index 0ace44d1d5ff..14aa686abe7d 100644 --- a/qa/scenarios/observability/otel-generation-config-watcher.yaml +++ b/qa/scenarios/observability/otel-generation-config-watcher.yaml @@ -33,6 +33,7 @@ scenario: - test/e2e/qa-lab/runtime/otel-test-support.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/otel-generation-config-watcher-runtime.ts config: requiredProviderMode: mock-openai diff --git a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml index ff0605161a15..de7c6aef2dd5 100644 --- a/qa/scenarios/plugins/mcp-plugin-tools-call.yaml +++ b/qa/scenarios/plugins/mcp-plugin-tools-call.yaml @@ -22,6 +22,7 @@ scenario: - src/mcp/plugin-tools-handlers.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts summary: Registers a fixture plugin, starts the real plugin-tools stdio server, and calls its tool with a real MCP client. args: diff --git a/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml b/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml index 825f80e2bb1f..bf61a3ee7052 100644 --- a/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml +++ b/qa/scenarios/plugins/voice-call-cli-rpc-agent-tool.yaml @@ -26,6 +26,7 @@ scenario: - test/e2e/qa-lab/runtime/voice-call-gateway.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/voice-call-gateway.ts summary: Starts a real Gateway and Voice Call plugin, runs the actual CLI plus RPC and tools.invoke entry points, then drives the runtime-issued realtime media stream through embedded-agent consult. args: diff --git a/qa/scenarios/runtime/active-talk-agent-run-status.yaml b/qa/scenarios/runtime/active-talk-agent-run-status.yaml index 3dc38f1d8eab..d1ce4acb4bf0 100644 --- a/qa/scenarios/runtime/active-talk-agent-run-status.yaml +++ b/qa/scenarios/runtime/active-talk-agent-run-status.yaml @@ -24,6 +24,7 @@ scenario: - test/e2e/qa-lab/runtime/media-talk-gateway.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/media-talk-gateway.ts summary: Starts a real Gateway and persistent WebChat client, creates a Talk session, starts an agent consult, and exercises status, steer, follow-up, and cancel RPCs. args: diff --git a/qa/scenarios/runtime/agent-run-identity-inspection.yaml b/qa/scenarios/runtime/agent-run-identity-inspection.yaml index eb4fc68761f4..34c1518a1548 100644 --- a/qa/scenarios/runtime/agent-run-identity-inspection.yaml +++ b/qa/scenarios/runtime/agent-run-identity-inspection.yaml @@ -30,6 +30,7 @@ scenario: - test/e2e/qa-lab/runtime/agent-run-identity-repeated-turn-child.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/agent-run-identity-inspection.ts summary: Starts an ephemeral Gateway and mock provider, runs local and repeated public-ingress turns, proves run ambiguity plus exact text/JSON selection, replaces the Gateway process, and compares normalized context bytes. args: diff --git a/qa/scenarios/runtime/diagnostic-events-boundary.yaml b/qa/scenarios/runtime/diagnostic-events-boundary.yaml index 3cd8e092d91d..abaafded380c 100644 --- a/qa/scenarios/runtime/diagnostic-events-boundary.yaml +++ b/qa/scenarios/runtime/diagnostic-events-boundary.yaml @@ -31,6 +31,7 @@ scenario: - test/e2e/qa-lab/runtime/diagnostic-events-boundary-runtime.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/diagnostic-events-boundary-runtime.ts config: requiredProviderMode: mock-openai diff --git a/qa/scenarios/runtime/gateway-loopback-lan-access.yaml b/qa/scenarios/runtime/gateway-loopback-lan-access.yaml index 1d4073eef24f..b2e345b4b953 100644 --- a/qa/scenarios/runtime/gateway-loopback-lan-access.yaml +++ b/qa/scenarios/runtime/gateway-loopback-lan-access.yaml @@ -25,6 +25,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-loopback-lan-access.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-loopback-lan-access.ts summary: Starts real loopback and LAN Gateway listeners, then probes their HTTP and WebSocket surfaces from loopback and the host's real LAN interface. args: diff --git a/qa/scenarios/runtime/gateway-smoke.yaml b/qa/scenarios/runtime/gateway-smoke.yaml index 73544fd3f247..186443f7a57e 100644 --- a/qa/scenarios/runtime/gateway-smoke.yaml +++ b/qa/scenarios/runtime/gateway-smoke.yaml @@ -23,6 +23,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-smoke.e2e.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts summary: Starts a real Gateway child and exercises the Gateway smoke client against its WebSocket and health RPC surfaces. args: diff --git a/qa/scenarios/runtime/gateway-ssh-tunnels.yaml b/qa/scenarios/runtime/gateway-ssh-tunnels.yaml index 2f8793efdb36..0394042eeca7 100644 --- a/qa/scenarios/runtime/gateway-ssh-tunnels.yaml +++ b/qa/scenarios/runtime/gateway-ssh-tunnels.yaml @@ -25,6 +25,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-ssh-tunnels.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-ssh-tunnels.ts summary: Starts an isolated real sshd and loopback Gateway with scenario-owned host trust, then exercises success, cleanup, host-key rejection, unreachable-daemon diagnostics, overlap, and forced termination. timeoutMs: 120000 diff --git a/qa/scenarios/runtime/gateway-tls-pinning.yaml b/qa/scenarios/runtime/gateway-tls-pinning.yaml index 8fe7e84ffe7f..d34efe7b94e8 100644 --- a/qa/scenarios/runtime/gateway-tls-pinning.yaml +++ b/qa/scenarios/runtime/gateway-tls-pinning.yaml @@ -25,6 +25,7 @@ scenario: - test/e2e/qa-lab/runtime/gateway-tls-pinning.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-tls-pinning.ts summary: Starts a real TLS Gateway and exercises exact-pin, wrong-pin, and plaintext mismatch behavior through the public GatewayClient. args: diff --git a/qa/scenarios/runtime/gateway-websocket-protocol-contracts.yaml b/qa/scenarios/runtime/gateway-websocket-protocol-contracts.yaml index c5745862778d..2012d03dd6de 100644 --- a/qa/scenarios/runtime/gateway-websocket-protocol-contracts.yaml +++ b/qa/scenarios/runtime/gateway-websocket-protocol-contracts.yaml @@ -28,6 +28,7 @@ scenario: - src/gateway/server-methods/nodes.read.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-websocket-protocol-contracts.ts summary: Starts a real Gateway child, drives raw WebSocket clients, and fetches a capability-scoped fixture plugin route. args: diff --git a/qa/scenarios/runtime/logging-file-boundary.yaml b/qa/scenarios/runtime/logging-file-boundary.yaml index 90ced80f4cdc..2bb829e72402 100644 --- a/qa/scenarios/runtime/logging-file-boundary.yaml +++ b/qa/scenarios/runtime/logging-file-boundary.yaml @@ -19,6 +19,7 @@ scenario: - test/e2e/qa-lab/runtime/logging-file-boundary-runtime.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/logging-file-boundary-runtime.ts summary: Runs the real logger with a tiny file cap and validates JSONL rotation and trace correlation. args: diff --git a/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml b/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml index 36f8eb763b91..2d87520cb441 100644 --- a/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml +++ b/qa/scenarios/runtime/mcp-gateway-connect-startup-retry.yaml @@ -23,6 +23,7 @@ scenario: - src/mcp/channel-bridge.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/gateway-mcp-real-transports.ts summary: Starts the real MCP client before a delayed real Gateway becomes ready and captures retry, connect-frame, and negotiated-protocol evidence. args: diff --git a/qa/scenarios/runtime/qa-otel-smoke.yaml b/qa/scenarios/runtime/qa-otel-smoke.yaml index cd58c5f4149a..c41e31120cbb 100644 --- a/qa/scenarios/runtime/qa-otel-smoke.yaml +++ b/qa/scenarios/runtime/qa-otel-smoke.yaml @@ -23,6 +23,7 @@ scenario: - extensions/diagnostics-otel/runtime-api.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts config: requiredProviderMode: mock-openai diff --git a/qa/scenarios/ui/tui-command-surfaces-pty.yaml b/qa/scenarios/ui/tui-command-surfaces-pty.yaml index 8db6da1746fd..84f29c497685 100644 --- a/qa/scenarios/ui/tui-command-surfaces-pty.yaml +++ b/qa/scenarios/ui/tui-command-surfaces-pty.yaml @@ -17,6 +17,7 @@ scenario: - src/tui/tui-pty-harness.e2e.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs independently matched real-PTY command surface assertions and authenticates their fresh Vitest results. timeoutMs: 180000 diff --git a/qa/scenarios/ui/tui-editor-input-pty.yaml b/qa/scenarios/ui/tui-editor-input-pty.yaml index 2982e801ff17..0ae7c1723c23 100644 --- a/qa/scenarios/ui/tui-editor-input-pty.yaml +++ b/qa/scenarios/ui/tui-editor-input-pty.yaml @@ -25,6 +25,7 @@ scenario: - src/tui/tui-submit.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs independently matched real-PTY editor assertions and authenticates their fresh Vitest results. timeoutMs: 180000 diff --git a/qa/scenarios/ui/tui-entrypoints-pty.yaml b/qa/scenarios/ui/tui-entrypoints-pty.yaml index 43ffd5002cec..d843a634a4bc 100644 --- a/qa/scenarios/ui/tui-entrypoints-pty.yaml +++ b/qa/scenarios/ui/tui-entrypoints-pty.yaml @@ -23,6 +23,7 @@ scenario: - test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs exact real-PTY TUI entrypoint assertions and authenticates their fresh Vitest results as QA evidence. timeoutMs: 300000 diff --git a/qa/scenarios/ui/tui-gateway-boundary-pty.yaml b/qa/scenarios/ui/tui-gateway-boundary-pty.yaml index 72f25d964df3..d0b1b808d4e4 100644 --- a/qa/scenarios/ui/tui-gateway-boundary-pty.yaml +++ b/qa/scenarios/ui/tui-gateway-boundary-pty.yaml @@ -17,6 +17,7 @@ scenario: codeRefs: [src/tui/tui-pty-local.e2e.test.ts] execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts timeoutMs: 600000 args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"] diff --git a/qa/scenarios/ui/tui-local-runtime-recovery-pty.yaml b/qa/scenarios/ui/tui-local-runtime-recovery-pty.yaml index 6fa269a5dbab..c5f4083c29fb 100644 --- a/qa/scenarios/ui/tui-local-runtime-recovery-pty.yaml +++ b/qa/scenarios/ui/tui-local-runtime-recovery-pty.yaml @@ -19,6 +19,7 @@ scenario: codeRefs: [src/tui/tui-pty-local.e2e.test.ts] execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts timeoutMs: 720000 args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"] diff --git a/qa/scenarios/ui/tui-local-shell-pty.yaml b/qa/scenarios/ui/tui-local-shell-pty.yaml index 37b5c8aa476d..db27a659d977 100644 --- a/qa/scenarios/ui/tui-local-shell-pty.yaml +++ b/qa/scenarios/ui/tui-local-shell-pty.yaml @@ -23,6 +23,7 @@ scenario: - src/tui/tui-pty-local.e2e.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs exact built local-TUI PTY assertions and authenticates their fresh Vitest results as QA evidence. timeoutMs: 360000 diff --git a/qa/scenarios/ui/tui-pty-evidence-producer-contract.yaml b/qa/scenarios/ui/tui-pty-evidence-producer-contract.yaml index 73fe726e7159..825b2098cefd 100644 --- a/qa/scenarios/ui/tui-pty-evidence-producer-contract.yaml +++ b/qa/scenarios/ui/tui-pty-evidence-producer-contract.yaml @@ -18,6 +18,7 @@ scenario: - src/tui/tui-pty-harness.e2e.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs configured TUI PTY assertions through the repository Vitest wrapper and verifies the fresh JSON report before emitting evidence. timeoutMs: 180000 diff --git a/qa/scenarios/ui/tui-session-management-pty.yaml b/qa/scenarios/ui/tui-session-management-pty.yaml index d4aede10b033..50201df7be53 100644 --- a/qa/scenarios/ui/tui-session-management-pty.yaml +++ b/qa/scenarios/ui/tui-session-management-pty.yaml @@ -17,6 +17,7 @@ scenario: codeRefs: [src/tui/tui-pty-local.e2e.test.ts] execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts timeoutMs: 600000 args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"] diff --git a/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml b/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml index cf9084ea365c..8c0fc2f03593 100644 --- a/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml +++ b/qa/scenarios/ui/tui-streaming-tool-cards-pty.yaml @@ -20,6 +20,7 @@ scenario: - src/tui/tui-pty-harness.e2e.test.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts timeoutMs: 180000 args: [--artifact-base, "${outputDir}", --scenario-id, "${scenarioId}"] diff --git a/qa/scenarios/ui/tui-terminal-safety-pty.yaml b/qa/scenarios/ui/tui-terminal-safety-pty.yaml index 25b499d264cf..e977833d4b9c 100644 --- a/qa/scenarios/ui/tui-terminal-safety-pty.yaml +++ b/qa/scenarios/ui/tui-terminal-safety-pty.yaml @@ -36,6 +36,7 @@ scenario: - src/tui/osc8-hyperlinks.ts execution: kind: script + parallelSafe: true path: test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts summary: Runs independently matched real-PTY rendering and output-safety assertions and authenticates their fresh Vitest results. timeoutMs: 180000 diff --git a/src/tui/tui-pty-local.e2e.test.ts b/src/tui/tui-pty-local.e2e.test.ts index d7691b6502e0..e1d7b77cba24 100644 --- a/src/tui/tui-pty-local.e2e.test.ts +++ b/src/tui/tui-pty-local.e2e.test.ts @@ -899,7 +899,11 @@ async function startGatewayModeTui( timeoutMs: LOCAL_OUTPUT_TIMEOUT_MS, read: () => { const screen = synchronizedFrameRows(run.output(), run)[0]?.join("\n") ?? ""; - return screen.includes(sessionAcknowledgement) && screen.includes("| idle") ? true : null; + return screen.includes(sessionAcknowledgement) && + screen.includes(scenario.modelId) && + screen.includes("| idle") + ? true + : null; }, onTimeout: () => new Error("adopted Gateway session did not reach an idle final screen"), }); @@ -1211,9 +1215,12 @@ describe("TUI PTY real backends", () => { const newOutput = fixture.run.visibleOutput().slice(newOffset); const createdKey = newOutput.match(/new session: (agent:main:tui-\S+)/)?.[1]; expect(createdKey).toBeDefined(); - const screen = - synchronizedFrameRows(fixture.run.output(), fixture.run)[0]?.join("\n") ?? ""; - expect(screen).toContain(`session ${createdKey!.split(":").at(-1)}`); + const sessionLabel = `session ${createdKey!.split(":").at(-1)}`; + await waitForSynchronizedFrameRows( + fixture.run, + (rows) => rows.some((row) => row.includes(sessionLabel)), + LOCAL_OUTPUT_TIMEOUT_MS, + ); const afterOffset = fixture.run.visibleOutput().length; await fixture.run.write("T03_LIFECYCLE_AFTER\r"); await waitForOutputAfter(fixture.run, reply, afterOffset); diff --git a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts index 46c47de57faa..1fd29be287ab 100644 --- a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts +++ b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.test.ts @@ -232,6 +232,7 @@ describe("TUI PTY evidence producer", () => { it("builds fake and local PTY commands with the required environment", () => { vi.stubEnv("OPENCLAW_TUI_PTY_INCLUDE_LOCAL", "1"); vi.stubEnv("OPENCLAW_TUI_PTY_USE_BUILT_CLI", "inherited"); + vi.stubEnv("OPENCLAW_VITEST_FS_MODULE_CACHE_PATH", "/shared/vitest-cache"); const fake = buildTuiPtyVitestCommand({ cases: [makeCase()], cliMode: "source", @@ -252,6 +253,9 @@ describe("TUI PTY evidence producer", () => { expect(fake.env.OPENCLAW_BEHAVIOR_EVIDENCE).toBe("1"); expect(fake.env.OPENCLAW_TUI_PTY_INCLUDE_LOCAL).toBeUndefined(); expect(fake.env.OPENCLAW_TUI_PTY_USE_BUILT_CLI).toBeUndefined(); + expect(fake.env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH).toBe( + path.join("/artifacts", "vitest-fs-module-cache"), + ); const oracle = buildTuiPtyVitestCommand({ cases: [makeCase({ testFile: ASSERTION_SUPPORT_FILE })], @@ -265,11 +269,17 @@ describe("TUI PTY evidence producer", () => { cases: [makeCase({ testFile: LOCAL_FILE })], cliMode: "built", repoRoot: "/repo", - reportPath: "/artifacts/report.json", + reportPath: "/artifacts-local/report.json", }); expect(local.args).toContain(LOCAL_FILE); expect(local.env.OPENCLAW_TUI_PTY_INCLUDE_LOCAL).toBe("1"); expect(local.env.OPENCLAW_TUI_PTY_USE_BUILT_CLI).toBe("1"); + expect(oracle.env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH).toBe( + fake.env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH, + ); + expect(local.env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH).not.toBe( + fake.env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH, + ); }); it("rejects wrong-file and unmatched-pattern reports", async () => { diff --git a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts index 725ae4f5fe3e..48af752569d4 100644 --- a/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts +++ b/test/e2e/qa-lab/tui/tui-pty-evidence-producer.ts @@ -267,6 +267,10 @@ export function buildTuiPtyVitestCommand(params: { const env: NodeJS.ProcessEnv = { ...process.env, OPENCLAW_BEHAVIOR_EVIDENCE: "1", + OPENCLAW_VITEST_FS_MODULE_CACHE_PATH: path.join( + path.dirname(params.reportPath), + "vitest-fs-module-cache", + ), }; if (usesLocalPty) { env.OPENCLAW_TUI_PTY_INCLUDE_LOCAL = "1";