From 89155e53316df87d626be044585c4698e4e8feb2 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Sat, 15 Aug 2026 19:48:20 +0800 Subject: [PATCH] fix(qa): publish readiness from final artifacts --- .../src/suite-run-isolated.cleanup.test.ts | 133 +++++++++++++++++- extensions/qa-lab/src/suite-run-isolated.ts | 1 - extensions/qa-lab/src/suite-run-standard.ts | 6 +- 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts index d2b581a2cf70..c7113dc715bb 100644 --- a/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts +++ b/extensions/qa-lab/src/suite-run-isolated.cleanup.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { createQaBusState } from "./bus-state.js"; import type { QaLabServerHandle } from "./lab-server.types.js"; import type { QaTransportAdapterFactory } from "./qa-transport-registry.js"; +import type { writeQaSuiteArtifacts } from "./suite-artifacts.js"; import { runQaFlowSuiteIsolated } from "./suite-run-isolated.js"; import { runQaFlowSuiteStandard } from "./suite-run-standard.js"; import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; @@ -25,7 +26,7 @@ const mocks = vi.hoisted(() => ({ getProcessRssBytes: () => null, stop: vi.fn(async () => {}), })), - writeQaSuiteArtifacts: vi.fn(async () => ({ + writeQaSuiteArtifacts: vi.fn(async () => ({ evidence: undefined, evidencePath: "/qa-output/qa-evidence.json", report: "", @@ -43,6 +44,27 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ vi.mock("./gateway-child.js", () => ({ startQaGatewayChild: mocks.startQaGatewayChild, })); +vi.mock("./crabline-transport.js", () => ({ + createQaCrablineTransportAdapter: vi.fn(async () => ({ + id: "telegram", + label: "Crabline Telegram", + accountId: "sut", + requiredPluginIds: [], + supportedActions: [], + sendInbound: vi.fn(async () => {}), + createGatewayConfig: () => ({}), + waitReady: vi.fn(async () => {}), + buildAgentDelivery: ({ target }: { target: string }) => ({ + channel: "telegram", + to: target, + replyChannel: "telegram", + replyTo: target, + }), + handleAction: vi.fn(async () => {}), + createReportNotes: () => [], + cleanup: vi.fn(async () => {}), + })), +})); vi.mock("./providers/server-runtime.js", () => ({ startQaProviderServer: vi.fn(async () => undefined), })); @@ -180,6 +202,115 @@ describe("isolated QA suite transport cleanup", () => { stderrWrite.mockRestore(); }); + it("keeps Crabline workers concurrent while publishing readiness only from the final aggregate", async () => { + const lab = createCleanupTestLab(); + const selection = { + capabilityMatrixPath: "crabline-fake-provider-capabilities.json", + channel: "telegram", + channelDriver: "crabline", + smokeArtifactPath: "crabline-fake-provider-smoke.json", + } as const; + let activeWorkers = 0; + let maxActiveWorkers = 0; + let releaseWorkers!: () => void; + const bothWorkersStarted = new Promise((resolve) => { + releaseWorkers = resolve; + }); + let releaseFirstScenario!: () => void; + const firstScenarioStarted = new Promise((resolve) => { + releaseFirstScenario = resolve; + }); + let releaseScenarioExecutions!: () => void; + const bothScenarioExecutionsStarted = new Promise((resolve) => { + releaseScenarioExecutions = resolve; + }); + const context = createCleanupTestContext(); + context.channelDriver = "crabline"; + context.concurrency = 2; + context.selectedScenarios = [ + makeQaSuiteTestScenario("first-crabline-scenario"), + makeQaSuiteTestScenario("second-crabline-scenario"), + ]; + const runScenario = vi + .fn() + .mockImplementation(async (_env, scenario) => { + if (scenario.id === "first-crabline-scenario") { + releaseFirstScenario(); + await bothScenarioExecutionsStarted; + } else { + releaseScenarioExecutions(); + } + return { + name: scenario.title, + status: "pass", + steps: [], + }; + }); + const runChild = vi.fn().mockImplementation(async (params) => { + if (!params) { + throw new Error("expected nested standard run params"); + } + activeWorkers += 1; + maxActiveWorkers = Math.max(maxActiveWorkers, activeWorkers); + if (activeWorkers === 2) { + releaseWorkers(); + } + await bothWorkersStarted; + const scenarioId = params?.scenarioIds?.[0] ?? "missing-scenario"; + if (scenarioId === "second-crabline-scenario") { + await firstScenarioStarted; + } + const scenario = context.selectedScenarios.find((candidate) => candidate.id === scenarioId); + if (!scenario) { + throw new Error(`missing scenario ${scenarioId}`); + } + try { + return await runQaFlowSuiteStandard( + params, + { + ...context, + startedAt: new Date("2026-08-04T00:00:01.000Z"), + outputDir: params.outputDir ?? `/qa-child/${scenarioId}`, + selectedScenarios: [scenario], + concurrency: 1, + }, + runScenario, + ); + } finally { + activeWorkers -= 1; + } + }); + + const result = await runQaFlowSuiteIsolated( + { + channelDriverSelection: selection, + channelId: "telegram", + lab, + startLab: async () => createCleanupTestLab(), + }, + context, + runChild, + ); + + expect(maxActiveWorkers).toBe(2); + expect(result.scenarios).toEqual([ + expect.objectContaining({ name: "first-crabline-scenario", status: "pass" }), + expect.objectContaining({ name: "second-crabline-scenario", status: "pass" }), + ]); + expect(runScenario).toHaveBeenCalledTimes(2); + expect(mocks.writeQaSuiteArtifacts).toHaveBeenCalledTimes(5); + for (const [nonFinalArtifacts] of mocks.writeQaSuiteArtifacts.mock.calls.slice(0, -1)) { + expect(nonFinalArtifacts).toMatchObject({ channel: "telegram", channelDriver: "crabline" }); + expect(nonFinalArtifacts.channelDriverSelection).toBeUndefined(); + } + const finalArtifacts = mocks.writeQaSuiteArtifacts.mock.calls.at(-1)?.[0]; + expect(finalArtifacts).toMatchObject({ + channel: "telegram", + channelDriver: "crabline", + channelDriverSelection: selection, + }); + }); + it("prints one generic completion after a real nested standard run and parent cleanup", async () => { const parentLab = createCleanupTestLab(); const childLab = createCleanupTestLab(); diff --git a/extensions/qa-lab/src/suite-run-isolated.ts b/extensions/qa-lab/src/suite-run-isolated.ts index 83d16bd65a85..ce17cb227d3f 100644 --- a/extensions/qa-lab/src/suite-run-isolated.ts +++ b/extensions/qa-lab/src/suite-run-isolated.ts @@ -114,7 +114,6 @@ export async function runQaFlowSuiteIsolated( concurrency, channel: params?.channelId ?? params?.channelDriverSelection?.channel ?? transport.id, channelDriver: transportFactoryResult.driver, - channelDriverSelection: params?.channelDriverSelection, isolatedWorkers: true, writeEvidenceFile: false, scenarioIds: diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts index 4fa98f50bdeb..8d2606f60c23 100644 --- a/extensions/qa-lab/src/suite-run-standard.ts +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -400,7 +400,11 @@ export async function runQaFlowSuiteStandard( concurrency, channel: params?.channelId ?? params?.channelDriverSelection?.channel ?? transport.id, channelDriver: transportFactoryResult.driver, - channelDriverSelection: params?.channelDriverSelection, + // Nested workers retain the selection for transport setup, but the outer + // aggregate alone owns readiness publication under the shared output tree. + channelDriverSelection: isQaSuiteNestedRun(params) + ? undefined + : params?.channelDriverSelection, isolatedWorkers: false, writeEvidenceFile: params?.writeEvidenceFile, // Same "filtered → executed list, unfiltered → null" convention as