fix(qa): publish readiness from final artifacts

This commit is contained in:
Dallin Romney
2026-08-15 19:48:20 +08:00
parent 94f042ba86
commit 89155e5331
3 changed files with 137 additions and 3 deletions
@@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { createQaBusState } from "./bus-state.js";
import type { QaLabServerHandle } from "./lab-server.types.js";
import type { QaTransportAdapterFactory } from "./qa-transport-registry.js";
import type { writeQaSuiteArtifacts } from "./suite-artifacts.js";
import { runQaFlowSuiteIsolated } from "./suite-run-isolated.js";
import { runQaFlowSuiteStandard } from "./suite-run-standard.js";
import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
@@ -25,7 +26,7 @@ const mocks = vi.hoisted(() => ({
getProcessRssBytes: () => null,
stop: vi.fn(async () => {}),
})),
writeQaSuiteArtifacts: vi.fn(async () => ({
writeQaSuiteArtifacts: vi.fn<typeof writeQaSuiteArtifacts>(async () => ({
evidence: undefined,
evidencePath: "/qa-output/qa-evidence.json",
report: "",
@@ -43,6 +44,27 @@ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
vi.mock("./gateway-child.js", () => ({
startQaGatewayChild: mocks.startQaGatewayChild,
}));
vi.mock("./crabline-transport.js", () => ({
createQaCrablineTransportAdapter: vi.fn(async () => ({
id: "telegram",
label: "Crabline Telegram",
accountId: "sut",
requiredPluginIds: [],
supportedActions: [],
sendInbound: vi.fn(async () => {}),
createGatewayConfig: () => ({}),
waitReady: vi.fn(async () => {}),
buildAgentDelivery: ({ target }: { target: string }) => ({
channel: "telegram",
to: target,
replyChannel: "telegram",
replyTo: target,
}),
handleAction: vi.fn(async () => {}),
createReportNotes: () => [],
cleanup: vi.fn(async () => {}),
})),
}));
vi.mock("./providers/server-runtime.js", () => ({
startQaProviderServer: vi.fn(async () => undefined),
}));
@@ -180,6 +202,115 @@ describe("isolated QA suite transport cleanup", () => {
stderrWrite.mockRestore();
});
it("keeps Crabline workers concurrent while publishing readiness only from the final aggregate", async () => {
const lab = createCleanupTestLab();
const selection = {
capabilityMatrixPath: "crabline-fake-provider-capabilities.json",
channel: "telegram",
channelDriver: "crabline",
smokeArtifactPath: "crabline-fake-provider-smoke.json",
} as const;
let activeWorkers = 0;
let maxActiveWorkers = 0;
let releaseWorkers!: () => void;
const bothWorkersStarted = new Promise<void>((resolve) => {
releaseWorkers = resolve;
});
let releaseFirstScenario!: () => void;
const firstScenarioStarted = new Promise<void>((resolve) => {
releaseFirstScenario = resolve;
});
let releaseScenarioExecutions!: () => void;
const bothScenarioExecutionsStarted = new Promise<void>((resolve) => {
releaseScenarioExecutions = resolve;
});
const context = createCleanupTestContext();
context.channelDriver = "crabline";
context.concurrency = 2;
context.selectedScenarios = [
makeQaSuiteTestScenario("first-crabline-scenario"),
makeQaSuiteTestScenario("second-crabline-scenario"),
];
const runScenario = vi
.fn<QaSuiteScenarioRunner>()
.mockImplementation(async (_env, scenario) => {
if (scenario.id === "first-crabline-scenario") {
releaseFirstScenario();
await bothScenarioExecutionsStarted;
} else {
releaseScenarioExecutions();
}
return {
name: scenario.title,
status: "pass",
steps: [],
};
});
const runChild = vi.fn<QaSuiteRunner>().mockImplementation(async (params) => {
if (!params) {
throw new Error("expected nested standard run params");
}
activeWorkers += 1;
maxActiveWorkers = Math.max(maxActiveWorkers, activeWorkers);
if (activeWorkers === 2) {
releaseWorkers();
}
await bothWorkersStarted;
const scenarioId = params?.scenarioIds?.[0] ?? "missing-scenario";
if (scenarioId === "second-crabline-scenario") {
await firstScenarioStarted;
}
const scenario = context.selectedScenarios.find((candidate) => candidate.id === scenarioId);
if (!scenario) {
throw new Error(`missing scenario ${scenarioId}`);
}
try {
return await runQaFlowSuiteStandard(
params,
{
...context,
startedAt: new Date("2026-08-04T00:00:01.000Z"),
outputDir: params.outputDir ?? `/qa-child/${scenarioId}`,
selectedScenarios: [scenario],
concurrency: 1,
},
runScenario,
);
} finally {
activeWorkers -= 1;
}
});
const result = await runQaFlowSuiteIsolated(
{
channelDriverSelection: selection,
channelId: "telegram",
lab,
startLab: async () => createCleanupTestLab(),
},
context,
runChild,
);
expect(maxActiveWorkers).toBe(2);
expect(result.scenarios).toEqual([
expect.objectContaining({ name: "first-crabline-scenario", status: "pass" }),
expect.objectContaining({ name: "second-crabline-scenario", status: "pass" }),
]);
expect(runScenario).toHaveBeenCalledTimes(2);
expect(mocks.writeQaSuiteArtifacts).toHaveBeenCalledTimes(5);
for (const [nonFinalArtifacts] of mocks.writeQaSuiteArtifacts.mock.calls.slice(0, -1)) {
expect(nonFinalArtifacts).toMatchObject({ channel: "telegram", channelDriver: "crabline" });
expect(nonFinalArtifacts.channelDriverSelection).toBeUndefined();
}
const finalArtifacts = mocks.writeQaSuiteArtifacts.mock.calls.at(-1)?.[0];
expect(finalArtifacts).toMatchObject({
channel: "telegram",
channelDriver: "crabline",
channelDriverSelection: selection,
});
});
it("prints one generic completion after a real nested standard run and parent cleanup", async () => {
const parentLab = createCleanupTestLab();
const childLab = createCleanupTestLab();
@@ -114,7 +114,6 @@ export async function runQaFlowSuiteIsolated(
concurrency,
channel: params?.channelId ?? params?.channelDriverSelection?.channel ?? transport.id,
channelDriver: transportFactoryResult.driver,
channelDriverSelection: params?.channelDriverSelection,
isolatedWorkers: true,
writeEvidenceFile: false,
scenarioIds:
+5 -1
View File
@@ -400,7 +400,11 @@ export async function runQaFlowSuiteStandard(
concurrency,
channel: params?.channelId ?? params?.channelDriverSelection?.channel ?? transport.id,
channelDriver: transportFactoryResult.driver,
channelDriverSelection: params?.channelDriverSelection,
// Nested workers retain the selection for transport setup, but the outer
// aggregate alone owns readiness publication under the shared output tree.
channelDriverSelection: isQaSuiteNestedRun(params)
? undefined
: params?.channelDriverSelection,
isolatedWorkers: false,
writeEvidenceFile: params?.writeEvidenceFile,
// Same "filtered → executed list, unfiltered → null" convention as