From 069cf888454a3fc937e948641fffd55e4e35f4c8 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 17 Aug 2026 08:23:55 -0700 Subject: [PATCH] fix(qa): isolate runtime parity readiness --- ...uite-runtime-parity-runner.cleanup.test.ts | 39 ++++++++++++ .../qa-lab/src/suite-runtime-parity-runner.ts | 63 ++++++++++--------- 2 files changed, 72 insertions(+), 30 deletions(-) diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts index 46862ccf0b95..f50446bb3119 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.cleanup.test.ts @@ -63,6 +63,27 @@ vi.mock("openclaw/plugin-sdk/agent-harness", () => ({ vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ fetchWithSsrFGuard: mocks.fetchWithSsrFGuard, })); +vi.mock("./crabline-transport.js", () => ({ + createQaCrablineTransportAdapter: vi.fn(async () => ({ + id: "telegram", + label: "Crabline Telegram", + accountId: "sut", + requiredPluginIds: [], + supportedActions: [], + sendInbound: vi.fn(async () => {}), + createGatewayConfig: () => ({}), + waitReady: vi.fn(async () => {}), + buildAgentDelivery: ({ target }: { target: string }) => ({ + channel: "telegram", + to: target, + replyChannel: "telegram", + replyTo: target, + }), + handleAction: vi.fn(async () => {}), + createReportNotes: () => [], + cleanup: vi.fn(async () => {}), + })), +})); vi.mock("./gateway-child.js", () => ({ startQaGatewayChild: mocks.startQaGatewayChild, })); @@ -254,7 +275,14 @@ describe("runtime parity suite transport cleanup", () => { }); it("prints one generic completion after real nested standard cells and parent cleanup", async () => { + mocks.writeQaSuiteArtifacts.mockClear(); const scenario = makeQaSuiteTestScenario("runtime-cleanup"); + const selection = { + capabilityMatrixPath: "crabline-fake-provider-capabilities.json", + channel: "telegram", + channelDriver: "crabline", + smokeArtifactPath: "crabline-fake-provider-smoke.json", + } as const; const parentLab = createCleanupTestLab(); const openClawLab = createCleanupTestLab(); const codexLab = createCleanupTestLab(); @@ -300,6 +328,8 @@ describe("runtime parity suite transport cleanup", () => { startedAt: new Date("2026-08-04T00:00:00.000Z"), providerMode: "mock-openai", transportId: "qa-channel", + channelId: "telegram", + channelDriverSelection: selection, primaryModel: "mock-openai/test-model", alternateModel: "mock-openai/test-model-alt", fastMode: true, @@ -317,6 +347,15 @@ describe("runtime parity suite transport cleanup", () => { .filter((line) => line.startsWith("[qa-suite] run complete")); expect(completionLines).toEqual(["[qa-suite] run complete"]); expect(runScenario).toHaveBeenCalledTimes(2); + expect(mocks.writeQaSuiteArtifacts).toHaveBeenCalledTimes(3); + for (const [cellArtifacts] of mocks.writeQaSuiteArtifacts.mock.calls.slice(0, -1)) { + expect(cellArtifacts.channelDriverSelection).toBeUndefined(); + } + expect(mocks.writeQaSuiteArtifacts.mock.calls.at(-1)?.[0]).toMatchObject({ + channel: "telegram", + channelDriver: "crabline", + channelDriverSelection: selection, + }); expect(openClawLab.stop).toHaveBeenCalledOnce(); expect(codexLab.stop).toHaveBeenCalledOnce(); expect(parentLab.stop).toHaveBeenCalledOnce(); diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.ts index 1878fe24ebe4..a9bc51b039a7 100644 --- a/extensions/qa-lab/src/suite-runtime-parity-runner.ts +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.ts @@ -35,6 +35,7 @@ import type { } from "./suite-types.js"; import { createQaSuiteTransportAdapter, + markQaSuiteNestedRun, requireQaSuiteStartLab, runQaSuiteCleanupSteps, throwQaSuiteCleanupErrors, @@ -153,39 +154,41 @@ export async function runQaRuntimeParitySuite(params: { runtime, ); const cellStartedAt = Date.now(); - const cellResult = await params.runQaFlowSuite({ - adapterFactories: params.adapterFactories, - channelId: params.channelId, - adapterOptions: params.adapterOptions, - repoRoot: params.repoRoot, - outputDir: cellOutputDir, - providerMode: params.providerMode, - transportId: params.transportId, - channelDriver: params.channelDriver ?? undefined, - channelDriverSelection: params.channelDriverSelection, - primaryModel: remapModelRefForForcedRuntime({ - modelRef: params.primaryModel, + const cellResult = await params.runQaFlowSuite( + markQaSuiteNestedRun({ + adapterFactories: params.adapterFactories, + channelId: params.channelId, + adapterOptions: params.adapterOptions, + repoRoot: params.repoRoot, + outputDir: cellOutputDir, providerMode: params.providerMode, + transportId: params.transportId, + channelDriver: params.channelDriver ?? undefined, + channelDriverSelection: params.channelDriverSelection, + primaryModel: remapModelRefForForcedRuntime({ + modelRef: params.primaryModel, + providerMode: params.providerMode, + forcedRuntime: runtime, + }), + alternateModel: remapModelRefForForcedRuntime({ + modelRef: params.alternateModel, + providerMode: params.providerMode, + forcedRuntime: runtime, + }), + fastMode: params.fastMode, + thinkingDefault: params.thinkingDefault, + claudeCliAuthMode: params.claudeCliAuthMode, + scenarioIds: [scenario.id], + concurrency: 1, + enabledPluginIds: params.enabledPluginIds, + startLab, + controlUiEnabled: params.controlUiEnabled ?? scenarioRequiresControlUi(scenario), + mutateConfig: params.mutateConfig, forcedRuntime: runtime, + captureRuntimeParityCell: true, + writeEvidenceFile: params.writeEvidenceFile, }), - alternateModel: remapModelRefForForcedRuntime({ - modelRef: params.alternateModel, - providerMode: params.providerMode, - forcedRuntime: runtime, - }), - fastMode: params.fastMode, - thinkingDefault: params.thinkingDefault, - claudeCliAuthMode: params.claudeCliAuthMode, - scenarioIds: [scenario.id], - concurrency: 1, - enabledPluginIds: params.enabledPluginIds, - startLab, - controlUiEnabled: params.controlUiEnabled ?? scenarioRequiresControlUi(scenario), - mutateConfig: params.mutateConfig, - forcedRuntime: runtime, - captureRuntimeParityCell: true, - writeEvidenceFile: params.writeEvidenceFile, - }); + ); for (const startedScenarioId of cellResult.startedScenarioIds) { startedScenarioIds.add(startedScenarioId); }