From 82fa956d5a22c65d0dd9f2bfc5ff964e45c9728e Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 17 Jul 2026 16:42:18 -0700 Subject: [PATCH] fix(qa): resolve profile channels through selected driver (#109350) * fix(qa): keep generic profiles off channel-specific lanes * fix(qa): resolve profile channels through selected driver --- extensions/qa-lab/src/cli.runtime.test.ts | 26 ++++++- extensions/qa-lab/src/cli.runtime.ts | 7 +- .../qa-lab/src/suite-launch.runtime.test.ts | 77 ++++++++++++++++++- extensions/qa-lab/src/suite-launch.runtime.ts | 53 +++++++++++-- 4 files changed, 151 insertions(+), 12 deletions(-) diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 4d46e62c1c31..f83d92fa331d 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -521,6 +521,23 @@ describe("qa cli runtime", () => { expect(suiteArgs.channelDriverSelection).toBeUndefined(); }); + it("keeps portable channel scenarios in driver-selected profile runs", async () => { + await runQaProfileCommand({ + repoRoot: "/tmp/openclaw-repo", + profile: "release", + surface: "channel-framework", + providerMode: "mock-openai", + scenarioIds: ["channel-chat-baseline", "telegram-help-command"], + }); + + const suiteArgs = mockFirstObjectArg(runQaSuite); + expect(suiteArgs.scenarioIds).toContain("channel-chat-baseline"); + expect(suiteArgs.scenarioIds).toContain("telegram-help-command"); + expect(suiteArgs.adapterFactories).toBe( + listLiveTransportQaAdapterFactories.mock.results[0]?.value, + ); + }); + it("runs the all profile through the live taxonomy profile path", async () => { await runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", @@ -767,14 +784,19 @@ describe("qa cli runtime", () => { expect(runQaSuite).not.toHaveBeenCalled(); }); - it("keeps live taxonomy metadata unchanged without an explicit adapter channel", async () => { + it("loads contributed adapters without preselecting a scenario channel", async () => { await runQaSuiteCommand({ channelDriver: "live", scenarioIds: ["channel-chat-baseline"], }); expect(runQaSuite).toHaveBeenCalledWith( - expect.not.objectContaining({ adapterFactories: expect.anything() }), + expect.objectContaining({ + adapterFactories: listLiveTransportQaAdapterFactories.mock.results[0]?.value, + }), + ); + expect(runQaSuite).toHaveBeenCalledWith( + expect.not.objectContaining({ channelId: expect.anything() }), ); }); diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 6dc2cbe38939..2daa98848039 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -994,7 +994,8 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { throw new Error("--channel override requires --channel-driver crabline or live."); } const liveChannelId = channelDriver === "live" ? opts.channel?.trim() : undefined; - const liveAdapterFactories = liveChannelId ? listLiveTransportQaAdapterFactories() : undefined; + const liveAdapterFactories = + channelDriver === "live" ? listLiveTransportQaAdapterFactories() : undefined; const liveAdapterFactory = liveChannelId ? liveAdapterFactories?.find((factory) => factory.id === liveChannelId) : undefined; @@ -1125,10 +1126,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { evidenceMode: opts.evidenceMode, transportId, channelDriver, - ...(liveChannelId + ...(liveAdapterFactories ? { adapterFactories: liveAdapterFactories, - channelId: liveChannelId, + ...(liveChannelId ? { channelId: liveChannelId } : {}), adapterOptions: { repoRoot, sutAccountId: opts.sutAccountId, diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 821d73d22dea..38d5fb20286b 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -186,7 +186,7 @@ describe("qa suite runtime launcher", () => { expect(runQaTestFileScenarios).not.toHaveBeenCalled(); }); - it("runs runtime-specific live scenarios in dedicated workers", async () => { + it("runs runtime-specific channel scenarios in dedicated workers", async () => { const repoRoot = await makeTempRepo("qa-suite-live-runtime-"); await runQaSuite({ repoRoot, @@ -215,6 +215,81 @@ describe("qa suite runtime launcher", () => { ); }); + it("partitions portable scenarios by channel for a pluggable driver", async () => { + const repoRoot = await makeTempRepo("qa-suite-pluggable-channels-"); + const adapterFactories = [ + { + id: "portable-driver", + matches: vi.fn(), + create: vi.fn(), + }, + ]; + + await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/pluggable-channels", + providerMode: "mock-openai", + channelDriver: "live", + adapterFactories, + scenarioIds: ["channel-chat-baseline", "telegram-help-command", "matrix-restart-resume"], + }); + + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "pluggable-channels"); + expect(runQaFlowSuite).toHaveBeenCalledTimes(3); + expect(runQaFlowSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories, + channelId: undefined, + outputDir: path.join(outputDir, "flow"), + scenarioIds: ["channel-chat-baseline"], + }), + ); + expect(runQaFlowSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories, + channelId: "telegram", + outputDir: path.join(outputDir, "flow", "telegram"), + scenarioIds: ["telegram-help-command"], + }), + ); + expect(runQaFlowSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories, + channelId: "matrix", + outputDir: path.join(outputDir, "flow", "matrix"), + scenarioIds: ["matrix-restart-resume"], + }), + ); + }); + + it("binds one portable channel scenario without an explicit channel override", async () => { + const adapterFactories = [ + { + id: "portable-driver", + matches: vi.fn(), + create: vi.fn(), + }, + ]; + + const result = await runQaSuite({ + repoRoot: process.cwd(), + providerMode: "mock-openai", + channelDriver: "live", + adapterFactories, + scenarioIds: ["telegram-help-command"], + }); + + expect(result.executionKind).toBe("suite"); + expect(runQaFlowSuite).toHaveBeenCalledTimes(1); + expect(runQaFlowSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories, + channelId: "telegram", + scenarioIds: ["telegram-help-command"], + }), + ); + }); + it("partitions mixed Crabline flow channels into one aggregate suite", async () => { const repoRoot = await makeTempRepo("qa-suite-crabline-channels-"); const defaultFlowImplementation = runQaFlowSuite.getMockImplementation(); diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index 225a85974aba..47d9242c03dc 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -93,6 +93,7 @@ type QaUnifiedPartitionTask = { type QaFlowChannelGroup = { channel: string | undefined; + channelId: string | undefined; channelDriverSelection: QaSuiteRunParams["channelDriverSelection"]; scenarios: QaSeedScenarioWithSource[]; }; @@ -132,10 +133,37 @@ async function resolveQaFlowChannelGroups( runParams: QaSuiteRunParams | undefined, scenarios: readonly QaSeedScenarioWithSource[], ): Promise { + if (runParams?.adapterFactories) { + const explicitChannel = runParams.channelId?.trim().toLowerCase(); + if (explicitChannel) { + return [ + { + channel: explicitChannel, + channelId: explicitChannel, + channelDriverSelection: runParams.channelDriverSelection, + scenarios: [...scenarios], + }, + ]; + } + const groups = new Map(); + for (const scenario of scenarios) { + const channel = normalizeQaSuiteScenarioChannel(scenario); + const group = groups.get(channel) ?? []; + group.push(scenario); + groups.set(channel, group); + } + return [...groups].map(([channel, groupedScenarios]) => ({ + channel, + channelId: channel, + channelDriverSelection: runParams.channelDriverSelection, + scenarios: groupedScenarios, + })); + } if (runParams?.channelDriver !== "crabline") { return [ { - channel: runParams?.channelDriverSelection?.channel, + channel: runParams?.channelId ?? runParams?.channelDriverSelection?.channel, + channelId: runParams?.channelId, channelDriverSelection: runParams?.channelDriverSelection, scenarios: [...scenarios], }, @@ -155,6 +183,7 @@ async function resolveQaFlowChannelGroups( return [ { channel: singleChannel, + channelId: undefined, channelDriverSelection: runParams.channelDriverSelection ?? resolveOpenClawCrablineChannelDriverSelection({ channel: singleChannel }), @@ -166,6 +195,7 @@ async function resolveQaFlowChannelGroups( // launch one flow partition per channel and aggregate them at this owner. return channels.map((channel) => ({ channel, + channelId: undefined, channelDriverSelection: resolveOpenClawCrablineChannelDriverSelection({ channel }), scenarios: scenarios.filter( (scenario) => @@ -196,10 +226,14 @@ async function resolveSuiteExecutionPlan( scenarios.push(scenario); testFileScenariosByKind.set(scenario.execution.kind, scenarios); } + const channelGroups = (await resolveQaFlowChannelGroups(params, flowScenarios)).filter( + (group) => group.scenarios.length > 0, + ); const requiresFlowPartitions = - (await resolveQaFlowChannelGroups(params, flowScenarios)).filter( - (group) => group.scenarios.length > 0, - ).length > 1 || + channelGroups.length > 1 || + channelGroups.some( + (group) => group.channelId !== undefined && group.channelId !== params?.channelId, + ) || (flowScenarios.length > 1 && flowScenarios.some(scenarioRequiresIsolatedQaSuiteWorker)); if (testFileScenariosByKind.size === 0 && !requiresFlowPartitions) { return { kind: "flow" }; @@ -560,12 +594,18 @@ async function runUnifiedQaSuite(params: { const ordinaryIsolatedFlowScenarios = isolatedFlowScenarios.filter( (scenario) => !runtimeScenarioSet.has(scenario), ); - const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency); + const usesContributedChannelDriver = Boolean( + channelGroup.channelId && params.runParams?.adapterFactories, + ); + const sharedFlowPartitions = partitionSharedFlowScenarios( + sharedFlowScenarios, + usesContributedChannelDriver ? 1 : concurrency, + ); // Channel-driver flow workers each launch a gateway plus transport harness. // Serializing their isolated workers keeps state-mutating smoke checks from // flaking under concurrent child gateways while preserving non-driver speed. const channelDriverFlowRequiresExclusiveWorkers = Boolean( - channelGroup.channelDriverSelection, + channelGroup.channelDriverSelection || usesContributedChannelDriver, ); const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers ? 1 @@ -633,6 +673,7 @@ async function runUnifiedQaSuite(params: { ? (partition.scenarios[0].execution.runtime ?? params.runParams?.forcedRuntime) : params.runParams?.forcedRuntime, concurrency: partition.concurrency, + channelId: channelGroup.channelId, channelDriverSelection: channelGroup.channelDriverSelection, workerStartStaggerMs: isolatedPartition ? (params.runParams?.workerStartStaggerMs ??