diff --git a/CHANGELOG.md b/CHANGELOG.md index ec6f1aadd05c..caee64b811b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- **QA profile channel execution:** partition mixed Crabline channel scenarios into one aggregate host suite so taxonomy-backed profile commands and evidence workflows no longer abort before execution. - **Plugin SDK API baseline:** cover every public entrypoint, preserve complete declaration shapes without source-line churn, and run baseline and export-surface guards from changed-file validation. - **SQLite terminal session recovery:** track physical transcript mutation time in the agent database so killed or timed-out main sessions rotate when transcript writes outlive the registry update, while preserving legacy transcript mtimes during doctor import. - **Gateway chat typecheck:** import chat event types from their owning protocol schema after the retired aggregate type module was removed, restoring full project typechecks. diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index e7c5f7716c94..9a5f67a39cd2 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -792,6 +792,56 @@ describe("qa cli runtime", () => { }); }); + it("defers mixed Crabline channels to the host suite launcher", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + channelDriver: "crabline", + scenarioIds: ["telegram-help-command", "matrix-restart-resume"], + }); + + expect(runQaSuite).toHaveBeenCalledWith( + expect.objectContaining({ + channelDriver: "crabline", + channelDriverSelection: undefined, + scenarioIds: ["telegram-help-command", "matrix-restart-resume"], + }), + ); + }); + + it("forwards resolved catalog scenarios for automatic mixed-channel host runs", async () => { + await runQaSuiteCommand({ + providerMode: "mock-openai", + channelDriver: "crabline", + }); + + const suiteArgs = mockFirstObjectArg(runQaSuite); + expect(suiteArgs.channelDriverSelection).toBeUndefined(); + expect(suiteArgs.scenarioIds).toEqual( + expect.arrayContaining(["telegram-help-command", "matrix-restart-resume"]), + ); + const scenarioById = new Map( + readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]), + ); + expect( + (suiteArgs.scenarioIds as string[]).every( + (scenarioId) => scenarioById.get(scenarioId)?.execution.kind === "flow", + ), + ).toBe(true); + }); + + it("keeps mixed Crabline channels unsupported on the Multipass runner", async () => { + await expect( + runQaSuiteCommand({ + providerMode: "mock-openai", + channelDriver: "crabline", + runner: "multipass", + scenarioIds: ["telegram-help-command", "matrix-restart-resume"], + }), + ).rejects.toThrow("Selected QA scenarios require multiple channels (telegram, matrix)"); + expect(runQaMultipass).not.toHaveBeenCalled(); + }); + it("passes Crabline channel-driver selection through to the multipass runner", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index c332e71d3fd3..f01e48af64a1 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -86,7 +86,11 @@ import { } from "./scorecard-taxonomy.js"; import { isQaSelfCheckSuccessful } from "./self-check.js"; import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js"; -import { resolveQaSuiteScenarioChannel, scenarioMatchesQaProviderLane } from "./suite-planning.js"; +import { + resolveQaSuiteScenarioChannel, + resolveQaSuiteScenarioChannels, + scenarioMatchesQaProviderLane, +} from "./suite-planning.js"; import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js"; import { buildTokenEfficiencyReport, @@ -956,22 +960,44 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { if (opts.preflight === true && runner !== "host") { throw new Error("--preflight requires --runner host."); } - const channelDriverSelection = + const channelDriverScenarios = channelDriver === "crabline" + ? selectQaScenarioDefinitionsForChannelResolution({ + scenarioIds, + providerMode, + primaryModel: primaryModel ?? defaultQaModelForMode(providerMode), + channelDriver, + claudeCliAuthMode, + }) + : []; + const channelDriverChannels = + channelDriver === "crabline" + ? resolveQaSuiteScenarioChannels({ + defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL, + explicitChannel: opts.channel, + scenarios: channelDriverScenarios, + }) + : []; + if (runner === "multipass" && channelDriverChannels.length > 1) { + resolveQaSuiteScenarioChannel({ + defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL, + explicitChannel: opts.channel, + scenarios: channelDriverScenarios, + }); + } + const [singleChannelDriverChannel] = channelDriverChannels; + const channelDriverSelection = + channelDriver === "crabline" && channelDriverChannels.length === 1 && singleChannelDriverChannel ? resolveOpenClawCrablineChannelDriverSelection({ - channel: resolveQaSuiteScenarioChannel({ - defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL, - explicitChannel: opts.channel, - scenarios: selectQaScenarioDefinitionsForChannelResolution({ - scenarioIds, - providerMode, - primaryModel: primaryModel ?? defaultQaModelForMode(providerMode), - channelDriver, - claudeCliAuthMode, - }), - }), + channel: singleChannelDriverChannel, }) : undefined; + const hostScenarioIds = + runner === "host" && channelDriverChannels.length > 1 && scenarioIds.length === 0 + ? channelDriverScenarios + .filter((scenario) => scenario.execution.kind === "flow") + .map((scenario) => scenario.id) + : scenarioIds; if ( runner === "host" && (opts.image !== undefined || @@ -1065,7 +1091,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { fastMode: opts.fastMode, ...(thinkingDefault ? { thinkingDefault } : {}), ...(claudeCliAuthMode ? { claudeCliAuthMode } : {}), - scenarioIds: liveChannelId ? liveScenarioIds : scenarioIds, + scenarioIds: liveChannelId ? liveScenarioIds : hostScenarioIds, ...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}), ...(liveChannelId ? { concurrency: 1 } diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 6ed471f871fd..1bd85f772748 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -132,6 +132,176 @@ describe("qa suite runtime launcher", () => { expect(runQaTestFileScenarios).not.toHaveBeenCalled(); }); + it("partitions mixed Crabline flow channels into one aggregate suite", async () => { + const repoRoot = await makeTempRepo("qa-suite-crabline-channels-"); + const defaultFlowImplementation = runQaFlowSuite.getMockImplementation(); + if (!defaultFlowImplementation) { + throw new Error("expected default QA flow suite mock implementation"); + } + runQaFlowSuite.mockImplementation(async (params) => { + const result = await defaultFlowImplementation(params); + const scenarioIds: readonly string[] = params?.scenarioIds ?? []; + result.evidence = { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: scenarioIds.map((scenarioId) => ({ + test: { + kind: "qa-scenario", + id: scenarioId, + title: scenarioId, + }, + coverage: [], + execution: { + runner: "host", + environment: { + ref: null, + os: "linux", + nodeVersion: "v24.0.0", + }, + provider: { + id: "mock-openai", + live: false, + model: { + name: "gpt-5.6-luna", + ref: "mock-openai/gpt-5.6-luna", + }, + fixture: "mock-openai", + }, + channel: { + id: params?.channelDriverSelection?.channel ?? "qa-channel", + live: false, + driver: "crabline", + }, + packageSource: { + kind: "source-checkout", + }, + artifacts: [ + { + kind: "report", + path: "qa-suite-report.md", + source: "qa-suite", + }, + ], + }, + result: { + status: "pass", + }, + })), + }; + return result; + }); + const result = await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/crabline-channels", + providerMode: "mock-openai", + channelDriver: "crabline", + scenarioIds: ["telegram-help-command", "matrix-restart-resume"], + }); + + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "crabline-channels"); + expect(result).toMatchObject({ + executionKind: "suite", + result: { + evidencePath: path.join(outputDir, "qa-evidence.json"), + summaryPath: path.join(outputDir, "qa-suite-summary.json"), + }, + }); + expect(runQaFlowSuite).toHaveBeenCalledTimes(2); + expect(runQaFlowSuite).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + outputDir: path.join(outputDir, "flow", "telegram"), + channelDriverSelection: expect.objectContaining({ channel: "telegram" }), + scenarioIds: ["telegram-help-command"], + }), + ); + expect(runQaFlowSuite).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + outputDir: path.join(outputDir, "flow", "matrix"), + channelDriverSelection: expect.objectContaining({ channel: "matrix" }), + scenarioIds: ["matrix-restart-resume"], + }), + ); + const summary = JSON.parse( + await fs.readFile(path.join(outputDir, "qa-suite-summary.json"), "utf8"), + ) as { run?: { channel?: unknown; channelDriver?: unknown; scenarioIds?: unknown } }; + expect(summary.run?.channelDriver).toBe("crabline"); + expect(summary.run?.channel).toBeNull(); + expect(summary.run?.scenarioIds).toEqual(["telegram-help-command", "matrix-restart-resume"]); + const evidence = JSON.parse( + await fs.readFile(path.join(outputDir, "qa-evidence.json"), "utf8"), + ) as { + entries?: Array<{ execution?: { artifacts?: Array<{ path?: unknown }> } }>; + }; + expect(evidence.entries?.map((entry) => entry.execution?.artifacts?.[0]?.path)).toEqual([ + ".artifacts/qa-e2e/crabline-channels/flow/telegram/qa-suite-report.md", + ".artifacts/qa-e2e/crabline-channels/flow/matrix/qa-suite-report.md", + ]); + }); + + it("preserves runtime parity options across mixed Crabline flow channels", async () => { + const repoRoot = await makeTempRepo("qa-suite-crabline-runtime-pair-"); + await runQaSuite({ + repoRoot, + outputDir: ".artifacts/qa-e2e/crabline-runtime-pair", + providerMode: "mock-openai", + channelDriver: "crabline", + runtimePair: ["openclaw", "codex"], + scenarioIds: ["telegram-help-command", "matrix-restart-resume"], + }); + + expect(runQaFlowSuite).toHaveBeenCalledTimes(2); + for (const call of runQaFlowSuite.mock.calls) { + expect(call[0]).toEqual( + expect.objectContaining({ + runtimePair: ["openclaw", "codex"], + }), + ); + } + const summary = JSON.parse( + await fs.readFile( + path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "crabline-runtime-pair", + "qa-suite-summary.json", + ), + "utf8", + ), + ) as { run?: { runtimePair?: unknown } }; + expect(summary.run?.runtimePair).toEqual(["openclaw", "codex"]); + await expect( + fs.access( + path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "crabline-runtime-pair", + "flow", + "telegram", + "qa-evidence.json", + ), + ), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + fs.access( + path.join( + repoRoot, + ".artifacts", + "qa-e2e", + "crabline-runtime-pair", + "flow", + "matrix", + "qa-evidence.json", + ), + ), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("routes selected Playwright scenarios to the Playwright scenario runner", async () => { const repoRoot = await makeTempRepo("qa-suite-launch-"); const result = await runQaSuite({ diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index a3384a67753c..efd99bcc1c55 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -1,8 +1,12 @@ // Qa Lab plugin module implements suite launch behavior. import fs from "node:fs/promises"; import path from "node:path"; +import { + OPENCLAW_CRABLINE_DEFAULT_CHANNEL, + resolveOpenClawCrablineChannelDriverSelection, +} from "@openclaw/crabline"; import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime"; -import { toRepoRelativePath } from "./cli-paths.js"; +import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js"; import { QA_EVIDENCE_FILENAME, QA_EVIDENCE_SUMMARY_KIND, @@ -23,6 +27,8 @@ import { } from "./scenario-catalog.js"; import { normalizeQaSuiteConcurrency, + normalizeQaSuiteScenarioChannel, + resolveQaSuiteScenarioChannels, resolveQaSuiteOutputDir, resolveQaSuiteWorkerStartStaggerMs, scenarioRequiresIsolatedQaSuiteWorker, @@ -89,6 +95,12 @@ type QaUnifiedPartitionTask = { weight: number; }; +type QaFlowChannelGroup = { + channel: string | undefined; + channelDriverSelection: QaSuiteRunParams["channelDriverSelection"]; + scenarios: QaSeedScenarioWithSource[]; +}; + async function loadQaLabServerRuntime() { const { startQaLabServer } = await import("./lab-server.js"); return startQaLabServer; @@ -120,6 +132,49 @@ function resolveRequestedScenarios(params: { }); } +function resolveQaFlowChannelGroups( + runParams: QaSuiteRunParams | undefined, + scenarios: readonly QaSeedScenarioWithSource[], +): QaFlowChannelGroup[] { + if (runParams?.channelDriver !== "crabline") { + return [ + { + channel: runParams?.channelDriverSelection?.channel, + channelDriverSelection: runParams?.channelDriverSelection, + scenarios: [...scenarios], + }, + ]; + } + const channels = resolveQaSuiteScenarioChannels({ + defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL, + explicitChannel: runParams.channelDriverSelection?.channel, + scenarios: [...scenarios], + }); + const [singleChannel] = channels; + if (channels.length === 1 && singleChannel) { + return [ + { + channel: singleChannel, + channelDriverSelection: + runParams.channelDriverSelection ?? + resolveOpenClawCrablineChannelDriverSelection({ channel: singleChannel }), + scenarios: [...scenarios], + }, + ]; + } + // One Crabline process serves one channel. Mixed logical suites therefore + // launch one flow partition per channel and aggregate them at this owner. + return channels.map((channel) => ({ + channel, + channelDriverSelection: resolveOpenClawCrablineChannelDriverSelection({ channel }), + scenarios: scenarios.filter( + (scenario) => + (normalizeQaSuiteScenarioChannel(scenario) ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL) === + channel, + ), + })); +} + function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan { const scenarioIds = params?.scenarioIds ?? []; if (scenarioIds.length === 0) { @@ -139,7 +194,10 @@ function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuit scenarios.push(scenario); testFileScenariosByKind.set(scenario.execution.kind, scenarios); } - if (testFileScenariosByKind.size === 0) { + const requiresChannelPartitions = + resolveQaFlowChannelGroups(params, flowScenarios).filter((group) => group.scenarios.length > 0) + .length > 1; + if (testFileScenariosByKind.size === 0 && !requiresChannelPartitions) { return { kind: "flow" }; } return { @@ -296,8 +354,26 @@ async function readQaSuiteEvidenceSummary(evidencePath: string) { async function resolveQaSuiteResultEvidenceSummary(result: { evidence?: QaEvidenceSummaryJson; evidencePath: string; + outputDir: string; + repoRoot: string; }) { - return result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath)); + const summary = result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath)); + const rebasedSummary = structuredClone(summary); + for (const entry of rebasedSummary.entries) { + if (!entry.execution) { + continue; + } + for (const artifact of entry.execution.artifacts) { + if (artifact.source !== "qa-suite" || !isRepoRootRelativeRef(artifact.path)) { + continue; + } + artifact.path = toRepoRelativePath( + result.repoRoot, + path.resolve(result.outputDir, artifact.path), + ); + } + } + return validateQaEvidenceSummaryJson(rebasedSummary); } function mergeQaEvidenceSummaries(params: { @@ -373,6 +449,7 @@ function renderUnifiedQaSuiteReport(params: { async function writeUnifiedQaSuiteArtifacts(params: { alternateModel: string; + channelDriver: QaSuiteRunParams["channelDriver"]; concurrency: number; evidence: QaEvidenceSummaryJson; fastMode: boolean; @@ -380,6 +457,7 @@ async function writeUnifiedQaSuiteArtifacts(params: { outputDir: string; primaryModel: string; providerMode: ReturnType; + runtimePair: QaSuiteRunParams["runtimePair"]; scenarioIds: readonly string[]; scenarios: readonly QaSuiteScenarioResult[]; startedAt: Date; @@ -395,12 +473,14 @@ async function writeUnifiedQaSuiteArtifacts(params: { }); const summary = buildQaSuiteSummaryJson({ alternateModel: params.alternateModel, + channelDriver: params.channelDriver, concurrency: params.concurrency, evidence: params.evidence, fastMode: params.fastMode, finishedAt: params.finishedAt, primaryModel: params.primaryModel, providerMode: params.providerMode, + runtimePair: params.runtimePair, scenarioIds: params.scenarioIds, scenarios: [...params.scenarios], startedAt: params.startedAt, @@ -422,7 +502,9 @@ async function runUnifiedQaSuite(params: { plan: Extract; runParams: QaSuiteRunParams | undefined; }): Promise { - rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams); + if (params.plan.testFileScenariosByKind.size > 0) { + rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams); + } const startedAt = new Date(); const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd()); const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir); @@ -438,9 +520,10 @@ async function runUnifiedQaSuite(params: { ? params.runParams.fastMode : isQaFastModeEnabled({ primaryModel, alternateModel }); const transportId = normalizeQaTransportId(params.runParams?.transportId); - const defaultConcurrency = params.runParams?.channelDriverSelection - ? 1 - : defaultQaSuiteConcurrencyForTransport(transportId); + const defaultConcurrency = + params.runParams?.channelDriver === "crabline" || params.runParams?.channelDriverSelection + ? 1 + : defaultQaSuiteConcurrencyForTransport(transportId); const concurrency = normalizeQaSuiteConcurrency( params.runParams?.concurrency, params.plan.scenarios.length, @@ -453,98 +536,116 @@ async function runUnifiedQaSuite(params: { const testFilePartitionTasks: QaUnifiedPartitionTask[] = []; const scriptPartitionTasks: QaUnifiedPartitionTask[] = []; if (params.plan.flowScenarios.length > 0) { - const sharedFlowScenarios = params.plan.flowScenarios.filter( - (scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario), - ); - const isolatedFlowScenarios = params.plan.flowScenarios.filter( - scenarioRequiresIsolatedQaSuiteWorker, - ); - const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency); - // Channel-driver flow workers each launch a gateway plus transport harness. - // Serializing their isolated workers keeps state-mutating smoke checks from - // flaking under concurrent child gateways while preserving non-driver speed. - const channelDriverFlowRequiresExclusiveWorkers = Boolean( - params.runParams?.channelDriverSelection, - ); - const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers - ? 1 - : MAX_ISOLATED_FLOW_CONCURRENCY; - const isolatedFlowConcurrency = Math.min( - concurrency, - isolatedFlowConcurrencyLimit, - isolatedFlowScenarios.length, - ); - const isolatedFlowPartitions = - isolatedFlowConcurrency === 1 && isolatedFlowScenarios.length > 1 - ? isolatedFlowScenarios.map((scenario, index) => ({ - kind: `isolated-${index + 1}`, - scenarios: [scenario], - concurrency: 1, - })) - : [ - { - kind: "isolated", - scenarios: isolatedFlowScenarios, - concurrency: isolatedFlowConcurrency, - }, - ]; - const flowPartitions = [ - ...sharedFlowPartitions.map((scenarios, index) => ({ - kind: sharedFlowPartitions.length === 1 ? "shared" : `shared-${index + 1}`, - scenarios, - concurrency: 1, - })), - ...isolatedFlowPartitions, - ].filter((partition) => partition.scenarios.length > 0); + const channelGroups = resolveQaFlowChannelGroups( + params.runParams, + params.plan.flowScenarios, + ).filter((group) => group.scenarios.length > 0); + const mixedChannelRun = channelGroups.length > 1; const runFlowSuite = await loadQaFlowSuiteRuntime(); - for (const partition of flowPartitions) { - const isolatedPartition = - partition.kind === "isolated" || partition.kind.startsWith("isolated-"); - const task = { - weight: - isolatedPartition && channelDriverFlowRequiresExclusiveWorkers - ? concurrency - : partition.concurrency, - run: async () => { - const result = await runFlowSuite({ - ...params.runParams, - outputDir: - flowPartitions.length === 1 - ? suitePartitionOutputDir(outputDir, "flow") - : flowSuitePartitionOutputDir(outputDir, partition.kind), - writeEvidenceFile: false, - providerMode, - primaryModel, - alternateModel, - fastMode, - concurrency: partition.concurrency, - workerStartStaggerMs: isolatedPartition - ? (params.runParams?.workerStartStaggerMs ?? - resolveQaSuiteWorkerStartStaggerMs( - partition.concurrency, - process.env, - ISOLATED_FLOW_WORKER_START_STAGGER_MS, - )) - : params.runParams?.workerStartStaggerMs, - scenarioIds: partition.scenarios.map((scenario) => scenario.id), - }); - const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = []; - for (const [index, scenario] of partition.scenarios.entries()) { - const scenarioResult = result.scenarios[index]; - if (scenarioResult) { - scenarioResults.push({ scenarioId: scenario.id, result: scenarioResult }); + for (const channelGroup of channelGroups) { + const sharedFlowScenarios = channelGroup.scenarios.filter( + (scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario), + ); + const isolatedFlowScenarios = channelGroup.scenarios.filter( + scenarioRequiresIsolatedQaSuiteWorker, + ); + const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency); + // Channel-driver flow workers each launch a gateway plus transport harness. + // Serializing their isolated workers keeps state-mutating smoke checks from + // flaking under concurrent child gateways while preserving non-driver speed. + const channelDriverFlowRequiresExclusiveWorkers = Boolean( + channelGroup.channelDriverSelection, + ); + const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers + ? 1 + : MAX_ISOLATED_FLOW_CONCURRENCY; + const isolatedFlowConcurrency = Math.min( + concurrency, + isolatedFlowConcurrencyLimit, + isolatedFlowScenarios.length, + ); + const isolatedFlowPartitions = + isolatedFlowConcurrency === 1 && isolatedFlowScenarios.length > 1 + ? isolatedFlowScenarios.map((scenario, index) => ({ + kind: `isolated-${index + 1}`, + scenarios: [scenario], + concurrency: 1, + })) + : [ + { + kind: "isolated", + scenarios: isolatedFlowScenarios, + concurrency: isolatedFlowConcurrency, + }, + ]; + const flowPartitions = [ + ...sharedFlowPartitions.map((scenarios, index) => ({ + kind: sharedFlowPartitions.length === 1 ? "shared" : `shared-${index + 1}`, + scenarios, + concurrency: 1, + })), + ...isolatedFlowPartitions, + ].filter((partition) => partition.scenarios.length > 0); + for (const partition of flowPartitions) { + const isolatedPartition = + partition.kind === "isolated" || partition.kind.startsWith("isolated-"); + const partitionName = [ + channelGroups.length > 1 ? channelGroup.channel : undefined, + flowPartitions.length > 1 ? partition.kind : undefined, + ] + .filter((part): part is string => Boolean(part)) + .join("-"); + const task = { + weight: + mixedChannelRun || (isolatedPartition && channelDriverFlowRequiresExclusiveWorkers) + ? concurrency + : partition.concurrency, + run: async () => { + const result = await runFlowSuite({ + ...params.runParams, + outputDir: partitionName + ? flowSuitePartitionOutputDir(outputDir, partitionName) + : suitePartitionOutputDir(outputDir, "flow"), + writeEvidenceFile: false, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency: partition.concurrency, + channelDriverSelection: channelGroup.channelDriverSelection, + workerStartStaggerMs: isolatedPartition + ? (params.runParams?.workerStartStaggerMs ?? + resolveQaSuiteWorkerStartStaggerMs( + partition.concurrency, + process.env, + ISOLATED_FLOW_WORKER_START_STAGGER_MS, + )) + : params.runParams?.workerStartStaggerMs, + scenarioIds: partition.scenarios.map((scenario) => scenario.id), + }); + const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = []; + for (const [index, scenario] of partition.scenarios.entries()) { + const scenarioResult = result.scenarios[index]; + if (scenarioResult) { + scenarioResults.push({ scenarioId: scenario.id, result: scenarioResult }); + } } - } - return { - evidenceSummaries: [await resolveQaSuiteResultEvidenceSummary(result)], - scenarioResults, - }; - }, - } satisfies QaUnifiedPartitionTask; - if (isolatedPartition) { - isolatedFlowPartitionTasks.push(task); - } else { - sharedFlowPartitionTasks.push(task); + return { + evidenceSummaries: [ + await resolveQaSuiteResultEvidenceSummary({ + ...result, + repoRoot, + }), + ], + scenarioResults, + }; + }, + } satisfies QaUnifiedPartitionTask; + if (isolatedPartition) { + isolatedFlowPartitionTasks.push(task); + } else { + sharedFlowPartitionTasks.push(task); + } } } } @@ -568,7 +669,12 @@ async function runUnifiedQaSuite(params: { }, scenarios: testFileScenarios, }); - testFileEvidenceSummaries.push(await resolveQaSuiteResultEvidenceSummary(result)); + testFileEvidenceSummaries.push( + await resolveQaSuiteResultEvidenceSummary({ + ...result, + repoRoot, + }), + ); testFileScenarioResults.push( ...result.results.map((scenarioResult) => ({ scenarioId: scenarioResult.scenario.id, @@ -636,6 +742,7 @@ async function runUnifiedQaSuite(params: { }); return await writeUnifiedQaSuiteArtifacts({ alternateModel, + channelDriver: params.runParams?.channelDriver, concurrency, evidence, fastMode, @@ -643,6 +750,7 @@ async function runUnifiedQaSuite(params: { outputDir, primaryModel, providerMode, + runtimePair: params.runParams?.runtimePair, scenarioIds: params.plan.scenarios.map((scenario) => scenario.id), scenarios, startedAt, diff --git a/extensions/qa-lab/src/suite-planning.test.ts b/extensions/qa-lab/src/suite-planning.test.ts index 31f8fc527239..086ec15f441d 100644 --- a/extensions/qa-lab/src/suite-planning.test.ts +++ b/extensions/qa-lab/src/suite-planning.test.ts @@ -12,7 +12,9 @@ import { collectQaSuiteTransportPolicy, mapQaSuiteWithConcurrency, normalizeQaSuiteConcurrency, + normalizeQaSuiteScenarioChannel, resolveQaSuiteScenarioChannel, + resolveQaSuiteScenarioChannels, resolveQaSuiteWorkerStartStaggerMs, resolveQaSuiteOutputDir, scenarioRequiresControlUi, @@ -33,6 +35,17 @@ function makePlaywrightQaSuiteTestScenario(id: string): ReturnType { + it("normalizes blank scenario channels as unpinned", () => { + expect( + normalizeQaSuiteScenarioChannel(makeQaSuiteTestScenario("blank-channel", { channel: " " })), + ).toBeUndefined(); + expect( + normalizeQaSuiteScenarioChannel( + makeQaSuiteTestScenario("matrix-channel", { channel: " Matrix " }), + ), + ).toBe("matrix"); + }); + it("normalizes suite concurrency to a bounded integer", () => { const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY; delete process.env.OPENCLAW_QA_SUITE_CONCURRENCY; @@ -310,6 +323,16 @@ describe("qa suite planning helpers", () => { ], }), ).toThrow("Selected QA scenarios require multiple channels"); + expect( + resolveQaSuiteScenarioChannels({ + defaultChannel: "telegram", + scenarios: [ + makeQaSuiteTestScenario("plain"), + makeQaSuiteTestScenario("matrix-flow", { channel: "matrix" }), + makeQaSuiteTestScenario("slack-flow", { channel: "slack" }), + ], + }), + ).toEqual(["telegram", "matrix", "slack"]); }); it("isolates flow scenarios with explicit suite isolation metadata", () => { diff --git a/extensions/qa-lab/src/suite-planning.ts b/extensions/qa-lab/src/suite-planning.ts index 915f14b6eece..cf51e12837f3 100644 --- a/extensions/qa-lab/src/suite-planning.ts +++ b/extensions/qa-lab/src/suite-planning.ts @@ -139,13 +139,17 @@ function selectQaFlowSuiteScenarios(params: { ); } +function normalizeQaSuiteScenarioChannel(scenario: QaSeedScenario) { + return scenario.execution.channel?.trim().toLowerCase() || undefined; +} + function listQaSuiteScenarioChannels( scenarios: ReturnType["scenarios"], ) { return [ ...new Set( scenarios - .map((scenario) => scenario.execution.channel?.trim().toLowerCase()) + .map(normalizeQaSuiteScenarioChannel) .filter((channel): channel is string => Boolean(channel)), ), ]; @@ -155,6 +159,21 @@ function resolveQaSuiteScenarioChannel(params: { defaultChannel: string; explicitChannel?: string | null; scenarios: ReturnType["scenarios"]; +}) { + const scenarioChannels = resolveQaSuiteScenarioChannels(params); + const [scenarioChannel] = scenarioChannels; + if (scenarioChannels.length === 1 && scenarioChannel) { + return scenarioChannel; + } + throw new Error( + `Selected QA scenarios require multiple channels (${scenarioChannels.join(", ")}); split the run by channel.`, + ); +} + +function resolveQaSuiteScenarioChannels(params: { + defaultChannel: string; + explicitChannel?: string | null; + scenarios: ReturnType["scenarios"]; }) { const scenarioChannels = listQaSuiteScenarioChannels(params.scenarios); const explicitChannel = params.explicitChannel?.trim().toLowerCase(); @@ -165,17 +184,20 @@ function resolveQaSuiteScenarioChannel(params: { `--channel ${explicitChannel} conflicts with selected scenario execution.channel ${conflictingChannels.join(", ")}.`, ); } - return explicitChannel; + return [explicitChannel]; } if (scenarioChannels.length === 0) { - return params.defaultChannel; + return [params.defaultChannel]; } if (scenarioChannels.length === 1) { - return scenarioChannels[0]; + return scenarioChannels; } - throw new Error( - `Selected QA scenarios require multiple channels (${scenarioChannels.join(", ")}); split the run by channel.`, + const hasUnpinnedScenario = params.scenarios.some( + (scenario) => !normalizeQaSuiteScenarioChannel(scenario), ); + return hasUnpinnedScenario && !scenarioChannels.includes(params.defaultChannel) + ? [params.defaultChannel, ...scenarioChannels] + : scenarioChannels; } function collectQaSuitePluginIds( @@ -460,7 +482,9 @@ export { collectQaSuitePluginIds, mapQaSuiteWithConcurrency, normalizeQaSuiteConcurrency, + normalizeQaSuiteScenarioChannel, resolveQaSuiteScenarioChannel, + resolveQaSuiteScenarioChannels, resolveQaSuiteWorkerStartStaggerMs, resolveQaSuiteOutputDir, scenarioRequiresControlUi, diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index e6b99527f9fb..7d1be90aeb55 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -791,6 +791,7 @@ async function runQaRuntimeParitySuite(params: { progressEnabled: boolean; scenarioIds?: readonly string[]; runtimePair: [RuntimeId, RuntimeId]; + writeEvidenceFile?: boolean; }) { const ownsLab = !params.lab; const startLab = requireQaSuiteStartLab(params.startLab); @@ -891,6 +892,7 @@ async function runQaRuntimeParitySuite(params: { controlUiEnabled: scenarioRequiresControlUi(scenario), forcedRuntime: runtime, captureRuntimeParityCell: true, + writeEvidenceFile: params.writeEvidenceFile, }); const scenarioResult = cellResult.scenarios[0] ?? @@ -981,6 +983,7 @@ async function runQaRuntimeParitySuite(params: { ? params.selectedScenarios.map((scenario) => scenario.id) : undefined, runtimePair: params.runtimePair, + writeEvidenceFile: params.writeEvidenceFile, }, ); lab.setLatestReport({ @@ -1368,6 +1371,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise