ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 04:47:03 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): run mixed channel suites

Browse Source
This commit is contained in:
Vincent Koc
2026-07-12 06:51:59 +02:00
parent 4139938dcf
commit 4040267af9
8 changed files with 524 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- **QA profile channel execution:** partition mixed Crabline channel scenarios into one aggregate host suite so taxonomy-backed profile commands and evidence workflows no longer abort before execution.
- **Plugin SDK API baseline:** cover every public entrypoint, preserve complete declaration shapes without source-line churn, and run baseline and export-surface guards from changed-file validation.
- **SQLite terminal session recovery:** track physical transcript mutation time in the agent database so killed or timed-out main sessions rotate when transcript writes outlive the registry update, while preserving legacy transcript mtimes during doctor import.
- **Gateway chat typecheck:** import chat event types from their owning protocol schema after the retired aggregate type module was removed, restoring full project typechecks.
extensions/qa-lab/src/cli.runtime.test.ts
+50
View File
@@ -792,6 +792,56 @@ describe("qa cli runtime", () => {
});
});
it("defers mixed Crabline channels to the host suite launcher", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "mock-openai",
channelDriver: "crabline",
scenarioIds: ["telegram-help-command", "matrix-restart-resume"],
});
expect(runQaSuite).toHaveBeenCalledWith(
expect.objectContaining({
channelDriver: "crabline",
channelDriverSelection: undefined,
scenarioIds: ["telegram-help-command", "matrix-restart-resume"],
}),
);
});
it("forwards resolved catalog scenarios for automatic mixed-channel host runs", async () => {
await runQaSuiteCommand({
providerMode: "mock-openai",
channelDriver: "crabline",
});
const suiteArgs = mockFirstObjectArg(runQaSuite);
expect(suiteArgs.channelDriverSelection).toBeUndefined();
expect(suiteArgs.scenarioIds).toEqual(
expect.arrayContaining(["telegram-help-command", "matrix-restart-resume"]),
);
const scenarioById = new Map(
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
);
expect(
(suiteArgs.scenarioIds as string[]).every(
(scenarioId) => scenarioById.get(scenarioId)?.execution.kind === "flow",
),
).toBe(true);
});
it("keeps mixed Crabline channels unsupported on the Multipass runner", async () => {
await expect(
runQaSuiteCommand({
providerMode: "mock-openai",
channelDriver: "crabline",
runner: "multipass",
scenarioIds: ["telegram-help-command", "matrix-restart-resume"],
}),
).rejects.toThrow("Selected QA scenarios require multiple channels (telegram, matrix)");
expect(runQaMultipass).not.toHaveBeenCalled();
});
it("passes Crabline channel-driver selection through to the multipass runner", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",
extensions/qa-lab/src/cli.runtime.ts
+40 -14
View File
@@ -86,7 +86,11 @@ import {
} from "./scorecard-taxonomy.js";
import { isQaSelfCheckSuccessful } from "./self-check.js";
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
import { resolveQaSuiteScenarioChannel, scenarioMatchesQaProviderLane } from "./suite-planning.js";
import {
resolveQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannels,
scenarioMatchesQaProviderLane,
} from "./suite-planning.js";
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
import {
buildTokenEfficiencyReport,
@@ -956,22 +960,44 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
if (opts.preflight === true && runner !== "host") {
throw new Error("--preflight requires --runner host.");
}
const channelDriverSelection =
const channelDriverScenarios =
channelDriver === "crabline"
? selectQaScenarioDefinitionsForChannelResolution({
scenarioIds,
providerMode,
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
channelDriver,
claudeCliAuthMode,
})
: [];
const channelDriverChannels =
channelDriver === "crabline"
? resolveQaSuiteScenarioChannels({
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
explicitChannel: opts.channel,
scenarios: channelDriverScenarios,
})
: [];
if (runner === "multipass" && channelDriverChannels.length > 1) {
resolveQaSuiteScenarioChannel({
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
explicitChannel: opts.channel,
scenarios: channelDriverScenarios,
});
}
const [singleChannelDriverChannel] = channelDriverChannels;
const channelDriverSelection =
channelDriver === "crabline" && channelDriverChannels.length === 1 && singleChannelDriverChannel
? resolveOpenClawCrablineChannelDriverSelection({
channel: resolveQaSuiteScenarioChannel({
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
explicitChannel: opts.channel,
scenarios: selectQaScenarioDefinitionsForChannelResolution({
scenarioIds,
providerMode,
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
channelDriver,
claudeCliAuthMode,
}),
}),
channel: singleChannelDriverChannel,
})
: undefined;
const hostScenarioIds =
runner === "host" && channelDriverChannels.length > 1 && scenarioIds.length === 0
? channelDriverScenarios
.filter((scenario) => scenario.execution.kind === "flow")
.map((scenario) => scenario.id)
: scenarioIds;
if (
runner === "host" &&
(opts.image !== undefined ||
@@ -1065,7 +1091,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
fastMode: opts.fastMode,
...(thinkingDefault ? { thinkingDefault } : {}),
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
scenarioIds: liveChannelId ? liveScenarioIds : scenarioIds,
scenarioIds: liveChannelId ? liveScenarioIds : hostScenarioIds,
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
...(liveChannelId
? { concurrency: 1 }
extensions/qa-lab/src/suite-launch.runtime.test.ts
+170
View File
@@ -132,6 +132,176 @@ describe("qa suite runtime launcher", () => {
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
});
it("partitions mixed Crabline flow channels into one aggregate suite", async () => {
const repoRoot = await makeTempRepo("qa-suite-crabline-channels-");
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
if (!defaultFlowImplementation) {
throw new Error("expected default QA flow suite mock implementation");
}
runQaFlowSuite.mockImplementation(async (params) => {
const result = await defaultFlowImplementation(params);
const scenarioIds: readonly string[] = params?.scenarioIds ?? [];
result.evidence = {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-06-14T00:00:00.000Z",
evidenceMode: "full",
entries: scenarioIds.map((scenarioId) => ({
test: {
kind: "qa-scenario",
id: scenarioId,
title: scenarioId,
},
coverage: [],
execution: {
runner: "host",
environment: {
ref: null,
os: "linux",
nodeVersion: "v24.0.0",
},
provider: {
id: "mock-openai",
live: false,
model: {
name: "gpt-5.6-luna",
ref: "mock-openai/gpt-5.6-luna",
},
fixture: "mock-openai",
},
channel: {
id: params?.channelDriverSelection?.channel ?? "qa-channel",
live: false,
driver: "crabline",
},
packageSource: {
kind: "source-checkout",
},
artifacts: [
{
kind: "report",
path: "qa-suite-report.md",
source: "qa-suite",
},
],
},
result: {
status: "pass",
},
})),
};
return result;
});
const result = await runQaSuite({
repoRoot,
outputDir: ".artifacts/qa-e2e/crabline-channels",
providerMode: "mock-openai",
channelDriver: "crabline",
scenarioIds: ["telegram-help-command", "matrix-restart-resume"],
});
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "crabline-channels");
expect(result).toMatchObject({
executionKind: "suite",
result: {
evidencePath: path.join(outputDir, "qa-evidence.json"),
summaryPath: path.join(outputDir, "qa-suite-summary.json"),
},
});
expect(runQaFlowSuite).toHaveBeenCalledTimes(2);
expect(runQaFlowSuite).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
outputDir: path.join(outputDir, "flow", "telegram"),
channelDriverSelection: expect.objectContaining({ channel: "telegram" }),
scenarioIds: ["telegram-help-command"],
}),
);
expect(runQaFlowSuite).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
outputDir: path.join(outputDir, "flow", "matrix"),
channelDriverSelection: expect.objectContaining({ channel: "matrix" }),
scenarioIds: ["matrix-restart-resume"],
}),
);
const summary = JSON.parse(
await fs.readFile(path.join(outputDir, "qa-suite-summary.json"), "utf8"),
) as { run?: { channel?: unknown; channelDriver?: unknown; scenarioIds?: unknown } };
expect(summary.run?.channelDriver).toBe("crabline");
expect(summary.run?.channel).toBeNull();
expect(summary.run?.scenarioIds).toEqual(["telegram-help-command", "matrix-restart-resume"]);
const evidence = JSON.parse(
await fs.readFile(path.join(outputDir, "qa-evidence.json"), "utf8"),
) as {
entries?: Array<{ execution?: { artifacts?: Array<{ path?: unknown }> } }>;
};
expect(evidence.entries?.map((entry) => entry.execution?.artifacts?.[0]?.path)).toEqual([
".artifacts/qa-e2e/crabline-channels/flow/telegram/qa-suite-report.md",
".artifacts/qa-e2e/crabline-channels/flow/matrix/qa-suite-report.md",
]);
});
it("preserves runtime parity options across mixed Crabline flow channels", async () => {
const repoRoot = await makeTempRepo("qa-suite-crabline-runtime-pair-");
await runQaSuite({
repoRoot,
outputDir: ".artifacts/qa-e2e/crabline-runtime-pair",
providerMode: "mock-openai",
channelDriver: "crabline",
runtimePair: ["openclaw", "codex"],
scenarioIds: ["telegram-help-command", "matrix-restart-resume"],
});
expect(runQaFlowSuite).toHaveBeenCalledTimes(2);
for (const call of runQaFlowSuite.mock.calls) {
expect(call[0]).toEqual(
expect.objectContaining({
runtimePair: ["openclaw", "codex"],
}),
);
}
const summary = JSON.parse(
await fs.readFile(
path.join(
repoRoot,
".artifacts",
"qa-e2e",
"crabline-runtime-pair",
"qa-suite-summary.json",
),
"utf8",
),
) as { run?: { runtimePair?: unknown } };
expect(summary.run?.runtimePair).toEqual(["openclaw", "codex"]);
await expect(
fs.access(
path.join(
repoRoot,
".artifacts",
"qa-e2e",
"crabline-runtime-pair",
"flow",
"telegram",
"qa-evidence.json",
),
),
).rejects.toMatchObject({ code: "ENOENT" });
await expect(
fs.access(
path.join(
repoRoot,
".artifacts",
"qa-e2e",
"crabline-runtime-pair",
"flow",
"matrix",
"qa-evidence.json",
),
),
).rejects.toMatchObject({ code: "ENOENT" });
});
it("routes selected Playwright scenarios to the Playwright scenario runner", async () => {
const repoRoot = await makeTempRepo("qa-suite-launch-");
const result = await runQaSuite({
extensions/qa-lab/src/suite-launch.runtime.ts
+206 -98
View File
@@ -1,8 +1,12 @@
// Qa Lab plugin module implements suite launch behavior.
import fs from "node:fs/promises";
import path from "node:path";
import {
OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
resolveOpenClawCrablineChannelDriverSelection,
} from "@openclaw/crabline";
import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime";
import { toRepoRelativePath } from "./cli-paths.js";
import { isRepoRootRelativeRef, toRepoRelativePath } from "./cli-paths.js";
import {
QA_EVIDENCE_FILENAME,
QA_EVIDENCE_SUMMARY_KIND,
@@ -23,6 +27,8 @@ import {
} from "./scenario-catalog.js";
import {
normalizeQaSuiteConcurrency,
normalizeQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannels,
resolveQaSuiteOutputDir,
resolveQaSuiteWorkerStartStaggerMs,
scenarioRequiresIsolatedQaSuiteWorker,
@@ -89,6 +95,12 @@ type QaUnifiedPartitionTask = {
weight: number;
};
type QaFlowChannelGroup = {
channel: string | undefined;
channelDriverSelection: QaSuiteRunParams["channelDriverSelection"];
scenarios: QaSeedScenarioWithSource[];
};
async function loadQaLabServerRuntime() {
const { startQaLabServer } = await import("./lab-server.js");
return startQaLabServer;
@@ -120,6 +132,49 @@ function resolveRequestedScenarios(params: {
});
}
function resolveQaFlowChannelGroups(
runParams: QaSuiteRunParams | undefined,
scenarios: readonly QaSeedScenarioWithSource[],
): QaFlowChannelGroup[] {
if (runParams?.channelDriver !== "crabline") {
return [
{
channel: runParams?.channelDriverSelection?.channel,
channelDriverSelection: runParams?.channelDriverSelection,
scenarios: [...scenarios],
},
];
}
const channels = resolveQaSuiteScenarioChannels({
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
explicitChannel: runParams.channelDriverSelection?.channel,
scenarios: [...scenarios],
});
const [singleChannel] = channels;
if (channels.length === 1 && singleChannel) {
return [
{
channel: singleChannel,
channelDriverSelection:
runParams.channelDriverSelection ??
resolveOpenClawCrablineChannelDriverSelection({ channel: singleChannel }),
scenarios: [...scenarios],
},
];
}
// One Crabline process serves one channel. Mixed logical suites therefore
// launch one flow partition per channel and aggregate them at this owner.
return channels.map((channel) => ({
channel,
channelDriverSelection: resolveOpenClawCrablineChannelDriverSelection({ channel }),
scenarios: scenarios.filter(
(scenario) =>
(normalizeQaSuiteScenarioChannel(scenario) ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL) ===
channel,
),
}));
}
function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan {
const scenarioIds = params?.scenarioIds ?? [];
if (scenarioIds.length === 0) {
@@ -139,7 +194,10 @@ function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuit
scenarios.push(scenario);
testFileScenariosByKind.set(scenario.execution.kind, scenarios);
}
if (testFileScenariosByKind.size === 0) {
const requiresChannelPartitions =
resolveQaFlowChannelGroups(params, flowScenarios).filter((group) => group.scenarios.length > 0)
.length > 1;
if (testFileScenariosByKind.size === 0 && !requiresChannelPartitions) {
return { kind: "flow" };
}
return {
@@ -296,8 +354,26 @@ async function readQaSuiteEvidenceSummary(evidencePath: string) {
async function resolveQaSuiteResultEvidenceSummary(result: {
evidence?: QaEvidenceSummaryJson;
evidencePath: string;
outputDir: string;
repoRoot: string;
}) {
return result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath));
const summary = result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath));
const rebasedSummary = structuredClone(summary);
for (const entry of rebasedSummary.entries) {
if (!entry.execution) {
continue;
}
for (const artifact of entry.execution.artifacts) {
if (artifact.source !== "qa-suite" || !isRepoRootRelativeRef(artifact.path)) {
continue;
}
artifact.path = toRepoRelativePath(
result.repoRoot,
path.resolve(result.outputDir, artifact.path),
);
}
}
return validateQaEvidenceSummaryJson(rebasedSummary);
}
function mergeQaEvidenceSummaries(params: {
@@ -373,6 +449,7 @@ function renderUnifiedQaSuiteReport(params: {
async function writeUnifiedQaSuiteArtifacts(params: {
alternateModel: string;
channelDriver: QaSuiteRunParams["channelDriver"];
concurrency: number;
evidence: QaEvidenceSummaryJson;
fastMode: boolean;
@@ -380,6 +457,7 @@ async function writeUnifiedQaSuiteArtifacts(params: {
outputDir: string;
primaryModel: string;
providerMode: ReturnType<typeof normalizeQaProviderMode>;
runtimePair: QaSuiteRunParams["runtimePair"];
scenarioIds: readonly string[];
scenarios: readonly QaSuiteScenarioResult[];
startedAt: Date;
@@ -395,12 +473,14 @@ async function writeUnifiedQaSuiteArtifacts(params: {
});
const summary = buildQaSuiteSummaryJson({
alternateModel: params.alternateModel,
channelDriver: params.channelDriver,
concurrency: params.concurrency,
evidence: params.evidence,
fastMode: params.fastMode,
finishedAt: params.finishedAt,
primaryModel: params.primaryModel,
providerMode: params.providerMode,
runtimePair: params.runtimePair,
scenarioIds: params.scenarioIds,
scenarios: [...params.scenarios],
startedAt: params.startedAt,
@@ -422,7 +502,9 @@ async function runUnifiedQaSuite(params: {
plan: Extract<QaSuiteExecutionPlan, { kind: "unified" }>;
runParams: QaSuiteRunParams | undefined;
}): Promise<QaUnifiedSuiteResult> {
rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams);
if (params.plan.testFileScenariosByKind.size > 0) {
rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams);
}
const startedAt = new Date();
const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd());
const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir);
@@ -438,9 +520,10 @@ async function runUnifiedQaSuite(params: {
? params.runParams.fastMode
: isQaFastModeEnabled({ primaryModel, alternateModel });
const transportId = normalizeQaTransportId(params.runParams?.transportId);
const defaultConcurrency = params.runParams?.channelDriverSelection
? 1
: defaultQaSuiteConcurrencyForTransport(transportId);
const defaultConcurrency =
params.runParams?.channelDriver === "crabline" || params.runParams?.channelDriverSelection
? 1
: defaultQaSuiteConcurrencyForTransport(transportId);
const concurrency = normalizeQaSuiteConcurrency(
params.runParams?.concurrency,
params.plan.scenarios.length,
@@ -453,98 +536,116 @@ async function runUnifiedQaSuite(params: {
const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
const scriptPartitionTasks: QaUnifiedPartitionTask[] = [];
if (params.plan.flowScenarios.length > 0) {
const sharedFlowScenarios = params.plan.flowScenarios.filter(
(scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
);
const isolatedFlowScenarios = params.plan.flowScenarios.filter(
scenarioRequiresIsolatedQaSuiteWorker,
);
const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency);
// Channel-driver flow workers each launch a gateway plus transport harness.
// Serializing their isolated workers keeps state-mutating smoke checks from
// flaking under concurrent child gateways while preserving non-driver speed.
const channelDriverFlowRequiresExclusiveWorkers = Boolean(
params.runParams?.channelDriverSelection,
);
const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers
? 1
: MAX_ISOLATED_FLOW_CONCURRENCY;
const isolatedFlowConcurrency = Math.min(
concurrency,
isolatedFlowConcurrencyLimit,
isolatedFlowScenarios.length,
);
const isolatedFlowPartitions =
isolatedFlowConcurrency === 1 && isolatedFlowScenarios.length > 1
? isolatedFlowScenarios.map((scenario, index) => ({
kind: `isolated-${index + 1}`,
scenarios: [scenario],
concurrency: 1,
}))
: [
{
kind: "isolated",
scenarios: isolatedFlowScenarios,
concurrency: isolatedFlowConcurrency,
},
];
const flowPartitions = [
...sharedFlowPartitions.map((scenarios, index) => ({
kind: sharedFlowPartitions.length === 1 ? "shared" : `shared-${index + 1}`,
scenarios,
concurrency: 1,
})),
...isolatedFlowPartitions,
].filter((partition) => partition.scenarios.length > 0);
const channelGroups = resolveQaFlowChannelGroups(
params.runParams,
params.plan.flowScenarios,
).filter((group) => group.scenarios.length > 0);
const mixedChannelRun = channelGroups.length > 1;
const runFlowSuite = await loadQaFlowSuiteRuntime();
for (const partition of flowPartitions) {
const isolatedPartition =
partition.kind === "isolated" || partition.kind.startsWith("isolated-");
const task = {
weight:
isolatedPartition && channelDriverFlowRequiresExclusiveWorkers
? concurrency
: partition.concurrency,
run: async () => {
const result = await runFlowSuite({
...params.runParams,
outputDir:
flowPartitions.length === 1
? suitePartitionOutputDir(outputDir, "flow")
: flowSuitePartitionOutputDir(outputDir, partition.kind),
writeEvidenceFile: false,
providerMode,
primaryModel,
alternateModel,
fastMode,
concurrency: partition.concurrency,
workerStartStaggerMs: isolatedPartition
? (params.runParams?.workerStartStaggerMs ??
resolveQaSuiteWorkerStartStaggerMs(
partition.concurrency,
process.env,
ISOLATED_FLOW_WORKER_START_STAGGER_MS,
))
: params.runParams?.workerStartStaggerMs,
scenarioIds: partition.scenarios.map((scenario) => scenario.id),
});
const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
for (const [index, scenario] of partition.scenarios.entries()) {
const scenarioResult = result.scenarios[index];
if (scenarioResult) {
scenarioResults.push({ scenarioId: scenario.id, result: scenarioResult });
for (const channelGroup of channelGroups) {
const sharedFlowScenarios = channelGroup.scenarios.filter(
(scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
);
const isolatedFlowScenarios = channelGroup.scenarios.filter(
scenarioRequiresIsolatedQaSuiteWorker,
);
const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency);
// Channel-driver flow workers each launch a gateway plus transport harness.
// Serializing their isolated workers keeps state-mutating smoke checks from
// flaking under concurrent child gateways while preserving non-driver speed.
const channelDriverFlowRequiresExclusiveWorkers = Boolean(
channelGroup.channelDriverSelection,
);
const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers
? 1
: MAX_ISOLATED_FLOW_CONCURRENCY;
const isolatedFlowConcurrency = Math.min(
concurrency,
isolatedFlowConcurrencyLimit,
isolatedFlowScenarios.length,
);
const isolatedFlowPartitions =
isolatedFlowConcurrency === 1 && isolatedFlowScenarios.length > 1
? isolatedFlowScenarios.map((scenario, index) => ({
kind: `isolated-${index + 1}`,
scenarios: [scenario],
concurrency: 1,
}))
: [
{
kind: "isolated",
scenarios: isolatedFlowScenarios,
concurrency: isolatedFlowConcurrency,
},
];
const flowPartitions = [
...sharedFlowPartitions.map((scenarios, index) => ({
kind: sharedFlowPartitions.length === 1 ? "shared" : `shared-${index + 1}`,
scenarios,
concurrency: 1,
})),
...isolatedFlowPartitions,
].filter((partition) => partition.scenarios.length > 0);
for (const partition of flowPartitions) {
const isolatedPartition =
partition.kind === "isolated" || partition.kind.startsWith("isolated-");
const partitionName = [
channelGroups.length > 1 ? channelGroup.channel : undefined,
flowPartitions.length > 1 ? partition.kind : undefined,
]
.filter((part): part is string => Boolean(part))
.join("-");
const task = {
weight:
mixedChannelRun || (isolatedPartition && channelDriverFlowRequiresExclusiveWorkers)
? concurrency
: partition.concurrency,
run: async () => {
const result = await runFlowSuite({
...params.runParams,
outputDir: partitionName
? flowSuitePartitionOutputDir(outputDir, partitionName)
: suitePartitionOutputDir(outputDir, "flow"),
writeEvidenceFile: false,
providerMode,
primaryModel,
alternateModel,
fastMode,
concurrency: partition.concurrency,
channelDriverSelection: channelGroup.channelDriverSelection,
workerStartStaggerMs: isolatedPartition
? (params.runParams?.workerStartStaggerMs ??
resolveQaSuiteWorkerStartStaggerMs(
partition.concurrency,
process.env,
ISOLATED_FLOW_WORKER_START_STAGGER_MS,
))
: params.runParams?.workerStartStaggerMs,
scenarioIds: partition.scenarios.map((scenario) => scenario.id),
});
const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
for (const [index, scenario] of partition.scenarios.entries()) {
const scenarioResult = result.scenarios[index];
if (scenarioResult) {
scenarioResults.push({ scenarioId: scenario.id, result: scenarioResult });
}
}
}
return {
evidenceSummaries: [await resolveQaSuiteResultEvidenceSummary(result)],
scenarioResults,
};
},
} satisfies QaUnifiedPartitionTask;
if (isolatedPartition) {
isolatedFlowPartitionTasks.push(task);
} else {
sharedFlowPartitionTasks.push(task);
return {
evidenceSummaries: [
await resolveQaSuiteResultEvidenceSummary({
...result,
repoRoot,
}),
],
scenarioResults,
};
},
} satisfies QaUnifiedPartitionTask;
if (isolatedPartition) {
isolatedFlowPartitionTasks.push(task);
} else {
sharedFlowPartitionTasks.push(task);
}
}
}
}
@@ -568,7 +669,12 @@ async function runUnifiedQaSuite(params: {
},
scenarios: testFileScenarios,
});
testFileEvidenceSummaries.push(await resolveQaSuiteResultEvidenceSummary(result));
testFileEvidenceSummaries.push(
await resolveQaSuiteResultEvidenceSummary({
...result,
repoRoot,
}),
);
testFileScenarioResults.push(
...result.results.map((scenarioResult) => ({
scenarioId: scenarioResult.scenario.id,
@@ -636,6 +742,7 @@ async function runUnifiedQaSuite(params: {
});
return await writeUnifiedQaSuiteArtifacts({
alternateModel,
channelDriver: params.runParams?.channelDriver,
concurrency,
evidence,
fastMode,
@@ -643,6 +750,7 @@ async function runUnifiedQaSuite(params: {
outputDir,
primaryModel,
providerMode,
runtimePair: params.runParams?.runtimePair,
scenarioIds: params.plan.scenarios.map((scenario) => scenario.id),
scenarios,
startedAt,
extensions/qa-lab/src/suite-planning.test.ts
+23
View File
@@ -12,7 +12,9 @@ import {
collectQaSuiteTransportPolicy,
mapQaSuiteWithConcurrency,
normalizeQaSuiteConcurrency,
normalizeQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannels,
resolveQaSuiteWorkerStartStaggerMs,
resolveQaSuiteOutputDir,
scenarioRequiresControlUi,
@@ -33,6 +35,17 @@ function makePlaywrightQaSuiteTestScenario(id: string): ReturnType<typeof makeQa
}
describe("qa suite planning helpers", () => {
it("normalizes blank scenario channels as unpinned", () => {
expect(
normalizeQaSuiteScenarioChannel(makeQaSuiteTestScenario("blank-channel", { channel: " " })),
).toBeUndefined();
expect(
normalizeQaSuiteScenarioChannel(
makeQaSuiteTestScenario("matrix-channel", { channel: " Matrix " }),
),
).toBe("matrix");
});
it("normalizes suite concurrency to a bounded integer", () => {
const previous = process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
delete process.env.OPENCLAW_QA_SUITE_CONCURRENCY;
@@ -310,6 +323,16 @@ describe("qa suite planning helpers", () => {
],
}),
).toThrow("Selected QA scenarios require multiple channels");
expect(
resolveQaSuiteScenarioChannels({
defaultChannel: "telegram",
scenarios: [
makeQaSuiteTestScenario("plain"),
makeQaSuiteTestScenario("matrix-flow", { channel: "matrix" }),
makeQaSuiteTestScenario("slack-flow", { channel: "slack" }),
],
}),
).toEqual(["telegram", "matrix", "slack"]);
});
it("isolates flow scenarios with explicit suite isolation metadata", () => {
extensions/qa-lab/src/suite-planning.ts
+30 -6
View File
@@ -139,13 +139,17 @@ function selectQaFlowSuiteScenarios(params: {
);
}
function normalizeQaSuiteScenarioChannel(scenario: QaSeedScenario) {
return scenario.execution.channel?.trim().toLowerCase() || undefined;
}
function listQaSuiteScenarioChannels(
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"],
) {
return [
...new Set(
scenarios
.map((scenario) => scenario.execution.channel?.trim().toLowerCase())
.map(normalizeQaSuiteScenarioChannel)
.filter((channel): channel is string => Boolean(channel)),
),
];
@@ -155,6 +159,21 @@ function resolveQaSuiteScenarioChannel(params: {
defaultChannel: string;
explicitChannel?: string | null;
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
}) {
const scenarioChannels = resolveQaSuiteScenarioChannels(params);
const [scenarioChannel] = scenarioChannels;
if (scenarioChannels.length === 1 && scenarioChannel) {
return scenarioChannel;
}
throw new Error(
`Selected QA scenarios require multiple channels (${scenarioChannels.join(", ")}); split the run by channel.`,
);
}
function resolveQaSuiteScenarioChannels(params: {
defaultChannel: string;
explicitChannel?: string | null;
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
}) {
const scenarioChannels = listQaSuiteScenarioChannels(params.scenarios);
const explicitChannel = params.explicitChannel?.trim().toLowerCase();
@@ -165,17 +184,20 @@ function resolveQaSuiteScenarioChannel(params: {
`--channel ${explicitChannel} conflicts with selected scenario execution.channel ${conflictingChannels.join(", ")}.`,
);
}
return explicitChannel;
return [explicitChannel];
}
if (scenarioChannels.length === 0) {
return params.defaultChannel;
return [params.defaultChannel];
}
if (scenarioChannels.length === 1) {
return scenarioChannels[0];
return scenarioChannels;
}
throw new Error(
`Selected QA scenarios require multiple channels (${scenarioChannels.join(", ")}); split the run by channel.`,
const hasUnpinnedScenario = params.scenarios.some(
(scenario) => !normalizeQaSuiteScenarioChannel(scenario),
);
return hasUnpinnedScenario && !scenarioChannels.includes(params.defaultChannel)
? [params.defaultChannel, ...scenarioChannels]
: scenarioChannels;
}
function collectQaSuitePluginIds(
@@ -460,7 +482,9 @@ export {
collectQaSuitePluginIds,
mapQaSuiteWithConcurrency,
normalizeQaSuiteConcurrency,
normalizeQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannel,
resolveQaSuiteScenarioChannels,
resolveQaSuiteWorkerStartStaggerMs,
resolveQaSuiteOutputDir,
scenarioRequiresControlUi,
extensions/qa-lab/src/suite.ts
+4
View File
@@ -791,6 +791,7 @@ async function runQaRuntimeParitySuite(params: {
progressEnabled: boolean;
scenarioIds?: readonly string[];
runtimePair: [RuntimeId, RuntimeId];
writeEvidenceFile?: boolean;
}) {
const ownsLab = !params.lab;
const startLab = requireQaSuiteStartLab(params.startLab);
@@ -891,6 +892,7 @@ async function runQaRuntimeParitySuite(params: {
controlUiEnabled: scenarioRequiresControlUi(scenario),
forcedRuntime: runtime,
captureRuntimeParityCell: true,
writeEvidenceFile: params.writeEvidenceFile,
});
const scenarioResult =
cellResult.scenarios[0] ??
@@ -981,6 +983,7 @@ async function runQaRuntimeParitySuite(params: {
? params.selectedScenarios.map((scenario) => scenario.id)
: undefined,
runtimePair: params.runtimePair,
writeEvidenceFile: params.writeEvidenceFile,
},
);
lab.setLatestReport({
@@ -1368,6 +1371,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
progressEnabled,
scenarioIds: params.scenarioIds,
runtimePair: params.runtimePair,
writeEvidenceFile: params.writeEvidenceFile,
});
}