From 9310586b092d24fc631135a6a4df99ef6ffaaff3 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Mon, 27 Jul 2026 14:05:14 +0800 Subject: [PATCH] fix(qa-lab): unify runner control plane (#112838) --- docs/concepts/qa-e2e-automation.md | 17 + extensions/qa-lab/runner-contract.ts | 60 ++ extensions/qa-lab/src/cli.runtime.test.ts | 38 +- extensions/qa-lab/src/cli.runtime.ts | 183 ++---- extensions/qa-lab/src/docker-harness.test.ts | 9 +- extensions/qa-lab/src/docker-harness.ts | 14 +- extensions/qa-lab/src/lab-server.test.ts | 367 +++++++++++- extensions/qa-lab/src/lab-server.ts | 158 ++++- extensions/qa-lab/src/profile-planning.ts | 146 +++++ extensions/qa-lab/src/run-config.test.ts | 542 +++++++++++++++++- extensions/qa-lab/src/run-config.ts | 434 ++++++++++++-- .../qa-lab/src/runtime-pair-lane-selection.ts | 91 +++ .../qa-lab/src/suite-launch.runtime.test.ts | 132 +++++ extensions/qa-lab/src/suite-launch.runtime.ts | 62 +- extensions/qa-lab/src/suite-progress.ts | 126 ++++ extensions/qa-lab/web/src/app.browser.test.ts | 424 ++++++++++++++ extensions/qa-lab/web/src/app.ts | 93 ++- extensions/qa-lab/web/src/http.ts | 17 +- extensions/qa-lab/web/src/styles.css | 1 + extensions/qa-lab/web/src/ui-render-shell.ts | 83 ++- extensions/qa-lab/web/src/ui-render.test.ts | 1 + extensions/qa-lab/web/src/ui-types.ts | 44 +- 22 files changed, 2738 insertions(+), 304 deletions(-) create mode 100644 extensions/qa-lab/runner-contract.ts create mode 100644 extensions/qa-lab/src/profile-planning.ts create mode 100644 extensions/qa-lab/src/runtime-pair-lane-selection.ts create mode 100644 extensions/qa-lab/src/suite-progress.ts create mode 100644 extensions/qa-lab/web/src/app.browser.test.ts diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 7fd64aee16ee..aa2a9e5ab6ae 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -110,6 +110,23 @@ the QA Lab page where an operator or automation loop can give the agent a QA mission, observe real channel behavior, and record what worked, failed, or stayed blocked. +The Runner's Scenarios panel can launch flow, Playwright, Vitest, and script +catalog entries together. **Profile** uses the taxonomy-owned membership plan; +checking scenarios creates an explicit override, while **Profile** in the +Scenarios panel returns to server-resolved profile membership. + +Config also exposes **Provider lane**, primary and alternate models, +**Execution channel**, **Channel driver**, **Evidence mode**, **Runtime pair**, +and **Runtime-pair lane** (`core`, `extended`, or `soak`). Provider/model, +runtime, and channel-driver choices remain independent: for example, Real +frontier providers can use the Crabline channel driver, and Synthetic (mock) can +use Real channels. The server resolves taxonomy membership, provider/model +eligibility, declared `execution.channel`, runtime-pair-lane membership, and +supported execution kinds before launch. The Run panel shows the selected +execution kinds plus explicit exclusions or errors. Unknown, empty explicit, +profile-incompatible, or lane-incompatible selections fail closed instead of +being replaced by a default suite. + For faster QA Lab UI iteration without rebuilding the Docker image each time, start the stack with a bind-mounted QA Lab bundle: diff --git a/extensions/qa-lab/runner-contract.ts b/extensions/qa-lab/runner-contract.ts new file mode 100644 index 000000000000..cadce331dc43 --- /dev/null +++ b/extensions/qa-lab/runner-contract.ts @@ -0,0 +1,60 @@ +import type { QaProviderMode } from "./src/providers/index.js"; +import type { QaRuntimePairLane, QaScenarioExecution } from "./src/scenario-catalog.js"; +import type { + QaScorecardChannelDriver, + QaScorecardEvidenceMode, +} from "./src/scorecard-taxonomy.js"; + +export type QaLabExecutionKind = NonNullable; + +export type QaLabRunSelection = { + profile: string; + channel: string | null; + channelDriver: QaScorecardChannelDriver; + evidenceMode: QaScorecardEvidenceMode; + providerMode: QaProviderMode; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + runtimePair: ["openclaw", "codex"] | null; + runtimePairLane: QaRuntimePairLane | null; + scenarioIds: string[] | null; +}; + +export type QaLabResolvedRunPlan = { + status: "ready" | "invalid"; + profile: string; + explicitScenarioSelection: boolean; + selectedScenarios: Array<{ + id: string; + title: string; + executionKind: QaLabExecutionKind; + declaredChannel: string | null; + effectiveChannel: string | null; + }>; + executionKinds: QaLabExecutionKind[]; + exclusions: Array<{ + scenarioId: string; + executionKind: QaLabExecutionKind; + reasons: string[]; + }>; + errors: string[]; +}; + +type QaLabRunArtifacts = { + outputDir: string; + evidencePath: string; + reportPath: string; + summaryPath: string; + watchUrl: string; +}; + +export type QaLabRunnerSnapshot = { + status: "idle" | "running" | "completed" | "failed"; + selection: QaLabRunSelection; + plan: QaLabResolvedRunPlan | null; + startedAt?: string; + finishedAt?: string; + artifacts: QaLabRunArtifacts | null; + error: string | null; +}; diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index c43fbd2e61eb..b0ae42931c14 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -556,7 +556,7 @@ describe("qa cli runtime", () => { expectWriteContains(stdoutWrite, "QA run profile: all; categories: 1; scenarios:"); }); - it("filters QA-channel-pinned scenarios from the Crabline smoke profile", async () => { + it("filters QA-channel-pinned scenarios from an implicit Crabline smoke profile", async () => { runQaSuite.mockImplementationOnce(async () => { await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8"); return flowSuiteRuntimeResult({ @@ -568,12 +568,11 @@ describe("qa cli runtime", () => { await runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", profile: "smoke-ci", - scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"], }); const suiteArgs = mockFirstObjectArg(runQaSuite); expect(suiteArgs.channelDriver).toBe("crabline"); - expect(suiteArgs.scenarioIds).toEqual(["channel-top-level-reply-shape"]); + expect(suiteArgs.scenarioIds).toContain("channel-top-level-reply-shape"); expect(suiteArgs.scenarioIds).not.toEqual( expect.arrayContaining([ "instruction-followthrough-repo-contract", @@ -603,6 +602,20 @@ describe("qa cli runtime", () => { ); }); + it("rejects explicit profile selections with an incompatible scenario", async () => { + await expect( + runQaProfileCommand({ + repoRoot: "/tmp/openclaw-repo", + profile: "smoke-ci", + scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"], + }), + ).rejects.toThrow( + "qa run --qa-profile smoke-ci cannot run explicitly selected scenario(s): control-ui-qa-channel-image-roundtrip (channelDriver=qa-channel).", + ); + + expect(runQaSuite).not.toHaveBeenCalled(); + }); + it("dispatches the Matrix restart scenario through the Crabline smoke profile", async () => { await runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", @@ -773,15 +786,20 @@ describe("qa cli runtime", () => { expect(runQaMultipass).not.toHaveBeenCalled(); }); - it("rejects runtime-pair execution for live adapters", async () => { - await expect( - runQaSuiteCommand({ + it("keeps runtime-pair execution independent from live adapters", async () => { + await runQaSuiteCommand({ + channelDriver: "live", + channel: "telegram", + runtimePair: "openclaw,codex", + }); + + expect(runQaSuite).toHaveBeenCalledWith( + expect.objectContaining({ channelDriver: "live", - channel: "telegram", - runtimePair: "openclaw,codex", + channelId: "telegram", + runtimePair: ["openclaw", "codex"], }), - ).rejects.toThrow("--runtime-pair is not supported with a live QA adapter."); - expect(runQaSuite).not.toHaveBeenCalled(); + ); }); it("loads contributed adapters without preselecting a scenario channel", async () => { diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index a0c9df072a94..e7f9ebe71152 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -46,6 +46,10 @@ import { startQaLabServer } from "./lab-server.js"; import { listLiveTransportQaAdapterFactories } from "./live-transports/cli.js"; import { runQaManualLane } from "./manual-lane.runtime.js"; import { runQaMultipass } from "./multipass.runtime.js"; +import { + resolveQaRunProfileExecutionSelection, + resolveQaRunProfileMembership, +} from "./profile-planning.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js"; import { QA_FRONTIER_PARITY_BASELINE_LABEL, @@ -68,6 +72,10 @@ import { type QaProviderMode, type QaProviderModeInput, } from "./run-config.js"; +import { + resolveQaRuntimePairLaneScenarioIds, + resolveQaRuntimePairScenarioSupport, +} from "./runtime-pair-lane-selection.js"; import type { RuntimeId } from "./runtime-parity.js"; import { QA_RUNTIME_PAIR_LANES, @@ -80,7 +88,6 @@ import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js" import { qaScorecardChannelDriverSchema, readQaScorecardTaxonomyReport, - type QaScorecardCategoryCoverageReport, type QaScorecardChannelDriver, type QaScorecardEvidenceMode, } from "./scorecard-taxonomy.js"; @@ -311,76 +318,6 @@ function parseQaRuntimePairLaneFilters(input: string[] | undefined): QaRuntimePa return rawValues as QaRuntimePairLane[]; } -function resolveQaRuntimePairLaneScenarioIds(params: { - channelDriver?: QaScorecardChannelDriver | null; - claudeCliAuthMode?: QaCliBackendAuthMode; - primaryModel: string; - providerMode: QaProviderMode; - scenarioIds: string[]; - runtimePairLanes: readonly QaRuntimePairLane[]; - runtimePair: boolean; -}): { - scenarioIds: string[]; - excludedLaneScenarios: string[]; - excludedNonFlowScenarios: string[]; -} { - if (params.runtimePairLanes.length === 0) { - return { - scenarioIds: params.scenarioIds, - excludedLaneScenarios: [], - excludedNonFlowScenarios: [], - }; - } - const laneSet = new Set(params.runtimePairLanes); - const matchingScenarios = readQaScenarioPack().scenarios.filter( - (scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane), - ); - if (matchingScenarios.length === 0) { - throw new Error( - `--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`, - ); - } - const compatibleScenarios = params.runtimePair - ? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow") - : matchingScenarios; - const laneCompatibleScenarios = compatibleScenarios.filter((scenario) => - scenarioMatchesQaProviderLane({ - scenario, - providerMode: params.providerMode, - primaryModel: params.primaryModel, - channelDriver: params.channelDriver, - channel: scenario.execution.channel, - claudeCliAuthMode: params.claudeCliAuthMode, - }), - ); - const excludedLaneScenarios = compatibleScenarios - .filter((scenario) => !laneCompatibleScenarios.includes(scenario)) - .map((scenario) => scenario.id); - const excludedNonFlowScenarios = params.runtimePair - ? matchingScenarios - .filter((scenario) => scenario.execution.kind !== "flow") - .map((scenario) => `${scenario.id} (${scenario.execution.kind})`) - : []; - if (compatibleScenarios.length === 0) { - throw new Error( - `--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.join(", ")}.`, - ); - } - if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) { - throw new Error( - `--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.join(", ")}.`, - ); - } - return { - scenarioIds: uniqueStrings([ - ...params.scenarioIds, - ...laneCompatibleScenarios.map((scenario) => scenario.id), - ]), - excludedLaneScenarios, - excludedNonFlowScenarios, - }; -} - function rejectNonFlowScenarioIds(params: { option: "--runner multipass" | "--runtime-pair"; scenarioIds: readonly string[]; @@ -392,12 +329,13 @@ function rejectNonFlowScenarioIds(params: { const scenarioById = new Map( readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]), ); - const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => { + const selectedScenarios = scenarioIds.flatMap((scenarioId) => { const scenario = scenarioById.get(scenarioId); - return scenario && scenario.execution.kind !== "flow" - ? [`${scenario.id} (${scenario.execution.kind})`] - : []; + return scenario ? [scenario] : []; }); + const nonFlowScenarios = resolveQaRuntimePairScenarioSupport( + selectedScenarios, + ).excludedScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`); if (nonFlowScenarios.length > 0) { throw new Error( `${params.option} requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`, @@ -742,43 +680,30 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { if (!profileReport) { throw new Error(`taxonomy.yaml does not define QA run profile ${profile}.`); } - const categories = scorecardReport.categories.filter((category) => - qaScorecardCategoryMatchesRunProfile(category, { + const membership = resolveQaRunProfileMembership( + { profile, surface: opts.surface, category: opts.category, - }), + scenarioIds: opts.scenarioIds, + }, + { scenarios: scenarioPack.scenarios, scorecardReport }, ); + const categories = membership.categories; if (categories.length === 0) { throw new Error(formatQaRunProfileNoMatchMessage(opts)); } - const scenarioBySourcePath = new Map( - scenarioPack.scenarios.map((scenario) => [scenario.sourcePath, scenario] as const), - ); - const taxonomyScenariosForCategories = uniqueStrings( - categories.flatMap((category) => category.scenarioRefs), - ) - .map((scenarioRef) => scenarioBySourcePath.get(scenarioRef)) - .filter((scenario): scenario is NonNullable => scenario !== undefined); const requestedScenarioIds = uniqueStrings( (opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean), ); - const taxonomyScenarios = - requestedScenarioIds.length === 0 - ? taxonomyScenariosForCategories - : taxonomyScenariosForCategories.filter((scenario) => - requestedScenarioIds.includes(scenario.id), - ); + const taxonomyScenarios = membership.selectedScenarios; if (requestedScenarioIds.length > 0 && taxonomyScenarios.length === 0) { throw new Error( `qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${requestedScenarioIds.join(",")}.`, ); } - const matchedScenarioIds = new Set(taxonomyScenarios.map((scenario) => scenario.id)); - const missingScenarioIds = requestedScenarioIds.filter( - (scenarioId) => !matchedScenarioIds.has(scenarioId), - ); + const missingScenarioIds = membership.excludedScenarioIds; if (missingScenarioIds.length > 0) { throw new Error( `qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${missingScenarioIds.join(",")}.`, @@ -787,29 +712,23 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) { const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile); const normalizedProviderMode = normalizeQaProviderMode(providerMode); const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode); - const scenarios = taxonomyScenarios.filter((scenario) => { - // qa-channel is the built-in harness channel, so another driver cannot implement it. - if ( - scenario.execution.channel === "qa-channel" && - profileReport.channelDriver !== "qa-channel" - ) { - return false; - } - const channel = - profileReport.channelDriver === "qa-channel" - ? "qa-channel" - : (scenario.execution.channel ?? - (profileReport.channelDriver === "crabline" - ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL - : undefined)); - return scenarioMatchesQaProviderLane({ - scenario, - providerMode: normalizedProviderMode, - primaryModel, - channelDriver: profileReport.channelDriver, - channel, - }); + const executionSelection = resolveQaRunProfileExecutionSelection({ + scenarios: taxonomyScenarios, + providerMode: normalizedProviderMode, + primaryModel, + channelDriver: profileReport.channelDriver, + defaultChannel: + profileReport.channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined, }); + if (requestedScenarioIds.length > 0 && executionSelection.excludedScenarios.length > 0) { + const exclusions = executionSelection.excludedScenarios + .map(({ scenario, reasons }) => `${scenario.id} (${reasons.join(", ")})`) + .join(", "); + throw new Error( + `qa run --qa-profile ${profile} cannot run explicitly selected scenario(s): ${exclusions}.`, + ); + } + const scenarios = executionSelection.selectedScenarios; if (scenarios.length === 0) { throw new Error( `qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`, @@ -898,25 +817,6 @@ function defaultQaRunProfileProviderMode(profile: string): QaProviderModeInput { return profile === "smoke-ci" ? "mock-openai" : DEFAULT_QA_LIVE_PROVIDER_MODE; } -function qaScorecardCategoryMatchesRunProfile( - category: QaScorecardCategoryCoverageReport, - opts: { profile: string; surface?: string; category?: string }, -): boolean { - if (!category.profiles.includes(opts.profile)) { - return false; - } - if (opts.surface?.trim()) { - const surface = opts.surface.trim(); - if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) { - return false; - } - } - if (opts.category?.trim() && category.id !== opts.category.trim()) { - return false; - } - return true; -} - function formatQaRunProfileNoMatchMessage( opts: Pick, ) { @@ -967,8 +867,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { }); const runtimePairLanes = parseQaRuntimePairLaneFilters(opts.runtimePairLane); const runtimePairLaneSelection = resolveQaRuntimePairLaneScenarioIds({ + channel: opts.channel, channelDriver, claudeCliAuthMode, + defaultChannel: channelDriver === "crabline" ? OPENCLAW_CRABLINE_DEFAULT_CHANNEL : undefined, primaryModel: primaryModel ?? defaultQaModelForMode(providerMode), providerMode, scenarioIds: explicitScenarioIds, @@ -981,12 +883,12 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { } if (runtimePairLaneSelection.excludedNonFlowScenarios.length > 0) { process.stderr.write( - `QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.join(", ")}\n`, + `QA runtime-pair lane selection excluded incompatible non-flow scenario(s): ${runtimePairLaneSelection.excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}\n`, ); } if (runtimePairLaneSelection.excludedLaneScenarios.length > 0) { process.stderr.write( - `QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.join(", ")}\n`, + `QA runtime-pair lane selection excluded lane-incompatible scenario(s): ${runtimePairLaneSelection.excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}\n`, ); } const allowFailures = opts.allowFailures === true; @@ -1064,9 +966,6 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { if (runner === "multipass" && liveChannelId) { throw new Error("--channel-driver live with --channel requires --runner host."); } - if (runtimePair && liveChannelId) { - throw new Error("--runtime-pair is not supported with a live QA adapter."); - } if (runner === "multipass") { rejectNonFlowScenarioIds({ option: "--runner multipass", scenarioIds }); const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking); diff --git a/extensions/qa-lab/src/docker-harness.test.ts b/extensions/qa-lab/src/docker-harness.test.ts index 66288512338e..69b199dcaa0e 100644 --- a/extensions/qa-lab/src/docker-harness.test.ts +++ b/extensions/qa-lab/src/docker-harness.test.ts @@ -75,6 +75,9 @@ describe("qa docker harness", () => { OPENCLAW_STATE_DIR: "/tmp/openclaw/state", }); expect(services["qa-lab"]?.volumes).toContain("./state:/opt/openclaw-scaffold:ro"); + expect(services["qa-lab"]?.volumes).toContain( + `${path.relative(outputDir, "/repo/openclaw/taxonomy.yaml").split(path.sep).join("/")}:/app/taxonomy.yaml:ro`, + ); expect(compose).toContain(' - "127.0.0.1:18889:18789"'); expect(compose).toContain(' - "127.0.0.1:43124:43123"'); expect(compose).toContain(":/opt/openclaw-qa-lab-ui:ro"); @@ -154,7 +157,7 @@ describe("qa docker harness", () => { expect(result.imageName).toBe("openclaw:qa-local-prebaked"); expect(calls).toEqual([ - "docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=qa-channel qa-lab -f Dockerfile . @/repo/openclaw", + "docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS=acpx qa-channel qa-lab -f Dockerfile . @/repo/openclaw", ]); }); @@ -177,10 +180,14 @@ describe("qa docker harness", () => { const compose = await readFile(path.join(outputDir, "docker-compose.qa.yml"), "utf8"); const services = parseComposeServices(compose); + expect(compose).toContain('OPENCLAW_EXTENSIONS: "acpx qa-channel qa-lab"'); expect(services["qa-mock-openai"]?.build?.context).toBe("../repo #hash"); expect(services["qa-lab"]?.volumes).toContain( "../repo #hash/extensions/qa-lab/web/dist:/opt/openclaw-qa-lab-ui:ro", ); + expect(services["qa-lab"]?.volumes).toContain( + "../repo #hash/taxonomy.yaml:/app/taxonomy.yaml:ro", + ); expect(services["openclaw-qa-gateway"]?.volumes).toContain( "../repo #hash:/opt/openclaw-repo:ro", ); diff --git a/extensions/qa-lab/src/docker-harness.ts b/extensions/qa-lab/src/docker-harness.ts index cb49bc06eeb7..fde64308c972 100644 --- a/extensions/qa-lab/src/docker-harness.ts +++ b/extensions/qa-lab/src/docker-harness.ts @@ -12,6 +12,9 @@ import { buildQaGatewayConfig } from "./qa-gateway-config.js"; const QA_LAB_INTERNAL_PORT = 43123; const QA_LAB_UI_OVERLAY_DIR = "/opt/openclaw-qa-lab-ui"; +// The QA config enables ACPX. Bake the external plugin so ephemeral Gateways do +// not block startup on a network install before their health deadline. +const QA_DOCKER_PLUGIN_SELECTION = "acpx qa-channel qa-lab"; function toPosixRelative(fromDir: string, toPath: string): string { return path.relative(fromDir, toPath).split(path.sep).join("/"); @@ -31,7 +34,7 @@ function renderImageBlock(params: { return ` image: ${params.imageName}\n`; } const context = toPosixRelative(params.outputDir, params.repoRoot) || "."; - return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "qa-channel qa-lab"\n`; + return ` build:\n context: ${yamlDoubleQuoted(context)}\n dockerfile: Dockerfile\n args:\n OPENCLAW_EXTENSIONS: "${QA_DOCKER_PLUGIN_SELECTION}"\n`; } function renderCompose(params: { @@ -46,6 +49,10 @@ function renderCompose(params: { }) { const imageBlock = renderImageBlock(params); const repoMount = toPosixRelative(params.outputDir, params.repoRoot) || "."; + const taxonomyMount = toPosixRelative( + params.outputDir, + path.join(params.repoRoot, "taxonomy.yaml"), + ); const qaLabUiMount = toPosixRelative( params.outputDir, path.join(params.repoRoot, "extensions", "qa-lab", "web", "dist"), @@ -84,6 +91,7 @@ ${imageBlock} pull_policy: never - "127.0.0.1:${params.qaLabPort}:${QA_LAB_INTERNAL_PORT}" volumes: - ./state:/opt/openclaw-scaffold:ro + - ${yamlDoubleQuoted(`${taxonomyMount}:/app/taxonomy.yaml:ro`)} ${params.bindUiDist ? ` - ${yamlDoubleQuoted(`${qaLabUiMount}:${QA_LAB_UI_OVERLAY_DIR}:ro`)}\n` : ""} healthcheck: test: - CMD @@ -190,7 +198,7 @@ Files: Suggested flow: 1. Build the prebaked image once: - - \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="qa-channel qa-lab" -f Dockerfile .\` + - \`docker build -t openclaw:qa-local-prebaked --build-arg OPENCLAW_EXTENSIONS="${QA_DOCKER_PLUGIN_SELECTION}" -f Dockerfile .\` 2. Start the stack: - \`docker compose -f docker-compose.qa.yml up${params.usePrebuiltImage ? "" : " --build"} -d\` 3. Open the QA dashboard: @@ -354,7 +362,7 @@ export async function buildQaDockerHarnessImage( "-t", imageName, "--build-arg", - "OPENCLAW_EXTENSIONS=qa-channel qa-lab", + `OPENCLAW_EXTENSIONS=${QA_DOCKER_PLUGIN_SELECTION}`, "-f", "Dockerfile", ".", diff --git a/extensions/qa-lab/src/lab-server.test.ts b/extensions/qa-lab/src/lab-server.test.ts index c622085de828..e290ba6d9058 100644 --- a/extensions/qa-lab/src/lab-server.test.ts +++ b/extensions/qa-lab/src/lab-server.test.ts @@ -14,6 +14,22 @@ const qaChannelMock = vi.hoisted(() => ({ startAccount: vi.fn(), })); +const suiteLaunchMock = vi.hoisted(() => ({ + runQaSuite: vi.fn(), +})); +const liveTransportMock = vi.hoisted(() => ({ + adapterFactories: [{ id: "live-test-factory", matches: vi.fn(), create: vi.fn() }], + listAdapterFactories: vi.fn(), +})); + +vi.mock("./suite-launch.runtime.js", () => ({ + runQaSuite: suiteLaunchMock.runQaSuite, +})); + +vi.mock("./live-transports/cli.js", () => ({ + listLiveTransportQaAdapterFactories: liveTransportMock.listAdapterFactories, +})); + vi.mock("openclaw/plugin-sdk/qa-channel", () => ({ qaChannelPlugin: { config: { @@ -161,6 +177,10 @@ async function startQaLabServerForTest(params?: QaLabServerStartParams) { } beforeEach(() => { + suiteLaunchMock.runQaSuite.mockReset(); + liveTransportMock.listAdapterFactories.mockReset(); + liveTransportMock.listAdapterFactories.mockReturnValue(liveTransportMock.adapterFactories); + liveTransportMock.adapterFactories[0]!.matches.mockReset(); qaChannelMock.resolveAccount.mockReset(); qaChannelMock.resolveAccount.mockImplementation((_cfg: unknown, accountId: string) => ({ accountId, @@ -186,10 +206,10 @@ beforeEach(() => { }); afterEach(async () => { - captureMock.reset(); while (cleanups.length > 0) { await cleanups.pop()?.(); } + captureMock.reset(); }); function isRetryableLocalFetchError(error: unknown) { @@ -324,6 +344,333 @@ async function createQaLabRepoRootFixture(params?: { } describe("qa-lab server", () => { + it("dispatches explicit mixed-kind selections through the suite planner", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + suiteLaunchMock.runQaSuite.mockResolvedValue({ + executionKind: "suite", + result: { + evidencePath: "/tmp/qa-evidence.json", + outputDir: "/tmp/qa-output", + report: "# QA report\n", + reportPath: "/tmp/qa-report.md", + scenarios: [], + summaryPath: "/tmp/qa-summary.json", + }, + }); + + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + channelDriver: "crabline", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + alternateModel: "openai/gpt-5.6-luna", + scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"], + }), + }); + + expect(response.status).toBe(202); + const launch = (await response.json()) as { + plan: { + executionKinds: string[]; + selectedScenarios: Array<{ + id: string; + declaredChannel: string | null; + effectiveChannel: string | null; + }>; + }; + }; + expect(launch.plan.executionKinds).toEqual(["flow", "playwright"]); + expect(launch.plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([ + "dm-chat-baseline", + "browser-talk-start-stop", + ]); + expect(launch.plan.selectedScenarios).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "dm-chat-baseline", + declaredChannel: null, + effectiveChannel: "telegram", + }), + expect.objectContaining({ + id: "browser-talk-start-stop", + declaredChannel: null, + effectiveChannel: null, + }), + ]), + ); + await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1)); + expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith( + expect.objectContaining({ + alternateModel: "openai/gpt-5.6-luna", + channelDriver: "crabline", + primaryModel: "openai/gpt-5.6-luna", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"], + }), + ); + expect(liveTransportMock.listAdapterFactories).not.toHaveBeenCalled(); + await vi.waitFor(async () => { + const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as { + runner: { + status: string; + selection: { scenarioIds: string[] }; + artifacts: { watchUrl: string }; + }; + }; + expect(bootstrap.runner.status).toBe("completed"); + expect(bootstrap.runner.artifacts.watchUrl).toBe(lab.baseUrl); + expect(bootstrap.runner.selection.scenarioIds).toEqual([ + "dm-chat-baseline", + "browser-talk-start-stop", + ]); + }); + }); + + it("keeps mock providers independent from real channel adapters", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + suiteLaunchMock.runQaSuite.mockResolvedValue({ + executionKind: "flow", + result: { + evidencePath: "/tmp/qa-evidence.json", + outputDir: "/tmp/qa-output", + report: "# QA report\n", + reportPath: "/tmp/qa-report.md", + scenarios: [], + summaryPath: "/tmp/qa-summary.json", + watchUrl: "http://runtime-watch.invalid", + }, + }); + + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + channelDriver: "live", + providerMode: "mock-openai", + scenarioIds: ["dm-chat-baseline"], + }), + }); + + expect(response.status).toBe(202); + await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1)); + expect(liveTransportMock.listAdapterFactories).toHaveBeenCalledTimes(1); + expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories: liveTransportMock.adapterFactories, + channelDriver: "live", + providerMode: "mock-openai", + }), + ); + await vi.waitFor(async () => { + const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as { + runner: { status: string; artifacts: { watchUrl: string } }; + }; + expect(bootstrap.runner.status).toBe("completed"); + expect(bootstrap.runner.artifacts.watchUrl).toBe("http://runtime-watch.invalid"); + }); + }); + + it("allows only one concurrent request to commit a resolved suite plan", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + let finishSuite: ((value: unknown) => void) | undefined; + suiteLaunchMock.runQaSuite.mockImplementation( + () => + new Promise((resolve) => { + finishSuite = resolve; + }), + ); + const request = () => + fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline"], + }), + }); + + const responses = await Promise.all([request(), request()]); + + expect( + responses.map((response) => response.status).toSorted((left, right) => left - right), + ).toEqual([202, 409]); + expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1); + finishSuite?.({ + executionKind: "flow", + result: { + evidencePath: "/tmp/qa-evidence.json", + outputDir: "/tmp/qa-output", + report: "# QA report\n", + reportPath: "/tmp/qa-report.md", + scenarios: [], + summaryPath: "/tmp/qa-summary.json", + }, + }); + await vi.waitFor(async () => { + const bootstrap = (await (await fetchWithRetry(`${lab.baseUrl}/api/bootstrap`)).json()) as { + runner: { status: string }; + }; + expect(bootstrap.runner.status).toBe("completed"); + }); + }); + + it("rejects empty and unknown explicit selections before dispatch", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + for (const scenarioIds of [[], ["missing-scenario"]]) { + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ scenarioIds }), + }); + expect(response.status).toBe(400); + } + expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled(); + }); + + it("returns the resolved runtime-pair-lane plan and launches it with independent live transport", async () => { + liveTransportMock.adapterFactories[0]!.matches.mockReturnValue(true); + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + suiteLaunchMock.runQaSuite.mockResolvedValue({ + executionKind: "flow", + result: { + evidencePath: "/tmp/qa-evidence.json", + outputDir: "/tmp/qa-output", + report: "# QA report\n", + reportPath: "/tmp/qa-report.md", + scenarios: [], + summaryPath: "/tmp/qa-summary.json", + watchUrl: lab.baseUrl, + }, + }); + + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "all", + channel: "telegram", + channelDriver: "live", + evidenceMode: "slim", + providerMode: "mock-openai", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "core", + }), + }); + + expect(response.status).toBe(202); + const payload = (await response.json()) as { + plan: { + executionKinds: string[]; + exclusions: Array<{ scenarioId: string }>; + selectedScenarios: Array<{ id: string }>; + }; + }; + expect(payload.plan.executionKinds).toEqual(["flow"]); + expect(payload.plan.selectedScenarios.map((scenario) => scenario.id)).toContain( + "runtime-first-hour-20-turn", + ); + expect(payload.plan.exclusions.map((exclusion) => exclusion.scenarioId)).toContain( + "codex-plugin-cold-install", + ); + await vi.waitFor(() => expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledTimes(1)); + expect(suiteLaunchMock.runQaSuite).toHaveBeenCalledWith( + expect.objectContaining({ + adapterFactories: liveTransportMock.adapterFactories, + channelDriver: "live", + channelId: "telegram", + evidenceMode: "slim", + providerMode: "mock-openai", + runtimePair: ["openclaw", "codex"], + }), + ); + }); + + it("returns explicit exclusions and errors without launching unsupported execution kinds", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "all", + channelDriver: "qa-channel", + providerMode: "live-frontier", + runtimePair: ["openclaw", "codex"], + scenarioIds: ["browser-talk-start-stop"], + }), + }); + + expect(response.status).toBe(400); + const payload = (await response.json()) as { + error: string; + plan: { + status: string; + exclusions: Array<{ scenarioId: string; reasons: string[] }>; + errors: string[]; + }; + }; + expect(payload.plan.status).toBe("invalid"); + expect(payload.plan.exclusions).toEqual([ + expect.objectContaining({ + scenarioId: "browser-talk-start-stop", + reasons: ["runtimePair requires execution.kind=flow"], + }), + ]); + expect(payload.error).toContain("Explicit QA scenario selection is not runnable"); + expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled(); + }); + + it("enforces explicit execution.channel through the shared suite channel planner", async () => { + const lab = await startQaLabServerForTest(); + cleanups.push(async () => { + await lab.stop(); + }); + + const response = await fetch(`${lab.baseUrl}/api/scenario/suite`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + profile: "all", + channel: "telegram", + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["matrix-room-block-streaming"], + }), + }); + + expect(response.status).toBe(400); + const payload = (await response.json()) as { + plan: { exclusions: Array<{ scenarioId: string; reasons: string[] }> }; + }; + expect(payload.plan.exclusions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ scenarioId: "matrix-room-block-streaming" }), + ]), + ); + expect(suiteLaunchMock.runQaSuite).not.toHaveBeenCalled(); + }); + it("cleans up capture state when embedded gateway setup fails", async () => { qaChannelMock.resolveAccount.mockImplementationOnce(() => { throw new Error("embedded setup failed"); @@ -396,7 +743,8 @@ describe("qa-lab server", () => { kickoffTask: string; scenarios: Array<{ id: string; title: string; execution?: { kind?: string } }>; defaults: { conversationId: string; senderId: string }; - runner: { status: string; selection: { providerMode: string; scenarioIds: string[] } }; + runner: { status: string; selection: { providerMode: string; scenarioIds: string[] | null } }; + runnerCatalog: { channels: string[]; profiles: Array<{ id: string }> }; }; expect(bootstrap.defaults.conversationId).toBe("qa-operator"); expect(bootstrap.defaults.senderId).toBe("qa-operator"); @@ -406,13 +754,14 @@ describe("qa-lab server", () => { expect(bootstrap.scenarios.length).toBeGreaterThanOrEqual(10); expect(bootstrap.scenarios.map((scenario) => scenario.id)).toContain("dm-chat-baseline"); expect(bootstrap.runner.status).toBe("idle"); - expect(bootstrap.runner.selection.providerMode).toBe("live-frontier"); - const flowScenarioIds = bootstrap.scenarios - .filter( - (scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow", - ) - .map((scenario) => scenario.id); - expect(bootstrap.runner.selection.scenarioIds).toEqual(flowScenarioIds); + expect(bootstrap.runner.selection.providerMode).toBe("mock-openai"); + expect(bootstrap.runner.selection.scenarioIds).toBeNull(); + expect(bootstrap.runnerCatalog.profiles.map((profile) => profile.id)).toEqual([ + "smoke-ci", + "release", + "all", + ]); + expect(bootstrap.runnerCatalog.channels).toContain("qa-channel"); const startupStatus = (await ( await fetchWithRetry(`${lab.baseUrl}/api/capture/startup-status`) diff --git a/extensions/qa-lab/src/lab-server.ts b/extensions/qa-lab/src/lab-server.ts index ae24eea4ee23..00b47682928b 100644 --- a/extensions/qa-lab/src/lab-server.ts +++ b/extensions/qa-lab/src/lab-server.ts @@ -50,12 +50,15 @@ import type { } from "./lab-server.types.js"; import type { QaRunnerModelOption } from "./model-catalog.runtime.js"; import { createQaChannelGatewayConfig } from "./qa-channel-transport.js"; +import type { QaTransportAdapterFactory } from "./qa-transport-registry.js"; import { createIdleQaRunnerSnapshot, createQaRunOutputDir, normalizeQaRunSelection, + resolveQaLabRunPlan, } from "./run-config.js"; import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; +import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js"; type QaLabBootstrapDefaults = { @@ -300,10 +303,53 @@ export async function startQaLabServer( let latestReport: QaLabLatestReport | null = null; let latestScenarioRun: QaLabScenarioRun | null = null; const scenarioCatalog = readQaBootstrapScenarioCatalog(); + const scorecardReport = readQaScorecardTaxonomyReport(scenarioCatalog.scenarios); + const runnerChannels = [ + ...new Set( + scenarioCatalog.scenarios + .map((scenario) => scenario.execution.channel) + .filter((channel): channel is string => Boolean(channel)), + ), + ].toSorted(); const bootstrapDefaults = createBootstrapDefaults(params?.autoKickoffTarget); let runnerModelOptions: QaRunnerModelOption[] = []; let runnerModelCatalogStatus: "loading" | "ready" | "failed" = "loading"; - let runnerSnapshot = createIdleQaRunnerSnapshot(scenarioCatalog.scenarios); + const resolveServerRunPlan = async ( + selection: ReturnType, + adapterFactories?: readonly QaTransportAdapterFactory[], + ) => { + const crabline = + selection.channelDriver === "crabline" ? await import("@openclaw/crabline") : undefined; + return resolveQaLabRunPlan({ + selection, + scenarios: scenarioCatalog.scenarios, + scorecardReport, + defaultChannel: + crabline?.OPENCLAW_CRABLINE_DEFAULT_CHANNEL ?? + (selection.channelDriver === "qa-channel" ? "qa-channel" : undefined), + ...(crabline + ? { + supportsChannel: (channel: string) => { + try { + crabline.resolveOpenClawCrablineChannelDriverSelection({ channel }); + return true; + } catch { + return false; + } + }, + } + : adapterFactories + ? { + supportsChannel: (channel: string) => + adapterFactories.some((factory) => + factory.matches({ channelId: channel, driver: "live" }), + ), + } + : {}), + }); + }; + let runnerSnapshot = createIdleQaRunnerSnapshot(scorecardReport.profiles); + runnerSnapshot.plan = await resolveServerRunPlan(runnerSnapshot.selection); let activeSuiteRun: Promise | null = null; let controlUiProxyTarget = params?.controlUiProxyTarget?.trim() ? new URL(params.controlUiProxyTarget) @@ -431,6 +477,8 @@ export async function startQaLabServer( runnerCatalog: { status: runnerModelCatalogStatus, real: runnerModelOptions, + profiles: scorecardReport.profiles, + channels: runnerChannels, }, }); return; @@ -681,17 +729,52 @@ export async function startQaLabServer( writeError(res, 409, "QA suite run already in progress"); return; } - const selection = normalizeQaRunSelection( - await readQaJsonBody(req), - scenarioCatalog.scenarios, - ); + let selection: ReturnType; + let plan: ReturnType; + let adapterFactories: readonly QaTransportAdapterFactory[] | undefined; + try { + selection = normalizeQaRunSelection( + await readQaJsonBody(req), + scenarioCatalog.scenarios, + scorecardReport.profiles, + ); + adapterFactories = + selection.channelDriver === "live" + ? (await import("./live-transports/cli.js")).listLiveTransportQaAdapterFactories() + : undefined; + plan = await resolveServerRunPlan(selection, adapterFactories); + } catch (error) { + writeError(res, 400, error); + return; + } + if (plan.status === "invalid") { + writeJson(res, 400, { + error: plan.errors.join(" "), + plan, + }); + return; + } + if (activeSuiteRun) { + writeError(res, 409, "QA suite run already in progress"); + return; + } state.reset(); latestReport = null; - latestScenarioRun = null; const startedAt = new Date().toISOString(); + latestScenarioRun = withQaLabRunCounts({ + kind: "suite", + status: "running", + startedAt, + scenarios: plan.selectedScenarios.map((scenario) => ({ + id: scenario.id, + name: scenario.title, + status: "pending", + })), + }); runnerSnapshot = { status: "running", selection, + plan, startedAt, finishedAt: undefined, artifacts: null, @@ -699,38 +782,86 @@ export async function startQaLabServer( }; activeSuiteRun = (async () => { try { - const { runQaFlowSuite } = await import("./suite.js"); - const result = await runQaFlowSuite({ + const [{ runQaSuite }, channelDriverSelection] = await Promise.all([ + import("./suite-launch.runtime.js"), + selection.channelDriver === "crabline" && selection.channel + ? import("@openclaw/crabline").then((module) => + module.resolveOpenClawCrablineChannelDriverSelection({ + channel: selection.channel!, + }), + ) + : Promise.resolve(undefined), + ]); + const runtimeResult = await runQaSuite({ lab: labHandle ?? undefined, startLab: startQaLabServer, + repoRoot, outputDir: createQaRunOutputDir(repoRoot), + channelDriver: selection.channelDriver, + ...(adapterFactories ? { adapterFactories } : {}), + ...(selection.channelDriver === "live" && selection.channel + ? { channelId: selection.channel } + : {}), + ...(channelDriverSelection ? { channelDriverSelection } : {}), + evidenceMode: selection.evidenceMode, providerMode: selection.providerMode, primaryModel: selection.primaryModel, alternateModel: selection.alternateModel, - scenarioIds: selection.scenarioIds, + fastMode: selection.fastMode, + scenarioIds: plan.selectedScenarios.map((scenario) => scenario.id), + ...(selection.runtimePair ? { runtimePair: selection.runtimePair } : {}), }); + const result = runtimeResult.result; + const finishedAt = new Date().toISOString(); + latestReport = { + outputPath: result.reportPath, + markdown: result.report, + generatedAt: finishedAt, + }; runnerSnapshot = { status: "completed", selection, + plan, startedAt, - finishedAt: new Date().toISOString(), + finishedAt, artifacts: { outputDir: result.outputDir, evidencePath: result.evidencePath, reportPath: result.reportPath, summaryPath: result.summaryPath, - watchUrl: result.watchUrl, + watchUrl: + "watchUrl" in result && typeof result.watchUrl === "string" + ? result.watchUrl + : (labHandle?.baseUrl ?? publicBaseUrl), }, error: null, }; } catch (error) { + const finishedAt = new Date().toISOString(); + const message = formatErrorMessage(error); + latestScenarioRun = withQaLabRunCounts({ + kind: "suite", + status: "completed", + startedAt, + finishedAt, + scenarios: (latestScenarioRun?.scenarios ?? []).map((scenario) => + scenario.status === "pending" || scenario.status === "running" + ? Object.assign({}, scenario, { + status: "fail" as const, + details: message, + finishedAt, + }) + : scenario, + ), + }); runnerSnapshot = { status: "failed", selection, + plan, startedAt, - finishedAt: new Date().toISOString(), + finishedAt, artifacts: null, - error: formatErrorMessage(error), + error: message, }; } finally { activeSuiteRun = null; @@ -738,6 +869,7 @@ export async function startQaLabServer( })(); writeJson(res, 202, { ok: true, + plan, runner: runnerSnapshot, }); return; diff --git a/extensions/qa-lab/src/profile-planning.ts b/extensions/qa-lab/src/profile-planning.ts new file mode 100644 index 000000000000..ca1cec82ac7e --- /dev/null +++ b/extensions/qa-lab/src/profile-planning.ts @@ -0,0 +1,146 @@ +// Qa Lab plugin module owns canonical taxonomy profile membership planning. +import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { QaCliBackendAuthMode } from "./gateway-child.js"; +import type { QaProviderMode } from "./model-selection.js"; +import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { describeQaProviderLaneMismatches } from "./scenario-lane.js"; +import { + readQaScorecardTaxonomyReport, + type QaScorecardCategoryCoverageReport, + type QaScorecardTaxonomyReport, + type QaScorecardChannelDriver, +} from "./scorecard-taxonomy.js"; + +type QaRunProfileMembership = { + categories: QaScorecardCategoryCoverageReport[]; + excludedScenarioIds: string[]; + profile: QaScorecardTaxonomyReport["profiles"][number]; + profileScenarios: QaSeedScenarioWithSource[]; + selectedScenarios: QaSeedScenarioWithSource[]; +}; + +type QaRunProfileExecutionSelection = { + excludedScenarios: Array<{ scenario: QaSeedScenarioWithSource; reasons: string[] }>; + selectedScenarios: QaSeedScenarioWithSource[]; +}; + +function categoryMatchesRunProfile( + category: QaScorecardCategoryCoverageReport, + opts: { profile: string; surface?: string; category?: string }, +): boolean { + if (!category.profiles.includes(opts.profile)) { + return false; + } + if (opts.surface?.trim()) { + const surface = opts.surface.trim(); + if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) { + return false; + } + } + return !opts.category?.trim() || category.id === opts.category.trim(); +} + +export function resolveQaRunProfileMembership( + opts: { + profile: string; + surface?: string; + category?: string; + scenarioIds?: readonly string[]; + }, + source?: { + scenarios?: QaSeedScenarioWithSource[]; + scorecardReport?: QaScorecardTaxonomyReport; + }, +): QaRunProfileMembership { + const scenarios = source?.scenarios ?? readQaScenarioPack().scenarios; + const scorecardReport = source?.scorecardReport ?? readQaScorecardTaxonomyReport(scenarios); + const profileId = opts.profile.trim(); + const profile = scorecardReport.profiles.find((entry) => entry.id === profileId); + if (!profile) { + const profileIds = scorecardReport.profiles.map((entry) => entry.id); + if (profileIds.length === 0) { + throw new Error("taxonomy.yaml does not define QA run profiles."); + } + throw new Error( + `QA run profile must be one of ${profileIds.join(", ")}, got "${opts.profile}".`, + ); + } + const categories = scorecardReport.categories.filter((category) => + categoryMatchesRunProfile(category, { + profile: profileId, + surface: opts.surface, + category: opts.category, + }), + ); + const scenarioBySourcePath = new Map( + scenarios.map((scenario) => [scenario.sourcePath, scenario] as const), + ); + const profileScenarios = uniqueStrings(categories.flatMap((category) => category.scenarioRefs)) + .map((scenarioRef) => scenarioBySourcePath.get(scenarioRef)) + .filter((scenario): scenario is QaSeedScenarioWithSource => scenario !== undefined); + const requestedScenarioIds = uniqueStrings( + (opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean), + ); + if (requestedScenarioIds.length === 0) { + return { + categories, + excludedScenarioIds: [], + profile, + profileScenarios, + selectedScenarios: profileScenarios, + }; + } + const requestedScenarioIdSet = new Set(requestedScenarioIds); + const selectedScenarios = profileScenarios.filter((scenario) => + requestedScenarioIdSet.has(scenario.id), + ); + const selectedScenarioIdSet = new Set(selectedScenarios.map((scenario) => scenario.id)); + return { + categories, + excludedScenarioIds: requestedScenarioIds.filter( + (scenarioId) => !selectedScenarioIdSet.has(scenarioId), + ), + profile, + profileScenarios, + selectedScenarios, + }; +} + +export function resolveQaRunProfileExecutionSelection(params: { + scenarios: readonly QaSeedScenarioWithSource[]; + providerMode: QaProviderMode; + primaryModel: string; + channelDriver: QaScorecardChannelDriver; + channel?: string | null; + defaultChannel?: string; + claudeCliAuthMode?: QaCliBackendAuthMode; +}): QaRunProfileExecutionSelection { + const selectedScenarios: QaSeedScenarioWithSource[] = []; + const excludedScenarios: QaRunProfileExecutionSelection["excludedScenarios"] = []; + for (const scenario of params.scenarios) { + const reasons: string[] = []; + // qa-channel is the built-in harness channel, so another driver cannot implement it. + if (scenario.execution.channel === "qa-channel" && params.channelDriver !== "qa-channel") { + reasons.push("channelDriver=qa-channel"); + } + reasons.push( + ...describeQaProviderLaneMismatches({ + scenario, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + channelDriver: params.channelDriver, + channel: + params.channelDriver === "qa-channel" + ? "qa-channel" + : (params.channel ?? scenario.execution.channel ?? params.defaultChannel), + claudeCliAuthMode: params.claudeCliAuthMode, + }), + ); + if (reasons.length > 0) { + excludedScenarios.push({ scenario, reasons: uniqueStrings(reasons) }); + } else { + selectedScenarios.push(scenario); + } + } + return { excludedScenarios, selectedScenarios }; +} diff --git a/extensions/qa-lab/src/run-config.test.ts b/extensions/qa-lab/src/run-config.test.ts index 5177f6cad61a..343b596280da 100644 --- a/extensions/qa-lab/src/run-config.test.ts +++ b/extensions/qa-lab/src/run-config.test.ts @@ -11,14 +11,29 @@ vi.mock("./model-selection.runtime.js", () => ({ defaultQaRuntimeModelForMode, })); import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js"; +import { + resolveQaRunProfileExecutionSelection, + resolveQaRunProfileMembership, +} from "./profile-planning.js"; import { createIdleQaRunnerSnapshot, createQaRunOutputDir, normalizeQaRunSelection, + resolveQaLabRunPlan, type QaProviderModeInput, } from "./run-config.js"; +import { readQaScenarioPack } from "./scenario-catalog.js"; +import { + readQaScorecardTaxonomyReport, + type QaScorecardTaxonomyReport, +} from "./scorecard-taxonomy.js"; const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier"); +const profiles: QaScorecardTaxonomyReport["profiles"] = [ + { id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] }, + { id: "release", evidenceMode: "full", channelDriver: "live", categoryIds: [] }, + { id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] }, +]; const scenarios = [ { @@ -58,17 +73,25 @@ describe("qa run config", () => { ); }); - it("creates a live-by-default selection that arms flow scenarios", () => { - expect(normalizeQaRunSelection({}, scenarios)).toEqual({ - providerMode: "live-frontier", - primaryModel: DEFAULT_LIVE_FRONTIER_MODEL, - alternateModel: DEFAULT_LIVE_FRONTIER_MODEL, - fastMode: true, - scenarioIds: ["dm-chat-baseline", "thread-lifecycle"], - }); + it("creates a canonical smoke-profile request without copying profile membership", () => { + const expected = { + profile: "smoke-ci", + channel: null, + channelDriver: "crabline", + evidenceMode: "slim", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + alternateModel: "mock-openai/gpt-5.6-luna-alt", + fastMode: false, + runtimePair: null, + runtimePairLane: null, + scenarioIds: null, + } as const; + expect(normalizeQaRunSelection({}, scenarios, profiles)).toEqual(expected); + expect(normalizeQaRunSelection({ profile: null }, scenarios, profiles)).toEqual(expected); }); - it("normalizes live selections and filters unknown scenario ids", () => { + it("normalizes live selections and deduplicates scenario ids", () => { expect( normalizeQaRunSelection( { @@ -76,15 +99,45 @@ describe("qa run config", () => { primaryModel: "openai/gpt-5.6-luna", alternateModel: "", fastMode: false, - scenarioIds: ["thread-lifecycle", "missing", "thread-lifecycle"], + scenarioIds: ["thread-lifecycle", "thread-lifecycle"], }, scenarios, + profiles, ), ).toEqual({ + profile: "all", + channel: null, + channelDriver: "live", + evidenceMode: "full", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", alternateModel: DEFAULT_LIVE_FRONTIER_MODEL, fastMode: true, + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["thread-lifecycle"], + }); + }); + + it("applies the canonical profile execution defaults to explicit scenario requests", () => { + expect( + normalizeQaRunSelection( + { profile: "all", scenarioIds: ["thread-lifecycle"] }, + scenarios, + profiles, + ), + ).toMatchObject({ + profile: "all", + channelDriver: "live", + evidenceMode: "full", + scenarioIds: ["thread-lifecycle"], + }); + expect( + normalizeQaRunSelection({ scenarioIds: ["thread-lifecycle"] }, scenarios, profiles), + ).toMatchObject({ + profile: "all", + channelDriver: "live", + evidenceMode: "full", scenarioIds: ["thread-lifecycle"], }); }); @@ -96,46 +149,464 @@ describe("qa run config", () => { providerMode: "live-openai", }, scenarios, + profiles, ), ).toThrow("unknown QA provider mode: live-openai"); }); - it("falls back to all scenarios when selection would otherwise be empty", () => { - const snapshot = createIdleQaRunnerSnapshot(scenarios); + it("keeps implicit profile membership server-owned and rejects an explicit empty selection", () => { + const snapshot = createIdleQaRunnerSnapshot(profiles); expect(snapshot.status).toBe("idle"); - expect(snapshot.selection.scenarioIds).toEqual(["dm-chat-baseline", "thread-lifecycle"]); - expect( - normalizeQaRunSelection( - { - scenarioIds: [], - }, - scenarios, - ).scenarioIds, - ).toEqual(["dm-chat-baseline", "thread-lifecycle"]); + expect(snapshot.selection.scenarioIds).toBeNull(); + expect(snapshot.plan).toBeNull(); + expect(() => normalizeQaRunSelection({ scenarioIds: [] }, scenarios, profiles)).toThrow( + "scenarioIds must be a non-empty array", + ); }); - it("filters non-flow scenarios from lab runner selections", () => { + it("preserves explicit non-flow scenarios for the mixed-kind suite planner", () => { expect( normalizeQaRunSelection( { scenarioIds: ["control-ui-chat-flow-playwright", "thread-lifecycle"], }, scenarios, + profiles, ).scenarioIds, - ).toEqual(["thread-lifecycle"]); + ).toEqual(["control-ui-chat-flow-playwright", "thread-lifecycle"]); + }); + + it("fails closed on unknown explicit scenario ids", () => { + expect(() => + normalizeQaRunSelection( + { + scenarioIds: ["thread-lifecycle", "missing"], + }, + scenarios, + profiles, + ), + ).toThrow("unknown QA scenario id(s): missing"); + }); + + it("fails closed on malformed explicit profiles", () => { + for (const profile of [false, ""] as const) { + expect(() => + normalizeQaRunSelection( + { + profile, + scenarioIds: ["thread-lifecycle"], + }, + scenarios, + profiles, + ), + ).toThrow("QA runner profile must be a non-empty string"); + } + }); + + it("normalizes the channel driver independently from the provider lane", () => { + expect( + normalizeQaRunSelection( + { + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline"], + }, + scenarios, + profiles, + ), + ).toMatchObject({ channelDriver: "crabline", providerMode: "live-frontier" }); + expect( + normalizeQaRunSelection( + { + channelDriver: "live", + providerMode: "mock-openai", + scenarioIds: ["dm-chat-baseline"], + }, + scenarios, + profiles, + ), + ).toMatchObject({ channelDriver: "live", providerMode: "mock-openai" }); + }); + + it("normalizes every server-owned control-plane axis", () => { + expect( + normalizeQaRunSelection( + { + profile: "all", + channel: " Telegram ", + channelDriver: "crabline", + evidenceMode: "slim", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "extended", + scenarioIds: ["dm-chat-baseline"], + }, + scenarios, + profiles, + ), + ).toMatchObject({ + profile: "all", + channel: "telegram", + channelDriver: "crabline", + evidenceMode: "slim", + providerMode: "live-frontier", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "extended", + scenarioIds: ["dm-chat-baseline"], + }); + }); + + it("rejects a reversed runtime pair instead of silently changing its order", () => { + expect(() => + normalizeQaRunSelection( + { + runtimePair: ["codex", "openclaw"], + }, + scenarios, + profiles, + ), + ).toThrow('runtimePair must be ["openclaw", "codex"]'); + }); + + it("shares implicit profile membership and eligibility with the canonical profile planner", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const selection = normalizeQaRunSelection({}, catalog.scenarios, scorecardReport.profiles); + const membership = resolveQaRunProfileMembership( + { profile: selection.profile }, + { scenarios: catalog.scenarios, scorecardReport }, + ); + const expected = resolveQaRunProfileExecutionSelection({ + scenarios: membership.selectedScenarios, + providerMode: selection.providerMode, + primaryModel: selection.primaryModel, + channelDriver: selection.channelDriver, + channel: selection.channel, + }); + + const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport }); + + expect(plan.status).toBe("ready"); + expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual( + expected.selectedScenarios.map((scenario) => scenario.id), + ); + }); + + it("resolves mixed execution kinds and reports runtime-pair-lane exclusions", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const mixedSelection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "qa-channel", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + const mixedPlan = resolveQaLabRunPlan({ + selection: mixedSelection, + scenarios: catalog.scenarios, + scorecardReport, + }); + expect(mixedPlan.status).toBe("ready"); + expect(mixedPlan.executionKinds).toEqual(["flow", "playwright"]); + + const pairSelection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "qa-channel", + providerMode: "mock-openai", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "core", + }, + catalog.scenarios, + scorecardReport.profiles, + ); + const pairPlan = resolveQaLabRunPlan({ + selection: pairSelection, + scenarios: catalog.scenarios, + scorecardReport, + }); + expect(pairPlan.status).toBe("ready"); + expect(pairPlan.selectedScenarios.map((scenario) => scenario.id)).toContain( + "runtime-first-hour-20-turn", + ); + expect(pairPlan.exclusions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + scenarioId: "codex-plugin-cold-install", + reasons: ["runtimePair requires execution.kind=flow"], + }), + ]), + ); + }); + + it("validates explicit runtime-pair-lane selections without expanding them", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const selectedId = "runtime-first-hour-20-turn"; + const selection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "qa-channel", + providerMode: "mock-openai", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "core", + scenarioIds: [selectedId], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + + const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport }); + + expect(plan.status).toBe("ready"); + expect(plan.selectedScenarios.map((scenario) => scenario.id)).toEqual([selectedId]); + + const outsideLaneSelection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "qa-channel", + providerMode: "mock-openai", + runtimePairLane: "core", + scenarioIds: ["dm-chat-baseline"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + const outsideLanePlan = resolveQaLabRunPlan({ + selection: outsideLaneSelection, + scenarios: catalog.scenarios, + scorecardReport, + }); + expect(outsideLanePlan.status).toBe("invalid"); + expect(outsideLanePlan.selectedScenarios).toEqual([]); + expect(outsideLanePlan.exclusions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + scenarioId: "dm-chat-baseline", + reasons: ["runtimePairLane=core"], + }), + ]), + ); + }); + + it("does not apply live-adapter eligibility to non-flow executions", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const selection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "live", + providerMode: "live-frontier", + scenarioIds: ["browser-talk-start-stop"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + + const plan = resolveQaLabRunPlan({ + selection, + scenarios: catalog.scenarios, + scorecardReport, + supportsChannel: () => false, + }); + + expect(plan.status).toBe("ready"); + expect(plan.selectedScenarios).toEqual([ + expect.objectContaining({ + id: "browser-talk-start-stop", + effectiveChannel: null, + }), + ]); + }); + + it("preserves canonical unresolved live dispatch and rejects unsupported driver channels", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const liveSelection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "live", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + const unresolvedLivePlan = resolveQaLabRunPlan({ + selection: liveSelection, + scenarios: catalog.scenarios, + scorecardReport, + supportsChannel: () => true, + }); + expect(unresolvedLivePlan.status).toBe("ready"); + expect(unresolvedLivePlan.selectedScenarios).toEqual([ + expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: null }), + ]); + + const crablineSelection = normalizeQaRunSelection( + { + profile: "all", + channel: "qa-channel", + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + const unsupportedCrablinePlan = resolveQaLabRunPlan({ + selection: crablineSelection, + scenarios: catalog.scenarios, + scorecardReport, + defaultChannel: "telegram", + supportsChannel: (channel) => channel === "telegram", + }); + expect(unsupportedCrablinePlan.status).toBe("invalid"); + expect(unsupportedCrablinePlan.exclusions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + scenarioId: "dm-chat-baseline", + reasons: ["unsupported crabline channel=qa-channel"], + }), + ]), + ); + }); + + it("fails closed when an explicit scenario conflicts with execution.channel", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const scenario = catalog.scenarios.find( + (entry) => entry.execution.channel && entry.execution.channel !== "qa-channel", + ); + expect(scenario).toBeDefined(); + const selection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "qa-channel", + providerMode: "live-frontier", + scenarioIds: [scenario!.id], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + + const plan = resolveQaLabRunPlan({ selection, scenarios: catalog.scenarios, scorecardReport }); + + expect(plan.status).toBe("invalid"); + expect(plan.exclusions).toEqual( + expect.arrayContaining([expect.objectContaining({ scenarioId: scenario!.id })]), + ); + expect(plan.errors.join(" ")).toContain("Explicit QA scenario selection is not runnable"); + }); + + it("returns an invalid resolved plan for a qa-channel execution override", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const selection = normalizeQaRunSelection( + { + profile: "all", + channel: "telegram", + channelDriver: "qa-channel", + providerMode: "mock-openai", + scenarioIds: ["dm-chat-baseline"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + + const plan = resolveQaLabRunPlan({ + selection, + scenarios: catalog.scenarios, + scorecardReport, + defaultChannel: "qa-channel", + }); + + expect(plan.status).toBe("invalid"); + expect(plan.errors).toContain( + "An execution channel requires channelDriver=crabline or channelDriver=live.", + ); + expect(plan.selectedScenarios).toEqual([ + expect.objectContaining({ id: "dm-chat-baseline", effectiveChannel: "qa-channel" }), + ]); + }); + + it("distinguishes declared constraints from the suite-resolved effective channel", () => { + const catalog = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(catalog.scenarios); + const selection = normalizeQaRunSelection( + { + profile: "all", + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"], + }, + catalog.scenarios, + scorecardReport.profiles, + ); + + const plan = resolveQaLabRunPlan({ + selection, + scenarios: catalog.scenarios, + scorecardReport, + defaultChannel: "telegram", + }); + + expect(plan.selectedScenarios).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "dm-chat-baseline", + declaredChannel: null, + effectiveChannel: "telegram", + }), + expect.objectContaining({ + id: "browser-talk-start-stop", + declaredChannel: null, + effectiveChannel: null, + }), + ]), + ); + }); + + it("rejects malformed requests and unknown channel drivers", () => { + expect(() => normalizeQaRunSelection(null, scenarios, profiles)).toThrow( + "request must be a JSON object", + ); + expect(() => + normalizeQaRunSelection({ channelDriver: "renamed-cli-policy" }, scenarios, profiles), + ).toThrow("unknown QA channel driver: renamed-cli-policy"); }); it("keeps idle snapshots on static defaults so startup does not inspect auth profiles", () => { defaultQaRuntimeModelForMode.mockReturnValue("openai/gpt-5.6-luna"); defaultQaRuntimeModelForMode.mockClear(); - const selection = createIdleQaRunnerSnapshot(scenarios).selection; - expect(selection.providerMode).toBe("live-frontier"); - expect(selection.primaryModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL); - expect(selection.alternateModel).toBe(DEFAULT_LIVE_FRONTIER_MODEL); + const selection = createIdleQaRunnerSnapshot(profiles).selection; + expect(selection.providerMode).toBe("mock-openai"); + expect(selection.primaryModel).toBe("mock-openai/gpt-5.6-luna"); + expect(selection.alternateModel).toBe("mock-openai/gpt-5.6-luna-alt"); expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled(); }); + it("fails closed when required canonical profiles are missing", () => { + const profilesWithoutSmoke = profiles.filter((profile) => profile.id !== "smoke-ci"); + const profilesWithoutAll = profiles.filter((profile) => profile.id !== "all"); + + expect(() => createIdleQaRunnerSnapshot(profilesWithoutSmoke)).toThrow( + "profile table does not define required profile: smoke-ci", + ); + expect(() => normalizeQaRunSelection({}, scenarios, profilesWithoutSmoke)).toThrow( + "profile table does not define required profile: smoke-ci", + ); + expect(() => + normalizeQaRunSelection({ scenarioIds: ["dm-chat-baseline"] }, scenarios, profilesWithoutAll), + ).toThrow("unknown QA run profile: all"); + }); + it("normalizes aimock selections", () => { expect( normalizeQaRunSelection( @@ -146,12 +617,19 @@ describe("qa run config", () => { scenarioIds: ["dm-chat-baseline"], }, scenarios, + profiles, ), ).toEqual({ + profile: "all", + channel: null, + channelDriver: "live", + evidenceMode: "full", providerMode: "aimock", primaryModel: "aimock/gpt-5.6-luna", alternateModel: "aimock/gpt-5.6-luna-alt", fastMode: false, + runtimePair: null, + runtimePairLane: null, scenarioIds: ["dm-chat-baseline"], }); }); @@ -185,12 +663,18 @@ describe("qa run config", () => { : defaultQaProviderModelForMode(mode as QaProviderModeInput, options), ); - expect(normalizeQaRunSelection({}, scenarios)).toEqual({ + expect(normalizeQaRunSelection({ profile: "release" }, scenarios, profiles)).toEqual({ + profile: "release", + channel: null, + channelDriver: "live", + evidenceMode: "full", providerMode: "live-frontier", primaryModel: "openai/gpt-5.6-luna", alternateModel: "openai/gpt-5.6-luna", fastMode: true, - scenarioIds: ["dm-chat-baseline", "thread-lifecycle"], + runtimePair: null, + runtimePairLane: null, + scenarioIds: null, }); }); }); diff --git a/extensions/qa-lab/src/run-config.ts b/extensions/qa-lab/src/run-config.ts index fa21f16255db..eca59e6f1e8f 100644 --- a/extensions/qa-lab/src/run-config.ts +++ b/extensions/qa-lab/src/run-config.ts @@ -2,8 +2,18 @@ import { randomUUID } from "node:crypto"; import path from "node:path"; import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { + QaLabExecutionKind, + QaLabResolvedRunPlan, + QaLabRunnerSnapshot, + QaLabRunSelection, +} from "../runner-contract.js"; import { defaultQaModelForMode as defaultStaticQaModelForMode } from "./model-selection.js"; import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js"; +import { + resolveQaRunProfileExecutionSelection, + resolveQaRunProfileMembership, +} from "./profile-planning.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider, @@ -11,35 +21,26 @@ import { normalizeQaProviderMode as normalizeQaProviderModeInput, type QaProviderMode, } from "./providers/index.js"; -import type { QaSeedScenario } from "./scenario-catalog.js"; +import { + resolveQaRuntimePairLaneScenarioIds, + resolveQaRuntimePairScenarioSupport, +} from "./runtime-pair-lane-selection.js"; +import { + qaRuntimePairLaneSchema, + type QaSeedScenario, + type QaSeedScenarioWithSource, +} from "./scenario-catalog.js"; +import { + qaScorecardChannelDriverSchema, + qaScorecardEvidenceModeSchema, + type QaScorecardTaxonomyReport, +} from "./scorecard-taxonomy.js"; +import { resolveQaSuiteScenarioChannels } from "./suite-planning.js"; export type { QaProviderMode } from "./model-selection.js"; export type { QaProviderModeInput } from "./providers/index.js"; -type QaLabRunSelection = { - providerMode: QaProviderMode; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - scenarioIds: string[]; -}; - -type QaLabRunArtifacts = { - outputDir: string; - evidencePath: string; - reportPath: string; - summaryPath: string; - watchUrl: string; -}; - -type QaLabRunnerSnapshot = { - status: "idle" | "running" | "completed" | "failed"; - selection: QaLabRunSelection; - startedAt?: string; - finishedAt?: string; - artifacts: QaLabRunArtifacts | null; - error: string | null; -}; +type QaLabRunProfileOption = QaScorecardTaxonomyReport["profiles"][number]; export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) { return defaultQaRuntimeModelForMode(mode, alternate ? { alternate: true } : undefined); @@ -51,26 +52,33 @@ function defaultStaticModelForMode(mode: QaProviderMode, alternate = false) { return defaultStaticQaModelForMode(mode, alternate ? { alternate: true } : undefined); } -function qaLabFlowScenarioIds(scenarios: QaSeedScenario[]) { - return scenarios - .filter( - (scenario) => scenario.execution?.kind === undefined || scenario.execution.kind === "flow", - ) - .map((scenario) => scenario.id); +function requireQaRunProfile(profiles: readonly QaLabRunProfileOption[], profileId: string) { + const profile = profiles.find((entry) => entry.id === profileId); + if (!profile) { + throw new Error(`QA runner profile table does not define required profile: ${profileId}`); + } + return profile; } function createDefaultQaRunSelection( - scenarios: QaSeedScenario[], + profiles: readonly QaLabRunProfileOption[], options?: { resolveDefaultModel?: QaDefaultModelResolver }, ): QaLabRunSelection { - const providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE; + const profile = requireQaRunProfile(profiles, "smoke-ci"); + const providerMode: QaProviderMode = "mock-openai"; const resolveDefaultModel = options?.resolveDefaultModel ?? defaultQaModelForMode; return { + profile: profile.id, + channel: null, + channelDriver: profile.channelDriver, + evidenceMode: profile.evidenceMode, providerMode, primaryModel: resolveDefaultModel(providerMode), alternateModel: resolveDefaultModel(providerMode, true), - fastMode: true, - scenarioIds: qaLabFlowScenarioIds(scenarios), + fastMode: getQaProvider(providerMode).kind === "live", + runtimePair: null, + runtimePairLane: null, + scenarioIds: null, }; } @@ -90,25 +98,143 @@ function normalizeModel(input: unknown, fallback: string) { return value || fallback; } -function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]) { - const defaultScenarioIds = qaLabFlowScenarioIds(scenarios); - const availableIds = new Set(defaultScenarioIds); - const requestedIds = Array.isArray(input) - ? input - .map((value) => (typeof value === "string" ? value.trim() : "")) - .filter((value) => value.length > 0) - : []; - const selectedIds = uniqueStrings(requestedIds.filter((id) => availableIds.has(id))); - return selectedIds.length > 0 ? selectedIds : defaultScenarioIds; +function normalizeScenarioIds(input: unknown, scenarios: QaSeedScenario[]): string[] | null { + if (input === undefined || input === null) { + return null; + } + if (!Array.isArray(input) || input.length === 0) { + throw new Error("QA runner scenarioIds must be a non-empty array"); + } + const requestedIds = input.map((value) => { + if (typeof value !== "string" || !value.trim()) { + throw new Error("QA runner scenarioIds must contain non-empty strings"); + } + return value.trim(); + }); + const selectedIds = uniqueStrings(requestedIds); + const availableIds = new Set(scenarios.map((scenario) => scenario.id)); + const unknownIds = selectedIds.filter((id) => !availableIds.has(id)); + if (unknownIds.length > 0) { + throw new Error(`unknown QA scenario id(s): ${unknownIds.join(", ")}`); + } + return selectedIds; +} + +function normalizeQaChannelDriver( + input: unknown, + fallback: QaLabRunSelection["channelDriver"], +): QaLabRunSelection["channelDriver"] { + if (input === undefined || input === null || input === "") { + return fallback; + } + const parsed = qaScorecardChannelDriverSchema.safeParse(input); + if (!parsed.success) { + const details = typeof input === "string" ? `: ${input}` : ""; + throw new Error(`unknown QA channel driver${details}`); + } + return parsed.data; +} + +function normalizeQaProfile( + input: unknown, + profiles: readonly QaLabRunProfileOption[], + fallbackProfile?: string, +) { + const fallback = fallbackProfile ?? requireQaRunProfile(profiles, "smoke-ci").id; + // Match the other optional runner controls: null means no override, while any + // concrete profile value must be a non-empty string or the request fails closed. + if (input !== undefined && input !== null && (typeof input !== "string" || !input.trim())) { + throw new Error("QA runner profile must be a non-empty string"); + } + const profile = typeof input === "string" ? input.trim() : fallback; + if (!profiles.some((entry) => entry.id === profile)) { + throw new Error( + `unknown QA run profile: ${profile}; expected one of ${profiles.map((entry) => entry.id).join(", ")}`, + ); + } + return profile; +} + +function normalizeQaChannel(input: unknown): string | null { + if (input === undefined || input === null || input === "") { + return null; + } + if (typeof input !== "string" || !input.trim()) { + throw new Error("QA runner channel must be a non-empty string"); + } + return input.trim().toLowerCase(); +} + +function normalizeQaEvidenceMode( + input: unknown, + fallback: QaLabRunSelection["evidenceMode"], +): QaLabRunSelection["evidenceMode"] { + if (input === undefined || input === null || input === "") { + return fallback; + } + const parsed = qaScorecardEvidenceModeSchema.safeParse(input); + if (!parsed.success) { + const details = typeof input === "string" ? `: ${input}` : ""; + throw new Error(`unknown QA evidence mode${details}`); + } + return parsed.data; +} + +function normalizeQaRuntimePair(input: unknown): QaLabRunSelection["runtimePair"] { + if (input === undefined || input === null) { + return null; + } + if ( + !Array.isArray(input) || + input.length !== 2 || + !input.every((runtime) => runtime === "openclaw" || runtime === "codex") + ) { + throw new Error('QA runner runtimePair must be ["openclaw", "codex"]'); + } + if (input[0] === input[1]) { + throw new Error("QA runner runtimePair must compare two different runtimes"); + } + if (input[0] !== "openclaw" || input[1] !== "codex") { + throw new Error('QA runner runtimePair must be ["openclaw", "codex"]'); + } + return ["openclaw", "codex"]; +} + +function normalizeQaRuntimePairLane(input: unknown): QaLabRunSelection["runtimePairLane"] { + if (input === undefined || input === null || input === "") { + return null; + } + const parsed = qaRuntimePairLaneSchema.safeParse(input); + if (!parsed.success) { + const details = typeof input === "string" ? `: ${input}` : ""; + throw new Error(`unknown QA runtime-pair lane${details}`); + } + return parsed.data; } export function normalizeQaRunSelection( input: unknown, scenarios: QaSeedScenario[], + profiles: readonly QaLabRunProfileOption[], ): QaLabRunSelection { - const payload = input && typeof input === "object" ? (input as Record) : {}; - const providerMode = normalizeQaProviderMode(payload.providerMode); + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new Error("QA runner request must be a JSON object"); + } + const payload = input as Record; + const profile = normalizeQaProfile( + payload.profile, + profiles, + Array.isArray(payload.scenarioIds) ? "all" : undefined, + ); + const profileDefaults = requireQaRunProfile(profiles, profile); + const providerMode = normalizeQaProviderMode( + payload.providerMode ?? (profile === "smoke-ci" ? "mock-openai" : undefined), + ); return { + profile, + channel: normalizeQaChannel(payload.channel), + channelDriver: normalizeQaChannelDriver(payload.channelDriver, profileDefaults.channelDriver), + evidenceMode: normalizeQaEvidenceMode(payload.evidenceMode, profileDefaults.evidenceMode), providerMode, primaryModel: normalizeModel(payload.primaryModel, defaultQaModelForMode(providerMode)), alternateModel: normalizeModel( @@ -116,16 +242,226 @@ export function normalizeQaRunSelection( defaultQaModelForMode(providerMode, true), ), fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true, + runtimePair: normalizeQaRuntimePair(payload.runtimePair), + runtimePairLane: normalizeQaRuntimePairLane(payload.runtimePairLane), scenarioIds: normalizeScenarioIds(payload.scenarioIds, scenarios), }; } -export function createIdleQaRunnerSnapshot(scenarios: QaSeedScenario[]): QaLabRunnerSnapshot { +function effectiveChannelForScenario(params: { + scenario: QaSeedScenarioWithSource; + selection: QaLabRunSelection; + defaultChannel?: string; +}): string | null { + // Catalog parsing defaults flow executions to kind="flow"; plans never consume raw YAML. + if (params.scenario.execution.kind !== "flow") { + return null; + } + const fallbackChannel = + params.defaultChannel ?? params.selection.channel ?? params.scenario.execution.channel; + if (!fallbackChannel) { + return null; + } + return ( + resolveQaSuiteScenarioChannels({ + defaultChannel: fallbackChannel, + explicitChannel: + params.selection.channelDriver === "qa-channel" ? null : params.selection.channel, + scenarios: [params.scenario], + })[0] ?? null + ); +} + +export function resolveQaLabRunPlan(params: { + selection: QaLabRunSelection; + scenarios: QaSeedScenarioWithSource[]; + scorecardReport: QaScorecardTaxonomyReport; + defaultChannel?: string; + supportsChannel?: (channel: string) => boolean; +}): QaLabResolvedRunPlan { + const { selection } = params; + const explicitScenarioSelection = selection.scenarioIds !== null; + const membership = resolveQaRunProfileMembership( + { + profile: selection.profile, + scenarioIds: selection.scenarioIds ?? undefined, + }, + { scenarios: params.scenarios, scorecardReport: params.scorecardReport }, + ); + const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario])); + const exclusions: QaLabResolvedRunPlan["exclusions"] = membership.excludedScenarioIds.map( + (scenarioId) => { + const scenario = scenarioById.get(scenarioId); + return { + scenarioId, + executionKind: scenario ? scenario.execution.kind : "flow", + reasons: [`not a member of profile ${selection.profile}`], + }; + }, + ); + let laneSelection: ReturnType; + const errors: string[] = []; + try { + laneSelection = resolveQaRuntimePairLaneScenarioIds({ + channel: selection.channel, + channelDriver: selection.channelDriver, + defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined, + primaryModel: selection.primaryModel, + providerMode: selection.providerMode, + scenarioIds: selection.runtimePairLane + ? [] + : membership.selectedScenarios.map((scenario) => scenario.id), + scenarios: membership.profileScenarios, + runtimePairLanes: selection.runtimePairLane ? [selection.runtimePairLane] : [], + runtimePair: selection.runtimePair !== null, + }); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + laneSelection = { + scenarioIds: [], + excludedLaneScenarios: [], + excludedNonFlowScenarios: [], + }; + } + exclusions.push( + ...laneSelection.excludedLaneScenarios.map((scenario) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons: ["does not match the selected provider/model/channel lane"], + })), + ...laneSelection.excludedNonFlowScenarios.map((scenario) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons: ["runtimePair requires execution.kind=flow"], + })), + ); + if (selection.runtimePairLane && explicitScenarioSelection) { + const laneScenarioIds = new Set(laneSelection.scenarioIds); + const alreadyExcludedIds = new Set(exclusions.map((exclusion) => exclusion.scenarioId)); + exclusions.push( + ...membership.selectedScenarios + .filter( + (scenario) => !laneScenarioIds.has(scenario.id) && !alreadyExcludedIds.has(scenario.id), + ) + .map((scenario) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons: [`runtimePairLane=${selection.runtimePairLane}`], + })), + ); + laneSelection = { + ...laneSelection, + scenarioIds: membership.selectedScenarios + .filter((scenario) => laneScenarioIds.has(scenario.id)) + .map((scenario) => scenario.id), + }; + } + const laneScenarios = laneSelection.scenarioIds.flatMap((scenarioId) => { + const scenario = scenarioById.get(scenarioId); + return scenario ? [scenario] : []; + }); + const profileExecution = resolveQaRunProfileExecutionSelection({ + scenarios: laneScenarios, + providerMode: selection.providerMode, + primaryModel: selection.primaryModel, + channelDriver: selection.channelDriver, + channel: selection.channel, + defaultChannel: selection.channelDriver === "crabline" ? params.defaultChannel : undefined, + }); + exclusions.push( + ...profileExecution.excludedScenarios.map(({ scenario, reasons }) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons, + })), + ); + const runtimePairSupport = selection.runtimePair + ? resolveQaRuntimePairScenarioSupport(profileExecution.selectedScenarios) + : { selectedScenarios: profileExecution.selectedScenarios, excludedScenarios: [] }; + exclusions.push( + ...runtimePairSupport.excludedScenarios.map((scenario) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons: ["runtimePair requires execution.kind=flow"], + })), + ); + let selectedScenarios = runtimePairSupport.selectedScenarios; + if (selection.channelDriver !== "qa-channel" && params.supportsChannel) { + const unsupportedChannelScenarios = selectedScenarios.flatMap((scenario) => { + const channel = effectiveChannelForScenario({ + scenario, + selection, + defaultChannel: params.defaultChannel, + }); + if (scenario.execution.kind !== "flow") { + return []; + } + return channel !== null && !params.supportsChannel?.(channel) ? [{ scenario, channel }] : []; + }); + const unsupportedIds = new Set(unsupportedChannelScenarios.map(({ scenario }) => scenario.id)); + exclusions.push( + ...unsupportedChannelScenarios.map(({ scenario, channel }) => ({ + scenarioId: scenario.id, + executionKind: scenario.execution.kind, + reasons: [`unsupported ${selection.channelDriver} channel=${channel}`], + })), + ); + selectedScenarios = selectedScenarios.filter((scenario) => !unsupportedIds.has(scenario.id)); + } + if (membership.categories.length === 0) { + errors.push(`QA run profile ${selection.profile} did not resolve any taxonomy categories.`); + } + if (selection.channel && selection.channelDriver === "qa-channel") { + errors.push("An execution channel requires channelDriver=crabline or channelDriver=live."); + } + const explicitScenarioIds = new Set(selection.scenarioIds ?? []); + const explicitExclusions = exclusions.filter((exclusion) => + explicitScenarioIds.has(exclusion.scenarioId), + ); + if (explicitScenarioSelection && explicitExclusions.length > 0) { + errors.push( + `Explicit QA scenario selection is not runnable: ${explicitExclusions + .map((exclusion) => `${exclusion.scenarioId} (${exclusion.reasons.join(", ")})`) + .join("; ")}.`, + ); + } + if (selectedScenarios.length === 0) { + errors.push("QA run plan selected no runnable scenarios."); + } + const executionKinds = uniqueStrings( + selectedScenarios.map((scenario) => scenario.execution.kind), + ) as QaLabExecutionKind[]; + return { + status: errors.length > 0 ? "invalid" : "ready", + profile: selection.profile, + explicitScenarioSelection, + selectedScenarios: selectedScenarios.map((scenario) => ({ + id: scenario.id, + title: scenario.title, + executionKind: scenario.execution.kind, + declaredChannel: scenario.execution.channel ?? null, + effectiveChannel: effectiveChannelForScenario({ + scenario, + selection, + defaultChannel: params.defaultChannel, + }), + })), + executionKinds, + exclusions, + errors: uniqueStrings(errors), + }; +} + +export function createIdleQaRunnerSnapshot( + profiles: readonly QaLabRunProfileOption[], + plan: QaLabResolvedRunPlan | null = null, +): QaLabRunnerSnapshot { return { status: "idle", - selection: createDefaultQaRunSelection(scenarios, { + selection: createDefaultQaRunSelection(profiles, { resolveDefaultModel: defaultStaticModelForMode, }), + plan, artifacts: null, error: null, }; diff --git a/extensions/qa-lab/src/runtime-pair-lane-selection.ts b/extensions/qa-lab/src/runtime-pair-lane-selection.ts new file mode 100644 index 000000000000..e42c75ab923d --- /dev/null +++ b/extensions/qa-lab/src/runtime-pair-lane-selection.ts @@ -0,0 +1,91 @@ +// Qa Lab plugin module owns canonical runtime-pair-lane scenario selection. +import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { QaCliBackendAuthMode } from "./gateway-child.js"; +import type { QaProviderMode } from "./model-selection.js"; +import { + readQaScenarioPack, + type QaRuntimePairLane, + type QaSeedScenarioWithSource, +} from "./scenario-catalog.js"; +import { scenarioMatchesQaProviderLane } from "./scenario-lane.js"; +import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js"; + +export function resolveQaRuntimePairScenarioSupport( + scenarios: readonly QaSeedScenarioWithSource[], +) { + return { + selectedScenarios: scenarios.filter((scenario) => scenario.execution.kind === "flow"), + excludedScenarios: scenarios.filter((scenario) => scenario.execution.kind !== "flow"), + }; +} + +export function resolveQaRuntimePairLaneScenarioIds(params: { + channel?: string | null; + channelDriver?: QaScorecardChannelDriver | null; + claudeCliAuthMode?: QaCliBackendAuthMode; + defaultChannel?: string; + primaryModel: string; + providerMode: QaProviderMode; + scenarioIds: string[]; + scenarios?: QaSeedScenarioWithSource[]; + runtimePairLanes: readonly QaRuntimePairLane[]; + runtimePair: boolean; +}): { + scenarioIds: string[]; + excludedLaneScenarios: QaSeedScenarioWithSource[]; + excludedNonFlowScenarios: QaSeedScenarioWithSource[]; +} { + if (params.runtimePairLanes.length === 0) { + return { + scenarioIds: params.scenarioIds, + excludedLaneScenarios: [], + excludedNonFlowScenarios: [], + }; + } + const laneSet = new Set(params.runtimePairLanes); + const matchingScenarios = (params.scenarios ?? readQaScenarioPack().scenarios).filter( + (scenario) => scenario.runtimePairLane && laneSet.has(scenario.runtimePairLane), + ); + if (matchingScenarios.length === 0) { + throw new Error( + `--runtime-pair-lane matched no scenarios for ${params.runtimePairLanes.join(", ")}.`, + ); + } + const compatibleScenarios = params.runtimePair + ? matchingScenarios.filter((scenario) => scenario.execution.kind === "flow") + : matchingScenarios; + const laneCompatibleScenarios = compatibleScenarios.filter((scenario) => + scenarioMatchesQaProviderLane({ + scenario, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + channelDriver: params.channelDriver, + channel: params.channel ?? scenario.execution.channel ?? params.defaultChannel, + claudeCliAuthMode: params.claudeCliAuthMode, + }), + ); + const excludedLaneScenarios = compatibleScenarios.filter( + (scenario) => !laneCompatibleScenarios.includes(scenario), + ); + const excludedNonFlowScenarios = params.runtimePair + ? matchingScenarios.filter((scenario) => scenario.execution.kind !== "flow") + : []; + if (compatibleScenarios.length === 0) { + throw new Error( + `--runtime-pair-lane matched no execution.kind: flow scenarios for ${params.runtimePairLanes.join(", ")}; incompatible scenario(s): ${excludedNonFlowScenarios.map((scenario) => `${scenario.id} (${scenario.execution.kind})`).join(", ")}.`, + ); + } + if (params.scenarioIds.length === 0 && laneCompatibleScenarios.length === 0) { + throw new Error( + `--runtime-pair-lane matched no scenarios for provider mode ${params.providerMode}; incompatible scenario(s): ${excludedLaneScenarios.map((scenario) => scenario.id).join(", ")}.`, + ); + } + return { + scenarioIds: uniqueStrings([ + ...params.scenarioIds, + ...laneCompatibleScenarios.map((scenario) => scenario.id), + ]), + excludedLaneScenarios, + excludedNonFlowScenarios, + }; +} diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 0ce5a543d25c..b46e7c3584cd 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { QaLabServerHandle } from "./lab-server.types.js"; import type { QaSuiteScenarioResult } from "./suite.js"; import type { QaTestFileScenario, @@ -542,7 +543,19 @@ describe("qa suite runtime launcher", () => { it("routes selected Playwright scenarios to the Playwright scenario runner", async () => { const repoRoot = await makeTempRepo("qa-suite-launch-"); + const setScenarioRun = vi.fn(); + const lab = { + baseUrl: "http://127.0.0.1:43124", + listenUrl: "http://127.0.0.1:43124", + runSelfCheck: vi.fn(), + setControlUi: vi.fn(), + setLatestReport: vi.fn(), + setScenarioRun, + state: {} as QaLabServerHandle["state"], + stop: vi.fn(), + } satisfies QaLabServerHandle; const result = await runQaSuite({ + lab, repoRoot, outputDir: ".artifacts/qa-e2e/scenario-test", scenarioIds: ["control-ui-chat-flow-playwright"], @@ -582,6 +595,14 @@ describe("qa suite runtime launcher", () => { kind: scenario.execution.kind, })), ).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]); + expect(setScenarioRun).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "completed", + scenarios: [ + expect.objectContaining({ id: "control-ui-chat-flow-playwright", status: "pass" }), + ], + }), + ); }); it("serializes test-file runner partitions in one checkout", async () => { @@ -699,6 +720,96 @@ describe("qa suite runtime launcher", () => { ); }); + it("aggregates mixed-kind progress through the parent lab", async () => { + const repoRoot = await makeTempRepo("qa-suite-mixed-progress-"); + const scenarioRuns: Array[0]> = []; + const lab = { + baseUrl: "http://127.0.0.1:43124", + listenUrl: "http://127.0.0.1:43124", + runSelfCheck: vi.fn(), + setControlUi: vi.fn(), + setLatestReport: vi.fn(), + setScenarioRun: vi.fn((run) => scenarioRuns.push(run)), + state: {} as QaLabServerHandle["state"], + stop: vi.fn(), + } satisfies QaLabServerHandle; + const defaultFlowImplementation = runQaFlowSuite.getMockImplementation(); + if (!defaultFlowImplementation) { + throw new Error("expected default QA flow suite mock implementation"); + } + runQaFlowSuite.mockImplementationOnce(async (params) => { + params?.lab?.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: "2026-07-23T00:00:00.000Z", + scenarios: [ + { id: "channel-chat-baseline", name: "channel-chat-baseline", status: "running" }, + ], + }); + params?.lab?.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: "2026-07-23T00:00:00.000Z", + scenarios: [ + { id: "not-a-selected-scenario", name: "channel-chat-baseline", status: "fail" }, + ], + }); + const result = await defaultFlowImplementation(params); + params?.lab?.setScenarioRun({ + kind: "suite", + status: "completed", + startedAt: "2026-07-23T00:00:00.000Z", + finishedAt: "2026-07-23T00:00:01.000Z", + scenarios: [{ id: "channel-chat-baseline", name: "channel-chat-baseline", status: "pass" }], + }); + return result; + }); + + await runQaSuite({ + lab, + repoRoot, + outputDir: ".artifacts/qa-e2e/mixed-progress", + scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"], + }); + + expect( + scenarioRuns.some( + (run) => + run?.status === "running" && + run.scenarios.some( + (scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "running", + ), + ), + ).toBe(true); + expect( + scenarioRuns.some((run) => + run?.scenarios.some( + (scenario) => scenario.id === "channel-chat-baseline" && scenario.status === "fail", + ), + ), + ).toBe(false); + expect( + scenarioRuns.some( + (run) => + run?.status === "running" && + run.scenarios.some( + (scenario) => + scenario.id === "control-ui-chat-flow-playwright" && scenario.status === "running", + ), + ), + ).toBe(true); + expect(scenarioRuns.at(-1)).toMatchObject({ + status: "completed", + scenarios: [ + { id: "channel-chat-baseline", status: "pass" }, + { id: "control-ui-chat-flow-playwright", status: "pass" }, + ], + }); + expect(lab.setLatestReport).toHaveBeenCalledWith( + expect.objectContaining({ outputPath: expect.stringMatching(/qa-suite-report\.md$/u) }), + ); + }); + it("keeps channel-driver unified flow partitions serial by default", async () => { const repoRoot = await makeTempRepo("qa-suite-crabline-serial-"); await runQaSuite({ @@ -870,6 +981,17 @@ describe("qa suite runtime launcher", () => { it("stops unified suite partitions after the first failed flow scenario", async () => { const repoRoot = await makeTempRepo("qa-suite-fail-fast-flow-"); + const scenarioRuns: Array[0]> = []; + const lab = { + baseUrl: "http://127.0.0.1:43124", + listenUrl: "http://127.0.0.1:43124", + runSelfCheck: vi.fn(), + setControlUi: vi.fn(), + setLatestReport: vi.fn(), + setScenarioRun: vi.fn((run) => scenarioRuns.push(run)), + state: {} as QaLabServerHandle["state"], + stop: vi.fn(), + } satisfies QaLabServerHandle; const defaultFlowImplementation = runQaFlowSuite.getMockImplementation(); if (!defaultFlowImplementation) { throw new Error("expected default QA flow suite mock implementation"); @@ -888,6 +1010,7 @@ describe("qa suite runtime launcher", () => { }); const result = await runQaSuite({ + lab, repoRoot, outputDir: ".artifacts/qa-e2e/fail-fast-flow", concurrency: 8, @@ -928,6 +1051,15 @@ describe("qa suite runtime launcher", () => { "docker-npm-onboard-channel-agent", ]); expect(summary.scenarios).toMatchObject([{ name: "dm-chat-baseline", status: "fail" }]); + expect(scenarioRuns.at(-1)).toMatchObject({ + status: "completed", + scenarios: [ + { id: "dm-chat-baseline", status: "fail" }, + { id: "group-visible-reply-tool", status: "pending" }, + { id: "control-ui-chat-flow-playwright", status: "pending" }, + { id: "docker-npm-onboard-channel-agent", status: "pending" }, + ], + }); }); it("stops pending flow and script partitions after a native scenario fails", async () => { diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index 23cb68e21956..bdc43ab0e77c 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -32,6 +32,7 @@ import { resolveQaSuiteWorkerStartStaggerMs, scenarioRequiresIsolatedQaSuiteWorker, } from "./suite-planning.js"; +import { createQaSuiteProgressController } from "./suite-progress.js"; import { buildQaSuiteSummaryJson, type QaSuiteResult, @@ -604,6 +605,14 @@ async function runUnifiedQaSuite(params: { const startedAt = new Date(); const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd()); const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir); + const progress = params.runParams?.lab + ? createQaSuiteProgressController({ + lab: params.runParams.lab, + scenarios: params.plan.scenarios, + startedAt: startedAt.toISOString(), + }) + : undefined; + progress?.start(); const providerMode = normalizeQaProviderMode( params.runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE, ); @@ -774,6 +783,13 @@ async function runUnifiedQaSuite(params: { } const result = await runFlowSuite({ ...params.runParams, + ...(progress + ? { + lab: progress.createPartitionLab( + partition.scenarios.map((scenario) => scenario.id), + ), + } + : {}), outputDir: partitionName ? flowSuitePartitionOutputDir(outputDir, partitionName) : suitePartitionOutputDir(outputDir, "flow"), @@ -852,6 +868,11 @@ async function runUnifiedQaSuite(params: { const testFileScenarioResults: QaUnifiedPartitionResult["scenarioResults"] = []; const testFileStartedScenarioIds: string[] = []; for (const [kind, testFileScenarios] of scenariosByKind) { + progress?.markRunning( + (failFast ? testFileScenarios.slice(0, 1) : testFileScenarios).map( + (scenario) => scenario.id, + ), + ); const result = await runQaTestFileSuiteFromRuntime({ runParams: { ...params.runParams, @@ -869,12 +890,12 @@ async function runUnifiedQaSuite(params: { repoRoot, }), ); - testFileScenarioResults.push( - ...result.results.map((scenarioResult) => ({ - scenarioId: scenarioResult.scenario.id, - result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot), - })), - ); + const scenarioResults = result.results.map((scenarioResult) => ({ + scenarioId: scenarioResult.scenario.id, + result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot), + })); + testFileScenarioResults.push(...scenarioResults); + progress?.recordResults(scenarioResults); const resultsByScenarioId = new Map( result.results.map((scenarioResult) => [scenarioResult.scenario.id, scenarioResult]), ); @@ -1049,7 +1070,7 @@ async function runUnifiedQaSuite(params: { } satisfies QaSuiteScenarioResult, ]; }); - return await writeUnifiedQaSuiteArtifacts({ + const unifiedResult = await writeUnifiedQaSuiteArtifacts({ alternateModel, channelDriver: params.runParams?.channelDriver, concurrency, @@ -1064,6 +1085,33 @@ async function runUnifiedQaSuite(params: { scenarios, startedAt, }); + const progressResults = params.plan.scenarios.flatMap((scenario) => { + const result = scenarioResultsById.get(scenario.id); + if (result) { + return [{ scenarioId: scenario.id, result }]; + } + if (failFast && !startedScenarioIds.has(scenario.id)) { + return []; + } + return [ + { + scenarioId: scenario.id, + result: { + name: scenario.title, + status: "fail" as const, + details: "suite partition returned no scenario result", + steps: [], + }, + }, + ]; + }); + progress?.complete(progressResults, finishedAt.toISOString()); + params.runParams?.lab?.setLatestReport({ + outputPath: unifiedResult.reportPath, + markdown: unifiedResult.report, + generatedAt: finishedAt.toISOString(), + }); + return unifiedResult; } export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise { diff --git a/extensions/qa-lab/src/suite-progress.ts b/extensions/qa-lab/src/suite-progress.ts new file mode 100644 index 000000000000..54a7b3e41306 --- /dev/null +++ b/extensions/qa-lab/src/suite-progress.ts @@ -0,0 +1,126 @@ +import type { + QaLabScenarioOutcome, + QaLabScenarioRun, + QaLabServerHandle, +} from "./lab-server.types.js"; + +type QaSuiteProgressScenario = { + id: string; + title: string; +}; + +type QaSuiteProgressResult = { + scenarioId: string; + result: { + name: string; + status: "pass" | "fail" | "skip"; + steps: QaLabScenarioOutcome["steps"]; + details?: string; + }; +}; + +function cloneOutcome(outcome: QaLabScenarioOutcome): QaLabScenarioOutcome { + return { + ...outcome, + ...(outcome.steps ? { steps: outcome.steps.map((step) => ({ ...step })) } : {}), + }; +} + +export function createQaSuiteProgressController(params: { + lab: QaLabServerHandle; + scenarios: readonly QaSuiteProgressScenario[]; + startedAt: string; +}) { + const outcomes = new Map( + params.scenarios.map((scenario) => [ + scenario.id, + { + id: scenario.id, + name: scenario.title, + status: "pending" as const, + }, + ]), + ); + + const emit = (status: QaLabScenarioRun["status"], finishedAt?: string) => { + params.lab.setScenarioRun({ + kind: "suite", + status, + startedAt: params.startedAt, + ...(finishedAt ? { finishedAt } : {}), + scenarios: params.scenarios.map((scenario) => cloneOutcome(outcomes.get(scenario.id)!)), + }); + }; + + const updateResult = (entry: QaSuiteProgressResult, finishedAt?: string) => { + const current = outcomes.get(entry.scenarioId); + if (!current) { + return; + } + outcomes.set(entry.scenarioId, { + ...current, + name: entry.result.name, + status: entry.result.status, + ...(entry.result.details ? { details: entry.result.details } : {}), + ...(entry.result.steps ? { steps: entry.result.steps } : {}), + ...(finishedAt ? { finishedAt } : {}), + }); + }; + + return { + start() { + emit("running"); + }, + markRunning(scenarioIds: readonly string[]) { + const startedAt = new Date().toISOString(); + for (const scenarioId of scenarioIds) { + const current = outcomes.get(scenarioId); + if (!current || current.status !== "pending") { + continue; + } + outcomes.set(scenarioId, { ...current, status: "running", startedAt }); + } + emit("running"); + }, + recordResults(entries: readonly QaSuiteProgressResult[]) { + const finishedAt = new Date().toISOString(); + for (const entry of entries) { + updateResult(entry, finishedAt); + } + emit("running"); + }, + createPartitionLab(scenarioIds: readonly string[]): QaLabServerHandle { + const partitionIds = new Set(scenarioIds); + return { + ...params.lab, + setScenarioRun(next) { + if (!next) { + return; + } + for (const nextOutcome of next.scenarios) { + if (!partitionIds.has(nextOutcome.id)) { + continue; + } + const current = outcomes.get(nextOutcome.id); + if (!current) { + continue; + } + outcomes.set(nextOutcome.id, { + ...current, + ...nextOutcome, + }); + } + emit("running"); + }, + // Child partition reports are incomplete. The unified owner publishes one aggregate. + setLatestReport() {}, + }; + }, + complete(entries: readonly QaSuiteProgressResult[], finishedAt: string) { + for (const entry of entries) { + updateResult(entry, finishedAt); + } + emit("completed", finishedAt); + }, + }; +} diff --git a/extensions/qa-lab/web/src/app.browser.test.ts b/extensions/qa-lab/web/src/app.browser.test.ts new file mode 100644 index 000000000000..47ed732933c4 --- /dev/null +++ b/extensions/qa-lab/web/src/app.browser.test.ts @@ -0,0 +1,424 @@ +/* @vitest-environment jsdom */ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { Bootstrap, RunnerSelection } from "./ui-types.js"; + +const httpMock = vi.hoisted(() => { + class QaLabHttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly payload: unknown, + ) { + super(message); + } + } + return { + getJson: vi.fn(), + getJsonNoStore: vi.fn(), + postJson: vi.fn(), + QaLabHttpError, + }; +}); + +vi.mock("./http.js", () => httpMock); + +import { createQaLabApp } from "./app.js"; + +const scenarios: Bootstrap["scenarios"] = [ + { + id: "dm-chat-baseline", + title: "DM baseline", + surface: "dm", + objective: "test DM", + successCriteria: ["reply"], + execution: { kind: "flow" }, + }, + { + id: "browser-talk-start-stop", + title: "Browser Talk start-stop", + surface: "control-ui", + objective: "test browser Talk", + successCriteria: ["playwright pass"], + execution: { kind: "playwright" }, + }, +]; + +function createBootstrap(selection: RunnerSelection): Bootstrap { + const selectedScenarioIds = selection.scenarioIds ?? scenarios.map((scenario) => scenario.id); + return { + baseUrl: "http://127.0.0.1:43124", + controlUiEmbeddedUrl: null, + controlUiUrl: null, + defaults: { + conversationId: "qa-operator", + conversationKind: "direct", + senderId: "qa-operator", + senderName: "QA Operator", + }, + kickoffTask: "Run QA", + latestReport: null, + runner: { + artifacts: null, + error: null, + plan: { + errors: [], + exclusions: [], + executionKinds: ["flow", "playwright"], + explicitScenarioSelection: selection.scenarioIds !== null, + profile: selection.profile, + selectedScenarios: scenarios + .filter((scenario) => selectedScenarioIds.includes(scenario.id)) + .map((scenario) => ({ + declaredChannel: null, + effectiveChannel: scenario.execution?.kind === "flow" ? "qa-channel" : null, + executionKind: scenario.execution?.kind ?? "flow", + id: scenario.id, + title: scenario.title, + })), + status: "ready", + }, + selection, + status: "idle", + }, + runnerCatalog: { + channels: ["matrix", "telegram"], + profiles: [ + { id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] }, + { id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] }, + ], + status: "ready", + real: [ + { + input: "text", + key: "openai/gpt-5.6-luna", + name: "GPT-5.6 Luna", + preferred: true, + provider: "openai", + }, + ], + }, + scenarios, + }; +} + +async function mountRunner(selection: RunnerSelection) { + let bootstrap = createBootstrap(selection); + httpMock.getJson.mockImplementation(async (url: string) => { + if (url === "/api/bootstrap") { + return bootstrap; + } + if (url === "/api/state") { + return { conversations: [], events: [], messages: [], threads: [] }; + } + if (url === "/api/report") { + return { report: null }; + } + if (url === "/api/outcomes") { + return { run: null }; + } + if (url === "/api/capture/sessions") { + return { sessions: [] }; + } + if (url === "/api/capture/startup-status") { + return { + status: { + gateway: { label: "Gateway", ok: true, url: "http://127.0.0.1:18789" }, + proxy: { label: "Proxy", ok: true, url: "http://127.0.0.1:7799" }, + qaLab: { label: "QA Lab", ok: true, url: bootstrap.baseUrl }, + }, + }; + } + throw new Error(`unexpected GET ${url}`); + }); + httpMock.getJsonNoStore.mockResolvedValue({ version: "test" }); + httpMock.postJson.mockImplementation(async (url: string, body: unknown) => { + if (url !== "/api/scenario/suite") { + throw new Error(`unexpected POST ${url}`); + } + const nextSelection = body as RunnerSelection; + bootstrap = createBootstrap(nextSelection); + return { runner: { selection: nextSelection } }; + }); + const root = document.createElement("div"); + document.body.append(root); + await createQaLabApp(root); + return root; +} + +function selectValue(root: HTMLElement, selector: string, value: string) { + const select = root.querySelector(selector); + if (!select) { + throw new Error(`missing select ${selector}`); + } + select.value = value; + select.dispatchEvent(new Event("change", { bubbles: true })); +} + +beforeEach(() => { + vi.useFakeTimers(); + const styles = document.createElement("style"); + styles.dataset.qaLabTestStyles = "true"; + styles.textContent = readFileSync( + path.join(process.cwd(), "extensions/qa-lab/web/src/styles.css"), + "utf8", + ); + document.head.append(styles); + httpMock.getJson.mockReset(); + httpMock.getJsonNoStore.mockReset(); + httpMock.postJson.mockReset(); + const storage = new Map(); + vi.stubGlobal("localStorage", { + clear: () => storage.clear(), + getItem: (key: string) => storage.get(key) ?? null, + key: (index: number) => [...storage.keys()][index] ?? null, + get length() { + return storage.size; + }, + removeItem: (key: string) => storage.delete(key), + setItem: (key: string, value: string) => storage.set(key, value), + }); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + callback(0); + return 1; + }); + vi.stubGlobal("matchMedia", () => ({ matches: false })); +}); + +afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + document.body.replaceChildren(); + document.querySelector("style[data-qa-lab-test-styles]")?.remove(); +}); + +describe("QA Lab runner browser interactions", () => { + it("keeps scenario rows from collapsing inside the scrolling list", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "qa-channel", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + const scroll = root.querySelector(".scenario-scroll"); + const row = root.querySelector(".scenario-item"); + expect(scroll).not.toBeNull(); + expect(row).not.toBeNull(); + expect(getComputedStyle(scroll!).overflowY).toBe("auto"); + expect(getComputedStyle(row!).flexShrink).toBe("0"); + }); + + it("submits live-provider and Crabline selections with non-flow scenarios", async () => { + const root = await mountRunner({ + alternateModel: "openai/gpt-5.6-luna", + channel: null, + channelDriver: "crabline", + evidenceMode: "full", + fastMode: true, + primaryModel: "openai/gpt-5.6-luna", + profile: "all", + providerMode: "live-frontier", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + root.querySelector("[data-action='select-all-scenarios']")?.click(); + root.querySelector("[data-action='run-suite']")?.click(); + + await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1)); + expect(httpMock.postJson).toHaveBeenCalledWith( + "/api/scenario/suite", + expect.objectContaining({ + channelDriver: "crabline", + providerMode: "live-frontier", + scenarioIds: ["dm-chat-baseline", "browser-talk-start-stop"], + }), + ); + }); + + it("changes to real channels without changing the mock provider lane", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "qa-channel", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + root.querySelector("[data-sidebar-panel='config']")?.click(); + selectValue(root, "#channel-driver", "live"); + selectValue(root, "#execution-channel", "telegram"); + root.querySelector("[data-action='run-suite']")?.click(); + + await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1)); + expect(httpMock.postJson).toHaveBeenCalledWith( + "/api/scenario/suite", + expect.objectContaining({ + channelDriver: "live", + channel: "telegram", + providerMode: "mock-openai", + }), + ); + }); + + it("submits profile, evidence, runtime-pair, lane, and channel controls", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "live", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + root.querySelector("[data-sidebar-panel='config']")?.click(); + selectValue(root, "#run-profile", "smoke-ci"); + selectValue(root, "#execution-channel", "telegram"); + selectValue(root, "#evidence-mode", "slim"); + selectValue(root, "#runtime-pair", "openclaw,codex"); + selectValue(root, "#runtime-pair-lane", "core"); + root.querySelector("[data-action='run-suite']")?.click(); + + await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1)); + expect(httpMock.postJson).toHaveBeenCalledWith( + "/api/scenario/suite", + expect.objectContaining({ + profile: "smoke-ci", + channel: "telegram", + channelDriver: "crabline", + evidenceMode: "slim", + runtimePair: ["openclaw", "codex"], + runtimePairLane: "core", + scenarioIds: null, + }), + ); + }); + + it("renders server-resolved exclusions and errors from a rejected launch", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "qa-channel", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + httpMock.postJson.mockRejectedValueOnce( + new httpMock.QaLabHttpError("selection rejected", 400, { + plan: { + errors: ["Explicit QA scenario selection is not runnable."], + exclusions: [ + { + executionKind: "flow", + reasons: ["channel=telegram"], + scenarioId: "dm-chat-baseline", + }, + ], + executionKinds: [], + explicitScenarioSelection: true, + profile: "all", + selectedScenarios: [], + status: "invalid", + }, + }), + ); + + root.querySelector("[data-action='run-suite']")?.click(); + + await vi.waitFor(() => expect(root.textContent).toContain("1 excluded")); + expect(root.textContent).toContain("Explicit QA scenario selection is not runnable"); + + await vi.advanceTimersByTimeAsync(1_000); + await vi.waitFor(() => expect(root.textContent).toContain("1 excluded")); + expect(root.textContent).toContain("Explicit QA scenario selection is not runnable"); + + root.querySelector("[data-sidebar-panel='config']")?.click(); + selectValue(root, "#evidence-mode", "slim"); + root.querySelector("[data-sidebar-panel='run']")?.click(); + expect(root.textContent).not.toContain("Explicit QA scenario selection is not runnable"); + expect(root.textContent).not.toContain("Resolved plan:"); + }); + + it("starts a dirty profile override without reusing the previous resolved plan", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "qa-channel", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + root.querySelector("[data-sidebar-panel='config']")?.click(); + selectValue(root, "#run-profile", "smoke-ci"); + root.querySelector("[data-sidebar-panel='scenarios']")?.click(); + root + .querySelector("[data-scenario-toggle-id='browser-talk-start-stop']") + ?.click(); + root.querySelector("[data-action='run-suite']")?.click(); + + await vi.waitFor(() => expect(httpMock.postJson).toHaveBeenCalledTimes(1)); + expect(httpMock.postJson).toHaveBeenCalledWith( + "/api/scenario/suite", + expect.objectContaining({ + profile: "smoke-ci", + scenarioIds: ["browser-talk-start-stop"], + }), + ); + }); + + it("disables launch when an explicit override becomes empty", async () => { + const root = await mountRunner({ + alternateModel: "mock-openai/gpt-5.6-luna-alt", + channel: null, + channelDriver: "qa-channel", + evidenceMode: "full", + fastMode: false, + primaryModel: "mock-openai/gpt-5.6-luna", + profile: "all", + providerMode: "mock-openai", + runtimePair: null, + runtimePairLane: null, + scenarioIds: ["dm-chat-baseline"], + }); + + root.querySelector("[data-scenario-toggle-id='dm-chat-baseline']")?.click(); + const runButton = root.querySelector("[data-action='run-suite']"); + expect(runButton?.disabled).toBe(true); + expect(runButton?.textContent).toContain("Run 0 scenarios"); + runButton?.click(); + expect(httpMock.postJson).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/qa-lab/web/src/app.ts b/extensions/qa-lab/web/src/app.ts index 0f1651a2b636..c89a0382a019 100644 --- a/extensions/qa-lab/web/src/app.ts +++ b/extensions/qa-lab/web/src/app.ts @@ -2,13 +2,14 @@ import { defaultQaModelForMode, isQaFastModeEnabled } from "../../model-selection.js"; import { normalizeCaptureSavedView, normalizeCaptureSavedViews } from "./capture-saved-view.js"; import { formatErrorMessage } from "./errors.js"; -import { getJson, getJsonNoStore, postJson } from "./http.js"; +import { getJson, getJsonNoStore, postJson, QaLabHttpError } from "./http.js"; import { conversationSelectionKey, findConversationBySelectionKey } from "./ui-conversation-key.js"; import { type Bootstrap, type EvidenceEnvelope, type OutcomesEnvelope, type ReportEnvelope, + type RunnerResolvedPlan, type RunnerSelection, type Snapshot, type TabId, @@ -84,6 +85,14 @@ function defaultModelsForProviderMode( }; } +function cloneRunnerSelection(selection: RunnerSelection): RunnerSelection { + return { + ...selection, + runtimePair: selection.runtimePair ? [...selection.runtimePair] : null, + scenarioIds: selection.scenarioIds ? [...selection.scenarioIds] : null, + }; +} + function detectTheme(): "light" | "dark" { const stored = localStorage.getItem("qa-lab-theme"); if (stored === "light" || stored === "dark") { @@ -207,6 +216,7 @@ export async function createQaLabApp(root: HTMLDivElement) { activeTab: initialUrl.pathname === "/evidence" || initialEvidencePath ? "evidence" : "chat", runnerDraft: null, runnerDraftDirty: false, + runnerPlanOverride: null, composer: { conversationKind: "direct", conversationId: "alice", @@ -250,6 +260,9 @@ export async function createQaLabApp(root: HTMLDivElement) { ra: state.bootstrap?.runner.startedAt, rf: state.bootstrap?.runner.finishedAt, re: state.bootstrap?.runner.error, + rpo: state.runnerPlanOverride + ? `${state.runnerPlanOverride.status}:${state.runnerPlanOverride.selectedScenarios.length}:${state.runnerPlanOverride.exclusions.length}:${state.runnerPlanOverride.errors.join("|")}` + : null, ss: state.scenarioRun?.status, sc: state.scenarioRun?.counts, so: state.scenarioRun?.scenarios.map((o) => o.status).join(","), @@ -341,10 +354,7 @@ export async function createQaLabApp(root: HTMLDivElement) { state.evidencePathDraft = bootstrap.runner.artifacts.evidencePath; } if (!state.runnerDraft || !state.runnerDraftDirty) { - state.runnerDraft = { - ...bootstrap.runner.selection, - scenarioIds: [...bootstrap.runner.selection.scenarioIds], - }; + state.runnerDraft = cloneRunnerSelection(bootstrap.runner.selection); state.runnerDraftDirty = false; } if (!state.selectedConversationKey) { @@ -497,13 +507,14 @@ export async function createQaLabApp(root: HTMLDivElement) { function updateRunnerDraft(mutator: (draft: RunnerSelection) => RunnerSelection) { const fallback = state.bootstrap?.runner.selection; if (!state.runnerDraft && fallback) { - state.runnerDraft = { ...fallback, scenarioIds: [...fallback.scenarioIds] }; + state.runnerDraft = cloneRunnerSelection(fallback); } if (!state.runnerDraft) { return; } state.runnerDraft = mutator(state.runnerDraft); state.runnerDraftDirty = true; + state.runnerPlanOverride = null; render(); } @@ -610,20 +621,32 @@ export async function createQaLabApp(root: HTMLDivElement) { const result = await postJson<{ runner: { selection: RunnerSelection } }>( "/api/scenario/suite", { + profile: state.runnerDraft.profile, + channel: state.runnerDraft.channel, + channelDriver: state.runnerDraft.channelDriver, + evidenceMode: state.runnerDraft.evidenceMode, providerMode: state.runnerDraft.providerMode, primaryModel: state.runnerDraft.primaryModel, alternateModel: state.runnerDraft.alternateModel, + fastMode: state.runnerDraft.fastMode, + runtimePair: state.runnerDraft.runtimePair, + runtimePairLane: state.runnerDraft.runtimePairLane, scenarioIds: state.runnerDraft.scenarioIds, }, ); - state.runnerDraft = { - ...result.runner.selection, - scenarioIds: [...result.runner.selection.scenarioIds], - }; + state.runnerDraft = cloneRunnerSelection(result.runner.selection); state.runnerDraftDirty = false; + state.runnerPlanOverride = null; state.activeTab = "chat"; await refresh(); } catch (error) { + if (error instanceof QaLabHttpError) { + const plan = (error.payload as { plan?: RunnerResolvedPlan } | null)?.plan; + if (plan) { + state.runnerPlanOverride = plan; + state.sidebarPanel = "run"; + } + } state.error = formatErrorMessage(error); render(); } finally { @@ -904,7 +927,7 @@ export async function createQaLabApp(root: HTMLDivElement) { root .querySelector("[data-action='clear-scenarios']") ?.addEventListener("click", () => { - updateRunnerDraft((d) => ({ ...d, scenarioIds: [] })); + updateRunnerDraft((d) => ({ ...d, scenarioIds: null })); }); /* Scenario toggles */ @@ -915,7 +938,13 @@ export async function createQaLabApp(root: HTMLDivElement) { return; } updateRunnerDraft((draft) => { - const selected = new Set(draft.scenarioIds); + const selected = new Set( + draft.scenarioIds ?? + (!state.runnerDraftDirty + ? state.bootstrap?.runner.plan?.selectedScenarios.map((scenario) => scenario.id) + : undefined) ?? + [], + ); if (node.checked) { selected.add(scenarioId); } else { @@ -930,6 +959,19 @@ export async function createQaLabApp(root: HTMLDivElement) { }); /* Config form */ + root.querySelector("#run-profile")?.addEventListener("change", (e) => { + const profile = (e.currentTarget as HTMLSelectElement).value; + const profileDefaults = state.bootstrap?.runnerCatalog.profiles.find( + (entry) => entry.id === profile, + ); + updateRunnerDraft((draft) => ({ + ...draft, + profile, + channelDriver: profileDefaults?.channelDriver ?? draft.channelDriver, + evidenceMode: profileDefaults?.evidenceMode ?? draft.evidenceMode, + scenarioIds: null, + })); + }); root.querySelector("#provider-mode")?.addEventListener("change", (e) => { const mode = (e.currentTarget as HTMLSelectElement).value === "live-frontier" @@ -941,6 +983,33 @@ export async function createQaLabApp(root: HTMLDivElement) { ...defaultModelsForProviderMode(mode, state.bootstrap), })); }); + root.querySelector("#channel-driver")?.addEventListener("change", (e) => { + const value = (e.currentTarget as HTMLSelectElement).value; + const channelDriver = value === "crabline" || value === "live" ? value : "qa-channel"; + updateRunnerDraft((draft) => ({ ...draft, channelDriver })); + }); + root.querySelector("#execution-channel")?.addEventListener("change", (e) => { + const channel = (e.currentTarget as HTMLSelectElement).value.trim() || null; + updateRunnerDraft((draft) => ({ ...draft, channel })); + }); + root.querySelector("#evidence-mode")?.addEventListener("change", (e) => { + const evidenceMode = + (e.currentTarget as HTMLSelectElement).value === "slim" ? "slim" : "full"; + updateRunnerDraft((draft) => ({ ...draft, evidenceMode })); + }); + root.querySelector("#runtime-pair")?.addEventListener("change", (e) => { + const runtimePair: RunnerSelection["runtimePair"] = + (e.currentTarget as HTMLSelectElement).value === "openclaw,codex" + ? ["openclaw", "codex"] + : null; + updateRunnerDraft((draft) => ({ ...draft, runtimePair })); + }); + root.querySelector("#runtime-pair-lane")?.addEventListener("change", (e) => { + const value = (e.currentTarget as HTMLSelectElement).value; + const runtimePairLane = + value === "core" || value === "extended" || value === "soak" ? value : null; + updateRunnerDraft((draft) => ({ ...draft, runtimePairLane })); + }); root.querySelector("#primary-model")?.addEventListener("change", (e) => { const primaryModel = (e.currentTarget as HTMLSelectElement).value; updateRunnerDraft((d) => ({ diff --git a/extensions/qa-lab/web/src/http.ts b/extensions/qa-lab/web/src/http.ts index e0e4241232d8..a0788f4c33e2 100644 --- a/extensions/qa-lab/web/src/http.ts +++ b/extensions/qa-lab/web/src/http.ts @@ -51,7 +51,22 @@ export async function postJson(path: string, body: unknown): Promise { response, path, ).catch(() => ({})); - throw new Error(payload.error || `${response.status} ${response.statusText}`); + throw new QaLabHttpError( + payload.error || `${response.status} ${response.statusText}`, + response.status, + payload, + ); } return await readJsonResponse(response, path); } + +export class QaLabHttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly payload: unknown, + ) { + super(message); + this.name = "QaLabHttpError"; + } +} diff --git a/extensions/qa-lab/web/src/styles.css b/extensions/qa-lab/web/src/styles.css index bdd532aba004..d19da60cbbd0 100644 --- a/extensions/qa-lab/web/src/styles.css +++ b/extensions/qa-lab/web/src/styles.css @@ -511,6 +511,7 @@ select { .scenario-item { display: flex; + flex-shrink: 0; align-items: center; gap: 10px; padding: 10px 12px; diff --git a/extensions/qa-lab/web/src/ui-render-shell.ts b/extensions/qa-lab/web/src/ui-render-shell.ts index 4dfd966b546d..e760372c9960 100644 --- a/extensions/qa-lab/web/src/ui-render-shell.ts +++ b/extensions/qa-lab/web/src/ui-render-shell.ts @@ -93,7 +93,17 @@ export function renderSidebar(state: UiState): string { const realModels = state.bootstrap?.runnerCatalog.real ?? []; const modelOptions = selection?.providerMode === "live-frontier" && realModels.length > 0 ? realModels : MOCK_MODELS; - const selectedIds = new Set(selection?.scenarioIds ?? []); + const plan = state.runnerPlanOverride ?? state.bootstrap?.runner.plan ?? null; + const resolvedIds = plan?.selectedScenarios.map((scenario) => scenario.id) ?? []; + const selectedIds = new Set( + selection?.scenarioIds ?? (state.runnerDraftDirty ? [] : resolvedIds), + ); + const profiles = state.bootstrap?.runnerCatalog.profiles ?? []; + const channels = state.bootstrap?.runnerCatalog.channels ?? []; + const hasRunnableSelection = + selection?.scenarioIds === null + ? Boolean(selection.profile) + : Boolean(selection?.scenarioIds.length); return `