diff --git a/.agents/skills/openclaw-qa-testing/SKILL.md b/.agents/skills/openclaw-qa-testing/SKILL.md index 28eae8a499ac..5df480350951 100644 --- a/.agents/skills/openclaw-qa-testing/SKILL.md +++ b/.agents/skills/openclaw-qa-testing/SKILL.md @@ -62,17 +62,6 @@ scenario through qa-channel, decodes the emitted protobuf spans, and verifies the exported trace names and privacy contract. It does not require Opik, Langfuse, or external collector credentials. -## Matrix live selection - -`pnpm openclaw qa matrix` derives its default set from flow scenarios that -explicitly declare Matrix channel eligibility. It has no named profiles or -runner-owned scenario-id list. Repeat `--scenario ` for a focused run, or -use `--shard /` to reproduce one deterministic CI partition. - -`QA-Lab - All Lanes` and release checks fan the same catalog-derived selection -across five balanced shards. Shards distribute work only; they do not define -semantic membership or priority. - ## QA credentials and 1Password - Use `op` only inside `tmux` for QA secret lookup in this repo. diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index 6e9fbcb08b9d..82a9dae5f4f1 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -452,18 +452,6 @@ jobs, followed by a report job that downloads both artifacts and runs first; inspect the report job when both lane summaries exist but the comparison fails. -### QA Lab Matrix selection - -`pnpm openclaw qa matrix` runs every flow scenario that explicitly declares -Matrix channel eligibility. There are no Matrix named profiles or runner-owned -scenario-id lists. Repeat `--scenario ` for an explicit subset, or use -`--shard /` to reproduce one deterministic CI partition. - -`QA-Lab - All Lanes` and `OpenClaw Release Checks` use the same catalog-derived -selection and standard artifacts. CI fans the selection across five balanced -shards; shard placement is operational only and does not define membership or -priority. - ### Reusable Live/E2E Checks `OpenClaw Live And E2E Checks (Reusable)` diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index 84fe92434a73..27c675a2047f 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -386,6 +386,16 @@ jobs: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + - name: Verify internal Matrix execution sharding support + shell: bash + run: | + set -euo pipefail + runtime_source="extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts" + if [[ ! -f "${runtime_source}" ]] || ! grep -Fq "OPENCLAW_QA_EXECUTION_SHARD" "${runtime_source}"; then + echo "Selected target does not support internal post-selection QA sharding; use a target that includes the current QA execution protocol." >&2 + exit 1 + fi + - name: Build private QA runtime env: NODE_OPTIONS: --max-old-space-size=12288 @@ -397,32 +407,17 @@ jobs: shell: bash env: OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_EXECUTION_SHARD: ${{ format('{0}/5', matrix.shard) }} run: | set -euo pipefail output_dir=".artifacts/qa-e2e/matrix-live-shard-${{ matrix.shard }}-of-5-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" mkdir -p "${output_dir}" - matrix_help="$(pnpm openclaw qa matrix --help 2>&1)" - if grep -Fq -- "--shard " <<<"${matrix_help}"; then - matrix_selection=(--shard "${{ matrix.shard }}/5") - else - legacy_profiles=(transport media e2ee-smoke e2ee-deep e2ee-cli) - shard_index="${{ matrix.shard }}" - if (( shard_index < 1 || shard_index > ${#legacy_profiles[@]} )); then - echo "Invalid Matrix shard index for legacy profile fallback: ${shard_index}" >&2 - exit 1 - fi - legacy_profile="${legacy_profiles[shard_index - 1]}" - echo "Selected target predates profile-free Matrix catalog sharding; using legacy profile '${legacy_profile}'." - matrix_selection=(--profile "${legacy_profile}") - fi - pnpm openclaw qa matrix \ --repo-root . \ --output-dir "${output_dir}" \ --provider-mode mock-openai \ - "${matrix_selection[@]}" \ --fast - name: Upload Matrix QA artifacts diff --git a/docs/ci.md b/docs/ci.md index 999806416542..792b6ae72914 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -568,8 +568,6 @@ QA Lab has dedicated CI lanes outside the main smart-scoped workflow. Agentic pa Scheduled, manual, and release Matrix checks use the deterministic mock provider so the live transport contract is isolated from model latency and normal provider-plugin startup. Telegram release checks use the same deterministic model boundary. The live transport gateway disables memory search because QA parity covers memory behavior separately; provider connectivity is covered by the separate live model, native provider, and Docker provider suites. -Scheduled, manual, and release Matrix gates use the shared QA Lab suite host and live adapter. Default membership comes from flow scenarios that explicitly declare Matrix channel eligibility; the runner and workflow keep no curated profile or scenario-id list. CI distributes that catalog-derived selection across five deterministic balanced shards so membership is order-independent and each job stays within its timeout. Focused local runs repeat `--scenario `. - `OpenClaw Release Checks` also runs the release-critical QA Lab lanes before release approval; its QA parity gate runs the candidate and baseline packs as parallel lane jobs, then downloads both artifacts into a small report job for the final parity comparison. For normal PRs, follow scoped CI/check evidence instead of treating parity as a required status. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 8364ba2096ec..7cb43f291a71 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -210,12 +210,7 @@ declares Matrix eligibility through `execution.channel` or `--fail-fast` for a shorter feedback loop or repeat `--scenario ` for an explicit subset, including portable scenarios with no channel restriction. -Declarative scenario metadata is the only default-membership source. The -Matrix runner has no named profiles or scenario-id allowlists. The run chooses -the channel driver; deterministic `--shard /` partitioning only -distributes the selected catalog and does not define semantic membership or -execution priority. -Their live implementations live under +Matrix live implementations live under `extensions/qa-lab/src/live-transports/matrix/scenarios/`. The adapter provisions a disposable Tuwunel homeserver in Docker (default @@ -230,7 +225,6 @@ Common options: | Flag | Default | Purpose | | ------------------------ | ----------------- | ------------------------------------------------------------------------------------ | | `--scenario ` | - | Select one scenario; repeatable. | -| `--shard ` | - | Run one deterministic, balanced partition of the selected Matrix catalog. | | `--fail-fast` | off | Stop after the first failed check or scenario. | | `--allow-failures` | off | Write artifacts without returning a failing exit code for scenario failures. | | `--provider-mode ` | `live-frontier` | Use `mock-openai` for deterministic dispatch or `live-frontier` for a live provider. | diff --git a/docs/help/testing.md b/docs/help/testing.md index 7d5b658416dd..a95ad8835070 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -174,10 +174,7 @@ checks keep exhaustive live/Docker soak behind `run_release_soak=true`; the from manual dispatch with the mock parity lane, live Matrix lane, Convex-managed live Telegram lane, and Convex-managed live Discord lane as parallel jobs. Scheduled QA and release checks run the catalog-derived Matrix -selection through the shared live adapter. The Matrix CLI and workflow have no -curated profiles: scenario channel eligibility defines default membership, -explicit `--scenario` flags narrow local runs, and CI fans the same selection -across deterministic shards. `OpenClaw Release Checks` runs parity plus the +selection through the shared live adapter. `OpenClaw Release Checks` runs parity plus the reusable Matrix live-adapter lane and Telegram lane before release approval. Release transport checks use `mock-openai/gpt-5.6-luna` so they stay deterministic and avoid normal provider-plugin startup. These live transport gateways diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index b9d0d23cc0e5..31ce362dd869 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -279,7 +279,7 @@ A legacy fallback correction tag may reuse base-package evidence only when the c Manually dispatch `Windows Node Release` only for recovery, and always pass an exact tag, never `latest`, plus the explicit `expected_installer_digests` JSON map from the approved source release. Website download links should target exact OpenClaw release asset URLs for the current stable release, or `releases/latest/download/...` only after verifying GitHub's latest redirect points at that same release; do not link only to the companion repo release page. -- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix catalog and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. The `QA-Lab - All Lanes` workflow derives Matrix membership from scenario channel eligibility and fans it across deterministic balanced shards to keep proof within per-job timeouts; there is no separate Matrix selector input. +- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix catalog and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. - Cross-OS install and upgrade runtime validation is part of public `OpenClaw Release Checks` and `Full Release Validation`, which call the reusable workflow `.github/workflows/openclaw-cross-os-release-checks-reusable.yml` directly. This split is intentional: keep the real npm release path short, deterministic, and artifact-focused, while slower live checks stay in their own lane so they do not stall or block publish. - Secret-bearing release checks should be dispatched through `Full Release Validation` or from the `main`/release workflow ref so workflow logic and secrets stay controlled. - `OpenClaw Release Checks` accepts a branch, tag, or full commit SHA as long as the resolved commit is reachable from an OpenClaw branch or release tag. diff --git a/extensions/qa-lab/src/execution-sharding.test.ts b/extensions/qa-lab/src/execution-sharding.test.ts new file mode 100644 index 000000000000..c6e5cff77cf0 --- /dev/null +++ b/extensions/qa-lab/src/execution-sharding.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; +import { + resolveQaExecutionShard, + selectQaExecutionShardScenarioIds, +} from "./execution-sharding.js"; + +describe("QA execution sharding", () => { + it("resolves only valid internal coordinates", () => { + expect(resolveQaExecutionShard(undefined)).toBeUndefined(); + expect(resolveQaExecutionShard(" 2/5 ")).toEqual({ + index: 2, + count: 5, + }); + expect(() => resolveQaExecutionShard("1", "QA execution")).toThrow( + "QA execution shard must use /", + ); + expect(() => resolveQaExecutionShard("0/5")).toThrow("1 <= index <= count"); + expect(() => resolveQaExecutionShard("6/5")).toThrow("1 <= index <= count"); + }); + + it("partitions a resolved semantic selection deterministically and evenly", () => { + const semanticScenarioIds = ["four", "one", "five", "two", "three", "six"]; + const shards = [1, 2, 3].map((index) => + selectQaExecutionShardScenarioIds(semanticScenarioIds, { index, count: 3 }), + ); + + expect(shards.flat().toSorted()).toEqual(semanticScenarioIds.toSorted()); + expect(new Set(shards.flat()).size).toBe(semanticScenarioIds.length); + expect(Math.max(...shards.map((shard) => shard.length))).toBe( + Math.min(...shards.map((shard) => shard.length)), + ); + expect( + selectQaExecutionShardScenarioIds(semanticScenarioIds.toReversed(), { + index: 1, + count: 3, + }), + ).toEqual(shards[0]); + }); + + it("fails when an execution worker receives no resolved scenarios", () => { + expect(() => + selectQaExecutionShardScenarioIds(["only-scenario"], { index: 2, count: 2 }), + ).toThrow("resolved no scenarios from 1 semantically selected scenario"); + }); +}); diff --git a/extensions/qa-lab/src/execution-sharding.ts b/extensions/qa-lab/src/execution-sharding.ts new file mode 100644 index 000000000000..7217f6e0e124 --- /dev/null +++ b/extensions/qa-lab/src/execution-sharding.ts @@ -0,0 +1,53 @@ +type QaExecutionShard = { + index: number; + count: number; +}; + +function compareStrings(left: string, right: string) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function hashQaExecutionKey(value: string) { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +export function resolveQaExecutionShard( + value: string | undefined, + label = "QA execution", +): QaExecutionShard | undefined { + const normalized = value?.trim(); + if (!normalized) { + return undefined; + } + const match = /^([1-9]\d*)\/([1-9]\d*)$/.exec(normalized); + const index = Number(match?.[1]); + const count = Number(match?.[2]); + if (!match || !Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index > count) { + throw new Error(`${label} shard must use / with 1 <= index <= count.`); + } + return { index, count }; +} + +export function selectQaExecutionShardScenarioIds( + scenarioIds: readonly string[], + shard: QaExecutionShard, +) { + // Hash ordering decouples execution placement from catalog order. Round-robin + // assignment keeps balanced workers without becoming another membership owner. + const orderedIds = [...new Set(scenarioIds)].toSorted((left, right) => { + const hashDelta = hashQaExecutionKey(left) - hashQaExecutionKey(right); + return hashDelta || compareStrings(left, right); + }); + const selectedIds = orderedIds.filter((_, index) => index % shard.count === shard.index - 1); + if (selectedIds.length === 0) { + throw new Error( + `QA execution shard ${shard.index}/${shard.count} resolved no scenarios from ${orderedIds.length} semantically selected scenario(s).`, + ); + } + return selectedIds; +} diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts index 10a1a390f37e..35e22d952f61 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts @@ -63,14 +63,13 @@ describe("QA Lab Matrix CLI registration", () => { "--model", "--alt-model", "--scenario", - "--shard", "--fast", "--fail-fast", "--sut-account", ]) { expect(optionNames).toContain(optionName); } - for (const optionName of ["--profile", "--credential-source", "--credential-role"]) { + for (const optionName of ["--profile", "--shard", "--credential-source", "--credential-role"]) { expect(optionNames).not.toContain(optionName); } }); @@ -86,8 +85,6 @@ describe("QA Lab Matrix CLI registration", () => { "matrix", "--scenario", "matrix-allowlist-hot-reload", - "--shard", - "1/1", ]); expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith( @@ -97,7 +94,6 @@ describe("QA Lab Matrix CLI registration", () => { options: expect.objectContaining({ providerMode: "live-frontier", scenarioIds: ["matrix-allowlist-hot-reload"], - shard: "1/1", }), }), ); diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.ts b/extensions/qa-lab/src/live-transports/matrix/cli.ts index cb9a784460d6..df38a51de3e7 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.ts @@ -8,7 +8,7 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { resolveMatrixQaScenarioIds } from "./scenario-selection.js"; +import { resolveCatalogLiveTransportQaScenarioIds } from "../shared/scenario-selection.js"; const DISABLE_MATRIX_QA_FORCE_EXIT_ENV = "OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT"; @@ -47,9 +47,11 @@ async function runQaMatrix(opts: LiveTransportQaCommandOptions) { laneLabel: "Matrix", options: opts, selectScenarioIds: (selection) => - resolveMatrixQaScenarioIds({ + resolveCatalogLiveTransportQaScenarioIds({ + channelId: "matrix", + primaryModel: selection.primaryModel, + providerMode: selection.providerMode, scenarioIds: selection.scenarioIds, - shard: opts.shard, }), }); }; @@ -84,7 +86,6 @@ export const matrixQaCliRegistration: LiveTransportQaCliRegistration = description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver", outputDirHelp: "Matrix QA artifact directory", scenarioHelp: "Run only the named Matrix QA scenario (repeatable)", - shardHelp: "Run one deterministic Matrix catalog shard as /", failFastHelp: "Stop after the first failed Matrix QA scenario", sutAccountHelp: "Temporary Matrix account id inside the QA gateway config", run: runQaMatrix, diff --git a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts deleted file mode 100644 index a67605cbd79d..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { readQaScenarioPack } from "../../scenario-catalog.js"; -import { resolveMatrixQaScenarioIds } from "./scenario-selection.js"; - -describe("QA Lab Matrix scenario selection", () => { - it("derives the default set from explicit Matrix channel eligibility", () => { - const catalog = readQaScenarioPack().scenarios; - const scenarioById = new Map(catalog.map((scenario) => [scenario.id, scenario] as const)); - const scenarioIds = resolveMatrixQaScenarioIds({}); - - expect(scenarioIds.length).toBeGreaterThan(0); - expect(new Set(scenarioIds).size).toBe(scenarioIds.length); - for (const scenarioId of scenarioIds) { - const scenario = scenarioById.get(scenarioId); - expect(scenario?.execution.kind, scenarioId).toBe("flow"); - expect( - scenario?.execution.channel === "matrix" || - (scenario?.execution.kind === "flow" && scenario.execution.channels?.includes("matrix")), - scenarioId, - ).toBe(true); - } - - const unrelatedScenario = catalog.find( - (scenario) => - scenario.execution.kind === "flow" && - scenario.execution.channel !== "matrix" && - !scenario.execution.channels?.includes("matrix"), - ); - expect(unrelatedScenario).toBeDefined(); - expect(scenarioIds).not.toContain(unrelatedScenario?.id); - }); - - it("preserves an explicit scenario subset without a named profile", () => { - const explicitScenarioIds = resolveMatrixQaScenarioIds({}).slice(0, 2).toReversed(); - - expect(resolveMatrixQaScenarioIds({ scenarioIds: explicitScenarioIds })).toEqual( - explicitScenarioIds, - ); - }); - - it("balances deterministic shards independently of input order", () => { - const scenarioIds = resolveMatrixQaScenarioIds({}); - const shardValues = ["1/5", "2/5", "3/5", "4/5", "5/5"] as const; - const shards = shardValues.map((shard) => resolveMatrixQaScenarioIds({ scenarioIds, shard })); - const sizes = shards.map((shard) => shard.length); - - expect(Math.max(...sizes) - Math.min(...sizes)).toBeLessThanOrEqual(1); - expect(shards.flat().toSorted()).toEqual(scenarioIds.toSorted()); - expect(new Set(shards.flat()).size).toBe(scenarioIds.length); - expect( - resolveMatrixQaScenarioIds({ scenarioIds: scenarioIds.toReversed(), shard: shardValues[0] }), - ).toEqual(shards[0]); - }); - - it("rejects invalid and empty shard selections honestly", () => { - expect(() => resolveMatrixQaScenarioIds({ scenarioIds: ["scenario"], shard: "0/5" })).toThrow( - "Expected /", - ); - expect(() => resolveMatrixQaScenarioIds({ scenarioIds: ["scenario"], shard: "2/5" })).toThrow( - "resolved no scenarios", - ); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts deleted file mode 100644 index ecbaa3689d1d..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { readQaScenarioPack } from "../../scenario-catalog.js"; - -const MATRIX_QA_CHANNEL_ID = "matrix"; - -type QaScenarioShard = { - index: number; - total: number; -}; - -function compareStrings(left: string, right: string) { - return left < right ? -1 : left > right ? 1 : 0; -} - -function hashScenarioId(id: string) { - let hash = 0x811c9dc5; - for (let index = 0; index < id.length; index += 1) { - hash ^= id.charCodeAt(index); - hash = Math.imul(hash, 0x01000193); - } - return hash >>> 0; -} - -function parseQaScenarioShard(value: string): QaScenarioShard { - const match = /^(\d+)\/(\d+)$/.exec(value.trim()); - const index = Number.parseInt(match?.[1] ?? "", 10); - const total = Number.parseInt(match?.[2] ?? "", 10); - if ( - !match || - !Number.isSafeInteger(index) || - !Number.isSafeInteger(total) || - index < 1 || - total < 1 || - index > total - ) { - throw new Error( - `Invalid Matrix QA shard "${value}". Expected / with 1 <= index <= total.`, - ); - } - return { index, total }; -} - -function shardMatrixQaScenarioIds(scenarioIds: readonly string[], shardValue: string): string[] { - const shard = parseQaScenarioShard(shardValue); - // Hash ordering makes shard membership independent of catalog/YAML order while - // round-robin assignment keeps every shard within one scenario of the others. - const orderedIds = [...new Set(scenarioIds)].toSorted((left, right) => { - const hashDelta = hashScenarioId(left) - hashScenarioId(right); - return hashDelta || compareStrings(left, right); - }); - const selectedIds = orderedIds.filter((_, index) => index % shard.total === shard.index - 1); - if (selectedIds.length === 0) { - throw new Error( - `Matrix QA shard ${shardValue} resolved no scenarios from ${orderedIds.length} selected scenario(s).`, - ); - } - return selectedIds; -} - -function listMatrixQaScenarioIds(): string[] { - return readQaScenarioPack() - .scenarios.filter( - (scenario) => - scenario.execution.kind === "flow" && - (scenario.execution.channel === MATRIX_QA_CHANNEL_ID || - scenario.execution.channels?.includes(MATRIX_QA_CHANNEL_ID)), - ) - .map((scenario) => scenario.id) - .toSorted(compareStrings); -} - -export function resolveMatrixQaScenarioIds(params: { - scenarioIds?: readonly string[]; - shard?: string; -}): string[] { - const scenarioIds = params.scenarioIds?.length - ? [...new Set(params.scenarioIds)] - : listMatrixQaScenarioIds(); - if (scenarioIds.length === 0) { - throw new Error("Matrix QA catalog selection resolved no scenarios."); - } - return params.shard ? shardMatrixQaScenarioIds(scenarioIds, params.shard) : scenarioIds; -} diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index 807d0c05f857..100a6920976b 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -13,7 +13,6 @@ export type LiveTransportQaCommandOptions = { allowFailures?: boolean; failFast?: boolean; profile?: string; - shard?: string; scenarioIds?: string[]; listScenarios?: boolean; sutAccountId?: string; @@ -33,7 +32,6 @@ type LiveTransportQaCommanderOptions = { allowFailures?: boolean; failFast?: boolean; profile?: string; - shard?: string; sutAccount?: string; credentialSource?: string; credentialRole?: string; @@ -53,7 +51,6 @@ type LiveTransportQaCliRegistrationOptions = { listScenariosHelp?: string; outputDirHelp: string; profileHelp?: string; - shardHelp?: string; failFastHelp?: string; allowFailuresHelp?: string; scenarioHelp: string; @@ -86,7 +83,6 @@ function mapCommanderOptions(opts: LiveTransportQaCommanderOptions): LiveTranspo allowFailures: opts.allowFailures, failFast: opts.failFast, profile: opts.profile, - shard: opts.shard, scenarioIds: opts.scenario, listScenarios: opts.listScenarios, sutAccountId: opts.sutAccount, @@ -123,9 +119,6 @@ function createSharedLiveTransportQaCliRegistration( if (params.profileHelp) { command.option("--profile ", params.profileHelp); } - if (params.shardHelp) { - command.option("--shard ", params.shardHelp); - } if (params.failFastHelp) { command.option("--fail-fast", params.failFastHelp, false); } diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts index cbffbd0c2ce0..00b659323490 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; const runQaSuiteCommand = vi.hoisted(() => vi.fn()); @@ -7,8 +7,19 @@ vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); import { runLiveTransportQaSuiteCommand } from "./live-transport-suite.runtime.js"; describe("live transport suite runtime", () => { + const originalExecutionShard = process.env.OPENCLAW_QA_EXECUTION_SHARD; + beforeEach(() => { vi.clearAllMocks(); + delete process.env.OPENCLAW_QA_EXECUTION_SHARD; + }); + + afterAll(() => { + if (originalExecutionShard === undefined) { + delete process.env.OPENCLAW_QA_EXECUTION_SHARD; + } else { + process.env.OPENCLAW_QA_EXECUTION_SHARD = originalExecutionShard; + } }); it("normalizes one live command into the shared suite host", async () => { @@ -70,6 +81,28 @@ describe("live transport suite runtime", () => { ); }); + it("applies execution sharding only after semantic scenario selection", async () => { + const selectScenarioIds = vi.fn(() => ["semantic-c", "semantic-a", "semantic-b"]); + process.env.OPENCLAW_QA_EXECUTION_SHARD = "2/2"; + + await runLiveTransportQaSuiteCommand({ + channelId: "slack", + defaultProviderMode: "live-frontier", + options: {}, + selectScenarioIds, + }); + + expect(selectScenarioIds).toHaveBeenCalledWith({ + profile: undefined, + primaryModel: expect.any(String), + providerMode: "live-frontier", + scenarioIds: undefined, + }); + const suiteArgs = runQaSuiteCommand.mock.calls[0]?.[0]; + expect(suiteArgs?.scenarioIds).toHaveLength(1); + expect(suiteArgs?.scenarioIds?.[0]).toMatch(/^semantic-/); + }); + it("rejects shared credentials for disposable transports", async () => { await expect( runLiveTransportQaSuiteCommand({ diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts index 4a52c4db175b..77f5263370e5 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts @@ -1,10 +1,17 @@ import type { LiveTransportQaCommandOptions } from "openclaw/plugin-sdk/qa-runtime"; import { runQaSuiteCommand } from "../../cli.runtime.js"; +import { + resolveQaExecutionShard, + selectQaExecutionShardScenarioIds, +} from "../../execution-sharding.js"; import type { QaProviderMode } from "../../providers/index.js"; -import { normalizeQaProviderMode } from "../../run-config.js"; +import { defaultQaModelForMode, normalizeQaProviderMode } from "../../run-config.js"; + +const QA_EXECUTION_SHARD_ENV = "OPENCLAW_QA_EXECUTION_SHARD"; type LiveTransportScenarioSelection = (params: { profile?: string; + primaryModel: string; providerMode: QaProviderMode; scenarioIds?: readonly string[]; }) => string[]; @@ -36,6 +43,14 @@ export async function runLiveTransportQaSuiteCommand(params: { options.providerMode === undefined ? params.defaultProviderMode : normalizeQaProviderMode(options.providerMode); + const primaryModel = options.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const selectedScenarioIds = params.selectScenarioIds({ + profile: options.profile, + primaryModel, + providerMode, + scenarioIds: options.scenarioIds, + }); + const executionShard = resolveQaExecutionShard(process.env[QA_EXECUTION_SHARD_ENV]); return runQaSuiteCommand({ repoRoot: options.repoRoot, outputDir: options.outputDir, @@ -48,11 +63,9 @@ export async function runLiveTransportQaSuiteCommand(params: { channelDriver: "live", channel: params.channelId, concurrency: 1, - scenarioIds: params.selectScenarioIds({ - profile: options.profile, - providerMode, - scenarioIds: options.scenarioIds, - }), + scenarioIds: executionShard + ? selectQaExecutionShardScenarioIds(selectedScenarioIds, executionShard) + : selectedScenarioIds, sutAccountId: options.sutAccountId, ...(params.credentialMode === "env-only" ? {} diff --git a/extensions/qa-lab/src/live-transports/shared/scenario-selection.test.ts b/extensions/qa-lab/src/live-transports/shared/scenario-selection.test.ts new file mode 100644 index 000000000000..a7af37d7f6c5 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/scenario-selection.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { scenarioDeclaresQaChannel } from "../../profile-planning.js"; +import { readQaScenarioPack } from "../../scenario-catalog.js"; +import { resolveCatalogLiveTransportQaScenarioIds } from "./scenario-selection.js"; + +const MOCK_LANE = { + providerMode: "mock-openai" as const, + primaryModel: "mock-openai/gpt-5.6-luna", +}; + +describe("catalog live transport QA scenario selection", () => { + it.each(["matrix", "telegram"] as const)( + "selects declared %s scenarios and preserves an explicit subset", + (channelId) => { + const scenarioIds = resolveCatalogLiveTransportQaScenarioIds({ + ...MOCK_LANE, + channelId, + }); + const explicitScenarioIds = scenarioIds.slice(0, 2).toReversed(); + const scenarioById = new Map( + readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario] as const), + ); + + expect(scenarioIds.length).toBeGreaterThan(1); + expect( + scenarioIds.every((scenarioId) => { + const scenario = scenarioById.get(scenarioId); + return ( + scenario?.execution.kind === "flow" && scenarioDeclaresQaChannel(scenario, channelId) + ); + }), + ).toBe(true); + expect( + resolveCatalogLiveTransportQaScenarioIds({ + ...MOCK_LANE, + channelId, + scenarioIds: explicitScenarioIds, + }), + ).toEqual(explicitScenarioIds); + }, + ); + + it.each([ + { channelId: "matrix", scenarioId: "whatsapp-whoami-command", mismatch: "channel=whatsapp" }, + { + channelId: "telegram", + scenarioId: "anthropic-opus-api-key-smoke", + mismatch: "provider=anthropic", + }, + ] as const)( + "rejects $scenarioId from the $channelId lane", + ({ channelId, scenarioId, mismatch }) => { + expect(() => + resolveCatalogLiveTransportQaScenarioIds({ + ...MOCK_LANE, + channelId, + scenarioIds: [scenarioId], + }), + ).toThrow(mismatch); + }, + ); + + it.each([ + { channelId: "matrix", scenarioId: "thread-follow-up" }, + { channelId: "telegram", scenarioId: "channel-message-flows" }, + ] as const)( + "keeps $scenarioId eligible through both $channelId drivers", + ({ channelId, scenarioId }) => { + const selectForDriver = (channelDriver: "crabline" | "live") => + resolveCatalogLiveTransportQaScenarioIds({ + ...MOCK_LANE, + channelId, + channelDriver, + scenarioIds: [scenarioId], + }); + + expect(selectForDriver("live")).toEqual([scenarioId]); + expect(selectForDriver("crabline")).toEqual([scenarioId]); + }, + ); +}); diff --git a/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts b/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts index 008e370b5c89..3c6bce4be5be 100644 --- a/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts +++ b/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts @@ -9,6 +9,35 @@ import { scenarioDeclaresQaChannel, } from "../../profile-planning.js"; import { readQaScenarioPack } from "../../scenario-catalog.js"; +import type { QaScorecardChannelDriver } from "../../scorecard-taxonomy.js"; +import { selectQaFlowSuiteScenarios } from "../../suite-planning.js"; + +export function resolveCatalogLiveTransportQaScenarioIds(params: { + channelId: string; + channelDriver?: QaScorecardChannelDriver; + primaryModel?: string; + providerMode: QaProviderModeInput; + scenarioIds?: readonly string[]; +}) { + const channelId = params.channelId.trim().toLowerCase(); + const catalog = readQaScenarioPack().scenarios; + const scenarios = params.scenarioIds?.length + ? catalog + : catalog.filter((scenario) => scenarioDeclaresQaChannel(scenario, channelId)); + const providerMode = normalizeQaProviderMode(params.providerMode); + const selectedScenarios = selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: params.scenarioIds?.length ? [...params.scenarioIds] : undefined, + providerMode, + primaryModel: params.primaryModel?.trim() || defaultQaModelForMode(providerMode), + channelDriver: params.channelDriver ?? "live", + channel: channelId, + }); + if (selectedScenarios.length === 0) { + throw new Error(`${channelId} QA catalog selection resolved no scenarios.`); + } + return selectedScenarios.map((scenario) => scenario.id); +} export function resolveLiveTransportQaScenarioIds(params: { channelId: string; diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 4973d548a4d3..2c72b25e851e 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -2632,12 +2632,15 @@ describe("package artifact reuse", () => { ); expect(matrixJob["continue-on-error"]).toBeUndefined(); expect(matrixJob.strategy?.matrix?.shard).toEqual([1, 2, 3, 4, 5]); - expect(workflowStep(matrixJob, "Run Matrix live lane").run).toContain( - '--shard "${{ matrix.shard }}/5"', - ); - expect(workflowStep(matrixJob, "Run Matrix live lane").run).toContain( - 'matrix_selection=(--shard "${{ matrix.shard }}/5")', - ); + expect( + workflowStep(matrixJob, "Verify internal Matrix execution sharding support").run, + ).toContain('grep -Fq "OPENCLAW_QA_EXECUTION_SHARD"'); + expect( + workflowStep(matrixJob, "Verify internal Matrix execution sharding support").run, + ).toContain("does not support internal post-selection QA sharding"); + expect(workflowStep(matrixJob, "Run Matrix live lane").env).toMatchObject({ + OPENCLAW_QA_EXECUTION_SHARD: "${{ format('{0}/5', matrix.shard) }}", + }); expect(readWorkflow(QA_LIVE_TRANSPORTS_WORKFLOW).jobs?.run_live_matrix_sharded).toBeUndefined(); expect(releaseTelegramWorkflow).toContain( 'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2', @@ -2645,23 +2648,23 @@ describe("package artifact reuse", () => { expect(workflowStep(matrixJob, "Run Matrix live lane").run).not.toContain("for attempt in"); expect(qaWorkflow).not.toContain("Matrix live lane failed on attempt"); expect(qaWorkflow).not.toContain("OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS"); - expect(qaWorkflow).toContain('matrix_selection=(--profile "${legacy_profile}")'); + expect(qaWorkflow).not.toContain("legacy_profiles"); + expect(qaWorkflow).not.toContain("matrix_selection"); + expect(qaWorkflow).not.toContain("--shard "); expect(qaWorkflow).not.toContain("--fail-fast"); }); - it("keeps modern Matrix shards and legacy profile partitions coverage-equivalent", () => { + it("keeps Matrix sharding internal to post-selection execution", () => { const matrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix"); - const run = workflowStep(matrixJob, "Run Matrix live lane").run; + const runStep = workflowStep(matrixJob, "Run Matrix live lane"); + const run = runStep.run; - expect(run).toContain('grep -Fq -- "--shard "'); - expect(run).toContain('matrix_selection=(--shard "${{ matrix.shard }}/5")'); - expect(run).toContain("legacy_profiles=(transport media e2ee-smoke e2ee-deep e2ee-cli)"); - expect(run).toContain('legacy_profile="${legacy_profiles[shard_index - 1]}"'); - expect(run).toContain('matrix_selection=(--profile "${legacy_profile}")'); - expect(run).toContain('"${matrix_selection[@]}" \\'); - expect(run).toContain( - "Selected target predates profile-free Matrix catalog sharding; using legacy profile", - ); + expect(runStep.env).toMatchObject({ + OPENCLAW_QA_EXECUTION_SHARD: "${{ format('{0}/5', matrix.shard) }}", + }); + expect(run).not.toContain("--shard"); + expect(run).not.toContain("--profile"); + expect(run).not.toContain("matrix_selection"); expect(matrixJob.strategy?.matrix?.shard).toEqual([1, 2, 3, 4, 5]); });