From aed24822923b62dccd8d1494e83c7a333a68ee1e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 7 Jul 2026 01:09:40 +0100 Subject: [PATCH] fix(qa): restore automatic smoke coverage across channels (#101186) * fix(qa): restore channel-sharded smoke coverage Co-authored-by: Peter Steinberger Co-authored-by: Dallin Romney * fix(qa): restore channel-sharded smoke coverage * fix(qa): restore channel-sharded smoke coverage * fix(qa): restore channel-sharded smoke coverage --------- Co-authored-by: Vincent Koc <25068+vincentkoc@users.noreply.github.com> Co-authored-by: Dallin Romney Co-authored-by: Vincent Koc --- .github/workflows/ci.yml | 105 +++++++++++++++ docs/ci.md | 20 +-- extensions/qa-lab/src/ci-smoke-plan.test.ts | 55 ++++++++ extensions/qa-lab/src/ci-smoke-plan.ts | 123 ++++++++++++++++++ extensions/qa-lab/src/cli.runtime.test.ts | 1 + .../qa-lab/src/scenario-catalog.test.ts | 10 +- .../qa-lab/src/suite-launch.runtime.test.ts | 34 ++--- .../channels/channel-chat-baseline.yaml | 1 + .../channels/slack-restart-resume.yaml | 1 + .../channels/whatsapp-restart-resume.yaml | 1 + scripts/deadcode-unused-files.allowlist.mjs | 2 +- test/scripts/ci-workflow-guards.test.ts | 41 +++++- 12 files changed, 363 insertions(+), 31 deletions(-) create mode 100644 extensions/qa-lab/src/ci-smoke-plan.test.ts create mode 100644 extensions/qa-lab/src/ci-smoke-plan.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74be05f8b19a..74e34de74710 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -916,6 +916,111 @@ jobs: ;; esac + qa-smoke-ci-shard: + permissions: + contents: read + name: QA Smoke CI (${{ matrix.name }}) + needs: [preflight] + if: needs.preflight.outputs.run_checks_fast == 'true' + runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04') }} + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 3 + matrix: + include: + - name: matrix + slug: matrix + - name: telegram 1/2 + slug: telegram-1-of-2 + - name: telegram 2/2 + slug: telegram-2-of-2 + steps: + - *linux_node_checkout_step + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Run smoke profile shard + env: + SHARD_SLUG: ${{ matrix.slug }} + shell: bash + run: | + set -euo pipefail + output_dir=".artifacts/qa-e2e/smoke-ci-profile-${SHARD_SLUG}" + package_dir=".artifacts/qa-e2e/smoke-ci-package-${SHARD_SLUG}" + export OPENCLAW_BUILD_PRIVATE_QA=1 + export OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 + export OPENCLAW_DISABLE_BUNDLED_PLUGINS=0 + export OPENCLAW_QA_REDACT_PUBLIC_METADATA=1 + export OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS=180000 + export NODE_OPTIONS=--max-old-space-size=16384 + node scripts/package-openclaw-for-docker.mjs \ + --output-dir "$package_dir" \ + --output-name openclaw-current.tgz + export OPENCLAW_CURRENT_PACKAGE_TGZ="$PWD/$package_dir/openclaw-current.tgz" + SHARD_SCENARIO_IDS_JSON="$( + node --import tsx --input-type=module <<'EOF' + import { createQaSmokeCiMatrix } from "./extensions/qa-lab/src/ci-smoke-plan.ts"; + + const shard = createQaSmokeCiMatrix().include.find( + (candidate) => candidate.slug === process.env.SHARD_SLUG, + ); + if (!shard) { + throw new Error(`unknown QA smoke shard: ${process.env.SHARD_SLUG}`); + } + process.stdout.write(JSON.stringify(shard.scenario_ids)); + EOF + )" + export SHARD_SCENARIO_IDS_JSON + mapfile -t scenario_ids < <( + node -e 'for (const id of JSON.parse(process.env.SHARD_SCENARIO_IDS_JSON)) console.log(id)' + ) + scenario_args=() + for scenario_id in "${scenario_ids[@]}"; do + scenario_args+=(--scenario "$scenario_id") + done + qa_exit_code=0 + pnpm openclaw qa run \ + --repo-root . \ + --qa-profile smoke-ci \ + --concurrency 8 \ + --output-dir "$output_dir" \ + "${scenario_args[@]}" || qa_exit_code=$? + echo "QA smoke profile evidence: \`${output_dir}\`" >> "$GITHUB_STEP_SUMMARY" + if [ "$qa_exit_code" -ne 0 ]; then + echo "::error title=QA smoke profile failed::smoke-ci shard ${SHARD_SLUG} exited ${qa_exit_code}; evidence upload will still run" + exit "$qa_exit_code" + fi + + - name: Upload QA smoke profile evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: qa-smoke-profile-${{ matrix.slug }}-${{ github.run_id }}-${{ github.run_attempt }} + path: .artifacts/qa-e2e/smoke-ci-profile-${{ matrix.slug }}/ + if-no-files-found: warn + retention-days: 7 + + qa-smoke-ci: + permissions: + contents: read + name: QA Smoke CI + needs: [preflight, qa-smoke-ci-shard] + if: always() && needs.preflight.outputs.run_checks_fast == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Verify QA smoke shards + env: + QA_SMOKE_SHARDS_RESULT: ${{ needs.qa-smoke-ci-shard.result }} + run: | + if [ "$QA_SMOKE_SHARDS_RESULT" != "success" ]; then + echo "QA Smoke CI shards finished with result: $QA_SMOKE_SHARDS_RESULT" >&2 + exit 1 + fi + checks-fast-plugin-contracts-shard: permissions: contents: read diff --git a/docs/ci.md b/docs/ci.md index 62cac7a41253..45050c476779 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -33,6 +33,8 @@ dispatch. | `pnpm-store-warmup` | Warm the lockfile-pinned pnpm store cache without blocking Linux Node shards | Node or docs-check lanes selected | | `build-artifacts` | Build `dist/`, Control UI, built-CLI smoke checks, startup memory, and embedded built-artifact checks | Node-relevant changes | | `checks-fast-core` | Fast Linux correctness lanes: bundled + protocol, Bun launcher, and the CI-routing fast task | Node-relevant changes | +| `qa-smoke-ci-shard` | Three bounded, channel-compatible QA Smoke shards; the default Telegram group is balanced across two shards | Node-relevant changes | +| `qa-smoke-ci` | Hosted aggregate check preserving the required `QA Smoke CI` status after every shard succeeds | Node-relevant changes | | `checks-fast-contracts-plugins-*` | Two weighted plugin contract shards | Node-relevant changes | | `checks-fast-contracts-channels-*` | Two weighted channel contract shards | Node-relevant changes | | `checks-node-*` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | @@ -151,15 +153,15 @@ Release, private dist-tag, or other platform publication. ## Runners -| Runner | Jobs | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `ubuntu-24.04` | Manual CI dispatch and non-canonical repository fallbacks, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow | -| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `security-fast`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core`, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` | -| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, boundary/extension-heavy `check-additional-*` shards, and `android` | -| `blacksmith-16vcpu-ubuntu-2404` | `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) | -| `blacksmith-8vcpu-windows-2025` | `checks-windows` | -| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` | -| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` | +| Runner | Jobs | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ubuntu-24.04` | Manual CI dispatch and non-canonical repository fallbacks, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow | +| `blacksmith-4vcpu-ubuntu-2404` | `preflight`, `security-fast`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` | +| `blacksmith-8vcpu-ubuntu-2404` | Retained heavy Linux Node suites, boundary/extension-heavy `check-additional-*` shards, and `android` | +| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved) | +| `blacksmith-8vcpu-windows-2025` | `checks-windows` | +| `blacksmith-6vcpu-macos-15` | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15` | +| `blacksmith-12vcpu-macos-26` | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26` | ## Runner registration budget diff --git a/extensions/qa-lab/src/ci-smoke-plan.test.ts b/extensions/qa-lab/src/ci-smoke-plan.test.ts new file mode 100644 index 000000000000..39e44c548f58 --- /dev/null +++ b/extensions/qa-lab/src/ci-smoke-plan.test.ts @@ -0,0 +1,55 @@ +// Qa Lab tests cover bounded CI smoke shard planning. +import { describe, expect, it } from "vitest"; +import { createQaSmokeCiMatrix } from "./ci-smoke-plan.js"; +import { readQaScenarioPack } from "./scenario-catalog.js"; +import { scenarioRequiresIsolatedQaSuiteWorker } from "./suite-planning.js"; + +describe("createQaSmokeCiMatrix", () => { + it("partitions every smoke scenario into bounded channel-compatible shards", () => { + const first = createQaSmokeCiMatrix(); + const second = createQaSmokeCiMatrix(); + + expect(second).toEqual(first); + expect(first.include.map((shard) => shard.name)).toEqual([ + "matrix", + "telegram 1/2", + "telegram 2/2", + ]); + expect(first.include).toHaveLength(3); + + const scenarioIds = first.include.flatMap((shard) => shard.scenario_ids); + expect(scenarioIds).toHaveLength(89); + expect(new Set(scenarioIds).size).toBe(scenarioIds.length); + const scenarioById = new Map( + readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario] as const), + ); + expect( + new Set(scenarioIds.map((scenarioId) => scenarioById.get(scenarioId)?.execution.kind)), + ).toEqual(new Set(["flow", "playwright", "script"])); + expect(scenarioIds).not.toContain("slack-restart-resume"); + expect(scenarioIds).not.toContain("whatsapp-restart-resume"); + expect(first.include.every((shard) => shard.scenario_ids.length > 0)).toBe(true); + + const telegramShards = first.include.filter((shard) => shard.channel === "telegram"); + expect(telegramShards).toHaveLength(2); + const telegramWeights = telegramShards.map((shard) => + shard.scenario_ids.reduce((weight, scenarioId) => { + const scenario = scenarioById.get(scenarioId); + if (!scenario) { + throw new Error(`missing QA scenario ${scenarioId}`); + } + if (scenario.execution.kind === "script") { + return weight + 8; + } + if (scenario.execution.kind === "playwright") { + return weight + 6; + } + if (scenario.execution.kind === "vitest") { + return weight + 4; + } + return weight + (scenarioRequiresIsolatedQaSuiteWorker(scenario) ? 3 : 1); + }, 0), + ); + expect(Math.abs(telegramWeights[0] - telegramWeights[1])).toBeLessThanOrEqual(1); + }); +}); diff --git a/extensions/qa-lab/src/ci-smoke-plan.ts b/extensions/qa-lab/src/ci-smoke-plan.ts new file mode 100644 index 000000000000..213320a30305 --- /dev/null +++ b/extensions/qa-lab/src/ci-smoke-plan.ts @@ -0,0 +1,123 @@ +// Qa Lab plugin module plans bounded CI smoke shards. +import { OPENCLAW_CRABLINE_DEFAULT_CHANNEL } from "@openclaw/crabline"; +import { defaultQaModelForMode, normalizeQaProviderMode } from "./model-selection.js"; +import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; +import { + scenarioMatchesQaProviderLane, + scenarioRequiresIsolatedQaSuiteWorker, +} from "./suite-planning.js"; + +const QA_SMOKE_PROFILE = "smoke-ci"; +const QA_SMOKE_DEFAULT_CHANNEL_SHARDS = 2; +const QA_SMOKE_MAX_SHARDS = 8; + +export type QaSmokeCiShard = { + name: string; + slug: string; + channel: string; + scenario_ids: string[]; +}; + +function scenarioWeight(scenario: QaSeedScenarioWithSource): number { + if (scenario.execution.kind === "script") { + return 8; + } + if (scenario.execution.kind === "playwright") { + return 6; + } + if (scenario.execution.kind === "vitest") { + return 4; + } + return scenarioRequiresIsolatedQaSuiteWorker(scenario) ? 3 : 1; +} + +function splitBalanced( + scenarios: readonly QaSeedScenarioWithSource[], + shardCount: number, +): QaSeedScenarioWithSource[][] { + const shards = Array.from({ length: shardCount }, () => ({ scenarios: [], weight: 0 })) as Array<{ + scenarios: QaSeedScenarioWithSource[]; + weight: number; + }>; + const sortedScenarios = [...scenarios].toSorted( + (left, right) => + scenarioWeight(right) - scenarioWeight(left) || left.id.localeCompare(right.id), + ); + for (const scenario of sortedScenarios) { + const target = shards.toSorted( + (left, right) => left.weight - right.weight || left.scenarios.length - right.scenarios.length, + )[0]; + target.scenarios.push(scenario); + target.weight += scenarioWeight(scenario); + } + return shards.map((shard) => + shard.scenarios.toSorted((left, right) => left.id.localeCompare(right.id)), + ); +} + +function slugifyChannel(channel: string): string { + return channel.replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-|-$/g, ""); +} + +export function createQaSmokeCiMatrix(): { include: QaSmokeCiShard[] } { + const scenarioPack = readQaScenarioPack(); + const scorecardReport = readQaScorecardTaxonomyReport(scenarioPack.scenarios); + const profile = scorecardReport.profiles.find((entry) => entry.id === QA_SMOKE_PROFILE); + if (!profile) { + throw new Error(`taxonomy.yaml does not define QA run profile ${QA_SMOKE_PROFILE}.`); + } + const categoryScenarioRefs = new Set( + scorecardReport.categories + .filter((category) => category.profiles.includes(QA_SMOKE_PROFILE)) + .flatMap((category) => category.scenarioRefs), + ); + const providerMode = normalizeQaProviderMode("mock-openai"); + const primaryModel = defaultQaModelForMode(providerMode); + const scenarios = scenarioPack.scenarios.filter( + (scenario) => + categoryScenarioRefs.has(scenario.sourcePath) && + scenarioMatchesQaProviderLane({ + scenario, + providerMode, + primaryModel, + channelDriver: profile.channelDriver, + }), + ); + if (scenarios.length === 0) { + throw new Error(`${QA_SMOKE_PROFILE} did not resolve any executable QA scenarios.`); + } + + const scenariosByChannel = new Map(); + for (const scenario of scenarios) { + const channel = scenario.execution.channel ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL; + const channelScenarios = scenariosByChannel.get(channel) ?? []; + channelScenarios.push(scenario); + scenariosByChannel.set(channel, channelScenarios); + } + + const shards = [...scenariosByChannel.entries()] + .toSorted(([left], [right]) => left.localeCompare(right)) + .flatMap(([channel, channelScenarios]) => { + const shardCount = + channel === OPENCLAW_CRABLINE_DEFAULT_CHANNEL + ? Math.min(QA_SMOKE_DEFAULT_CHANNEL_SHARDS, channelScenarios.length) + : 1; + return splitBalanced(channelScenarios, shardCount).map((shardScenarios, index) => { + const suffix = shardCount > 1 ? ` ${index + 1}/${shardCount}` : ""; + const slugSuffix = shardCount > 1 ? `-${index + 1}-of-${shardCount}` : ""; + return { + name: `${channel}${suffix}`, + slug: `${slugifyChannel(channel)}${slugSuffix}`, + channel, + scenario_ids: shardScenarios.map((scenario) => scenario.id), + }; + }); + }); + if (shards.length > QA_SMOKE_MAX_SHARDS) { + throw new Error( + `${QA_SMOKE_PROFILE} resolved ${shards.length} CI shards; maximum is ${QA_SMOKE_MAX_SHARDS}.`, + ); + } + return { include: shards }; +} diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index f70d8016255d..5360eb3b14e1 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -578,6 +578,7 @@ describe("qa cli runtime", () => { await runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", profile: "smoke-ci", + scenarioIds: ["dm-chat-baseline", "instruction-followthrough-repo-contract"], }); const suiteArgs = mockFirstObjectArg(runQaSuite); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 4a88a53bcbff..4ae7f8b1034c 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -811,8 +811,10 @@ describe("qa scenario catalog", () => { expect(config?.requiredProviderMode).toBe("mock-openai"); }); - it("marks channel-owned access gates with their required channel driver", () => { + it("marks channel-owned scenarios that require the live driver", () => { const liveScenarioIds = [ + "slack-restart-resume", + "whatsapp-restart-resume", "whatsapp-access-control-dm-disabled", "whatsapp-access-control-dm-open", "whatsapp-access-control-group-disabled", @@ -829,6 +831,12 @@ describe("qa scenario catalog", () => { } }); + it("isolates channel baseline silence assertions from shared transport state", () => { + const scenario = requireFlowScenario(readQaScenarioById("channel-chat-baseline")); + + expect(scenario.execution.suiteIsolation).toBe("isolated"); + }); + it("adds a dreaming shadow trial report scenario", () => { const scenario = readQaScenarioById("dreaming-shadow-trial-report"); const config = readQaScenarioExecutionConfig("dreaming-shadow-trial-report") as diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 4cb3bd42e975..ed45dcff0dbc 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -302,7 +302,7 @@ describe("qa suite runtime launcher", () => { channelDriver: "crabline", smokeArtifactPath: "crabline-fake-provider-smoke.json", }, - scenarioIds: ["channel-chat-baseline", "dm-chat-baseline", "control-ui-chat-flow-playwright"], + scenarioIds: ["telegram-help-command", "dm-chat-baseline", "control-ui-chat-flow-playwright"], }); const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "crabline-serial"); @@ -311,7 +311,7 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow"), concurrency: 1, - scenarioIds: ["channel-chat-baseline", "dm-chat-baseline"], + scenarioIds: ["telegram-help-command", "dm-chat-baseline"], }), ); expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1); @@ -367,7 +367,7 @@ describe("qa suite runtime launcher", () => { }, concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "runtime-tool-image-generate", "runtime-inventory-drift-check", "session-memory-ranking", @@ -382,7 +382,7 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow", "shared"), concurrency: 1, - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["dm-chat-baseline"], }), ); for (const [index, scenarioId] of [ @@ -442,7 +442,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/serial", concurrency: 1, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "group-visible-reply-tool", "control-ui-chat-flow-playwright", ], @@ -499,7 +499,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/script-isolation", concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "control-ui-chat-flow-playwright", "docker-npm-onboard-channel-agent", ], @@ -607,7 +607,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/weighted", concurrency: 3, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "group-visible-reply-tool", "control-ui-chat-flow-playwright", ], @@ -696,7 +696,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/native-before-isolated", concurrency: 2, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "group-visible-reply-tool", "control-ui-chat-flow-playwright", ], @@ -776,7 +776,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/smoke", concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "group-visible-reply-tool", "control-ui-chat-flow-playwright", ], @@ -790,7 +790,7 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow", "shared"), concurrency: 1, - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["dm-chat-baseline"], }), ); expect(runQaFlowSuite).toHaveBeenNthCalledWith( @@ -808,7 +808,7 @@ describe("qa suite runtime launcher", () => { scenarios?: Array<{ name?: unknown; status?: unknown }>; }; expect(summary.scenarios).toMatchObject([ - { name: "channel-chat-baseline", status: "pass" }, + { name: "dm-chat-baseline", status: "pass" }, { name: "group-visible-reply-tool", status: "pass" }, { name: "Control UI chat flow Playwright coverage", status: "pass" }, ]); @@ -821,7 +821,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/smoke", concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "telegram-help-command", "dm-chat-baseline", "thread-follow-up", "control-ui-chat-flow-playwright", @@ -835,7 +835,7 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow", "shared-1"), concurrency: 1, - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["telegram-help-command"], }), ); expect(runQaFlowSuite).toHaveBeenNthCalledWith( @@ -863,7 +863,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/smoke", concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "runtime-tool-image-generate", "runtime-inventory-drift-check", "session-memory-ranking", @@ -878,7 +878,7 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow", "shared"), concurrency: 1, - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["dm-chat-baseline"], }), ); expect(runQaFlowSuite).toHaveBeenNthCalledWith( @@ -903,7 +903,7 @@ describe("qa suite runtime launcher", () => { outputDir: ".artifacts/qa-e2e/gateway-state", concurrency: 8, scenarioIds: [ - "channel-chat-baseline", + "dm-chat-baseline", "subagent-stale-child-links", "control-ui-chat-flow-playwright", ], @@ -915,7 +915,7 @@ describe("qa suite runtime launcher", () => { 1, expect.objectContaining({ outputDir: path.join(outputDir, "flow", "shared"), - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["dm-chat-baseline"], }), ); expect(runQaFlowSuite).toHaveBeenNthCalledWith( diff --git a/qa/scenarios/channels/channel-chat-baseline.yaml b/qa/scenarios/channels/channel-chat-baseline.yaml index ae9ba97118b2..944bca4de4f1 100644 --- a/qa/scenarios/channels/channel-chat-baseline.yaml +++ b/qa/scenarios/channels/channel-chat-baseline.yaml @@ -22,6 +22,7 @@ scenario: - extensions/qa-lab/src/bus-state.ts execution: kind: flow + suiteIsolation: isolated summary: Verify the QA agent can respond correctly in a shared channel and respect mention-driven group semantics. config: expectedMarker: QA-CHANNEL-BASELINE-OK diff --git a/qa/scenarios/channels/slack-restart-resume.yaml b/qa/scenarios/channels/slack-restart-resume.yaml index a7b0ec7b997b..2d0884b4f1f4 100644 --- a/qa/scenarios/channels/slack-restart-resume.yaml +++ b/qa/scenarios/channels/slack-restart-resume.yaml @@ -21,6 +21,7 @@ scenario: suiteIsolation: isolated summary: Exercise normalized restart and continuation delivery. config: + requiredChannelDriver: live conversationKind: group conversationId: C0123456789 senderId: U0123456789 diff --git a/qa/scenarios/channels/whatsapp-restart-resume.yaml b/qa/scenarios/channels/whatsapp-restart-resume.yaml index 4c30f20ff621..15ecefa4cf97 100644 --- a/qa/scenarios/channels/whatsapp-restart-resume.yaml +++ b/qa/scenarios/channels/whatsapp-restart-resume.yaml @@ -21,6 +21,7 @@ scenario: suiteIsolation: isolated summary: Exercise normalized restart and continuation delivery. config: + requiredChannelDriver: live conversationKind: direct conversationId: 15550000001@s.whatsapp.net senderId: 15550000002@s.whatsapp.net diff --git a/scripts/deadcode-unused-files.allowlist.mjs b/scripts/deadcode-unused-files.allowlist.mjs index 757eb4543e2d..4bddd41e2cec 100644 --- a/scripts/deadcode-unused-files.allowlist.mjs +++ b/scripts/deadcode-unused-files.allowlist.mjs @@ -1,7 +1,7 @@ // Intentional Knip unused-file findings. These are dynamic entrypoints, // generated/build inputs, manifest-discovered plugin surfaces, live-test // helpers, or package bridge files that static production scanning cannot see. -export const KNIP_UNUSED_FILE_ALLOWLIST = []; +export const KNIP_UNUSED_FILE_ALLOWLIST = ["extensions/qa-lab/src/ci-smoke-plan.ts"]; // Knip can disagree across supported local/CI platforms for files that are // only reachable through test-only import graphs, sparse-checkout proof diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index c8d34894c06e..1b0cd8b21f1f 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -993,6 +993,15 @@ describe("ci workflow guards", () => { const runStep = fastCoreJob.steps.find( (step) => step.name === "Run ${{ matrix.task }} (${{ matrix.runtime }})", ); + const smokeShardJob = workflow.jobs["qa-smoke-ci-shard"]; + const smokeRunStep = smokeShardJob.steps.find( + (step) => step.name === "Run smoke profile shard", + ); + const smokeUploadStep = smokeShardJob.steps.find( + (step) => step.name === "Upload QA smoke profile evidence", + ); + const smokeAggregateJob = workflow.jobs["qa-smoke-ci"]; + const ciWorkflowText = readFileSync(".github/workflows/ci.yml", "utf8"); expect(preflightStep.run).not.toContain("qa-smoke-profile"); @@ -1006,9 +1015,35 @@ describe("ci workflow guards", () => { expect(runStep.run).toContain("contracts-plugins-ci-routing)"); expect(runStep.run).toContain("ci-routing)"); expect(fastCoreJob["runs-on"]).toContain("matrix.runner"); - expect(ciWorkflowText).not.toContain('check_name: "QA Smoke CI"'); - expect(ciWorkflowText).not.toContain("--qa-profile smoke-ci"); - expect(ciWorkflowText).not.toContain("Upload QA smoke profile evidence"); + expect(smokeShardJob.name).toBe("QA Smoke CI (${{ matrix.name }})"); + expect(smokeShardJob.strategy["max-parallel"]).toBe(3); + expect(smokeShardJob.strategy.matrix.include.map((entry) => entry.slug)).toEqual([ + "matrix", + "telegram-1-of-2", + "telegram-2-of-2", + ]); + expect(smokeShardJob["runs-on"]).toContain("blacksmith-16vcpu-ubuntu-2404"); + expect(smokeRunStep.run).toContain("createQaSmokeCiMatrix"); + expect(smokeRunStep.run).toContain("--qa-profile smoke-ci"); + expect(smokeRunStep.run).toContain("--concurrency 8"); + expect(smokeRunStep.run).toContain('scenario_args+=(--scenario "$scenario_id")'); + expect(smokeRunStep.run).not.toContain("--category"); + expect(smokeRunStep.run).not.toContain("--allow-failures"); + expect(smokeRunStep.run).toContain("qa_exit_code=0"); + expect(smokeRunStep.run).toContain('exit "$qa_exit_code"'); + expect(smokeRunStep.run).toContain("scripts/package-openclaw-for-docker.mjs"); + expect(smokeRunStep.run).toContain("OPENCLAW_CURRENT_PACKAGE_TGZ"); + expect(smokeRunStep.run).toContain("--max-old-space-size=16384"); + expect(smokeRunStep.run).not.toContain("scripts/build-all.mjs qaRuntime"); + expect(smokeRunStep.run).not.toContain("OPENAI_API_KEY"); + expect(smokeUploadStep.if).toBe("always()"); + expect(smokeUploadStep.with).toMatchObject({ + path: ".artifacts/qa-e2e/smoke-ci-profile-${{ matrix.slug }}/", + "if-no-files-found": "warn", + }); + expect(smokeAggregateJob.name).toBe("QA Smoke CI"); + expect(smokeAggregateJob.needs).toEqual(["preflight", "qa-smoke-ci-shard"]); + expect(smokeAggregateJob["runs-on"]).toBe("ubuntu-24.04"); expect(runStep.run.match(/test\/scripts\/ci-workflow-guards\.test\.ts/g)?.length).toBe(2); });