fix(qa): restore automatic smoke coverage across channels (#101186)

* fix(qa): restore channel-sharded smoke coverage Co-authored-by: Peter Steinberger <steipete@gmail.com> Co-authored-by: Dallin Romney <dallinromney@gmail.com> * fix(qa): restore channel-sharded smoke coverage * fix(qa): restore channel-sharded smoke coverage * fix(qa): restore channel-sharded smoke coverage --------- Co-authored-by: Vincent Koc <25068+vincentkoc@users.noreply.github.com> Co-authored-by: Dallin Romney <dallinromney@gmail.com> Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
2026-08-12 21:53:00 -06:00 · 2026-07-07 01:09:40 +01:00
parent 736c6743c1
commit aed2482292
12 changed files with 363 additions and 31 deletions
@@ -916,6 +916,111 @@ jobs:
              ;;
          esac

+  qa-smoke-ci-shard:
+    permissions:
+      contents: read
+    name: QA Smoke CI (${{ matrix.name }})
+    needs: [preflight]
+    if: needs.preflight.outputs.run_checks_fast == 'true'
+    runs-on: ${{ github.event_name == 'workflow_dispatch' && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04') }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        include:
+          - name: matrix
+            slug: matrix
+          - name: telegram 1/2
+            slug: telegram-1-of-2
+          - name: telegram 2/2
+            slug: telegram-2-of-2
+    steps:
+      - *linux_node_checkout_step
+      - name: Setup Node environment
+        uses: ./.github/actions/setup-node-env
+        with:
+          install-bun: "false"
+
+      - name: Run smoke profile shard
+        env:
+          SHARD_SLUG: ${{ matrix.slug }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          output_dir=".artifacts/qa-e2e/smoke-ci-profile-${SHARD_SLUG}"
+          package_dir=".artifacts/qa-e2e/smoke-ci-package-${SHARD_SLUG}"
+          export OPENCLAW_BUILD_PRIVATE_QA=1
+          export OPENCLAW_ENABLE_PRIVATE_QA_CLI=1
+          export OPENCLAW_DISABLE_BUNDLED_PLUGINS=0
+          export OPENCLAW_QA_REDACT_PUBLIC_METADATA=1
+          export OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS=180000
+          export NODE_OPTIONS=--max-old-space-size=16384
+          node scripts/package-openclaw-for-docker.mjs \
+            --output-dir "$package_dir" \
+            --output-name openclaw-current.tgz
+          export OPENCLAW_CURRENT_PACKAGE_TGZ="$PWD/$package_dir/openclaw-current.tgz"
+          SHARD_SCENARIO_IDS_JSON="$(
+            node --import tsx --input-type=module <<'EOF'
+          import { createQaSmokeCiMatrix } from "./extensions/qa-lab/src/ci-smoke-plan.ts";
+
+          const shard = createQaSmokeCiMatrix().include.find(
+            (candidate) => candidate.slug === process.env.SHARD_SLUG,
+          );
+          if (!shard) {
+            throw new Error(`unknown QA smoke shard: ${process.env.SHARD_SLUG}`);
+          }
+          process.stdout.write(JSON.stringify(shard.scenario_ids));
+          EOF
+          )"
+          export SHARD_SCENARIO_IDS_JSON
+          mapfile -t scenario_ids < <(
+            node -e 'for (const id of JSON.parse(process.env.SHARD_SCENARIO_IDS_JSON)) console.log(id)'
+          )
+          scenario_args=()
+          for scenario_id in "${scenario_ids[@]}"; do
+            scenario_args+=(--scenario "$scenario_id")
+          done
+          qa_exit_code=0
+          pnpm openclaw qa run \
+            --repo-root . \
+            --qa-profile smoke-ci \
+            --concurrency 8 \
+            --output-dir "$output_dir" \
+            "${scenario_args[@]}" || qa_exit_code=$?
+          echo "QA smoke profile evidence: \`${output_dir}\`" >> "$GITHUB_STEP_SUMMARY"
+          if [ "$qa_exit_code" -ne 0 ]; then
+            echo "::error title=QA smoke profile failed::smoke-ci shard ${SHARD_SLUG} exited ${qa_exit_code}; evidence upload will still run"
+            exit "$qa_exit_code"
+          fi
+
+      - name: Upload QA smoke profile evidence
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: qa-smoke-profile-${{ matrix.slug }}-${{ github.run_id }}-${{ github.run_attempt }}
+          path: .artifacts/qa-e2e/smoke-ci-profile-${{ matrix.slug }}/
+          if-no-files-found: warn
+          retention-days: 7
+
+  qa-smoke-ci:
+    permissions:
+      contents: read
+    name: QA Smoke CI
+    needs: [preflight, qa-smoke-ci-shard]
+    if: always() && needs.preflight.outputs.run_checks_fast == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - name: Verify QA smoke shards
+        env:
+          QA_SMOKE_SHARDS_RESULT: ${{ needs.qa-smoke-ci-shard.result }}
+        run: |
+          if [ "$QA_SMOKE_SHARDS_RESULT" != "success" ]; then
+            echo "QA Smoke CI shards finished with result: $QA_SMOKE_SHARDS_RESULT" >&2
+            exit 1
+          fi
+
  checks-fast-plugin-contracts-shard:
    permissions:
      contents: read
@@ -33,6 +33,8 @@ dispatch.
 | `pnpm-store-warmup`                | Warm the lockfile-pinned pnpm store cache without blocking Linux Node shards                                                                                                                       | Node or docs-check lanes selected                   |
 | `build-artifacts`                  | Build `dist/`, Control UI, built-CLI smoke checks, startup memory, and embedded built-artifact checks                                                                                              | Node-relevant changes                               |
 | `checks-fast-core`                 | Fast Linux correctness lanes: bundled + protocol, Bun launcher, and the CI-routing fast task                                                                                                       | Node-relevant changes                               |
+| `qa-smoke-ci-shard`                | Three bounded, channel-compatible QA Smoke shards; the default Telegram group is balanced across two shards                                                                                        | Node-relevant changes                               |
+| `qa-smoke-ci`                      | Hosted aggregate check preserving the required `QA Smoke CI` status after every shard succeeds                                                                                                     | Node-relevant changes                               |
 | `checks-fast-contracts-plugins-*`  | Two weighted plugin contract shards                                                                                                                                                                | Node-relevant changes                               |
 | `checks-fast-contracts-channels-*` | Two weighted channel contract shards                                                                                                                                                               | Node-relevant changes                               |
 | `checks-node-*`                    | Core Node test shards, excluding channel, bundled, contract, and extension lanes                                                                                                                   | Node-relevant changes                               |
@@ -151,15 +153,15 @@ Release, private dist-tag, or other platform publication.

 ## Runners

-| Runner                          | Jobs                                                                                                                                                                                                                                                                            |
-| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ubuntu-24.04`                  | Manual CI dispatch and non-canonical repository fallbacks, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow                                                                       |
-| `blacksmith-4vcpu-ubuntu-2404`  | `preflight`, `security-fast`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core`, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` |
-| `blacksmith-8vcpu-ubuntu-2404`  | Retained heavy Linux Node suites, boundary/extension-heavy `check-additional-*` shards, and `android`                                                                                                                                                                           |
-| `blacksmith-16vcpu-ubuntu-2404` | `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved)                                                                                                                                                              |
-| `blacksmith-8vcpu-windows-2025` | `checks-windows`                                                                                                                                                                                                                                                                |
-| `blacksmith-6vcpu-macos-15`     | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15`                                                                                                                                                                                                              |
-| `blacksmith-12vcpu-macos-26`    | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26`                                                                                                                                                                                             |
+| Runner                          | Jobs                                                                                                                                                                                                                                                                                               |
+| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ubuntu-24.04`                  | Manual CI dispatch and non-canonical repository fallbacks, the QA Smoke aggregate, CodeQL security and quality scans, workflow-sanity, labeler, auto-response, the standalone Docs workflow, and the whole Install Smoke workflow                                                                  |
+| `blacksmith-4vcpu-ubuntu-2404`  | `preflight`, `security-fast`, `pnpm-store-warmup`, `native-i18n`, `checks-fast-core` except QA Smoke CI, plugin/channel contract shards, most bundled/lower-weight Linux Node shards, `check-*` lanes except `check-lint`, selected `check-additional-*` shards, `check-docs`, and `skills-python` |
+| `blacksmith-8vcpu-ubuntu-2404`  | Retained heavy Linux Node suites, boundary/extension-heavy `check-additional-*` shards, and `android`                                                                                                                                                                                              |
+| `blacksmith-16vcpu-ubuntu-2404` | Automatic QA Smoke CI shards, `build-artifacts` in CI and Testbox, and `check-lint` (CPU-sensitive enough that 8 vCPU cost more than they saved)                                                                                                                                                   |
+| `blacksmith-8vcpu-windows-2025` | `checks-windows`                                                                                                                                                                                                                                                                                   |
+| `blacksmith-6vcpu-macos-15`     | `macos-node` on `openclaw/openclaw`; forks fall back to `macos-15`                                                                                                                                                                                                                                 |
+| `blacksmith-12vcpu-macos-26`    | `macos-swift` and `ios-build` on `openclaw/openclaw`; forks fall back to `macos-26`                                                                                                                                                                                                                |

 ## Runner registration budget

@@ -0,0 +1,55 @@
+// Qa Lab tests cover bounded CI smoke shard planning.
+import { describe, expect, it } from "vitest";
+import { createQaSmokeCiMatrix } from "./ci-smoke-plan.js";
+import { readQaScenarioPack } from "./scenario-catalog.js";
+import { scenarioRequiresIsolatedQaSuiteWorker } from "./suite-planning.js";
+
+describe("createQaSmokeCiMatrix", () => {
+  it("partitions every smoke scenario into bounded channel-compatible shards", () => {
+    const first = createQaSmokeCiMatrix();
+    const second = createQaSmokeCiMatrix();
+
+    expect(second).toEqual(first);
+    expect(first.include.map((shard) => shard.name)).toEqual([
+      "matrix",
+      "telegram 1/2",
+      "telegram 2/2",
+    ]);
+    expect(first.include).toHaveLength(3);
+
+    const scenarioIds = first.include.flatMap((shard) => shard.scenario_ids);
+    expect(scenarioIds).toHaveLength(89);
+    expect(new Set(scenarioIds).size).toBe(scenarioIds.length);
+    const scenarioById = new Map(
+      readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario] as const),
+    );
+    expect(
+      new Set(scenarioIds.map((scenarioId) => scenarioById.get(scenarioId)?.execution.kind)),
+    ).toEqual(new Set(["flow", "playwright", "script"]));
+    expect(scenarioIds).not.toContain("slack-restart-resume");
+    expect(scenarioIds).not.toContain("whatsapp-restart-resume");
+    expect(first.include.every((shard) => shard.scenario_ids.length > 0)).toBe(true);
+
+    const telegramShards = first.include.filter((shard) => shard.channel === "telegram");
+    expect(telegramShards).toHaveLength(2);
+    const telegramWeights = telegramShards.map((shard) =>
+      shard.scenario_ids.reduce((weight, scenarioId) => {
+        const scenario = scenarioById.get(scenarioId);
+        if (!scenario) {
+          throw new Error(`missing QA scenario ${scenarioId}`);
+        }
+        if (scenario.execution.kind === "script") {
+          return weight + 8;
+        }
+        if (scenario.execution.kind === "playwright") {
+          return weight + 6;
+        }
+        if (scenario.execution.kind === "vitest") {
+          return weight + 4;
+        }
+        return weight + (scenarioRequiresIsolatedQaSuiteWorker(scenario) ? 3 : 1);
+      }, 0),
+    );
+    expect(Math.abs(telegramWeights[0] - telegramWeights[1])).toBeLessThanOrEqual(1);
+  });
+});
@@ -0,0 +1,123 @@
+// Qa Lab plugin module plans bounded CI smoke shards.
+import { OPENCLAW_CRABLINE_DEFAULT_CHANNEL } from "@openclaw/crabline";
+import { defaultQaModelForMode, normalizeQaProviderMode } from "./model-selection.js";
+import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js";
+import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js";
+import {
+  scenarioMatchesQaProviderLane,
+  scenarioRequiresIsolatedQaSuiteWorker,
+} from "./suite-planning.js";
+
+const QA_SMOKE_PROFILE = "smoke-ci";
+const QA_SMOKE_DEFAULT_CHANNEL_SHARDS = 2;
+const QA_SMOKE_MAX_SHARDS = 8;
+
+export type QaSmokeCiShard = {
+  name: string;
+  slug: string;
+  channel: string;
+  scenario_ids: string[];
+};
+
+function scenarioWeight(scenario: QaSeedScenarioWithSource): number {
+  if (scenario.execution.kind === "script") {
+    return 8;
+  }
+  if (scenario.execution.kind === "playwright") {
+    return 6;
+  }
+  if (scenario.execution.kind === "vitest") {
+    return 4;
+  }
+  return scenarioRequiresIsolatedQaSuiteWorker(scenario) ? 3 : 1;
+}
+
+function splitBalanced(
+  scenarios: readonly QaSeedScenarioWithSource[],
+  shardCount: number,
+): QaSeedScenarioWithSource[][] {
+  const shards = Array.from({ length: shardCount }, () => ({ scenarios: [], weight: 0 })) as Array<{
+    scenarios: QaSeedScenarioWithSource[];
+    weight: number;
+  }>;
+  const sortedScenarios = [...scenarios].toSorted(
+    (left, right) =>
+      scenarioWeight(right) - scenarioWeight(left) || left.id.localeCompare(right.id),
+  );
+  for (const scenario of sortedScenarios) {
+    const target = shards.toSorted(
+      (left, right) => left.weight - right.weight || left.scenarios.length - right.scenarios.length,
+    )[0];
+    target.scenarios.push(scenario);
+    target.weight += scenarioWeight(scenario);
+  }
+  return shards.map((shard) =>
+    shard.scenarios.toSorted((left, right) => left.id.localeCompare(right.id)),
+  );
+}
+
+function slugifyChannel(channel: string): string {
+  return channel.replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-|-$/g, "");
+}
+
+export function createQaSmokeCiMatrix(): { include: QaSmokeCiShard[] } {
+  const scenarioPack = readQaScenarioPack();
+  const scorecardReport = readQaScorecardTaxonomyReport(scenarioPack.scenarios);
+  const profile = scorecardReport.profiles.find((entry) => entry.id === QA_SMOKE_PROFILE);
+  if (!profile) {
+    throw new Error(`taxonomy.yaml does not define QA run profile ${QA_SMOKE_PROFILE}.`);
+  }
+  const categoryScenarioRefs = new Set(
+    scorecardReport.categories
+      .filter((category) => category.profiles.includes(QA_SMOKE_PROFILE))
+      .flatMap((category) => category.scenarioRefs),
+  );
+  const providerMode = normalizeQaProviderMode("mock-openai");
+  const primaryModel = defaultQaModelForMode(providerMode);
+  const scenarios = scenarioPack.scenarios.filter(
+    (scenario) =>
+      categoryScenarioRefs.has(scenario.sourcePath) &&
+      scenarioMatchesQaProviderLane({
+        scenario,
+        providerMode,
+        primaryModel,
+        channelDriver: profile.channelDriver,
+      }),
+  );
+  if (scenarios.length === 0) {
+    throw new Error(`${QA_SMOKE_PROFILE} did not resolve any executable QA scenarios.`);
+  }
+
+  const scenariosByChannel = new Map<string, QaSeedScenarioWithSource[]>();
+  for (const scenario of scenarios) {
+    const channel = scenario.execution.channel ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL;
+    const channelScenarios = scenariosByChannel.get(channel) ?? [];
+    channelScenarios.push(scenario);
+    scenariosByChannel.set(channel, channelScenarios);
+  }
+
+  const shards = [...scenariosByChannel.entries()]
+    .toSorted(([left], [right]) => left.localeCompare(right))
+    .flatMap(([channel, channelScenarios]) => {
+      const shardCount =
+        channel === OPENCLAW_CRABLINE_DEFAULT_CHANNEL
+          ? Math.min(QA_SMOKE_DEFAULT_CHANNEL_SHARDS, channelScenarios.length)
+          : 1;
+      return splitBalanced(channelScenarios, shardCount).map((shardScenarios, index) => {
+        const suffix = shardCount > 1 ? ` ${index + 1}/${shardCount}` : "";
+        const slugSuffix = shardCount > 1 ? `-${index + 1}-of-${shardCount}` : "";
+        return {
+          name: `${channel}${suffix}`,
+          slug: `${slugifyChannel(channel)}${slugSuffix}`,
+          channel,
+          scenario_ids: shardScenarios.map((scenario) => scenario.id),
+        };
+      });
+    });
+  if (shards.length > QA_SMOKE_MAX_SHARDS) {
+    throw new Error(
+      `${QA_SMOKE_PROFILE} resolved ${shards.length} CI shards; maximum is ${QA_SMOKE_MAX_SHARDS}.`,
+    );
+  }
+  return { include: shards };
+}
@@ -578,6 +578,7 @@ describe("qa cli runtime", () => {
    await runQaProfileCommand({
      repoRoot: "/tmp/openclaw-repo",
      profile: "smoke-ci",
+      scenarioIds: ["dm-chat-baseline", "instruction-followthrough-repo-contract"],
    });

    const suiteArgs = mockFirstObjectArg(runQaSuite);
@@ -811,8 +811,10 @@ describe("qa scenario catalog", () => {
    expect(config?.requiredProviderMode).toBe("mock-openai");
  });

-  it("marks channel-owned access gates with their required channel driver", () => {
+  it("marks channel-owned scenarios that require the live driver", () => {
    const liveScenarioIds = [
+      "slack-restart-resume",
+      "whatsapp-restart-resume",
      "whatsapp-access-control-dm-disabled",
      "whatsapp-access-control-dm-open",
      "whatsapp-access-control-group-disabled",
@@ -829,6 +831,12 @@ describe("qa scenario catalog", () => {
    }
  });

+  it("isolates channel baseline silence assertions from shared transport state", () => {
+    const scenario = requireFlowScenario(readQaScenarioById("channel-chat-baseline"));
+
+    expect(scenario.execution.suiteIsolation).toBe("isolated");
+  });
+
  it("adds a dreaming shadow trial report scenario", () => {
    const scenario = readQaScenarioById("dreaming-shadow-trial-report");
    const config = readQaScenarioExecutionConfig("dreaming-shadow-trial-report") as
@@ -302,7 +302,7 @@ describe("qa suite runtime launcher", () => {
        channelDriver: "crabline",
        smokeArtifactPath: "crabline-fake-provider-smoke.json",
      },
-      scenarioIds: ["channel-chat-baseline", "dm-chat-baseline", "control-ui-chat-flow-playwright"],
+      scenarioIds: ["telegram-help-command", "dm-chat-baseline", "control-ui-chat-flow-playwright"],
    });

    const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "crabline-serial");
@@ -311,7 +311,7 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow"),
        concurrency: 1,
-        scenarioIds: ["channel-chat-baseline", "dm-chat-baseline"],
+        scenarioIds: ["telegram-help-command", "dm-chat-baseline"],
      }),
    );
    expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
@@ -367,7 +367,7 @@ describe("qa suite runtime launcher", () => {
      },
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "runtime-tool-image-generate",
        "runtime-inventory-drift-check",
        "session-memory-ranking",
@@ -382,7 +382,7 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow", "shared"),
        concurrency: 1,
-        scenarioIds: ["channel-chat-baseline"],
+        scenarioIds: ["dm-chat-baseline"],
      }),
    );
    for (const [index, scenarioId] of [
@@ -442,7 +442,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/serial",
      concurrency: 1,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "group-visible-reply-tool",
        "control-ui-chat-flow-playwright",
      ],
@@ -499,7 +499,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/script-isolation",
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "control-ui-chat-flow-playwright",
        "docker-npm-onboard-channel-agent",
      ],
@@ -607,7 +607,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/weighted",
      concurrency: 3,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "group-visible-reply-tool",
        "control-ui-chat-flow-playwright",
      ],
@@ -696,7 +696,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/native-before-isolated",
      concurrency: 2,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "group-visible-reply-tool",
        "control-ui-chat-flow-playwright",
      ],
@@ -776,7 +776,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/smoke",
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "group-visible-reply-tool",
        "control-ui-chat-flow-playwright",
      ],
@@ -790,7 +790,7 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow", "shared"),
        concurrency: 1,
-        scenarioIds: ["channel-chat-baseline"],
+        scenarioIds: ["dm-chat-baseline"],
      }),
    );
    expect(runQaFlowSuite).toHaveBeenNthCalledWith(
@@ -808,7 +808,7 @@ describe("qa suite runtime launcher", () => {
      scenarios?: Array<{ name?: unknown; status?: unknown }>;
    };
    expect(summary.scenarios).toMatchObject([
-      { name: "channel-chat-baseline", status: "pass" },
+      { name: "dm-chat-baseline", status: "pass" },
      { name: "group-visible-reply-tool", status: "pass" },
      { name: "Control UI chat flow Playwright coverage", status: "pass" },
    ]);
@@ -821,7 +821,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/smoke",
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "telegram-help-command",
        "dm-chat-baseline",
        "thread-follow-up",
        "control-ui-chat-flow-playwright",
@@ -835,7 +835,7 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow", "shared-1"),
        concurrency: 1,
-        scenarioIds: ["channel-chat-baseline"],
+        scenarioIds: ["telegram-help-command"],
      }),
    );
    expect(runQaFlowSuite).toHaveBeenNthCalledWith(
@@ -863,7 +863,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/smoke",
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "runtime-tool-image-generate",
        "runtime-inventory-drift-check",
        "session-memory-ranking",
@@ -878,7 +878,7 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow", "shared"),
        concurrency: 1,
-        scenarioIds: ["channel-chat-baseline"],
+        scenarioIds: ["dm-chat-baseline"],
      }),
    );
    expect(runQaFlowSuite).toHaveBeenNthCalledWith(
@@ -903,7 +903,7 @@ describe("qa suite runtime launcher", () => {
      outputDir: ".artifacts/qa-e2e/gateway-state",
      concurrency: 8,
      scenarioIds: [
-        "channel-chat-baseline",
+        "dm-chat-baseline",
        "subagent-stale-child-links",
        "control-ui-chat-flow-playwright",
      ],
@@ -915,7 +915,7 @@ describe("qa suite runtime launcher", () => {
      1,
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow", "shared"),
-        scenarioIds: ["channel-chat-baseline"],
+        scenarioIds: ["dm-chat-baseline"],
      }),
    );
    expect(runQaFlowSuite).toHaveBeenNthCalledWith(
@@ -22,6 +22,7 @@ scenario:
    - extensions/qa-lab/src/bus-state.ts
  execution:
    kind: flow
+    suiteIsolation: isolated
    summary: Verify the QA agent can respond correctly in a shared channel and respect mention-driven group semantics.
    config:
      expectedMarker: QA-CHANNEL-BASELINE-OK
@@ -21,6 +21,7 @@ scenario:
    suiteIsolation: isolated
    summary: Exercise normalized restart and continuation delivery.
    config:
+      requiredChannelDriver: live
      conversationKind: group
      conversationId: C0123456789
      senderId: U0123456789
@@ -21,6 +21,7 @@ scenario:
    suiteIsolation: isolated
    summary: Exercise normalized restart and continuation delivery.
    config:
+      requiredChannelDriver: live
      conversationKind: direct
      conversationId: 15550000001@s.whatsapp.net
      senderId: 15550000002@s.whatsapp.net
@@ -1,7 +1,7 @@
 // Intentional Knip unused-file findings. These are dynamic entrypoints,
 // generated/build inputs, manifest-discovered plugin surfaces, live-test
 // helpers, or package bridge files that static production scanning cannot see.
-export const KNIP_UNUSED_FILE_ALLOWLIST = [];
+export const KNIP_UNUSED_FILE_ALLOWLIST = ["extensions/qa-lab/src/ci-smoke-plan.ts"];

 // Knip can disagree across supported local/CI platforms for files that are
 // only reachable through test-only import graphs, sparse-checkout proof
@@ -993,6 +993,15 @@ describe("ci workflow guards", () => {
    const runStep = fastCoreJob.steps.find(
      (step) => step.name === "Run ${{ matrix.task }} (${{ matrix.runtime }})",
    );
+    const smokeShardJob = workflow.jobs["qa-smoke-ci-shard"];
+    const smokeRunStep = smokeShardJob.steps.find(
+      (step) => step.name === "Run smoke profile shard",
+    );
+    const smokeUploadStep = smokeShardJob.steps.find(
+      (step) => step.name === "Upload QA smoke profile evidence",
+    );
+    const smokeAggregateJob = workflow.jobs["qa-smoke-ci"];
+
    const ciWorkflowText = readFileSync(".github/workflows/ci.yml", "utf8");

    expect(preflightStep.run).not.toContain("qa-smoke-profile");
@@ -1006,9 +1015,35 @@ describe("ci workflow guards", () => {
    expect(runStep.run).toContain("contracts-plugins-ci-routing)");
    expect(runStep.run).toContain("ci-routing)");
    expect(fastCoreJob["runs-on"]).toContain("matrix.runner");
-    expect(ciWorkflowText).not.toContain('check_name: "QA Smoke CI"');
-    expect(ciWorkflowText).not.toContain("--qa-profile smoke-ci");
-    expect(ciWorkflowText).not.toContain("Upload QA smoke profile evidence");
+    expect(smokeShardJob.name).toBe("QA Smoke CI (${{ matrix.name }})");
+    expect(smokeShardJob.strategy["max-parallel"]).toBe(3);
+    expect(smokeShardJob.strategy.matrix.include.map((entry) => entry.slug)).toEqual([
+      "matrix",
+      "telegram-1-of-2",
+      "telegram-2-of-2",
+    ]);
+    expect(smokeShardJob["runs-on"]).toContain("blacksmith-16vcpu-ubuntu-2404");
+    expect(smokeRunStep.run).toContain("createQaSmokeCiMatrix");
+    expect(smokeRunStep.run).toContain("--qa-profile smoke-ci");
+    expect(smokeRunStep.run).toContain("--concurrency 8");
+    expect(smokeRunStep.run).toContain('scenario_args+=(--scenario "$scenario_id")');
+    expect(smokeRunStep.run).not.toContain("--category");
+    expect(smokeRunStep.run).not.toContain("--allow-failures");
+    expect(smokeRunStep.run).toContain("qa_exit_code=0");
+    expect(smokeRunStep.run).toContain('exit "$qa_exit_code"');
+    expect(smokeRunStep.run).toContain("scripts/package-openclaw-for-docker.mjs");
+    expect(smokeRunStep.run).toContain("OPENCLAW_CURRENT_PACKAGE_TGZ");
+    expect(smokeRunStep.run).toContain("--max-old-space-size=16384");
+    expect(smokeRunStep.run).not.toContain("scripts/build-all.mjs qaRuntime");
+    expect(smokeRunStep.run).not.toContain("OPENAI_API_KEY");
+    expect(smokeUploadStep.if).toBe("always()");
+    expect(smokeUploadStep.with).toMatchObject({
+      path: ".artifacts/qa-e2e/smoke-ci-profile-${{ matrix.slug }}/",
+      "if-no-files-found": "warn",
+    });
+    expect(smokeAggregateJob.name).toBe("QA Smoke CI");
+    expect(smokeAggregateJob.needs).toEqual(["preflight", "qa-smoke-ci-shard"]);
+    expect(smokeAggregateJob["runs-on"]).toBe("ubuntu-24.04");
    expect(runStep.run.match(/test\/scripts\/ci-workflow-guards\.test\.ts/g)?.length).toBe(2);
  });