test(qa-lab): report live transport coverage lanes

CHANGELOG.md

@@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai
 - Agents/tools: remove the old sender-owner tool gating path so configured tools stay visible for trusted sessions while command and channel-action auth still carry real sender identity.
 - QA-Lab: add curated mock JSONL replay fixtures and first-drift reporting for runtime-parity audits. (#80323, refs #80176) Thanks @100yenadmin.
 - QA-Lab: replace generic evidence framing in seeded scenario prompts with concrete observed QA behavior.
+- QA-Lab: list live transport lane membership in the coverage report so real transport checks stay separate from seeded qa-channel scenarios.
 - QA-Lab: include the optional 100-turn runtime parity soak in release-soak artifacts so long-run Codex/Pi transcript drift stays visible outside the default gate. (#80395) Thanks @100yenadmin.
 - QA-Lab: add a live-only long-context progress watchdog scenario for Codex app-server timeout and stalled-run sentinels. (#80323) Thanks @100yenadmin.
 - QA-Lab: tag gateway restart recovery and streaming final-integrity scenarios as live-only runtime parity lanes. (#80323) Thanks @100yenadmin.

extensions/qa-lab/src/coverage-report.test.ts

@@ -12,6 +12,12 @@ describe("qa coverage report", () => {
     expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0);
     expect(inventory.overlappingCoverage.length).toBeGreaterThan(0);
     expect(inventory.missingCoverage).toStrictEqual([]);
+    expect(inventory.liveTransportLanes.map((lane) => lane.transportId)).toEqual([
+      "discord",
+      "slack",
+      "telegram",
+      "whatsapp",
+    ]);
     expect(inventory.byTheme.memory.map((feature) => feature.id)).toContain("memory.recall");
     expect(inventory.bySurface.memory.map((feature) => feature.id)).toContain("memory.recall");
   });
@@ -27,5 +33,10 @@ describe("qa coverage report", () => {
     expect(report).toContain("memory.recall");
     expect(report).toContain("primary: memory-recall (qa/scenarios/memory/memory-recall.md)");
     expect(report).toContain("secondary: active-memory-preprompt-recall");
+    expect(report).toContain("## Live Transport Lanes");
+    expect(report).toContain(
+      "- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume",
+    );
+    expect(report).toContain("thread-follow-up: slack-thread-follow-up");
   });
 });

extensions/qa-lab/src/coverage-report.ts

@@ -1,3 +1,7 @@
+import {
+  buildLiveTransportCoverageLaneSummaries,
+  type LiveTransportCoverageLaneSummary,
+} from "./live-transports/shared/live-transport-scenarios.js";
 import type { QaSeedScenarioWithSource } from "./scenario-catalog.js";
 
 type QaCoverageScenarioSummary = {
@@ -30,6 +34,7 @@ type QaCoverageInventory = {
   missingCoverage: QaCoverageScenarioSummary[];
   byTheme: Record<string, QaCoverageFeatureSummary[]>;
   bySurface: Record<string, QaCoverageFeatureSummary[]>;
+  liveTransportLanes: LiveTransportCoverageLaneSummary[];
 };
 
 function scenarioTheme(sourcePath: string) {
@@ -132,6 +137,7 @@ export function buildQaCoverageInventory(
     missingCoverage,
     byTheme,
     bySurface,
+    liveTransportLanes: buildLiveTransportCoverageLaneSummaries(),
   };
 }
 
@@ -144,6 +150,28 @@ function pushFeatureLines(lines: string[], features: readonly QaCoverageFeatureS
   }
 }
 
+function pushLiveTransportLines(
+  lines: string[],
+  lanes: readonly LiveTransportCoverageLaneSummary[],
+) {
+  for (const lane of lanes) {
+    const members = lane.members
+      .map((member) =>
+        member.scenarioId
+          ? `${member.standardId}: ${member.scenarioId}`
+          : `${member.standardId}: always-on`,
+      )
+      .join(", ");
+    const missing =
+      lane.baselineMissingStandardScenarioIds.length > 0
+        ? lane.baselineMissingStandardScenarioIds.join(", ")
+        : "none";
+    lines.push(
+      `- ${lane.transportId} (${lane.commandName}): ${members}; missing baseline: ${missing}`,
+    );
+  }
+}
+
 export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): string {
   const lines: string[] = [
     "# QA Coverage Inventory",
@@ -172,6 +200,12 @@ export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory):
     lines.push("");
   }
 
+  if (inventory.liveTransportLanes.length > 0) {
+    lines.push("## Live Transport Lanes", "");
+    pushLiveTransportLines(lines, inventory.liveTransportLanes);
+    lines.push("");
+  }
+
   if (inventory.overlappingCoverage.length > 0) {
     lines.push("## Overlap", "");
     pushFeatureLines(lines, inventory.overlappingCoverage);

extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, it } from "vitest";
+import { __testing as discordTesting } from "../discord/discord-live.runtime.js";
+import { __testing as slackTesting } from "../slack/slack-live.runtime.js";
+import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js";
+import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js";
 import {
   LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+  buildLiveTransportCoverageLaneSummaries,
   collectLiveTransportStandardScenarioCoverage,
   findMissingLiveTransportStandardScenarios,
   selectLiveTransportScenarios,
@@ -73,4 +78,39 @@ describe("live transport scenario helpers", () => {
       }),
     ).toEqual(["allowlist-block", "top-level-reply-shape"]);
   });
+
+  it("summarizes live transport lane membership for coverage reports", () => {
+    const lanes = buildLiveTransportCoverageLaneSummaries();
+
+    expect(lanes.map((lane) => lane.transportId)).toEqual([
+      "discord",
+      "slack",
+      "telegram",
+      "whatsapp",
+    ]);
+    expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({
+      standardId: "canary",
+    });
+    expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({
+      standardId: "thread-follow-up",
+      scenarioId: "slack-thread-follow-up",
+    });
+    expect(
+      lanes.find((lane) => lane.transportId === "discord")?.baselineMissingStandardScenarioIds,
+    ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]);
+  });
+
+  it("keeps coverage report lane summaries aligned with runtime lanes", () => {
+    const lanes = new Map(
+      buildLiveTransportCoverageLaneSummaries().map((lane) => [
+        lane.transportId,
+        lane.standardScenarioIds,
+      ]),
+    );
+
+    expect(lanes.get("discord")).toEqual(discordTesting.DISCORD_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("slack")).toEqual(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("telegram")).toEqual(telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS);
+    expect(lanes.get("whatsapp")).toEqual(whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS);
+  });
 });

extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts

@@ -1,4 +1,4 @@
-type LiveTransportStandardScenarioId =
+export type LiveTransportStandardScenarioId =
   | "canary"
   | "mention-gating"
   | "allowlist-block"
@@ -22,6 +22,26 @@ type LiveTransportStandardScenarioDefinition = {
   title: string;
 };
 
+export type LiveTransportCoverageMember = {
+  scenarioId?: string;
+  standardId: LiveTransportStandardScenarioId;
+};
+
+export type LiveTransportCoverageLane = {
+  commandName: string;
+  members: readonly LiveTransportCoverageMember[];
+  transportId: string;
+};
+
+export type LiveTransportCoverageLaneSummary = {
+  baselineMissingStandardScenarioIds: LiveTransportStandardScenarioId[];
+  commandName: string;
+  memberCount: number;
+  members: LiveTransportCoverageMember[];
+  standardScenarioIds: LiveTransportStandardScenarioId[];
+  transportId: string;
+};
+
 const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [
   {
     id: "canary",
@@ -79,6 +99,48 @@ export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTranspo
     "restart-resume",
   ] as const;
 
+export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [
+  {
+    transportId: "discord",
+    commandName: "discord",
+    members: [
+      { standardId: "canary", scenarioId: "discord-canary" },
+      { standardId: "mention-gating", scenarioId: "discord-mention-gating" },
+    ],
+  },
+  {
+    transportId: "slack",
+    commandName: "slack",
+    members: [
+      { standardId: "canary", scenarioId: "slack-canary" },
+      { standardId: "mention-gating", scenarioId: "slack-mention-gating" },
+      { standardId: "allowlist-block", scenarioId: "slack-allowlist-block" },
+      { standardId: "top-level-reply-shape", scenarioId: "slack-top-level-reply-shape" },
+      { standardId: "restart-resume", scenarioId: "slack-restart-resume" },
+      { standardId: "thread-follow-up", scenarioId: "slack-thread-follow-up" },
+      { standardId: "thread-isolation", scenarioId: "slack-thread-isolation" },
+    ],
+  },
+  {
+    transportId: "telegram",
+    commandName: "telegram",
+    members: [
+      { standardId: "canary" },
+      { standardId: "help-command", scenarioId: "telegram-help-command" },
+      { standardId: "mention-gating", scenarioId: "telegram-mention-gating" },
+    ],
+  },
+  {
+    transportId: "whatsapp",
+    commandName: "whatsapp",
+    members: [
+      { standardId: "canary", scenarioId: "whatsapp-canary" },
+      { standardId: "allowlist-block", scenarioId: "whatsapp-pairing-block" },
+      { standardId: "mention-gating", scenarioId: "whatsapp-mention-gating" },
+    ],
+  },
+] as const;
+
 const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set(
   LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id),
 );
@@ -146,3 +208,31 @@ export function findMissingLiveTransportStandardScenarios(params: {
   const covered = new Set(params.coveredStandardScenarioIds);
   return params.expectedStandardScenarioIds.filter((id) => !covered.has(id));
 }
+
+export function buildLiveTransportCoverageLaneSummaries(
+  lanes: readonly LiveTransportCoverageLane[] = LIVE_TRANSPORT_COVERAGE_LANES,
+): LiveTransportCoverageLaneSummary[] {
+  return lanes
+    .map((lane) => {
+      const standardScenarioIds = collectLiveTransportStandardScenarioCoverage({
+        scenarios: lane.members.map((member) => ({
+          id: member.scenarioId ?? `${lane.transportId}:${member.standardId}`,
+          standardId: member.standardId,
+          timeoutMs: 0,
+          title: member.standardId,
+        })),
+      });
+      return {
+        baselineMissingStandardScenarioIds: findMissingLiveTransportStandardScenarios({
+          coveredStandardScenarioIds: standardScenarioIds,
+          expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
+        }),
+        commandName: lane.commandName,
+        memberCount: lane.members.length,
+        members: [...lane.members],
+        standardScenarioIds,
+        transportId: lane.transportId,
+      };
+    })
+    .toSorted((left, right) => left.transportId.localeCompare(right.transportId));
+}