diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ead344159e6..3779448e6023 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai - Agents/tools: remove the old sender-owner tool gating path so configured tools stay visible for trusted sessions while command and channel-action auth still carry real sender identity. - QA-Lab: add curated mock JSONL replay fixtures and first-drift reporting for runtime-parity audits. (#80323, refs #80176) Thanks @100yenadmin. - QA-Lab: replace generic evidence framing in seeded scenario prompts with concrete observed QA behavior. +- QA-Lab: list live transport lane membership in the coverage report so real transport checks stay separate from seeded qa-channel scenarios. - QA-Lab: include the optional 100-turn runtime parity soak in release-soak artifacts so long-run Codex/Pi transcript drift stays visible outside the default gate. (#80395) Thanks @100yenadmin. - QA-Lab: add a live-only long-context progress watchdog scenario for Codex app-server timeout and stalled-run sentinels. (#80323) Thanks @100yenadmin. - QA-Lab: tag gateway restart recovery and streaming final-integrity scenarios as live-only runtime parity lanes. (#80323) Thanks @100yenadmin. diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index 2b95a62a76da..3ad7e9abea98 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -12,6 +12,12 @@ describe("qa coverage report", () => { expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0); expect(inventory.overlappingCoverage.length).toBeGreaterThan(0); expect(inventory.missingCoverage).toStrictEqual([]); + expect(inventory.liveTransportLanes.map((lane) => lane.transportId)).toEqual([ + "discord", + "slack", + "telegram", + "whatsapp", + ]); expect(inventory.byTheme.memory.map((feature) => feature.id)).toContain("memory.recall"); expect(inventory.bySurface.memory.map((feature) => feature.id)).toContain("memory.recall"); }); @@ -27,5 +33,10 @@ describe("qa coverage report", () => { expect(report).toContain("memory.recall"); expect(report).toContain("primary: memory-recall (qa/scenarios/memory/memory-recall.md)"); expect(report).toContain("secondary: active-memory-preprompt-recall"); + expect(report).toContain("## Live Transport Lanes"); + expect(report).toContain( + "- telegram (telegram): canary: always-on, help-command: telegram-help-command, mention-gating: telegram-mention-gating; missing baseline: allowlist-block, top-level-reply-shape, restart-resume", + ); + expect(report).toContain("thread-follow-up: slack-thread-follow-up"); }); }); diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index 6efaa4af46e2..5b6297a801d2 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -1,3 +1,7 @@ +import { + buildLiveTransportCoverageLaneSummaries, + type LiveTransportCoverageLaneSummary, +} from "./live-transports/shared/live-transport-scenarios.js"; import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; type QaCoverageScenarioSummary = { @@ -30,6 +34,7 @@ type QaCoverageInventory = { missingCoverage: QaCoverageScenarioSummary[]; byTheme: Record; bySurface: Record; + liveTransportLanes: LiveTransportCoverageLaneSummary[]; }; function scenarioTheme(sourcePath: string) { @@ -132,6 +137,7 @@ export function buildQaCoverageInventory( missingCoverage, byTheme, bySurface, + liveTransportLanes: buildLiveTransportCoverageLaneSummaries(), }; } @@ -144,6 +150,28 @@ function pushFeatureLines(lines: string[], features: readonly QaCoverageFeatureS } } +function pushLiveTransportLines( + lines: string[], + lanes: readonly LiveTransportCoverageLaneSummary[], +) { + for (const lane of lanes) { + const members = lane.members + .map((member) => + member.scenarioId + ? `${member.standardId}: ${member.scenarioId}` + : `${member.standardId}: always-on`, + ) + .join(", "); + const missing = + lane.baselineMissingStandardScenarioIds.length > 0 + ? lane.baselineMissingStandardScenarioIds.join(", ") + : "none"; + lines.push( + `- ${lane.transportId} (${lane.commandName}): ${members}; missing baseline: ${missing}`, + ); + } +} + export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): string { const lines: string[] = [ "# QA Coverage Inventory", @@ -172,6 +200,12 @@ export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): lines.push(""); } + if (inventory.liveTransportLanes.length > 0) { + lines.push("## Live Transport Lanes", ""); + pushLiveTransportLines(lines, inventory.liveTransportLanes); + lines.push(""); + } + if (inventory.overlappingCoverage.length > 0) { lines.push("## Overlap", ""); pushFeatureLines(lines, inventory.overlappingCoverage); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index 79da20e60174..cca81d170592 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; +import { __testing as discordTesting } from "../discord/discord-live.runtime.js"; +import { __testing as slackTesting } from "../slack/slack-live.runtime.js"; +import { __testing as telegramTesting } from "../telegram/telegram-live.runtime.js"; +import { __testing as whatsAppTesting } from "../whatsapp/whatsapp-live.runtime.js"; import { LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + buildLiveTransportCoverageLaneSummaries, collectLiveTransportStandardScenarioCoverage, findMissingLiveTransportStandardScenarios, selectLiveTransportScenarios, @@ -73,4 +78,39 @@ describe("live transport scenario helpers", () => { }), ).toEqual(["allowlist-block", "top-level-reply-shape"]); }); + + it("summarizes live transport lane membership for coverage reports", () => { + const lanes = buildLiveTransportCoverageLaneSummaries(); + + expect(lanes.map((lane) => lane.transportId)).toEqual([ + "discord", + "slack", + "telegram", + "whatsapp", + ]); + expect(lanes.find((lane) => lane.transportId === "telegram")?.members).toContainEqual({ + standardId: "canary", + }); + expect(lanes.find((lane) => lane.transportId === "slack")?.members).toContainEqual({ + standardId: "thread-follow-up", + scenarioId: "slack-thread-follow-up", + }); + expect( + lanes.find((lane) => lane.transportId === "discord")?.baselineMissingStandardScenarioIds, + ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); + }); + + it("keeps coverage report lane summaries aligned with runtime lanes", () => { + const lanes = new Map( + buildLiveTransportCoverageLaneSummaries().map((lane) => [ + lane.transportId, + lane.standardScenarioIds, + ]), + ); + + expect(lanes.get("discord")).toEqual(discordTesting.DISCORD_QA_STANDARD_SCENARIO_IDS); + expect(lanes.get("slack")).toEqual(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS); + expect(lanes.get("telegram")).toEqual(telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS); + expect(lanes.get("whatsapp")).toEqual(whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS); + }); }); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index 88bf702b121e..ad2325803f14 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -1,4 +1,4 @@ -type LiveTransportStandardScenarioId = +export type LiveTransportStandardScenarioId = | "canary" | "mention-gating" | "allowlist-block" @@ -22,6 +22,26 @@ type LiveTransportStandardScenarioDefinition = { title: string; }; +export type LiveTransportCoverageMember = { + scenarioId?: string; + standardId: LiveTransportStandardScenarioId; +}; + +export type LiveTransportCoverageLane = { + commandName: string; + members: readonly LiveTransportCoverageMember[]; + transportId: string; +}; + +export type LiveTransportCoverageLaneSummary = { + baselineMissingStandardScenarioIds: LiveTransportStandardScenarioId[]; + commandName: string; + memberCount: number; + members: LiveTransportCoverageMember[]; + standardScenarioIds: LiveTransportStandardScenarioId[]; + transportId: string; +}; + const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ { id: "canary", @@ -79,6 +99,48 @@ export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTranspo "restart-resume", ] as const; +export const LIVE_TRANSPORT_COVERAGE_LANES: readonly LiveTransportCoverageLane[] = [ + { + transportId: "discord", + commandName: "discord", + members: [ + { standardId: "canary", scenarioId: "discord-canary" }, + { standardId: "mention-gating", scenarioId: "discord-mention-gating" }, + ], + }, + { + transportId: "slack", + commandName: "slack", + members: [ + { standardId: "canary", scenarioId: "slack-canary" }, + { standardId: "mention-gating", scenarioId: "slack-mention-gating" }, + { standardId: "allowlist-block", scenarioId: "slack-allowlist-block" }, + { standardId: "top-level-reply-shape", scenarioId: "slack-top-level-reply-shape" }, + { standardId: "restart-resume", scenarioId: "slack-restart-resume" }, + { standardId: "thread-follow-up", scenarioId: "slack-thread-follow-up" }, + { standardId: "thread-isolation", scenarioId: "slack-thread-isolation" }, + ], + }, + { + transportId: "telegram", + commandName: "telegram", + members: [ + { standardId: "canary" }, + { standardId: "help-command", scenarioId: "telegram-help-command" }, + { standardId: "mention-gating", scenarioId: "telegram-mention-gating" }, + ], + }, + { + transportId: "whatsapp", + commandName: "whatsapp", + members: [ + { standardId: "canary", scenarioId: "whatsapp-canary" }, + { standardId: "allowlist-block", scenarioId: "whatsapp-pairing-block" }, + { standardId: "mention-gating", scenarioId: "whatsapp-mention-gating" }, + ], + }, +] as const; + const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), ); @@ -146,3 +208,31 @@ export function findMissingLiveTransportStandardScenarios(params: { const covered = new Set(params.coveredStandardScenarioIds); return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); } + +export function buildLiveTransportCoverageLaneSummaries( + lanes: readonly LiveTransportCoverageLane[] = LIVE_TRANSPORT_COVERAGE_LANES, +): LiveTransportCoverageLaneSummary[] { + return lanes + .map((lane) => { + const standardScenarioIds = collectLiveTransportStandardScenarioCoverage({ + scenarios: lane.members.map((member) => ({ + id: member.scenarioId ?? `${lane.transportId}:${member.standardId}`, + standardId: member.standardId, + timeoutMs: 0, + title: member.standardId, + })), + }); + return { + baselineMissingStandardScenarioIds: findMissingLiveTransportStandardScenarios({ + coveredStandardScenarioIds: standardScenarioIds, + expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + }), + commandName: lane.commandName, + memberCount: lane.members.length, + members: [...lane.members], + standardScenarioIds, + transportId: lane.transportId, + }; + }) + .toSorted((left, right) => left.transportId.localeCompare(right.transportId)); +}