diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 1ef9bb2583f2..14b89a230de5 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -4272d9b4edc35fbdf9e1980c109845f59283a5c1e120235fb158a9db3c054b01 plugin-sdk-api-baseline.json -e24313a7fadaad09b58c1b683e2906973ff01d8b1bb3f403b9cef30bdcb89ee6 plugin-sdk-api-baseline.jsonl +80d54b82c6cc4808cd95f747aea0b010234a0cae99fed4c8881a2c458414eca0 plugin-sdk-api-baseline.json +5b75842a255d7c89a6412e6a3f5f485c25e860d08b10519e75963269c0ed343d plugin-sdk-api-baseline.jsonl diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index af513d4c9efd..c203838194c0 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -266,6 +266,10 @@ The doctor checks Convex broker env, validates endpoint settings, and verifies a Live transport lanes share one contract instead of each inventing their own scenario list shape. `qa-channel` is the broad synthetic product-behavior suite and is not part of the live transport coverage matrix. +Live transport runners should import the shared scenario ids, baseline +coverage helpers, and scenario-selection helper from +`openclaw/plugin-sdk/qa-live-transport-scenarios`. + | Lane | Canary | Mention gating | Bot-to-bot | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration | | -------- | ------ | -------------- | ---------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- | | Matrix | x | x | x | x | x | x | x | x | x | | | diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 732bff70bdfc..d1fee7c75d87 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -220,6 +220,7 @@ and pairing-path families. | `plugin-sdk/lazy-runtime` | Lazy runtime import/binding helpers such as `createLazyRuntimeModule`, `createLazyRuntimeMethod`, and `createLazyRuntimeSurface` | | `plugin-sdk/process-runtime` | Process exec helpers | | `plugin-sdk/cli-runtime` | CLI formatting, wait, version, argument-invocation, and lazy command-group helpers | + | `plugin-sdk/qa-live-transport-scenarios` | Shared live transport QA scenario ids, baseline coverage helpers, and scenario-selection helper | | `plugin-sdk/gateway-method-runtime` | Reserved Gateway method dispatch helper for plugin HTTP routes that declare `contracts.gatewayMethodDispatch: ["authenticated-request"]` | | `plugin-sdk/gateway-runtime` | Gateway client, event-loop-ready client start helper, gateway CLI RPC, gateway protocol errors, and channel-status patch helpers | | `plugin-sdk/config-contracts` | Focused type-only config surface for plugin config shapes such as `OpenClawConfig` and channel/provider config types | diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index ad5ca3816bb9..59ea801df8d9 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -14,12 +14,13 @@ import { } from "./live-transport-scenarios.js"; describe("live transport scenario helpers", () => { - it("keeps package-mode helpers off private QA SDK exports", () => { + it("uses the public live transport scenario SDK seam", () => { const source = fs.readFileSync( fileURLToPath(new URL("./live-transport-scenarios.ts", import.meta.url)), "utf8", ); + expect(source).toContain("openclaw/plugin-sdk/qa-live-transport-scenarios"); expect(source).not.toContain("openclaw/plugin-sdk/qa-runtime"); }); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts index 475107f2aab6..8a79dc708b7c 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.ts @@ -1,151 +1,19 @@ -export type LiveTransportStandardScenarioId = - | "canary" - | "mention-gating" - | "allowlist-block" - | "top-level-reply-shape" - | "restart-resume" - | "thread-follow-up" - | "thread-isolation" - | "reaction-observation" - | "help-command"; +import { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + type LiveTransportScenarioDefinition, + type LiveTransportStandardScenarioId, +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; -export type LiveTransportScenarioDefinition = { - id: TId; - standardId?: LiveTransportStandardScenarioId; - timeoutMs: number; - title: string; -}; - -type LiveTransportStandardScenarioDefinition = { - description: string; - id: LiveTransportStandardScenarioId; - title: string; -}; - -const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ - { - id: "canary", - title: "Transport canary", - description: "The lane can trigger one known-good reply on the real transport.", - }, - { - id: "mention-gating", - title: "Mention gating", - description: "Messages without the required mention do not trigger a reply.", - }, - { - id: "allowlist-block", - title: "Sender allowlist block", - description: "Non-allowlisted senders do not trigger a reply.", - }, - { - id: "top-level-reply-shape", - title: "Top-level reply shape", - description: "Top-level replies stay top-level when the lane is configured that way.", - }, - { - id: "restart-resume", - title: "Restart resume", - description: "The lane still responds after a gateway restart.", - }, - { - id: "thread-follow-up", - title: "Thread follow-up", - description: "Threaded prompts receive threaded replies with the expected relation metadata.", - }, - { - id: "thread-isolation", - title: "Thread isolation", - description: "Fresh top-level prompts stay out of prior threads.", - }, - { - id: "reaction-observation", - title: "Reaction observation", - description: "Reaction events are observed and normalized correctly.", - }, - { - id: "help-command", - title: "Help command", - description: "The transport-specific help command path replies successfully.", - }, -] as const; - -export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = - [ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ] as const; - -const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( - LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), -); - -function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { - for (const id of ids) { - if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { - throw new Error(`unknown live transport standard scenario id: ${id}`); - } - } -} - -export function selectLiveTransportScenarios(params: { - ids?: string[]; - laneLabel: string; - scenarios: readonly TDefinition[]; -}) { - if (!params.ids || params.ids.length === 0) { - return [...params.scenarios]; - } - const requested = new Set(params.ids); - const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); - const missingIds = [...requested].filter( - (id) => !selected.some((scenario) => scenario.id === id), - ); - if (missingIds.length > 0) { - throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); - } - return selected; -} - -export function collectLiveTransportStandardScenarioCoverage(params: { - alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; - scenarios: readonly LiveTransportScenarioDefinition[]; -}) { - const coverage: LiveTransportStandardScenarioId[] = []; - const seen = new Set(); - const append = (id: LiveTransportStandardScenarioId | undefined) => { - if (!id || seen.has(id)) { - return; - } - seen.add(id); - coverage.push(id); - }; - - assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); - for (const id of params.alwaysOnStandardScenarioIds ?? []) { - append(id); - } - for (const scenario of params.scenarios) { - if (scenario.standardId) { - assertKnownStandardScenarioIds([scenario.standardId]); - } - append(scenario.standardId); - } - return coverage; -} - -export function findMissingLiveTransportStandardScenarios(params: { - coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; - expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; -}) { - assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); - assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); - const covered = new Set(params.coveredStandardScenarioIds); - return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); -} +export { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, + type LiveTransportStandardScenarioId, +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; export type LiveTransportCoverageMember = { scenarioId?: string; @@ -214,14 +82,13 @@ export function buildLiveTransportCoverageLaneSummaries( ): LiveTransportCoverageLaneSummary[] { return lanes .map((lane) => { - const standardScenarioIds = collectLiveTransportStandardScenarioCoverage({ - scenarios: lane.members.map((member) => ({ - id: member.scenarioId ?? `${lane.transportId}:${member.standardId}`, - standardId: member.standardId, - timeoutMs: 0, - title: member.standardId, - })), - }); + const scenarios: LiveTransportScenarioDefinition[] = lane.members.map((member) => ({ + id: member.scenarioId ?? `${lane.transportId}:${member.standardId}`, + standardId: member.standardId, + timeoutMs: 0, + title: member.standardId, + })); + const standardScenarioIds = collectLiveTransportStandardScenarioCoverage({ scenarios }); return { baselineMissingStandardScenarioIds: findMissingLiveTransportStandardScenarios({ coveredStandardScenarioIds: standardScenarioIds, diff --git a/extensions/qa-matrix/src/shared/live-transport-scenarios.ts b/extensions/qa-matrix/src/shared/live-transport-scenarios.ts index c91ad2ebb15f..38accb872893 100644 --- a/extensions/qa-matrix/src/shared/live-transport-scenarios.ts +++ b/extensions/qa-matrix/src/shared/live-transport-scenarios.ts @@ -5,4 +5,4 @@ export { selectLiveTransportScenarios, type LiveTransportScenarioDefinition, type LiveTransportStandardScenarioId, -} from "openclaw/plugin-sdk/qa-runtime"; +} from "openclaw/plugin-sdk/qa-live-transport-scenarios"; diff --git a/package.json b/package.json index 03ea31cc3a41..42f0bb50d716 100644 --- a/package.json +++ b/package.json @@ -1069,6 +1069,10 @@ "types": "./dist/plugin-sdk/qa-runner-runtime.d.ts", "default": "./dist/plugin-sdk/qa-runner-runtime.js" }, + "./plugin-sdk/qa-live-transport-scenarios": { + "types": "./dist/plugin-sdk/qa-live-transport-scenarios.d.ts", + "default": "./dist/plugin-sdk/qa-live-transport-scenarios.js" + }, "./plugin-sdk/memory-core": { "types": "./dist/plugin-sdk/memory-core.d.ts", "default": "./dist/plugin-sdk/memory-core.js" diff --git a/scripts/lib/plugin-sdk-doc-metadata.ts b/scripts/lib/plugin-sdk-doc-metadata.ts index 342e7f12f8e6..2f8e4be3892d 100644 --- a/scripts/lib/plugin-sdk-doc-metadata.ts +++ b/scripts/lib/plugin-sdk-doc-metadata.ts @@ -101,6 +101,9 @@ export const pluginSdkDocMetadata = { "runtime-store": { category: "runtime", }, + "qa-live-transport-scenarios": { + category: "utilities", + }, "agent-runtime": { category: "runtime", }, diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 7ca39a755ff7..65417510a487 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -246,6 +246,7 @@ "persistent-dedupe", "keyed-async-queue", "qa-runner-runtime", + "qa-live-transport-scenarios", "memory-core", "memory-core-engine-runtime", "memory-core-host-engine-embeddings", diff --git a/src/plugin-sdk/qa-live-transport-scenarios.ts b/src/plugin-sdk/qa-live-transport-scenarios.ts new file mode 100644 index 000000000000..b2d67fa94686 --- /dev/null +++ b/src/plugin-sdk/qa-live-transport-scenarios.ts @@ -0,0 +1,148 @@ +export type LiveTransportStandardScenarioId = + | "canary" + | "mention-gating" + | "allowlist-block" + | "top-level-reply-shape" + | "restart-resume" + | "thread-follow-up" + | "thread-isolation" + | "reaction-observation" + | "help-command"; + +export type LiveTransportScenarioDefinition = { + id: TId; + standardId?: LiveTransportStandardScenarioId; + timeoutMs: number; + title: string; +}; + +type LiveTransportStandardScenarioDefinition = { + description: string; + id: LiveTransportStandardScenarioId; + title: string; +}; + +const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ + { + id: "canary", + title: "Transport canary", + description: "The lane can trigger one known-good reply on the real transport.", + }, + { + id: "mention-gating", + title: "Mention gating", + description: "Messages without the required mention do not trigger a reply.", + }, + { + id: "allowlist-block", + title: "Sender allowlist block", + description: "Non-allowlisted senders do not trigger a reply.", + }, + { + id: "top-level-reply-shape", + title: "Top-level reply shape", + description: "Top-level replies stay top-level when the lane is configured that way.", + }, + { + id: "restart-resume", + title: "Restart resume", + description: "The lane still responds after a gateway restart.", + }, + { + id: "thread-follow-up", + title: "Thread follow-up", + description: "Threaded prompts receive threaded replies with the expected relation metadata.", + }, + { + id: "thread-isolation", + title: "Thread isolation", + description: "Fresh top-level prompts stay out of prior threads.", + }, + { + id: "reaction-observation", + title: "Reaction observation", + description: "Reaction events are observed and normalized correctly.", + }, + { + id: "help-command", + title: "Help command", + description: "The transport-specific help command path replies successfully.", + }, +] as const; + +export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = + [ + "canary", + "mention-gating", + "allowlist-block", + "top-level-reply-shape", + "restart-resume", + ] as const; + +const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( + LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), +); + +function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { + for (const id of ids) { + if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { + throw new Error(`unknown live transport standard scenario id: ${id}`); + } + } +} + +export function selectLiveTransportScenarios(params: { + ids?: string[]; + laneLabel: string; + scenarios: readonly TDefinition[]; +}) { + if (!params.ids || params.ids.length === 0) { + return [...params.scenarios]; + } + const requested = new Set(params.ids); + const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); + const missingIds = [...requested].filter( + (id) => !selected.some((scenario) => scenario.id === id), + ); + if (missingIds.length > 0) { + throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); + } + return selected; +} + +export function collectLiveTransportStandardScenarioCoverage(params: { + alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; + scenarios: readonly LiveTransportScenarioDefinition[]; +}) { + const coverage: LiveTransportStandardScenarioId[] = []; + const seen = new Set(); + const append = (id: LiveTransportStandardScenarioId | undefined) => { + if (!id || seen.has(id)) { + return; + } + seen.add(id); + coverage.push(id); + }; + + assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); + for (const id of params.alwaysOnStandardScenarioIds ?? []) { + append(id); + } + for (const scenario of params.scenarios) { + if (scenario.standardId) { + assertKnownStandardScenarioIds([scenario.standardId]); + } + append(scenario.standardId); + } + return coverage; +} + +export function findMissingLiveTransportStandardScenarios(params: { + coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; + expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; +}) { + assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); + assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); + const covered = new Set(params.coveredStandardScenarioIds); + return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); +} diff --git a/src/plugin-sdk/qa-runtime.ts b/src/plugin-sdk/qa-runtime.ts index d63ef053e3a3..9de44d531c13 100644 --- a/src/plugin-sdk/qa-runtime.ts +++ b/src/plugin-sdk/qa-runtime.ts @@ -221,154 +221,14 @@ export type QaReportScenario = { steps?: QaReportCheck[]; }; -export type LiveTransportStandardScenarioId = - | "canary" - | "mention-gating" - | "allowlist-block" - | "top-level-reply-shape" - | "restart-resume" - | "thread-follow-up" - | "thread-isolation" - | "reaction-observation" - | "help-command"; - -export type LiveTransportScenarioDefinition = { - id: TId; - standardId?: LiveTransportStandardScenarioId; - timeoutMs: number; - title: string; -}; - -type LiveTransportStandardScenarioDefinition = { - description: string; - id: LiveTransportStandardScenarioId; - title: string; -}; - -const LIVE_TRANSPORT_STANDARD_SCENARIOS: readonly LiveTransportStandardScenarioDefinition[] = [ - { - id: "canary", - title: "Transport canary", - description: "The lane can trigger one known-good reply on the real transport.", - }, - { - id: "mention-gating", - title: "Mention gating", - description: "Messages without the required mention do not trigger a reply.", - }, - { - id: "allowlist-block", - title: "Sender allowlist block", - description: "Non-allowlisted senders do not trigger a reply.", - }, - { - id: "top-level-reply-shape", - title: "Top-level reply shape", - description: "Top-level replies stay top-level when the lane is configured that way.", - }, - { - id: "restart-resume", - title: "Restart resume", - description: "The lane still responds after a gateway restart.", - }, - { - id: "thread-follow-up", - title: "Thread follow-up", - description: "Threaded prompts receive threaded replies with the expected relation metadata.", - }, - { - id: "thread-isolation", - title: "Thread isolation", - description: "Fresh top-level prompts stay out of prior threads.", - }, - { - id: "reaction-observation", - title: "Reaction observation", - description: "Reaction events are observed and normalized correctly.", - }, - { - id: "help-command", - title: "Help command", - description: "The transport-specific help command path replies successfully.", - }, -] as const; - -export const LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS: readonly LiveTransportStandardScenarioId[] = - [ - "canary", - "mention-gating", - "allowlist-block", - "top-level-reply-shape", - "restart-resume", - ] as const; - -const LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET = new Set( - LIVE_TRANSPORT_STANDARD_SCENARIOS.map((scenario) => scenario.id), -); - -function assertKnownStandardScenarioIds(ids: readonly LiveTransportStandardScenarioId[]) { - for (const id of ids) { - if (!LIVE_TRANSPORT_STANDARD_SCENARIO_ID_SET.has(id)) { - throw new Error(`unknown live transport standard scenario id: ${id}`); - } - } -} - -export function selectLiveTransportScenarios(params: { - ids?: string[]; - laneLabel: string; - scenarios: readonly TDefinition[]; -}) { - if (!params.ids || params.ids.length === 0) { - return [...params.scenarios]; - } - const requested = new Set(params.ids); - const selected = params.scenarios.filter((scenario) => params.ids?.includes(scenario.id)); - const missingIds = [...requested].filter( - (id) => !selected.some((scenario) => scenario.id === id), - ); - if (missingIds.length > 0) { - throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); - } - return selected; -} - -export function collectLiveTransportStandardScenarioCoverage(params: { - alwaysOnStandardScenarioIds?: readonly LiveTransportStandardScenarioId[]; - scenarios: readonly LiveTransportScenarioDefinition[]; -}) { - const coverage: LiveTransportStandardScenarioId[] = []; - const seen = new Set(); - const append = (id: LiveTransportStandardScenarioId | undefined) => { - if (!id || seen.has(id)) { - return; - } - seen.add(id); - coverage.push(id); - }; - - assertKnownStandardScenarioIds(params.alwaysOnStandardScenarioIds ?? []); - for (const id of params.alwaysOnStandardScenarioIds ?? []) { - append(id); - } - for (const scenario of params.scenarios) { - if (scenario.standardId) { - assertKnownStandardScenarioIds([scenario.standardId]); - } - append(scenario.standardId); - } - return coverage; -} - -export function findMissingLiveTransportStandardScenarios(params: { - coveredStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; - expectedStandardScenarioIds: readonly LiveTransportStandardScenarioId[]; -}) { - assertKnownStandardScenarioIds(params.coveredStandardScenarioIds); - assertKnownStandardScenarioIds(params.expectedStandardScenarioIds); - const covered = new Set(params.coveredStandardScenarioIds); - return params.expectedStandardScenarioIds.filter((id) => !covered.has(id)); -} +export { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + collectLiveTransportStandardScenarioCoverage, + findMissingLiveTransportStandardScenarios, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, + type LiveTransportStandardScenarioId, +} from "./qa-live-transport-scenarios.js"; export type QaDockerRunCommand = ( command: string,