diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 9f2a9039ee1f..68961097a974 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -81,7 +81,7 @@ on: - qa-parity - qa-live live_suite_filter: - description: Optional exact live/E2E suite id, or comma-separated QA live lane ids (qa-live-matrix, qa-live-telegram, qa-live-discord, qa-live-whatsapp, qa-live-slack); blank runs all selected live suites + description: Optional exact live/E2E suite id, or comma-separated QA live lane ids (qa-live-matrix, qa-live-buzz, qa-live-telegram, qa-live-discord, qa-live-whatsapp, qa-live-slack); blank runs all selected live suites required: false default: "" type: string @@ -146,6 +146,7 @@ jobs: repo_live_suite_filter: ${{ steps.inputs.outputs.repo_live_suite_filter }} cross_os_suite_filter: ${{ steps.inputs.outputs.cross_os_suite_filter }} qa_live_matrix_enabled: ${{ steps.inputs.outputs.qa_live_matrix_enabled }} + qa_live_buzz_enabled: ${{ steps.inputs.outputs.qa_live_buzz_enabled }} qa_live_telegram_enabled: ${{ steps.inputs.outputs.qa_live_telegram_enabled }} qa_live_discord_enabled: ${{ steps.inputs.outputs.qa_live_discord_enabled }} qa_live_whatsapp_enabled: ${{ steps.inputs.outputs.qa_live_whatsapp_enabled }} @@ -333,6 +334,7 @@ jobs: run: | set -euo pipefail qa_live_matrix_enabled=true + qa_live_buzz_enabled=true qa_live_telegram_enabled=true qa_live_discord_ci_enabled="$(printf '%s' "$RELEASE_QA_DISCORD_LIVE_CI_ENABLED" | tr '[:upper:]' '[:lower:]')" if [[ "$qa_live_discord_ci_enabled" != "true" && "$qa_live_discord_ci_enabled" != "1" && "$qa_live_discord_ci_enabled" != "yes" ]]; then @@ -408,6 +410,7 @@ jobs: qa_filter_seen=false repo_filter_tokens=() matrix_selected=false + buzz_selected=false telegram_selected=false discord_selected=false whatsapp_selected=false @@ -424,6 +427,7 @@ jobs: qa-live|qa-live-all|qa-all) qa_filter_seen=true matrix_selected=true + buzz_selected=true telegram_selected=true discord_selected="$qa_live_discord_ci_enabled" whatsapp_selected="$qa_live_whatsapp_ci_enabled" @@ -435,6 +439,7 @@ jobs: qa-live-non-slack|qa-non-slack|non-slack|no-slack|without-slack) qa_filter_seen=true matrix_selected=true + buzz_selected=true telegram_selected=true discord_selected="$qa_live_discord_ci_enabled" whatsapp_selected="$qa_live_whatsapp_ci_enabled" @@ -445,6 +450,10 @@ jobs: qa_filter_seen=true matrix_selected=true ;; + qa-live-buzz|qa-buzz|buzz) + qa_filter_seen=true + buzz_selected=true + ;; qa-live-telegram|qa-telegram|telegram) qa_filter_seen=true telegram_selected=true @@ -482,6 +491,7 @@ jobs: if [[ "$qa_filter_seen" == "true" ]]; then qa_live_matrix_enabled="$matrix_selected" + qa_live_buzz_enabled="$buzz_selected" qa_live_telegram_enabled="$telegram_selected" qa_live_discord_enabled="$discord_selected" qa_live_whatsapp_enabled="$whatsapp_selected" @@ -503,6 +513,7 @@ jobs: printf 'repo_live_suite_filter=%s\n' "$repo_live_suite_filter" printf 'cross_os_suite_filter=%s\n' "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT" printf 'qa_live_matrix_enabled=%s\n' "$qa_live_matrix_enabled" + printf 'qa_live_buzz_enabled=%s\n' "$qa_live_buzz_enabled" printf 'qa_live_telegram_enabled=%s\n' "$qa_live_telegram_enabled" printf 'qa_live_discord_enabled=%s\n' "$qa_live_discord_enabled" printf 'qa_live_whatsapp_enabled=%s\n' "$qa_live_whatsapp_enabled" @@ -551,7 +562,7 @@ jobs: if [[ -n "${RELEASE_CROSS_OS_SUITE_FILTER// }" ]]; then echo "- Cross-OS suite filter: \`${RELEASE_CROSS_OS_SUITE_FILTER}\`" fi - echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`" + echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Buzz \`${{ steps.inputs.outputs.qa_live_buzz_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`" if [[ -n "${RELEASE_PACKAGE_SPEC// }" ]]; then echo "- Release package spec: \`${RELEASE_PACKAGE_SPEC}\`" fi @@ -1655,6 +1666,20 @@ jobs: run_matrix: true fail_fast: ${{ fromJSON(needs.resolve_target.outputs.fail_fast) }} + qa_live_buzz_release_checks: + name: Run QA Lab live Buzz lane + needs: [resolve_target] + if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_buzz_enabled == 'true' + permissions: + contents: read + pull-requests: read + uses: ./.github/workflows/qa-live-transports-convex.yml + with: + ref: ${{ needs.resolve_target.outputs.revision }} + expected_sha: ${{ needs.resolve_target.outputs.revision }} + run_buzz: true + buzz_scenario: channel-canary,channel-mention-gating + # The dispatched child owns Telegram evidence/status artifacts; this blocking job # carries its exact conclusion into the parent summary without copying secrets or artifacts. qa_live_telegram_release_checks: @@ -2108,6 +2133,7 @@ jobs: - qa_lab_runtime_parity_release_checks - runtime_tool_coverage_release_checks - qa_live_release_checks + - qa_live_buzz_release_checks - qa_live_telegram_release_checks - qa_live_discord_release_checks - qa_live_whatsapp_release_checks @@ -2143,6 +2169,7 @@ jobs: QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }} RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: ${{ needs.runtime_tool_coverage_release_checks.result }} QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result }} + QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_buzz_release_checks.result }} QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }} QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }} QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }} @@ -2169,6 +2196,7 @@ jobs: "qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}" "runtime_tool_coverage_release_checks=${RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT}" "qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}" + "qa_live_buzz_release_checks=${QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT}" "qa_live_telegram_release_checks=${QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT}" "qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}" "qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}" diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index cc0cee9598b1..e5096edfdbc2 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -18,6 +18,16 @@ on: required: false default: false type: boolean + run_buzz: + description: Run the Buzz live lane + required: false + default: false + type: boolean + buzz_scenario: + description: Optional comma-separated Buzz scenario ids + required: false + default: "" + type: string run_matrix: description: Run the Matrix live lane required: false @@ -55,6 +65,15 @@ on: required: true default: main type: string + run_buzz: + description: Run the Buzz live lane (requires a pooled Buzz credential) + required: false + default: false + type: boolean + buzz_scenario: + description: Optional comma-separated Buzz scenario ids + required: false + type: string scenario: description: Optional comma-separated Telegram scenario ids required: false @@ -427,6 +446,95 @@ jobs: retention-days: 14 if-no-files-found: error + run_live_buzz: + name: Run Buzz live QA lane with Convex leases + needs: [authorize_actor, validate_selected_ref] + if: inputs.run_buzz + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 60 + concurrency: + group: qa-live-buzz-shared + cancel-in-progress: false + environment: qa-live-shared + steps: + - name: Checkout selected ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + install-bun: "true" + + - name: Validate required Buzz QA credential env + env: + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + shell: bash + run: | + set -euo pipefail + + for key in OPENCLAW_QA_CONVEX_SITE_URL OPENCLAW_QA_CONVEX_SECRET_CI; do + if [[ -z "${!key:-}" ]]; then + echo "Missing required ${key}." >&2 + exit 1 + fi + done + + - name: Build private QA runtime + env: + NODE_OPTIONS: --max-old-space-size=12288 + run: pnpm build + + - name: Run Buzz live lane + id: run_lane + shell: bash + env: + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000" + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000" + INPUT_SCENARIO: ${{ inputs.buzz_scenario || '' }} + run: | + set -euo pipefail + + output_dir=".artifacts/qa-e2e/buzz-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + scenario_args=() + + if [[ -n "${INPUT_SCENARIO// }" ]]; then + IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}" + for raw in "${raw_scenarios[@]}"; do + scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [[ -n "${scenario}" ]]; then + scenario_args+=(--scenario "${scenario}") + fi + done + fi + + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + pnpm openclaw qa buzz \ + --repo-root . \ + --output-dir "${output_dir}" \ + --provider-mode mock-openai \ + --credential-source convex \ + --credential-role ci \ + "${scenario_args[@]}" + + - name: Upload Buzz QA artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.expected_sha != '' && format('release-qa-live-buzz-{0}-{1}', inputs.expected_sha, github.run_attempt) || format('qa-live-buzz-{0}-{1}', github.run_id, github.run_attempt) }} + path: ${{ steps.run_lane.outputs.output_dir }} + retention-days: 14 + if-no-files-found: error + run_live_telegram: name: Run Telegram live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] diff --git a/docs/channels/buzz.md b/docs/channels/buzz.md index 51aa3510c800..d71fc6a9fc74 100644 --- a/docs/channels/buzz.md +++ b/docs/channels/buzz.md @@ -338,6 +338,32 @@ openclaw message send \ For a full round trip, have an allowed Buzz user mention the bot and confirm that OpenClaw replies in the room. +### QA Lab round trip + +Source checkouts can exercise the production Buzz channel path with two +dedicated test identities: + +```bash +pnpm openclaw qa buzz \ + --credential-file /secure/path/buzz-qa-credentials.json \ + --provider-mode mock-openai +``` + +The command runs a real relay canary and mention-gating check while using the +deterministic mock model. The private JSON credential +file contains `relayUrl`, `roomId`, `driverPrivateKey`, and `sutPrivateKey`, plus +optional `driverAuthTag` and `sutAuthTag` values for closed relays. Both test +public keys must be room members, and the SUT public key must have the **Bot** +role. A closed relay may require both public keys to be enrolled separately. +Use `--credential-source convex` for pooled QA credentials. + +Use `wss://` for hosted relays. Plaintext `ws://` credential URLs are accepted +only for loopback development relays. + +Never use a human owner or admin private key. Private keys and optional +authorization values are parent-harness secrets and must not appear in logs, +artifacts, screenshots, shell history, or source control. + ## Rotate the bot identity Bot identity rotation requires admin approval for the new public key: diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index cafa53de2712..07a5f7c31f7f 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -47,6 +47,7 @@ script aliases; both forms work. | `qa aimock` | Start only the AIMock provider server. | | `qa mock-openai` | Start only the scenario-aware `mock-openai` provider server. | | `qa credentials doctor` / `add` / `list` / `remove` | Manage the shared Convex credential pool. | +| `qa buzz` | Live transport lane against a real Buzz relay room with dedicated driver and SUT identities. | | `qa discord` | Live transport lane against a real private Discord guild channel. | | `qa matrix` | QA Lab Matrix catalog scenarios against a disposable Tuwunel homeserver. See [Matrix live lane](#matrix-live-lane). | | `qa slack` | Live transport lane against a real private Slack channel. | @@ -291,6 +292,7 @@ decision still comes from the Discord REST oracle. For the other transport-real smoke lanes: ```bash +pnpm openclaw qa buzz pnpm openclaw qa discord pnpm openclaw qa slack pnpm openclaw qa telegram @@ -299,8 +301,8 @@ pnpm openclaw qa whatsapp They target a pre-existing real channel with two bots or accounts (driver + SUT). Required env vars, scenario lists, output artifacts, and the Convex -credential pool for those four transports are documented in -[Discord, Slack, Telegram, and WhatsApp QA reference](#discord-slack-telegram-and-whatsapp-qa-reference) +credential pool for those five transports are documented in +[Buzz, Discord, Slack, Telegram, and WhatsApp QA reference](#buzz-discord-slack-telegram-and-whatsapp-qa-reference) below. ### Mantis Slack desktop and visual-task runners @@ -453,16 +455,16 @@ guest: env-based provider keys, the QA live provider config path, and `CODEX_HOME` when present. Keep `--output-dir` under the repo root so the guest can write back through the mounted workspace. -## Discord, Slack, Telegram, and WhatsApp QA reference +## Buzz, Discord, Slack, Telegram, and WhatsApp QA reference The Matrix adapter uses the disposable Docker-backed lane documented above. -Discord, Slack, Telegram, and WhatsApp run against pre-existing real +Buzz, Discord, Slack, Telegram, and WhatsApp run against pre-existing real transports, so their reference lives here. ### Shared CLI flags -These lanes register through -`extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and +These lanes register through the shared QA runner CLI contract. Transport +plugins may own the registration while QA Lab remains the suite host. They accept the same flags: | Flag | Default | Description | @@ -471,11 +473,12 @@ accept the same flags: | `--output-dir ` | `/.artifacts/qa-e2e/-` | Where reports, summaries, evidence, transport-specific artifacts, and the output log are written. Relative paths resolve against `--repo-root`. | | `--repo-root ` | `process.cwd()` | Repository root when invoking from a neutral cwd. | | `--sut-account ` | `sut` | Temporary account id inside the QA gateway config. | -| `--provider-mode ` | `live-frontier` | `mock-openai`, `aimock`, or `live-frontier`. | +| `--provider-mode ` | `live-frontier` (Buzz: `mock-openai`) | `mock-openai`, `aimock`, or `live-frontier`. | | `--model ` / `--alt-model ` | provider default | Primary/alternate model refs. | | `--fast` | off | Provider fast mode where supported. | -| `--credential-source ` | `env` | See [Convex credential pool](#convex-credential-pool). | +| `--credential-source ` | `env` (Buzz: `file`) | Existing lanes use `env` or `convex`; Buzz uses `file` or `convex`. See [Convex credential pool](#convex-credential-pool). | | `--credential-role ` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. | +| `--credential-file ` | - | Buzz-only JSON credential file for local runs. | | `--allow-failures` | off | Write artifacts without returning a failing exit code when scenarios fail. | Each lane exits non-zero on any failed scenario. `--allow-failures` writes @@ -483,6 +486,40 @@ artifacts without setting a failing exit code. Telegram also accepts `--list-scenarios` to print available scenario ids and exit; the other lanes do not expose that flag. +### Buzz QA + +```bash +pnpm openclaw qa buzz \ + --credential-file /secure/path/buzz-qa-credentials.json +``` + +Targets one real Buzz room with two dedicated Nostr identities. The driver +publishes inbound room events; the SUT identity is configured in the child +OpenClaw Gateway and its outbound events are observed from the relay. The +default `mock-openai` provider proves the real Buzz transport without requiring +a model-provider credential. + +Local runs use `--credential-file ` with a private JSON file containing +`relayUrl`, `roomId`, `driverPrivateKey`, and `sutPrivateKey`. Closed relays may +also need `driverAuthTag` and `sutAuthTag`. Relative paths resolve from +`--repo-root`. Hosted relays must use `wss://`; plaintext `ws://` is accepted +only for loopback development relays. + +Both identities must be members of the dedicated room, and the SUT public key +must have the **Bot** role. A hosted closed relay may also require both public +keys to be enrolled as relay members. Use dedicated QA identities only; never +use a human owner or admin private key. Keep all private keys and authorization +values out of logs, command lines, artifacts, screenshots, and source control. + +The default scenarios are: + +- `channel-canary` +- `channel-mention-gating` + +Each run writes `qa-suite-report.md`, `qa-suite-summary.json`, and +`qa-evidence.json` under the selected output directory. The report identifies +the real Buzz relay path but omits credential values. + ### Telegram QA ```bash @@ -1024,15 +1061,20 @@ Output artifacts: ### Convex credential pool -Discord, Slack, Telegram, and WhatsApp lanes can lease credentials from a +Buzz, Discord, Slack, Telegram, and WhatsApp lanes can lease credentials from a shared Convex pool instead of reading the env vars above. Pass `--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`); QA Lab acquires an exclusive lease, heartbeats it for the duration of the -run, and releases it on shutdown. Pool kinds are `"discord"`, `"slack"`, -`"telegram"`, and `"whatsapp"`. +run, and releases it on shutdown. Pool kinds are `"buzz"`, `"discord"`, +`"slack"`, `"telegram"`, and `"whatsapp"`. Payload shapes the broker validates on `admin/add`: +- Buzz (`kind: "buzz"`): `{ relayUrl: string, roomId: string, +driverPrivateKey: string, sutPrivateKey: string, driverAuthTag?: string, +sutAuthTag?: string }` - `relayUrl` must use `wss://`, with `ws://` allowed only + for loopback relays; `roomId` must be a channel UUID, and the identities must + be distinct. - Discord (`kind: "discord"`): `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string }`. - Telegram (`kind: "telegram"`): `{ groupId: string, driverToken: string, diff --git a/docs/docs_map.md b/docs/docs_map.md index c6ed073662a9..bfcbc9512f0f 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -313,6 +313,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Manual configuration - H3: Bot key storage - H2: Verify the connection + - H3: QA Lab round trip - H2: Rotate the bot identity - H2: Current limits and roadmap - H2: Troubleshooting @@ -2955,8 +2956,9 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H3: Mantis Slack desktop and visual-task runners - H3: Credential pool health check - H2: Canonical scenario coverage - - H2: Discord, Slack, Telegram, and WhatsApp QA reference + - H2: Buzz, Discord, Slack, Telegram, and WhatsApp QA reference - H3: Shared CLI flags + - H3: Buzz QA - H3: Telegram QA - H3: Discord QA - H3: Slack QA diff --git a/docs/help/testing.md b/docs/help/testing.md index a95ad8835070..e75f05c97efa 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -400,6 +400,18 @@ gh workflow run package-acceptance.yml --ref main \ - `pnpm openclaw qa aimock` - Starts only the local AIMock provider server for direct protocol smoke testing. +- `pnpm openclaw qa buzz` + - Runs the Buzz live QA lane against a real relay room using dedicated driver + and SUT identities. + - Local runs use `--credential-file ` with `relayUrl`, `roomId`, + `driverPrivateKey`, and `sutPrivateKey`. Closed relays may also need + `driverAuthTag` and `sutAuthTag`. Hosted relays require `wss://`; `ws://` is + accepted only for loopback development relays. + - Defaults to `mock-openai` and runs canary and mention-gating scenarios + through the real Buzz plugin path. + - Supports `--credential-source convex` with a pooled `kind: "buzz"` row. + Both public keys must be relay/room members, and the SUT must have the + **Bot** room role. Never use a human owner or admin private key. - `pnpm openclaw qa matrix` - Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel homeserver. Source-checkout only - packaged installs do not ship @@ -488,8 +500,8 @@ drift; the per-lane coverage matrix lives in When `--credential-source convex` (or `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`) is enabled for live transport QA, QA lab acquires an exclusive lease from a Convex-backed pool, heartbeats that lease while the lane is running, and -releases the lease on shutdown. The section name predates Discord, Slack, and -WhatsApp support; the lease contract is shared across kinds. +releases the lease on shutdown. The section name predates Buzz, Discord, Slack, +and WhatsApp support; the lease contract is shared across kinds. Reference Convex project scaffold: `qa/convex-credential-broker/` @@ -575,6 +587,7 @@ Payload shape for Telegram real-user kind: Broker-validated multi-channel payloads: +- Buzz: `{ relayUrl: string, roomId: string, driverPrivateKey: string, sutPrivateKey: string, driverAuthTag?: string, sutAuthTag?: string }` - Discord: `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string, voiceChannelId?: string }` - WhatsApp: `{ driverPhoneE164: string, sutPhoneE164: string, driverAuthArchiveBase64: string, sutAuthArchiveBase64: string, groupJid?: string }` diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index 8bd7675b1b7a..3259529ada5e 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -124,7 +124,7 @@ it before dispatching. A narrower `rerun_group` skips this preflight. | Docker assets preflight | **Job:** `Verify Docker runtime image assets`
**Child workflow:** none
**Proves:** the `runtime-assets` Docker build target still succeeds before any other stage dispatches. Runs only for `rerun_group=all`.
**Rerun:** rerun the umbrella with `rerun_group=all`. | | Vitest and normal CI | **Job:** `Run normal full CI`
**Child workflow:** `CI`
**Proves:** manual full CI graph against the target ref, including Linux Node lanes, bundled plugin shards, plugin and channel contract shards, Node 22 compatibility, `check-*`, `check-additional-*`, built-artifact smoke checks, docs checks, Python skills, Windows, macOS, Control UI i18n, and Android via the umbrella.
**Rerun:** `rerun_group=ci`. | | Plugin prerelease | **Job:** `Run plugin prerelease validation`
**Child workflow:** `Plugin Prerelease`
**Proves:** release-only plugin static checks, agentic plugin coverage, full plugin batch shards, plugin prerelease Docker lanes, and a non-blocking `plugin-inspector-advisory` artifact for compatibility triage.
**Rerun:** `rerun_group=plugin-prerelease`. | -| Release checks | **Job:** `Run release/live/Docker/QA validation`
**Child workflow:** `OpenClaw Release Checks`
**Proves:** install smoke, cross-OS package checks, Package Acceptance, QA Lab parity, live Matrix and Telegram, plus gated advisory Discord, WhatsApp, and Slack lanes. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks; beta can opt in with `run_release_soak=true`.
**Rerun:** `rerun_group=release-checks` or a narrower release-checks handle. | +| Release checks | **Job:** `Run release/live/Docker/QA validation`
**Child workflow:** `OpenClaw Release Checks`
**Proves:** install smoke, cross-OS package checks, Package Acceptance, QA Lab parity, live Matrix, Buzz, and Telegram, plus gated advisory Discord, WhatsApp, and Slack lanes. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks; beta can opt in with `run_release_soak=true`.
**Rerun:** `rerun_group=release-checks` or a narrower release-checks handle. | | Package Telegram | **Job:** `Run package Telegram E2E`
**Child workflow:** `NPM Telegram Beta E2E`
**Proves:** a focused published-package Telegram E2E when `release_package_spec` or `npm_telegram_package_spec` is set. Full candidate validation uses the canonical Package Acceptance Telegram E2E instead.
**Rerun:** `rerun_group=npm-telegram` with `release_package_spec` or `npm_telegram_package_spec`. | | Product performance | **Job:** `Run product performance evidence`
**Child workflow:** `OpenClaw Performance`
**Proves:** release-profile performance run (`profile=release`, `repeat=3`, `fail_on_regression=true`, `publish_reports=false`) against the target SHA. Kova output stays in workflow artifacts and the child must prove its report publisher was skipped. Required (blocking) only for `rerun_group=all` or `rerun_group=performance`; not required for narrower rerun groups.
**Rerun:** `rerun_group=performance`. | | Umbrella verifier | **Job:** `Verify full validation`
**Child workflow:** none
**Proves:** re-checks recorded child run conclusions and appends slowest-job tables from child workflows.
**Rerun:** rerun only this job after rerunning a failed child to green. | @@ -175,6 +175,7 @@ artifact when package or Docker-facing stages need it. | QA runtime parity | **Job:** `Verify QA Lab runtime-pair lanes`
**Backing workflow:** direct job
**Tests:** the canonical core `openclaw`/`codex` lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex --runtime-pair-lane core`) and, with `run_release_soak=true`, the soak lane. Advisory: individual lane jobs do not block the release-check verifier.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`
**Backing workflow:** direct job
**Tests:** dynamic tool drift between `openclaw` and `codex` in the canonical core runtime-pair lane (`pnpm openclaw qa coverage --tools`), using that lane's output. Blocking: this job is not advisory-overridable.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA live Matrix | **Job:** `Run QA Live Matrix catalog`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** catalog-derived YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment, distributed across deterministic shards.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. | +| QA live Buzz | **Job:** `Run QA Lab live Buzz lane`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** signed canary and mention-gating round trips through the real Buzz plugin using dedicated Convex-leased identities and a hosted relay room.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-buzz` for a focused Buzz rerun. | | QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** trusted `OpenClaw Release Telegram QA` dispatch
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | | QA live Discord | **Job:** `Run QA Lab live Discord lane`
**Backing workflow:** direct advisory job
**Tests:** live Discord QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_DISCORD_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-discord`. | | QA live WhatsApp | **Job:** `Run QA Lab live WhatsApp lane`
**Backing workflow:** direct advisory job
**Tests:** live WhatsApp QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_WHATSAPP_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-whatsapp`. | @@ -251,7 +252,7 @@ Use `rerun_group` to avoid repeating unrelated release boxes: | `package` | Package Acceptance. | | `qa` | QA parity plus QA live lanes. | | `qa-parity` | QA parity lanes and report only. | -| `qa-live` | QA live Matrix/Telegram plus gated Discord, WhatsApp, and Slack lanes when enabled. | +| `qa-live` | QA live Matrix, Buzz, and Telegram plus gated Discord, WhatsApp, and Slack lanes when enabled. | | `npm-telegram` | Published-package Telegram E2E; requires `release_package_spec` or `npm_telegram_package_spec`. | | `performance` | Product performance evidence only. | @@ -264,8 +265,8 @@ Valid filter ids are defined in the reusable live/E2E workflow, including `live-codex-harness-docker`. For a focused QA transport rerun, set `rerun_group=qa-live` and use the -canonical selector `qa-live-matrix`, `qa-live-telegram`, `qa-live-discord`, -`qa-live-whatsapp`, or `qa-live-slack`. +canonical selector `qa-live-matrix`, `qa-live-buzz`, `qa-live-telegram`, +`qa-live-discord`, `qa-live-whatsapp`, or `qa-live-slack`. The `live-gateway-advisory-docker` handle is an aggregate rerun handle for its three provider shards, so it still fans out to all advisory Docker gateway jobs. @@ -310,8 +311,8 @@ Useful artifacts: - Docker release-path artifacts under `.artifacts/docker-tests/` - Package Acceptance `package-under-test` and Docker acceptance artifacts - Cross-OS release-check artifacts for each OS and suite -- QA parity, runtime parity, and selected Matrix, Telegram, Discord, WhatsApp, - or Slack artifacts +- QA parity, runtime parity, and selected Matrix, Buzz, Telegram, Discord, + WhatsApp, or Slack artifacts ## Workflow files diff --git a/extensions/buzz/openclaw.plugin.json b/extensions/buzz/openclaw.plugin.json index a05ed833bc00..0bc835558c12 100644 --- a/extensions/buzz/openclaw.plugin.json +++ b/extensions/buzz/openclaw.plugin.json @@ -6,6 +6,12 @@ "onStartup": false }, "channels": ["buzz"], + "qaRunners": [ + { + "commandName": "buzz", + "description": "Run the Buzz live QA lane against a dedicated relay room" + } + ], "channelConfigs": { "buzz": { "label": "Buzz", diff --git a/extensions/buzz/runtime-api.ts b/extensions/buzz/runtime-api.ts index 3d23a44a74aa..5ba53e680da9 100644 --- a/extensions/buzz/runtime-api.ts +++ b/extensions/buzz/runtime-api.ts @@ -1,2 +1,6 @@ +import { buzzQaCliRegistration } from "./src/qa/cli.js"; + export type { ChannelGatewayContext } from "openclaw/plugin-sdk/channel-contract"; export type { ChannelPlugin, PluginRuntime } from "openclaw/plugin-sdk/core"; + +export const qaRunnerCliRegistrations = [buzzQaCliRegistration]; diff --git a/extensions/buzz/src/qa/adapter.runtime.test.ts b/extensions/buzz/src/qa/adapter.runtime.test.ts new file mode 100644 index 000000000000..45914366312c --- /dev/null +++ b/extensions/buzz/src/qa/adapter.runtime.test.ts @@ -0,0 +1,343 @@ +import type { + QaBusInboundMessageInput, + QaBusMessage, +} from "openclaw/plugin-sdk/qa-channel-protocol"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { parseBuzzQaCredentialPayload } from "./credentials.js"; + +const credentials = parseBuzzQaCredentialPayload({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: "01".repeat(32), + sutPrivateKey: "02".repeat(32), +}); + +const release = vi.hoisted(() => vi.fn(async () => {})); +const heartbeatStop = vi.hoisted(() => vi.fn(async () => {})); +const acquireQaCredentialLease = vi.hoisted(() => + vi.fn(async (_options: unknown) => ({ + source: "env" as const, + kind: "buzz", + payload: credentials, + heartbeatIntervalMs: 0, + leaseTtlMs: 0, + heartbeat: vi.fn(async () => {}), + release, + })), +); +const startQaCredentialLeaseHeartbeat = vi.hoisted(() => + vi.fn((_lease: unknown) => ({ + getFailure: () => null, + stop: heartbeatStop, + throwIfFailed: vi.fn(), + })), +); +const readBuzzQaCredentialFile = vi.hoisted(() => vi.fn()); +const sendMessage = vi.hoisted(() => + vi.fn(async () => ({ eventId: "native-inbound", timestamp: 1_750_000_000_000 })), +); +const closeRelay = vi.hoisted(() => vi.fn(async () => {})); +const relayDriverState = vi.hoisted(() => ({ + onMessage: undefined as + | ((message: { + id: string; + senderPubkey: string; + text: string; + channelId: string; + createdAt: number; + threadId?: string; + replyToId?: string; + mentionedPubkeys: string[]; + }) => Promise) + | undefined, +})); +const createBuzzQaRelayDriver = vi.hoisted(() => + vi.fn(async (params: { onMessage: NonNullable }) => { + relayDriverState.onMessage = params.onMessage; + return { + assertHealthy: vi.fn(), + close: closeRelay, + sendMessage, + }; + }), +); + +vi.mock("./credentials.js", async (importOriginal) => ({ + ...(await importOriginal()), + readBuzzQaCredentialFile, +})); +vi.mock("./relay-client.js", () => ({ createBuzzQaRelayDriver })); + +import { createBuzzQaTransportAdapter } from "./adapter.runtime.js"; + +const credentialHost: Parameters[0]["credentials"] = { + acquire: async (options) => (await acquireQaCredentialLease(options)) as never, + startHeartbeat: startQaCredentialLeaseHeartbeat, +}; + +describe("Buzz QA transport adapter", () => { + beforeEach(() => { + vi.clearAllMocks(); + relayDriverState.onMessage = undefined; + sendMessage.mockResolvedValue({ + eventId: "native-inbound", + timestamp: 1_750_000_000_000, + }); + readBuzzQaCredentialFile.mockResolvedValue(credentials); + }); + + it("uses private file credentials by default", async () => { + await createBuzzQaTransportAdapter({ + adapterOptions: { + credentialFile: "private/buzz-qa.json", + repoRoot: "/repo", + }, + channelId: "buzz", + credentials: credentialHost, + driver: "live", + messages: { + addInboundMessage: vi.fn(), + addOutboundMessage: vi.fn(), + editMessage: vi.fn(), + }, + outputDir: ".artifacts/qa-e2e/buzz", + }); + + expect(readBuzzQaCredentialFile).toHaveBeenCalledWith({ + filePath: "private/buzz-qa.json", + repoRoot: "/repo", + }); + expect(acquireQaCredentialLease).toHaveBeenCalledWith( + expect.objectContaining({ kind: "buzz", source: "env" }), + ); + }); + + it("sends a portable mentioned message through the native Buzz relay driver", async () => { + const addInboundMessage = vi.fn(async (input) => ({ + ...input, + id: "bus-inbound", + direction: "inbound" as const, + timestamp: input.timestamp ?? 1_750_000_000_000, + })); + const adapter = await createBuzzQaTransportAdapter({ + adapterOptions: { credentialSource: "convex", sutAccountId: "sut" }, + channelId: "buzz", + credentials: credentialHost, + driver: "live", + messages: { + addInboundMessage, + addOutboundMessage: vi.fn(), + editMessage: vi.fn(), + }, + outputDir: ".artifacts/qa-e2e/buzz", + }); + + const message = await adapter.sendInbound({ + accountId: "sut", + conversation: { id: "qa-routing-primary", kind: "group" }, + senderId: "driver", + senderName: "QA Driver", + text: "@openclaw reply exactly: QA-CHANNEL-CANARY-OK", + }); + + expect(sendMessage).toHaveBeenCalledWith({ + mentionSut: true, + text: "@openclaw reply exactly: QA-CHANNEL-CANARY-OK", + }); + expect(addInboundMessage).toHaveBeenCalledWith( + expect.objectContaining({ + accountId: "sut", + senderId: credentials.driverPublicKey, + timestamp: 1_750_000_000_000, + }), + ); + expect(message.id).toBe("bus-inbound"); + + const gateway = { + call: vi.fn(async () => ({ + channelAccounts: { buzz: [{ accountId: "default", running: true }] }, + })), + }; + await adapter.waitReady({ gateway }); + expect(gateway.call).toHaveBeenCalledWith( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + ); + }); + + it("maps native Buzz reply relations back to portable thread ids", async () => { + let inboundIndex = 0; + let outboundIndex = 0; + const addInboundMessage = vi.fn(async (input) => ({ + ...input, + id: `bus-inbound-${++inboundIndex}`, + direction: "inbound" as const, + timestamp: input.timestamp ?? 1_750_000_000_000, + })); + const addOutboundMessage = vi.fn(async (input) => ({ + ...input, + id: `bus-outbound-${++outboundIndex}`, + direction: "outbound" as const, + conversation: { id: "main", kind: "group" as const }, + })); + sendMessage + .mockResolvedValueOnce({ eventId: "native-root", timestamp: 1_750_000_000_000 }) + .mockResolvedValueOnce({ eventId: "native-follow-up", timestamp: 1_750_000_001_000 }); + const adapter = await createBuzzQaTransportAdapter({ + adapterOptions: { credentialSource: "convex", sutAccountId: "sut" }, + channelId: "buzz", + credentials: credentialHost, + driver: "live", + messages: { + addInboundMessage, + addOutboundMessage, + editMessage: vi.fn(), + }, + outputDir: ".artifacts/qa-e2e/buzz", + }); + + const root = await adapter.sendInbound({ + accountId: "sut", + conversation: { id: "main", kind: "group" }, + senderId: "driver", + senderName: "QA Driver", + text: "@openclaw root", + }); + await relayDriverState.onMessage?.({ + id: "native-sut-root", + senderPubkey: credentials.sutPublicKey, + text: "root reply", + channelId: credentials.roomId, + createdAt: 1_750_000_000, + threadId: "native-root", + replyToId: "native-root", + mentionedPubkeys: [], + }); + const followUp = await adapter.sendInbound({ + accountId: "sut", + conversation: { id: "main", kind: "group" }, + senderId: "driver", + senderName: "QA Driver", + text: "@openclaw follow-up", + threadId: root.id, + }); + await relayDriverState.onMessage?.({ + id: "native-sut-follow-up", + senderPubkey: credentials.sutPublicKey, + text: "thread reply", + channelId: credentials.roomId, + createdAt: 1_750_000_001, + threadId: "native-root", + replyToId: "native-follow-up", + mentionedPubkeys: [], + }); + + expect(sendMessage).toHaveBeenLastCalledWith({ + mentionSut: true, + text: "@openclaw follow-up", + threadId: "native-root", + }); + expect(addOutboundMessage).toHaveBeenLastCalledWith( + expect.objectContaining({ + threadId: root.id, + replyToId: followUp.id, + }), + ); + }); + + it("waits for a racing inbound id mapping before recording reply relations", async () => { + let releaseInbound = () => {}; + const inboundGate = new Promise((resolve) => { + releaseInbound = resolve; + }); + const addInboundMessage = vi.fn(async (input: QaBusInboundMessageInput) => { + await inboundGate; + return { + ...input, + id: "bus-inbound", + accountId: input.accountId ?? "sut", + direction: "inbound", + timestamp: input.timestamp ?? 1_750_000_000_000, + reactions: [], + } satisfies QaBusMessage; + }); + const addOutboundMessage = vi.fn(async (input) => ({ + ...input, + id: "bus-outbound", + direction: "outbound" as const, + conversation: { id: "main", kind: "group" as const }, + })); + const adapter = await createBuzzQaTransportAdapter({ + adapterOptions: { credentialSource: "convex", sutAccountId: "sut" }, + channelId: "buzz", + credentials: credentialHost, + driver: "live", + messages: { + addInboundMessage, + addOutboundMessage, + editMessage: vi.fn(), + }, + outputDir: ".artifacts/qa-e2e/buzz", + }); + + const inboundPromise = adapter.sendInbound({ + accountId: "sut", + conversation: { id: "main", kind: "group" }, + senderId: "driver", + senderName: "QA Driver", + text: "@openclaw root", + }); + await vi.waitFor(() => expect(addInboundMessage).toHaveBeenCalledOnce()); + const outboundPromise = relayDriverState.onMessage?.({ + id: "native-sut-reply", + senderPubkey: credentials.sutPublicKey, + text: "fast reply", + channelId: credentials.roomId, + createdAt: 1_750_000_000, + threadId: "native-inbound", + replyToId: "native-inbound", + mentionedPubkeys: [], + }); + expect(addOutboundMessage).not.toHaveBeenCalled(); + + releaseInbound(); + await inboundPromise; + await outboundPromise; + + expect(addOutboundMessage).toHaveBeenCalledWith( + expect.objectContaining({ + threadId: "bus-inbound", + replyToId: "bus-inbound", + }), + ); + }); + + it("closes relay observation before stopping and releasing the credential lease", async () => { + const adapter = await createBuzzQaTransportAdapter({ + adapterOptions: { credentialSource: "convex" }, + channelId: "buzz", + credentials: credentialHost, + driver: "live", + messages: { + addInboundMessage: vi.fn(), + addOutboundMessage: vi.fn(), + editMessage: vi.fn(), + }, + outputDir: ".artifacts/qa-e2e/buzz", + }); + + await adapter.cleanup?.(); + await adapter.cleanupAfterGatewayStop?.(); + + expect(closeRelay).toHaveBeenCalledOnce(); + expect(heartbeatStop).toHaveBeenCalledOnce(); + expect(release).toHaveBeenCalledOnce(); + expect(closeRelay.mock.invocationCallOrder[0]).toBeLessThan( + heartbeatStop.mock.invocationCallOrder[0] ?? 0, + ); + expect(heartbeatStop.mock.invocationCallOrder[0]).toBeLessThan( + release.mock.invocationCallOrder[0] ?? 0, + ); + }); +}); diff --git a/extensions/buzz/src/qa/adapter.runtime.ts b/extensions/buzz/src/qa/adapter.runtime.ts new file mode 100644 index 000000000000..772d56db1e33 --- /dev/null +++ b/extensions/buzz/src/qa/adapter.runtime.ts @@ -0,0 +1,230 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime"; +import type { BuzzInboundMessage } from "../message-event.js"; +import { buildBuzzTarget } from "../target.js"; +import { parseBuzzQaCredentialPayload, readBuzzQaCredentialFile } from "./credentials.js"; +import { createBuzzQaRelayDriver } from "./relay-client.js"; + +type AdapterFactory = NonNullable; +type FactoryContext = Parameters[0]; +type AdapterDefinition = Awaited> & { + cleanupAfterGatewayStop?: () => Promise; +}; + +const BUZZ_GATEWAY_ACCOUNT_ID = "default"; +const BUZZ_MESSAGE_ID_MAPPING_TIMEOUT_MS = 5_000; + +function isBuzzMention(text: string) { + return /(^|\s)@openclaw\b/iu.test(text); +} + +async function waitForBuzzChannelRunning(params: { + accountId: string; + gateway: Parameters[0]["gateway"]; + pollIntervalMs?: number; + timeoutMs?: number; +}) { + const timeoutMs = params.timeoutMs ?? 60_000; + const pollIntervalMs = params.pollIntervalMs ?? 500; + const startedAt = Date.now(); + let lastStatus: unknown; + while (Date.now() - startedAt < timeoutMs) { + const payload = (await params.gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { channelAccounts?: Record>> }; + const status = payload.channelAccounts?.buzz?.find( + (entry) => entry.accountId === params.accountId, + ); + lastStatus = status; + if (status?.running === true && status.restartPending !== true) { + return; + } + await sleep(pollIntervalMs); + } + throw new Error( + `buzz account "${params.accountId}" did not become ready; last status: ${JSON.stringify(lastStatus)}`, + ); +} + +export async function createBuzzQaTransportAdapter( + context: FactoryContext, +): Promise { + const options = context.adapterOptions ?? {}; + const requestedCredentialSource = options.credentialSource?.trim().toLowerCase() || "file"; + if (requestedCredentialSource !== "file" && requestedCredentialSource !== "convex") { + throw new Error('Buzz QA credential source must be "file" or "convex".'); + } + const credentialFile = options.credentialFile?.trim(); + if (requestedCredentialSource === "file" && !credentialFile) { + throw new Error("Buzz QA file credentials require --credential-file ."); + } + if (requestedCredentialSource === "file" && options.credentialRole?.trim()) { + throw new Error("Buzz QA --credential-role is only valid with --credential-source convex."); + } + if (requestedCredentialSource === "convex" && credentialFile) { + throw new Error( + "Buzz QA --credential-file cannot be combined with --credential-source convex.", + ); + } + const fileCredentials = + requestedCredentialSource === "file" && credentialFile + ? await readBuzzQaCredentialFile({ filePath: credentialFile, repoRoot: options.repoRoot }) + : undefined; + const lease = await context.credentials.acquire({ + kind: "buzz", + source: requestedCredentialSource === "convex" ? "convex" : "env", + role: options.credentialRole, + resolveEnvPayload: () => { + if (!fileCredentials) { + throw new Error("Buzz QA file credentials are unavailable."); + } + return fileCredentials; + }, + parsePayload: parseBuzzQaCredentialPayload, + }); + const heartbeat = context.credentials.startHeartbeat(lease); + const credentials = lease.payload; + const accountId = options.sutAccountId?.trim() || "sut"; + const nativeMessageIds = new Map(); + const busMessageIds = new Map(); + let logicalConversationId = credentials.roomId; + let relayDriver: Awaited>; + + const resolveBusMessageId = async (nativeId: string | undefined) => { + if (!nativeId) { + return undefined; + } + const startedAt = Date.now(); + while (Date.now() - startedAt < BUZZ_MESSAGE_ID_MAPPING_TIMEOUT_MS) { + const busId = busMessageIds.get(nativeId); + if (busId) { + return busId; + } + // A fast relay response can arrive before sendInbound records the + // published event's portable id. Preserve native reply relations until + // that canonical mapping is available instead of dropping them. + await sleep(10); + } + throw new Error(`Buzz QA could not resolve the portable id for native message ${nativeId}.`); + }; + + const recordOutbound = async (message: BuzzInboundMessage) => { + const [threadId, replyToId] = await Promise.all([ + resolveBusMessageId(message.threadId), + resolveBusMessageId(message.replyToId), + ]); + const outbound = await context.messages.addOutboundMessage({ + accountId, + to: `group:${logicalConversationId}`, + senderId: credentials.sutPublicKey, + text: message.text, + timestamp: message.createdAt * 1_000, + threadId, + replyToId, + }); + nativeMessageIds.set(outbound.id, message.id); + busMessageIds.set(message.id, outbound.id); + }; + + try { + relayDriver = await createBuzzQaRelayDriver({ credentials, onMessage: recordOutbound }); + } catch (error) { + try { + await heartbeat.stop(); + } finally { + await lease.release(); + } + throw error; + } + + return { + id: "buzz", + label: "Buzz live", + accountId, + requiredPluginIds: ["buzz"], + supportedActions: [], + assertTransportHealthy() { + heartbeat.throwIfFailed(); + relayDriver.assertHealthy(); + }, + async sendInbound(input) { + heartbeat.throwIfFailed(); + relayDriver.assertHealthy(); + logicalConversationId = input.conversation.id; + const sent = await relayDriver.sendMessage({ + text: input.text, + mentionSut: isBuzzMention(input.text), + ...(input.threadId ? { threadId: nativeMessageIds.get(input.threadId) } : {}), + ...(input.replyToId ? { replyToId: nativeMessageIds.get(input.replyToId) } : {}), + }); + const inbound = await context.messages.addInboundMessage({ + ...input, + accountId, + senderId: credentials.driverPublicKey, + timestamp: sent.timestamp, + }); + nativeMessageIds.set(inbound.id, sent.eventId); + busMessageIds.set(sent.eventId, inbound.id); + return inbound; + }, + resetTransport() { + logicalConversationId = credentials.roomId; + nativeMessageIds.clear(); + busMessageIds.clear(); + }, + createGatewayConfig: () => + ({ + channels: { + buzz: { + enabled: true, + relayUrl: credentials.relayUrl, + privateKey: credentials.sutPrivateKey, + ...(credentials.sutAuthTag ? { authTag: credentials.sutAuthTag } : {}), + groupPolicy: "allowlist", + groupAllowFrom: [credentials.driverPublicKey], + groups: { + [credentials.roomId]: { + enabled: true, + requireMention: options.transportPolicy?.requireGroupMention ?? true, + }, + }, + defaultTo: buildBuzzTarget(credentials.roomId), + }, + }, + }) as Pick, + waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) => + await waitForBuzzChannelRunning({ + // Buzz is currently single-account; the QA bus keeps its portable SUT + // label while Gateway status reports the plugin's canonical account id. + accountId: BUZZ_GATEWAY_ACCOUNT_ID, + gateway, + timeoutMs, + pollIntervalMs, + }), + buildAgentDelivery: () => ({ + channel: "buzz", + to: buildBuzzTarget(credentials.roomId), + replyChannel: "buzz", + replyTo: buildBuzzTarget(credentials.roomId), + }), + async handleAction() { + throw new Error("Buzz live QA adapter does not implement transport actions"); + }, + createReportNotes: () => [ + "Runs through a real authenticated Buzz relay room; credential values are omitted.", + ], + async cleanup() { + await relayDriver.close(); + }, + async cleanupAfterGatewayStop() { + try { + await heartbeat.stop(); + } finally { + await lease.release(); + } + }, + }; +} diff --git a/extensions/buzz/src/qa/cli.test.ts b/extensions/buzz/src/qa/cli.test.ts new file mode 100644 index 000000000000..430ceabb03b0 --- /dev/null +++ b/extensions/buzz/src/qa/cli.test.ts @@ -0,0 +1,51 @@ +import { Command } from "commander"; +import type { LiveTransportQaSuiteCommandOptions } from "openclaw/plugin-sdk/qa-runtime"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const runLiveTransportQaSuiteCommand = vi.hoisted(() => + vi.fn<(params: LiveTransportQaSuiteCommandOptions) => Promise>(async () => {}), +); + +vi.mock("openclaw/plugin-sdk/qa-runtime", async (importOriginal) => ({ + ...(await importOriginal()), + runLiveTransportQaSuiteCommand, +})); + +import { buzzQaCliRegistration } from "./cli.js"; + +describe("Buzz QA CLI", () => { + beforeEach(() => { + runLiveTransportQaSuiteCommand.mockClear(); + }); + + it("runs the portable canary and mention-gating scenarios", async () => { + const qa = new Command(); + buzzQaCliRegistration.register(qa); + + await qa.parseAsync([ + "node", + "openclaw", + "buzz", + "--provider-mode", + "mock-openai", + "--credential-file", + "/secure/buzz-qa.json", + ]); + + const params = runLiveTransportQaSuiteCommand.mock.calls[0]?.[0]; + expect(params).toMatchObject({ + channelId: "buzz", + defaultProviderMode: "mock-openai", + options: { + providerMode: "mock-openai", + credentialFile: "/secure/buzz-qa.json", + }, + }); + expect( + params?.selectScenarioIds({ + primaryModel: "openai/gpt-5.4", + providerMode: "mock-openai", + }), + ).toEqual(["channel-canary", "channel-mention-gating"]); + }); +}); diff --git a/extensions/buzz/src/qa/cli.ts b/extensions/buzz/src/qa/cli.ts new file mode 100644 index 000000000000..61f0fff4cb78 --- /dev/null +++ b/extensions/buzz/src/qa/cli.ts @@ -0,0 +1,49 @@ +import { + createLazyCliRuntimeLoader, + createLiveTransportQaCliRegistration, + runLiveTransportQaSuiteCommand, + type LiveTransportQaCliRegistration, + type LiveTransportQaCommandOptions, +} from "openclaw/plugin-sdk/qa-runtime"; + +const DEFAULT_BUZZ_QA_SCENARIOS = ["channel-canary", "channel-mention-gating"] as const; + +const loadBuzzQaAdapterRuntime = createLazyCliRuntimeLoader( + () => import("./adapter.runtime.js"), +); + +async function runQaBuzz(options: LiveTransportQaCommandOptions) { + await runLiveTransportQaSuiteCommand({ + channelId: "buzz", + defaultProviderMode: "mock-openai", + options, + selectScenarioIds: ({ scenarioIds }) => + scenarioIds?.length ? [...scenarioIds] : [...DEFAULT_BUZZ_QA_SCENARIOS], + }); +} + +export const buzzQaCliRegistration: LiveTransportQaCliRegistration = + createLiveTransportQaCliRegistration({ + commandName: "buzz", + credentialFileHelp: "JSON credential file for local Buzz QA", + adapterFactory: { + id: "buzz", + matches: ({ channelId, driver }) => channelId === "buzz" && driver === "live", + async create(context) { + return await (await loadBuzzQaAdapterRuntime()).createBuzzQaTransportAdapter(context); + }, + }, + credentialOptions: { + sourceDescription: "Credential source for Buzz QA: file or convex (default: file)", + roleDescription: + "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)", + }, + defaultProviderMode: "mock-openai", + description: "Run the Buzz live QA lane against a dedicated relay room", + providerModeHelp: "Provider mode: mock-openai, aimock, or live-frontier", + outputDirHelp: "Buzz QA artifact directory", + allowFailuresHelp: "Write artifacts without setting a failing exit code when scenarios fail", + scenarioHelp: "Run only the named Buzz QA scenario (repeatable)", + sutAccountHelp: "Normalized Buzz SUT account id in QA artifacts", + run: runQaBuzz, + }); diff --git a/extensions/buzz/src/qa/credentials.test.ts b/extensions/buzz/src/qa/credentials.test.ts new file mode 100644 index 000000000000..b0b6f870cf53 --- /dev/null +++ b/extensions/buzz/src/qa/credentials.test.ts @@ -0,0 +1,131 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { parseBuzzQaCredentialPayload, readBuzzQaCredentialFile } from "./credentials.js"; + +const DRIVER_PRIVATE_KEY = "01".repeat(32); +const SUT_PRIVATE_KEY = "02".repeat(32); +const tempDirs: string[] = []; + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map(async (dir) => await fs.rm(dir, { force: true, recursive: true })), + ); +}); + +describe("Buzz QA credentials", () => { + it("accepts a strict two-identity room payload", () => { + const credentials = parseBuzzQaCredentialPayload({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + driverAuthTag: '["auth","driver","conditions","signature"]', + sutAuthTag: '["auth","sut","conditions","signature"]', + }); + + expect(credentials).toMatchObject({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + }); + expect(credentials.driverPublicKey).toMatch(/^[a-f0-9]{64}$/u); + expect(credentials.sutPublicKey).toMatch(/^[a-f0-9]{64}$/u); + expect(credentials.driverPublicKey).not.toBe(credentials.sutPublicKey); + }); + + it.each(["ws://localhost:8080", "ws://127.0.0.1:8080", "ws://[::1]:8080"])( + "allows plaintext loopback relay URL %s", + (relayUrl) => { + expect( + parseBuzzQaCredentialPayload({ + relayUrl, + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + }).relayUrl, + ).toBe(relayUrl); + }, + ); + + it("rejects plaintext remote relay URLs", () => { + expect(() => + parseBuzzQaCredentialPayload({ + relayUrl: "ws://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + }), + ).toThrow("Buzz QA credentials are missing or malformed."); + }); + + it("never includes credential values in validation errors", () => { + const privateKey = "not-a-private-key-value"; + const authTag = "not-an-auth-tag-value"; + + expect(() => + parseBuzzQaCredentialPayload({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: privateKey, + sutPrivateKey: SUT_PRIVATE_KEY, + driverAuthTag: authTag, + }), + ).toThrow("Buzz QA credentials are missing or malformed."); + try { + parseBuzzQaCredentialPayload({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: privateKey, + sutPrivateKey: SUT_PRIVATE_KEY, + driverAuthTag: authTag, + }); + } catch (error) { + expect(String(error)).not.toContain(privateKey); + expect(String(error)).not.toContain(authTag); + } + }); + + it("reads a repo-relative private JSON credential file", async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-buzz-qa-")); + tempDirs.push(repoRoot); + const filePath = path.join("private", "buzz.json"); + await fs.mkdir(path.join(repoRoot, "private")); + await fs.writeFile( + path.join(repoRoot, filePath), + JSON.stringify({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + }), + { mode: 0o600 }, + ); + + await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).resolves.toMatchObject({ + relayUrl: "wss://relay.qa.example", + driverPrivateKey: DRIVER_PRIVATE_KEY, + sutPrivateKey: SUT_PRIVATE_KEY, + }); + }); + + it("does not echo malformed file contents", async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-buzz-qa-")); + tempDirs.push(repoRoot); + const filePath = "buzz.json"; + const secret = "private-key-material-must-not-leak"; + await fs.writeFile(path.join(repoRoot, filePath), `{${secret}`, { mode: 0o600 }); + + await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).rejects.not.toThrow(secret); + await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).rejects.toThrow( + "is not valid JSON", + ); + try { + await readBuzzQaCredentialFile({ filePath, repoRoot }); + } catch (error) { + expect((error as Error & { cause?: unknown }).cause).toBeUndefined(); + } + }); +}); diff --git a/extensions/buzz/src/qa/credentials.ts b/extensions/buzz/src/qa/credentials.ts new file mode 100644 index 000000000000..262f0873c1e0 --- /dev/null +++ b/extensions/buzz/src/qa/credentials.ts @@ -0,0 +1,87 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { isLoopbackHost } from "openclaw/plugin-sdk/ssrf-runtime"; +import { z } from "zod"; +import { parseBuzzAuthTag } from "../relay-auth.js"; +import { parseBuzzTarget } from "../target.js"; +import { resolveBuzzPublicKey } from "../types.js"; + +function isSafeBuzzQaRelayUrl(value: string): boolean { + try { + const relayUrl = new URL(value); + return ( + relayUrl.protocol === "wss:" || + (relayUrl.protocol === "ws:" && isLoopbackHost(relayUrl.hostname)) + ); + } catch { + return false; + } +} + +const buzzQaCredentialPayloadSchema = z + .object({ + // QA credentials may contain relay auth tags, so plaintext WebSockets must + // stay on loopback and never cross a network boundary. + relayUrl: z.string().url().refine(isSafeBuzzQaRelayUrl), + roomId: z.string().min(1), + driverPrivateKey: z.string().min(1), + sutPrivateKey: z.string().min(1), + driverAuthTag: z.string().optional(), + sutAuthTag: z.string().optional(), + }) + .strict(); + +export type BuzzQaCredentials = z.output & { + driverPublicKey: string; + sutPublicKey: string; +}; + +export function parseBuzzQaCredentialPayload(payload: unknown): BuzzQaCredentials { + const parsed = buzzQaCredentialPayloadSchema.safeParse(payload); + if (!parsed.success) { + throw new Error("Buzz QA credentials are missing or malformed."); + } + let roomId: string; + let driverPublicKey: string; + let sutPublicKey: string; + try { + roomId = parseBuzzTarget(parsed.data.roomId); + driverPublicKey = resolveBuzzPublicKey(parsed.data.driverPrivateKey); + sutPublicKey = resolveBuzzPublicKey(parsed.data.sutPrivateKey); + parseBuzzAuthTag(parsed.data.driverAuthTag ?? ""); + parseBuzzAuthTag(parsed.data.sutAuthTag ?? ""); + } catch { + throw new Error("Buzz QA credentials are missing or malformed."); + } + if (driverPublicKey === sutPublicKey) { + throw new Error("Buzz QA requires distinct driver and SUT identities."); + } + return { + ...parsed.data, + roomId, + driverPublicKey, + sutPublicKey, + }; +} + +export async function readBuzzQaCredentialFile(params: { + filePath: string; + repoRoot?: string; +}): Promise { + const resolvedPath = path.resolve(params.repoRoot ?? process.cwd(), params.filePath); + let raw: string; + try { + raw = await fs.readFile(resolvedPath, "utf8"); + } catch (error) { + throw new Error(`Unable to read Buzz QA credential file ${resolvedPath}.`, { cause: error }); + } + let payload: unknown; + try { + payload = JSON.parse(raw); + } catch { + // JSON.parse errors may include source snippets, so never retain the + // secret-bearing parser error as a cause. + throw new Error(`Buzz QA credential file ${resolvedPath} is not valid JSON.`); + } + return parseBuzzQaCredentialPayload(payload); +} diff --git a/extensions/buzz/src/qa/relay-client.test.ts b/extensions/buzz/src/qa/relay-client.test.ts new file mode 100644 index 000000000000..d53e90030d10 --- /dev/null +++ b/extensions/buzz/src/qa/relay-client.test.ts @@ -0,0 +1,134 @@ +import { getPublicKey, type Event, type Filter } from "nostr-tools"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { parseBuzzQaCredentialPayload } from "./credentials.js"; + +const relayMocks = vi.hoisted(() => ({ + auth: vi.fn(async () => "ok"), + close: vi.fn(), + connect: vi.fn(async () => {}), + publish: vi.fn(async () => "ok"), + replayedMessage: undefined as Event | undefined, + subscriptions: [] as Array<{ + filter: Filter; + handlers: { + onevent?: (event: Event) => void; + oneose?: () => void; + onclose?: (reason: string) => void; + }; + }>, +})); + +vi.mock("nostr-tools", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + Relay: class { + onauth?: (template: unknown) => Promise; + auth = relayMocks.auth; + close = relayMocks.close; + connect = relayMocks.connect; + publish = relayMocks.publish; + + subscribe( + filters: Filter[], + handlers: (typeof relayMocks.subscriptions)[number]["handlers"], + ) { + const filter = filters[0] ?? {}; + relayMocks.subscriptions.push({ filter, handlers }); + if (filter.kinds?.includes(39002)) { + handlers.onevent?.({ + id: "membership", + kind: 39002, + pubkey: "f".repeat(64), + created_at: 1_750_000_000, + content: "", + sig: "e".repeat(128), + tags: [ + ["d", "123e4567-e89b-42d3-a456-426614174000"], + [ + "p", + getPublicKey(Uint8Array.from(Buffer.from("01".repeat(32), "hex"))), + "", + "member", + ], + ["p", getPublicKey(Uint8Array.from(Buffer.from("02".repeat(32), "hex"))), "", "bot"], + ], + }); + handlers.oneose?.(); + } else { + if (relayMocks.replayedMessage) { + handlers.onevent?.(relayMocks.replayedMessage); + } + handlers.oneose?.(); + } + return { close: vi.fn() }; + } + }, + }; +}); + +import { createBuzzQaRelayDriver } from "./relay-client.js"; + +const credentials = parseBuzzQaCredentialPayload({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: "01".repeat(32), + sutPrivateKey: "02".repeat(32), +}); + +describe("Buzz QA relay driver", () => { + beforeEach(() => { + vi.clearAllMocks(); + relayMocks.subscriptions.length = 0; + relayMocks.replayedMessage = undefined; + }); + + it("authenticates, verifies membership, and publishes a native mentioned thread event", async () => { + const driver = await createBuzzQaRelayDriver({ + credentials, + onMessage: vi.fn(async () => {}), + }); + + const sent = await driver.sendMessage({ + text: "@openclaw hello", + mentionSut: true, + threadId: "root-event", + replyToId: "parent-event", + }); + + expect(sent.eventId).toMatch(/^[a-f0-9]{64}$/u); + expect(relayMocks.publish).toHaveBeenCalledWith( + expect.objectContaining({ + id: sent.eventId, + kind: 9, + content: "@openclaw hello", + tags: [ + ["h", credentials.roomId], + ["e", "root-event", "", "root"], + ["e", "parent-event", "", "reply"], + ["p", credentials.sutPublicKey], + ], + }), + ); + await driver.close(); + expect(relayMocks.close).toHaveBeenCalledOnce(); + }); + + it("ignores retained SUT messages before the observer reaches live events", async () => { + const onMessage = vi.fn(async () => {}); + relayMocks.replayedMessage = { + id: "retained-sut-message", + kind: 9, + pubkey: credentials.sutPublicKey, + created_at: 1_750_000_000, + content: "old response", + sig: "e".repeat(128), + tags: [["h", credentials.roomId]], + }; + + const driver = await createBuzzQaRelayDriver({ credentials, onMessage }); + + expect(onMessage).not.toHaveBeenCalled(); + await driver.close(); + }); +}); diff --git a/extensions/buzz/src/qa/relay-client.ts b/extensions/buzz/src/qa/relay-client.ts new file mode 100644 index 000000000000..b0f0223969d9 --- /dev/null +++ b/extensions/buzz/src/qa/relay-client.ts @@ -0,0 +1,245 @@ +import { Relay, finalizeEvent, type Event } from "nostr-tools"; +import { + buildBuzzMessageTags, + parseBuzzMessageEvent, + type BuzzInboundMessage, +} from "../message-event.js"; +import { authenticateBuzzRelay, createBuzzAuthSigner, parseBuzzAuthTag } from "../relay-auth.js"; +import { + BUZZ_ROOM_MEMBERSHIP_KIND, + isNewerBuzzRoomMembership, + parseBuzzRoomMembershipEvent, + type BuzzRoomMembership, +} from "../room-membership.js"; +import { decodeBuzzPrivateKey } from "../types.js"; +import type { BuzzQaCredentials } from "./credentials.js"; + +const BUZZ_MESSAGE_KIND = 9; +const MEMBERSHIP_TIMEOUT_MS = 10_000; +const OBSERVER_READY_TIMEOUT_MS = 10_000; + +type BuzzQaRelayDriver = { + assertHealthy(): void; + close(): Promise; + sendMessage(input: { + text: string; + mentionSut: boolean; + threadId?: string; + replyToId?: string; + }): Promise<{ eventId: string; timestamp: number }>; +}; + +async function loadBuzzQaRoomMembership(params: { + relay: Relay; + roomId: string; +}): Promise { + return await new Promise((resolve, reject) => { + let latest: BuzzRoomMembership | undefined; + let settled = false; + const subscriptionRef: { current?: ReturnType } = {}; + const finish = (error?: Error) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timeout); + subscriptionRef.current?.close("membership loaded"); + if (error) { + reject(error); + } else if (latest) { + resolve(latest); + } else { + reject(new Error(`Buzz QA room ${params.roomId} has no membership roster.`)); + } + }; + const timeout = setTimeout( + () => finish(new Error(`Timed out loading Buzz QA room ${params.roomId} membership.`)), + MEMBERSHIP_TIMEOUT_MS, + ); + subscriptionRef.current = params.relay.subscribe( + [{ kinds: [BUZZ_ROOM_MEMBERSHIP_KIND], "#d": [params.roomId], limit: 1 }], + { + onevent: (event) => { + const membership = parseBuzzRoomMembershipEvent(event); + if ( + membership?.roomId === params.roomId && + isNewerBuzzRoomMembership(membership, latest) + ) { + latest = membership; + } + }, + oneose: () => finish(), + onclose: (reason) => { + if (reason !== "membership loaded") { + finish(new Error(`Buzz QA membership subscription closed: ${reason}`)); + } + }, + }, + ); + if (settled) { + subscriptionRef.current.close("membership loaded"); + } + }); +} + +function assertBuzzQaMembership(membership: BuzzRoomMembership, credentials: BuzzQaCredentials) { + if (!membership.members.has(credentials.driverPublicKey)) { + throw new Error( + `Buzz QA driver ${credentials.driverPublicKey} is not a member of room ${credentials.roomId}.`, + ); + } + if ( + !membership.members.has(credentials.sutPublicKey) || + membership.roles.get(credentials.sutPublicKey) !== "bot" + ) { + throw new Error( + `Buzz QA SUT ${credentials.sutPublicKey} must have the Bot role in room ${credentials.roomId}.`, + ); + } +} + +export async function createBuzzQaRelayDriver(params: { + credentials: BuzzQaCredentials; + onMessage: (message: BuzzInboundMessage) => Promise; +}): Promise { + const credentials = params.credentials; + const secretKey = decodeBuzzPrivateKey(credentials.driverPrivateKey); + const relay = new Relay(credentials.relayUrl, { enableReconnect: false }); + const lifecycleAbort = new AbortController(); + const signAuth = createBuzzAuthSigner({ + secretKey, + authTag: parseBuzzAuthTag(credentials.driverAuthTag ?? ""), + }); + let transportError: Error | undefined; + let messageQueue = Promise.resolve(); + const observedEventIds = new Set(); + try { + await relay.connect({ abort: lifecycleAbort.signal }); + await authenticateBuzzRelay({ relay, signAuth, signal: lifecycleAbort.signal }); + relay.onauth = signAuth; + assertBuzzQaMembership( + await loadBuzzQaRoomMembership({ relay, roomId: credentials.roomId }), + credentials, + ); + } catch (error) { + lifecycleAbort.abort(error); + relay.close(); + throw error; + } + + let observerReady = false; + let resolveObserverReady: (() => void) | undefined; + let rejectObserverReady: ((error: Error) => void) | undefined; + const observerReadyPromise = new Promise((resolve, reject) => { + resolveObserverReady = resolve; + rejectObserverReady = reject; + }); + const observerReadyTimeout = setTimeout(() => { + rejectObserverReady?.(new Error("Timed out waiting for the Buzz QA message observer.")); + }, OBSERVER_READY_TIMEOUT_MS); + let subscription: ReturnType; + try { + subscription = relay.subscribe( + [ + { + kinds: [BUZZ_MESSAGE_KIND], + authors: [credentials.sutPublicKey], + "#h": [credentials.roomId], + since: Math.floor(Date.now() / 1_000) - 5, + }, + ], + { + onevent: (event: Event) => { + if (!observerReady) { + return; + } + if (observedEventIds.has(event.id)) { + return; + } + observedEventIds.add(event.id); + const message = parseBuzzMessageEvent(event); + if ( + !message || + message.channelId !== credentials.roomId || + message.senderPubkey !== credentials.sutPublicKey + ) { + return; + } + messageQueue = messageQueue + .then(async () => await params.onMessage(message)) + .catch((error: unknown) => { + transportError = error instanceof Error ? error : new Error(String(error)); + }); + }, + oneose: () => { + observerReady = true; + clearTimeout(observerReadyTimeout); + resolveObserverReady?.(); + }, + onclose: (reason) => { + if (!observerReady) { + clearTimeout(observerReadyTimeout); + rejectObserverReady?.( + new Error(`Buzz QA message observer closed before it was ready: ${reason}`), + ); + return; + } + if (reason !== "shutdown" && reason !== "relay connection closed by us") { + transportError = new Error(`Buzz QA message subscription closed: ${reason}`); + } + }, + }, + ); + } catch (error) { + clearTimeout(observerReadyTimeout); + lifecycleAbort.abort(error); + relay.close(); + throw error; + } + try { + await observerReadyPromise; + } catch (error) { + lifecycleAbort.abort(error); + subscription.close("shutdown"); + relay.close(); + throw error; + } + + return { + assertHealthy() { + if (transportError) { + throw transportError; + } + }, + async sendMessage(input) { + if (transportError) { + throw transportError; + } + const tags = buildBuzzMessageTags({ + channelId: credentials.roomId, + threadId: input.threadId, + replyToId: input.replyToId, + }); + if (input.mentionSut) { + tags.push(["p", credentials.sutPublicKey]); + } + const event = finalizeEvent( + { + kind: BUZZ_MESSAGE_KIND, + content: input.text, + created_at: Math.floor(Date.now() / 1_000), + tags, + }, + secretKey, + ); + await relay.publish(event); + return { eventId: event.id, timestamp: event.created_at * 1_000 }; + }, + async close() { + lifecycleAbort.abort(new Error("Buzz QA relay driver closed")); + subscription.close("shutdown"); + relay.close(); + await messageQueue; + }, + }; +} diff --git a/extensions/qa-lab/runtime-api.ts b/extensions/qa-lab/runtime-api.ts index 20e6d37057a3..a014764997bd 100644 --- a/extensions/qa-lab/runtime-api.ts +++ b/extensions/qa-lab/runtime-api.ts @@ -41,6 +41,11 @@ export { setQaChannelRuntime, } from "./src/runtime-api.js"; export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js"; +export { runLiveTransportQaSuiteCommand } from "./src/live-transports/shared/live-transport-suite.runtime.js"; +export { + acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat, +} from "./src/live-transports/shared/credential-lease.runtime.js"; export { createQaChannelDriverLifecycle, runQaChannelDriverLifecycleScenarios, diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 2fd79799e9af..8168ca123e77 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -160,6 +160,7 @@ export type QaSuiteCommandOptions = QaScenarioRunCommandOptions & { runtimePair?: string; runtimePairLane?: string[]; sutAccountId?: string; + credentialFile?: string; credentialSource?: string; credentialRole?: string; explicitScenarioSelection?: boolean; @@ -1019,6 +1020,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { adapterOptions: { repoRoot, sutAccountId: opts.sutAccountId, + credentialFile: opts.credentialFile, credentialSource: opts.credentialSource, credentialRole: opts.credentialRole, explicitScenarioSelection: diff --git a/extensions/qa-lab/src/gateway-log-redaction.test.ts b/extensions/qa-lab/src/gateway-log-redaction.test.ts index 4d39be2aa44e..e9fe235cdba9 100644 --- a/extensions/qa-lab/src/gateway-log-redaction.test.ts +++ b/extensions/qa-lab/src/gateway-log-redaction.test.ts @@ -26,6 +26,40 @@ describe("gateway log redaction", () => { expect(formatQaGatewayLogsForError(raw)).not.toContain(token); }); + it("redacts Buzz QA private keys and authorization tags", () => { + const privateKey = "01".repeat(32); + const authTag = '["auth","pubkey","conditions","signature"]'; + const raw = [ + `privateKey: ${privateKey}`, + `authTag: ${authTag}`, + `{"privateKey":"${privateKey}"}`, + JSON.stringify({ authTag }), + `channels.buzz.authTag: ${authTag}`, + [ + "channels.buzz.authTag: [", + ' "auth",', + ' "pubkey",', + ' "conditions",', + ' "signature"', + "]", + ].join("\n"), + [ + "channels.buzz.authTag: [", + " 'auth',", + " 'pubkey',", + " 'conditions',", + " 'escaped\\'signature'", + "]", + ].join("\n"), + ].join("\n"); + + const redacted = redactQaGatewayDebugText(raw); + expect(redacted).not.toContain(privateKey); + expect(redacted).not.toContain(authTag); + expect(redacted).not.toContain("signature"); + expect(redacted.match(//gu)).toHaveLength(7); + }); + it("neutralizes GitHub workflow commands at every line boundary", () => { const raw = [ "::set-output name=output_dir::/tmp/attacker", diff --git a/extensions/qa-lab/src/gateway-log-redaction.ts b/extensions/qa-lab/src/gateway-log-redaction.ts index 0271a33d703f..a5a1434e4f1b 100644 --- a/extensions/qa-lab/src/gateway-log-redaction.ts +++ b/extensions/qa-lab/src/gateway-log-redaction.ts @@ -20,6 +20,8 @@ const QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS = Object.freeze([ "client_secret", "cookie", "driverToken", + "privateKey", + "authTag", "sutToken", "leaseToken", "refreshToken", @@ -58,21 +60,41 @@ function redactSecretEnvKeyPattern(text: string, pattern: RegExp) { function redactSecretValueKey(text: string, key: string) { const escapedKey = escapeRegExp(key); + const valuePattern = `[^\\s"';,]+|"(?:\\\\.|[^"\\\\])*"|'[^']*'`; return text .replace(new RegExp(`([?#&]${escapedKey}=)[^&\\s]+`, "gi"), "$1") .replace( - new RegExp(`(^|\\s)(--${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"), + new RegExp(`(^|\\s)(--${escapedKey})(\\s*[=:]\\s*)(${valuePattern})`, "gi"), `$1$2$3`, ) .replace( - new RegExp(`(^|[^\\w?#&-])(${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"), + new RegExp(`(^|[^\\w?#&-])(${escapedKey})(\\s*[=:]\\s*)(${valuePattern})`, "gi"), `$1$2$3`, ) - .replace(new RegExp(`("${escapedKey}"\\s*:\\s*)"[^"]*"`, "gi"), `$1""`); + .replace(new RegExp(`("${escapedKey}"\\s*:\\s*)"(?:\\\\.|[^"\\\\])*"`, "gi"), `$1""`); +} + +function redactStructuredSecretLine(text: string, key: string) { + const escapedKey = escapeRegExp(key); + return text.replace( + new RegExp(`(^|[\\r\\n])(\\s*"?${escapedKey}"?\\s*[:=]\\s*).*$`, "gim"), + `$1$2`, + ); +} + +function redactStructuredSecretArray(text: string, key: string) { + const escapedKey = escapeRegExp(key); + const quotedString = `"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'`; + const quotedStringArray = `\\[\\s*(?:${quotedString})(?:\\s*,\\s*(?:${quotedString}))*\\s*\\]`; + return text.replace( + new RegExp(`("?${escapedKey}"?\\s*[:=]\\s*)${quotedStringArray}`, "gi"), + `$1`, + ); } export function redactQaGatewayDebugText(text: string) { let redacted = redactSensitiveText(redactTelegramBotTokens(text), { mode: "tools" }); + redacted = redactStructuredSecretArray(redacted, "authTag"); for (const key of QA_GATEWAY_DEBUG_SECRET_HEADER_KEYS) { const escapedKey = escapeRegExp(key); redacted = redacted.replace( @@ -97,6 +119,9 @@ export function redactQaGatewayDebugText(text: string) { for (const key of QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS) { redacted = redactSecretValueKey(redacted, key); } + for (const key of ["authTag", "privateKey"]) { + redacted = redactStructuredSecretLine(redacted, key); + } return redacted .replaceAll(/\bsk-ant-oat01-[A-Za-z0-9_-]+\b/g, "") .replaceAll(/\bBearer\s+[^\s"'<>]{8,}/gi, "Bearer ") diff --git a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts index 327d63410603..9e8cb9ccaa8b 100644 --- a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts +++ b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts @@ -91,6 +91,10 @@ describe("live transport adapter factories", () => { expect.objectContaining({ adapterOptions, channelId, + credentials: { + acquire: expect.any(Function), + startHeartbeat: expect.any(Function), + }, driver: "live", messages: expect.objectContaining({ addInboundMessage: expect.any(Function), diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts index 24d3a0ca1d0c..3c498bc724cf 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const runQaSuiteCommand = vi.hoisted(() => vi.fn()); @@ -8,9 +8,14 @@ import { runLiveTransportQaSuiteCommand } from "./live-transport-suite.runtime.j describe("live transport suite runtime", () => { beforeEach(() => { + vi.stubEnv("OPENCLAW_QA_CREDENTIAL_SOURCE", ""); vi.clearAllMocks(); }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("normalizes one live command into the shared suite host", async () => { await runLiveTransportQaSuiteCommand({ channelId: "slack", @@ -23,6 +28,7 @@ describe("live transport suite runtime", () => { fastMode: true, allowFailures: true, failFast: true, + credentialFile: "/secure/slack-qa.json", credentialSource: " convex ", credentialRole: " ci ", sutAccountId: "slack-sut", @@ -49,6 +55,7 @@ describe("live transport suite runtime", () => { concurrency: 1, scenarioIds: ["slack-canary"], sutAccountId: "slack-sut", + credentialFile: "/secure/slack-qa.json", credentialSource: "convex", credentialRole: "ci", explicitScenarioSelection: false, @@ -71,6 +78,21 @@ describe("live transport suite runtime", () => { ); }); + it("normalizes the shared credential source environment override", async () => { + vi.stubEnv("OPENCLAW_QA_CREDENTIAL_SOURCE", " convex "); + + await runLiveTransportQaSuiteCommand({ + channelId: "buzz", + defaultProviderMode: "mock-openai", + options: {}, + selectScenarioIds: () => ["channel-canary"], + }); + + expect(runQaSuiteCommand).toHaveBeenCalledWith( + expect.objectContaining({ credentialSource: "convex" }), + ); + }); + it("rejects shared credentials for disposable transports", async () => { await expect( runLiveTransportQaSuiteCommand({ diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts index 0da70b564601..14dff12084b6 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts @@ -20,10 +20,11 @@ export async function runLiveTransportQaSuiteCommand(params: { selectScenarioIds: LiveTransportScenarioSelection; }) { const options = params.options; + const credentialSource = + options.credentialSource?.trim() || process.env.OPENCLAW_QA_CREDENTIAL_SOURCE?.trim(); if (params.credentialMode === "env-only") { const laneLabel = params.laneLabel ?? params.channelId; - const credentialSource = options.credentialSource?.trim().toLowerCase(); - if (credentialSource && credentialSource !== "env") { + if (credentialSource && credentialSource.toLowerCase() !== "env") { throw new Error( `QA Lab ${laneLabel} supports only --credential-source env${params.envCredentialReason ? ` because ${params.envCredentialReason}` : "."}`, ); @@ -58,10 +59,11 @@ export async function runLiveTransportQaSuiteCommand(params: { concurrency: 1, scenarioIds: selectedScenarioIds, sutAccountId: options.sutAccountId, + ...(options.credentialFile ? { credentialFile: options.credentialFile } : {}), ...(params.credentialMode === "env-only" ? {} : { - credentialSource: options.credentialSource?.trim(), + credentialSource, credentialRole: options.credentialRole?.trim(), }), explicitScenarioSelection: Boolean(options.scenarioIds?.length), diff --git a/extensions/qa-lab/src/qa-transport-registry.ts b/extensions/qa-lab/src/qa-transport-registry.ts index 60cb4e0b14e9..570ce9e61df5 100644 --- a/extensions/qa-lab/src/qa-transport-registry.ts +++ b/extensions/qa-lab/src/qa-transport-registry.ts @@ -1,6 +1,10 @@ import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime"; // Qa Lab plugin module implements qa transport registry behavior. import type { QaBusState } from "./bus-state.js"; +import { + acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat, +} from "./live-transports/shared/credential-lease.runtime.js"; import { createQaChannelTransport, QA_CHANNEL_DEFAULT_SUITE_CONCURRENCY, @@ -102,6 +106,10 @@ function createQaTransportAdapterFactoryRegistry( const definition = await factory.create({ adapterOptions: context.adapterOptions, channelId: context.channelId, + credentials: { + acquire: acquireQaCredentialLease, + startHeartbeat: startQaCredentialLeaseHeartbeat, + }, driver: context.driver, messages: { addInboundMessage: (input) => context.state.addInboundMessage(input), diff --git a/qa/convex-credential-broker/README.md b/qa/convex-credential-broker/README.md index 7e5b3242c5d5..e2d3238a04f5 100644 --- a/qa/convex-credential-broker/README.md +++ b/qa/convex-credential-broker/README.md @@ -59,13 +59,17 @@ Maintainers can manage rows without using the Convex dashboard: ```bash pnpm openclaw qa credentials add \ - --kind telegram \ - --payload-file qa/telegram-credential.json + --kind buzz \ + --payload-file qa/buzz-credential.json pnpm openclaw qa credentials add \ --kind discord \ --payload-file qa/discord-credential.json +pnpm openclaw qa credentials add \ + --kind telegram \ + --payload-file qa/telegram-credential.json + pnpm openclaw qa credentials list --kind telegram pnpm openclaw qa credentials remove --credential-id @@ -146,6 +150,18 @@ For `kind: "telegram"`, broker `admin/add` validates that payload includes: - non-empty `driverToken` - non-empty `sutToken` +For `kind: "buzz"`, broker `admin/add` validates that payload includes: + +- `relayUrl` as a `wss://` URL, or `ws://` only for a loopback relay +- `roomId` as a channel UUID +- valid, distinct `driverPrivateKey` and `sutPrivateKey` values in nsec or + 64-character hex form +- optional `driverAuthTag` and `sutAuthTag` values matching the four-string + Buzz authorization tag JSON shape + +Use dedicated QA identities only. Never add a human owner or admin private key +to the shared pool. + For `kind: "telegram-user"`, broker `admin/add` validates one exclusive real-user credential for both the TDLib CLI driver and the Telegram Desktop visual witness: diff --git a/qa/convex-credential-broker/convex/payload_validation.ts b/qa/convex-credential-broker/convex/payload_validation.ts index 9d45d78fff88..be4e31c3bd17 100644 --- a/qa/convex-credential-broker/convex/payload_validation.ts +++ b/qa/convex-credential-broker/convex/payload_validation.ts @@ -1,4 +1,6 @@ // Payload Validation module supports OpenClaw QA credential workflows. +import { getPublicKey, nip19 } from "nostr-tools"; + class CredentialPayloadValidationError extends Error { code: string; httpStatus: number; @@ -15,6 +17,9 @@ type PayloadValidationFailureFactory = (httpStatus: number, code: string, messag const DISCORD_SNOWFLAKE_RE = /^\d{17,20}$/u; const E164_RE = /^\+[1-9]\d{6,14}$/u; +const BUZZ_ROOM_ID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu; +const BUZZ_PRIVATE_KEY_HEX_RE = /^[0-9a-f]{64}$/iu; const SHA256_HEX_RE = /^[a-f0-9]{64}$/u; const TELEGRAM_CHAT_ID_RE = /^-?\d+$/u; const TELEGRAM_USER_ID_RE = /^\d+$/u; @@ -65,6 +70,132 @@ function requireDiscordSnowflakePayloadString( return value; } +function decodeBuzzPrivateKey(value: string) { + if (BUZZ_PRIVATE_KEY_HEX_RE.test(value)) { + const bytes = value.match(/.{2}/gu); + if (bytes?.length === 32) { + return Uint8Array.from(bytes.map((byte) => Number.parseInt(byte, 16))); + } + } + const decoded = nip19.decode(value); + if (decoded.type !== "nsec") { + throw new Error("not a Buzz private key"); + } + return decoded.data; +} + +function requireBuzzPrivateKey( + payload: Record, + key: "driverPrivateKey" | "sutPrivateKey", + createFailure: PayloadValidationFailureFactory, +) { + const value = requirePayloadString(payload, key, "buzz", createFailure); + try { + return { value, publicKey: getPublicKey(decodeBuzzPrivateKey(value)) }; + } catch { + throwPayloadError( + createFailure, + `Credential payload for kind "buzz" must include "${key}" as an nsec or 64-character hex private key.`, + ); + } +} + +function requireBuzzAuthTag( + payload: Record, + key: "driverAuthTag" | "sutAuthTag", + createFailure: PayloadValidationFailureFactory, +) { + const value = requirePayloadString(payload, key, "buzz", createFailure); + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + parsed = undefined; + } + if ( + !Array.isArray(parsed) || + parsed.length !== 4 || + parsed[0] !== "auth" || + parsed.some((entry) => typeof entry !== "string") + ) { + throwPayloadError( + createFailure, + `Credential payload for kind "buzz" must include "${key}" as an auth tag JSON array.`, + ); + } + return value; +} + +function normalizeBuzzCredentialPayload( + payload: Record, + createFailure: PayloadValidationFailureFactory, +) { + const kind = "buzz"; + const relayUrl = requirePayloadString(payload, "relayUrl", kind, createFailure); + let parsedRelayUrl: URL | undefined; + try { + parsedRelayUrl = new URL(relayUrl); + } catch { + parsedRelayUrl = undefined; + } + const relayProtocol = parsedRelayUrl?.protocol; + const relayUsesSafeTransport = + relayProtocol === "wss:" || + (relayProtocol === "ws:" && isBuzzLoopbackHostname(parsedRelayUrl?.hostname ?? "")); + if (!relayUsesSafeTransport) { + throwPayloadError( + createFailure, + 'Credential payload for kind "buzz" must include "relayUrl" using wss:// (ws:// is allowed only for loopback).', + ); + } + const roomId = requirePayloadString(payload, "roomId", kind, createFailure).toLowerCase(); + if (!BUZZ_ROOM_ID_RE.test(roomId)) { + throwPayloadError( + createFailure, + 'Credential payload for kind "buzz" must include "roomId" as a channel UUID.', + ); + } + const driverIdentity = requireBuzzPrivateKey(payload, "driverPrivateKey", createFailure); + const sutIdentity = requireBuzzPrivateKey(payload, "sutPrivateKey", createFailure); + if (driverIdentity.publicKey === sutIdentity.publicKey) { + throwPayloadError( + createFailure, + 'Credential payload for kind "buzz" must use distinct driver and SUT identities.', + ); + } + const optionalString = (key: "driverAuthTag" | "sutAuthTag") => { + if (payload[key] === undefined) { + return undefined; + } + return requireBuzzAuthTag(payload, key, createFailure); + }; + const driverAuthTag = optionalString("driverAuthTag"); + const sutAuthTag = optionalString("sutAuthTag"); + + return { + relayUrl, + roomId, + driverPrivateKey: driverIdentity.value, + sutPrivateKey: sutIdentity.value, + ...(driverAuthTag ? { driverAuthTag } : {}), + ...(sutAuthTag ? { sutAuthTag } : {}), + } satisfies Record; +} + +function isBuzzLoopbackHostname(hostname: string): boolean { + const normalized = hostname.toLowerCase().replace(/^\[|\]$/gu, ""); + if (normalized === "localhost" || normalized === "::1") { + return true; + } + const ipv4 = normalized.startsWith("::ffff:") ? normalized.slice("::ffff:".length) : normalized; + const octets = ipv4.split("."); + return ( + octets.length === 4 && + octets[0] === "127" && + octets.every((octet) => /^\d{1,3}$/u.test(octet) && Number(octet) <= 255) + ); +} + function normalizeTelegramCredentialPayload( payload: Record, createFailure: PayloadValidationFailureFactory, @@ -263,6 +394,7 @@ const credentialPayloadNormalizers: Record< createFailure: PayloadValidationFailureFactory, ) => Record > = { + buzz: normalizeBuzzCredentialPayload, discord: normalizeDiscordCredentialPayload, telegram: normalizeTelegramCredentialPayload, "telegram-user": normalizeTelegramUserCredentialPayload, diff --git a/qa/convex-credential-broker/package.json b/qa/convex-credential-broker/package.json index 0ec78e13c09d..3a82bdd43a1c 100644 --- a/qa/convex-credential-broker/package.json +++ b/qa/convex-credential-broker/package.json @@ -10,6 +10,7 @@ "dev": "convex dev" }, "dependencies": { - "convex": "1.42.3" + "convex": "1.42.3", + "nostr-tools": "2.24.1" } } diff --git a/src/plugin-sdk/qa-runner-runtime.ts b/src/plugin-sdk/qa-runner-runtime.ts index 4eeb345b4470..77fc1b86c532 100644 --- a/src/plugin-sdk/qa-runner-runtime.ts +++ b/src/plugin-sdk/qa-runner-runtime.ts @@ -29,6 +29,7 @@ type QaRunnerAdapterOptions = { repoRoot?: string; scenarioIds?: readonly string[]; sutAccountId?: string; + credentialFile?: string; credentialSource?: string; credentialRole?: string; transportPolicy?: QaRunnerTransportPolicy; @@ -40,6 +41,46 @@ type QaRunnerMessageRecorder = { editMessage: (input: QaBusEditMessageInput) => QaBusMessage | Promise; }; +type QaRunnerCredentialLease = { + credentialId?: string; + heartbeat(): Promise; + heartbeatIntervalMs: number; + kind: string; + leaseToken?: string; + leaseTtlMs: number; + ownerId?: string; + payload: TPayload; + release(): Promise; + role?: "ci" | "maintainer"; + source: "convex" | "env"; +}; + +type QaRunnerCredentialLeaseOptions = { + kind: string; + parsePayload: (payload: unknown) => TPayload; + resolveEnvPayload: () => TPayload; + role?: string; + source?: string; +}; + +type QaRunnerCredentialHeartbeat = { + getFailure(): Error | null; + stop(): Promise; + throwIfFailed(): void; +}; + +type QaRunnerCredentialHost = { + acquire( + options: QaRunnerCredentialLeaseOptions, + ): Promise>; + startHeartbeat( + lease: Pick< + QaRunnerCredentialLease, + "heartbeat" | "heartbeatIntervalMs" | "kind" | "source" + >, + ): QaRunnerCredentialHeartbeat; +}; + type QaRunnerTransportFlowPreparationInput = { config: Record; scenarioId: string; @@ -147,6 +188,7 @@ type QaRunnerTransportFactory = { create: (context: { adapterOptions?: QaRunnerAdapterOptions; channelId: string; + credentials: QaRunnerCredentialHost; driver: string; messages: QaRunnerMessageRecorder; outputDir: string; diff --git a/src/plugin-sdk/qa-runtime.test.ts b/src/plugin-sdk/qa-runtime.test.ts index dcf8bccdd9c9..cf9db96e138e 100644 --- a/src/plugin-sdk/qa-runtime.test.ts +++ b/src/plugin-sdk/qa-runtime.test.ts @@ -118,6 +118,30 @@ describe("plugin-sdk qa-runtime", () => { expect(module.isQaRuntimeAvailable()).toBe(false); }); + it("runs a plugin-owned transport through the private QA suite host", async () => { + const runLiveTransportQaSuiteCommand = vi.fn(async () => {}); + loadBundledPluginPublicSurfaceModuleSync.mockReturnValue({ + runLiveTransportQaSuiteCommand, + }); + const module = await import("./qa-runtime.js"); + const options = { providerMode: "mock-openai" }; + const selectScenarioIds = vi.fn(() => ["channel-canary"]); + + await module.runLiveTransportQaSuiteCommand({ + channelId: "buzz", + defaultProviderMode: "mock-openai", + options, + selectScenarioIds, + }); + + expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith({ + channelId: "buzz", + defaultProviderMode: "mock-openai", + options, + selectScenarioIds, + }); + }); + it("registers shared live transport QA CLI options", async () => { const module = await import("./qa-runtime.js"); const run = vi.fn(async () => {}); @@ -126,6 +150,7 @@ describe("plugin-sdk qa-runtime", () => { module .createLiveTransportQaCliRegistration({ commandName: "telegram", + credentialFileHelp: "Private JSON credential file", credentialOptions: { sourceDescription: "Credential source for Telegram QA", roleDescription: "Credential role for Telegram QA", @@ -172,6 +197,8 @@ describe("plugin-sdk qa-runtime", () => { "--fail-fast", "--sut-account", "sut-2", + "--credential-file", + "/secure/telegram-qa.json", "--credential-source", "convex", "--credential-role", @@ -191,6 +218,7 @@ describe("plugin-sdk qa-runtime", () => { scenarioIds: ["alpha", "beta"], listScenarios: true, sutAccountId: "sut-2", + credentialFile: "/secure/telegram-qa.json", credentialSource: "convex", credentialRole: "maintainer", }); diff --git a/src/plugin-sdk/qa-runtime.ts b/src/plugin-sdk/qa-runtime.ts index d25d6b606950..1e3fe34732ca 100644 --- a/src/plugin-sdk/qa-runtime.ts +++ b/src/plugin-sdk/qa-runtime.ts @@ -24,6 +24,7 @@ type QaRuntimeSurface = { }, ) => string; startQaLiveLaneGateway: (...args: unknown[]) => Promise; + runLiveTransportQaSuiteCommand: (params: LiveTransportQaSuiteCommandOptions) => Promise; }; function isMissingQaRuntimeError(error: unknown) { @@ -71,10 +72,31 @@ export type LiveTransportQaCommandOptions = { scenarioIds?: string[]; listScenarios?: boolean; sutAccountId?: string; + credentialFile?: string; credentialSource?: string; credentialRole?: string; }; +export type LiveTransportQaSuiteCommandOptions = { + channelId: string; + credentialMode?: "env-only" | "shared-lease"; + defaultProviderMode: string; + envCredentialReason?: string; + laneLabel?: string; + options: LiveTransportQaCommandOptions; + selectScenarioIds: (params: { + profile?: string; + primaryModel: string; + providerMode: string; + scenarioIds?: readonly string[]; + }) => string[]; +}; + +/** Run a plugin-owned transport adapter through QA Lab's shared suite host. */ +export async function runLiveTransportQaSuiteCommand(params: LiveTransportQaSuiteCommandOptions) { + return await loadQaRuntimeModule().runLiveTransportQaSuiteCommand(params); +} + type LiveTransportQaCommanderOptions = { repoRoot?: string; outputDir?: string; @@ -88,6 +110,7 @@ type LiveTransportQaCommanderOptions = { failFast?: boolean; profile?: string; sutAccount?: string; + credentialFile?: string; credentialSource?: string; credentialRole?: string; }; @@ -104,6 +127,7 @@ export type LiveTransportQaCredentialCliOptions = { /** Declarative command metadata and runner used to install a live-transport QA CLI. */ export type LiveTransportQaCliRegistrationOptions = { commandName: string; + credentialFileHelp?: string; credentialOptions?: LiveTransportQaCredentialCliOptions; defaultProviderMode: string; description: string; @@ -149,6 +173,7 @@ function mapLiveTransportQaCommanderOptions( scenarioIds: opts.scenario, listScenarios: opts.listScenarios, sutAccountId: opts.sutAccount, + credentialFile: opts.credentialFile, credentialSource: opts.credentialSource, credentialRole: opts.credentialRole, }; @@ -177,6 +202,10 @@ function registerLiveTransportQaCli( command.option("--sut-account ", params.sutAccountHelp, "sut"); + if (params.credentialFileHelp) { + command.option("--credential-file ", params.credentialFileHelp); + } + if (params.listScenariosHelp) { command.option("--list-scenarios", params.listScenariosHelp, false); } diff --git a/test/qa-convex-credential-payload-validation.test.ts b/test/qa-convex-credential-payload-validation.test.ts index cfab5e235dfc..9c342fa6478e 100644 --- a/test/qa-convex-credential-payload-validation.test.ts +++ b/test/qa-convex-credential-payload-validation.test.ts @@ -2,7 +2,106 @@ import { describe, expect, it } from "vitest"; import { normalizeCredentialPayloadForKind } from "../qa/convex-credential-broker/convex/payload_validation.js"; +const BUZZ_DRIVER_PRIVATE_KEY = "01".repeat(32); +const BUZZ_SUT_PRIVATE_KEY = "02".repeat(32); +const BUZZ_DRIVER_NSEC = "nsec1qyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqstywftw"; + describe("QA Convex credential payload validation", () => { + it("normalizes Buzz credential payloads", () => { + expect( + normalizeCredentialPayloadForKind("buzz", { + relayUrl: " wss://relay.qa.example ", + roomId: " 123E4567-E89B-42D3-A456-426614174000 ", + driverPrivateKey: ` ${BUZZ_DRIVER_PRIVATE_KEY} `, + sutPrivateKey: ` ${BUZZ_SUT_PRIVATE_KEY} `, + driverAuthTag: ' ["auth","driver","conditions","signature"] ', + ignored: true, + }), + ).toEqual({ + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY, + sutPrivateKey: BUZZ_SUT_PRIVATE_KEY, + driverAuthTag: '["auth","driver","conditions","signature"]', + }); + }); + + it.each(["ws://localhost:8080", "ws://127.0.0.1:8080", "ws://[::1]:8080"])( + "allows plaintext loopback Buzz relay URL %s", + (relayUrl) => { + expect( + normalizeCredentialPayloadForKind("buzz", { + relayUrl, + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY, + sutPrivateKey: BUZZ_SUT_PRIVATE_KEY, + }), + ).toMatchObject({ relayUrl }); + }, + ); + + it("rejects plaintext remote Buzz relay URLs", () => { + expect(() => + normalizeCredentialPayloadForKind("buzz", { + relayUrl: "ws://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY, + sutPrivateKey: BUZZ_SUT_PRIVATE_KEY, + }), + ).toThrow(/wss:\/\//u); + }); + + it("rejects malformed Buzz credential payloads without echoing values", () => { + const privateKey = BUZZ_DRIVER_PRIVATE_KEY; + const invalidRelay = "https://relay.qa.example/private-path"; + expect(() => + normalizeCredentialPayloadForKind("buzz", { + relayUrl: invalidRelay, + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: privateKey, + sutPrivateKey: privateKey, + }), + ).toThrow(/wss:\/\//u); + try { + normalizeCredentialPayloadForKind("buzz", { + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: privateKey, + sutPrivateKey: privateKey, + }); + } catch (error) { + expect(String(error)).not.toContain(privateKey); + } + }); + + it("rejects runtime-invalid Buzz secrets and equivalent key encodings", () => { + expect(() => + normalizeCredentialPayloadForKind("buzz", { + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY, + sutPrivateKey: BUZZ_DRIVER_NSEC, + }), + ).toThrow(/distinct driver and SUT identities/u); + expect(() => + normalizeCredentialPayloadForKind("buzz", { + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: "not-a-private-key", + sutPrivateKey: BUZZ_SUT_PRIVATE_KEY, + }), + ).toThrow(/nsec or 64-character hex private key/u); + expect(() => + normalizeCredentialPayloadForKind("buzz", { + relayUrl: "wss://relay.qa.example", + roomId: "123e4567-e89b-42d3-a456-426614174000", + driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY, + sutPrivateKey: BUZZ_SUT_PRIVATE_KEY, + driverAuthTag: "not-an-auth-tag", + }), + ).toThrow(/auth tag JSON array/u); + }); + it("normalizes Discord credential payloads", () => { expect( normalizeCredentialPayloadForKind("discord", { diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 0c3b2ec5e2d0..fb539addc943 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -394,6 +394,7 @@ function runReleaseChecksSummary(params: { QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT: "skipped", QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: "skipped", QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: "skipped", + QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: "skipped", @@ -2590,7 +2591,7 @@ describe("package artifact reuse", () => { fail_fast: "${{ fromJSON(needs.resolve_target.outputs.fail_fast) }}", run_matrix: true, }); - for (const lane of ["mock_parity", "telegram", "discord", "whatsapp", "slack"]) { + for (const lane of ["mock_parity", "buzz", "telegram", "discord", "whatsapp", "slack"]) { expect(releaseJob.with?.[`run_${lane}`]).toBeUndefined(); } expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_mock_parity").if).toBe( @@ -2645,6 +2646,30 @@ describe("package artifact reuse", () => { ); }); + it("routes release Buzz through the QA Lab selector", () => { + const releaseJob = workflowJob(RELEASE_CHECKS_WORKFLOW, "qa_live_buzz_release_checks"); + + expect(releaseJob.uses).toBe("./.github/workflows/qa-live-transports-convex.yml"); + expect(releaseJob.secrets).toBeUndefined(); + expect(releaseJob.permissions).toEqual({ contents: "read", "pull-requests": "read" }); + expect(releaseJob.if).toContain('contains(fromJSON(\'["all","qa","qa-live"]\')'); + expect(releaseJob.if).toContain("needs.resolve_target.outputs.qa_live_buzz_enabled == 'true'"); + expect(releaseJob.with).toMatchObject({ + buzz_scenario: "channel-canary,channel-mention-gating", + expected_sha: "${{ needs.resolve_target.outputs.revision }}", + run_buzz: true, + }); + expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_buzz").if).toBe("inputs.run_buzz"); + expect( + workflowStep( + workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_buzz"), + "Upload Buzz QA artifacts", + ).with?.name, + ).toBe( + "${{ inputs.expected_sha != '' && format('release-qa-live-buzz-{0}-{1}', inputs.expected_sha, github.run_attempt) || format('qa-live-buzz-{0}-{1}', github.run_id, github.run_attempt) }}", + ); + }); + it("runs live transport lanes nightly while release checks stay gated", () => { const releaseWorkflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8"); const qaWorkflow = readFileSync(QA_LIVE_TRANSPORTS_WORKFLOW, "utf8"); @@ -2674,6 +2699,7 @@ describe("package artifact reuse", () => { "always() && steps.run_lane.outputs.output_dir != ''", ], ["run_live_matrix", "Upload Matrix QA artifacts", "always()"], + ["run_live_buzz", "Upload Buzz QA artifacts", "always()"], ["run_live_telegram", "Upload Telegram QA artifacts", "always()"], ["run_live_discord", "Upload Discord QA artifacts", "always()"], ["run_live_whatsapp", "Upload WhatsApp QA artifacts", "always()"], @@ -3431,6 +3457,7 @@ describe("package artifact reuse", () => { const verifyStep = workflowStep(summary, "Verify release check results"); expect(verifyStep.env).toMatchObject({ + QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: "${{ needs.qa_live_buzz_release_checks.result }}", QA_LIVE_RELEASE_CHECKS_RESULT: "${{ needs.qa_live_release_checks.result }}", RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}", RELEASE_CHECK_RUN_ID: "${{ github.run_id }}", @@ -3454,6 +3481,7 @@ describe("package artifact reuse", () => { "::warning::${name} ended with ${result}; Tideclaw alpha treats non-package-safety release-check lanes as advisory.", "::error::${name} ended with ${result}", '"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"', + '"qa_live_buzz_release_checks=${QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT}"', ]); expect(verifyStep.run).not.toContain("qa_live_matrix_release_checks"); expect(verifyStep.run).not.toContain(