mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
test(buzz): add live QA Lab channel lane (#116298)
* feat(buzz): add live QA runner * ci(qa): add gated Buzz live lane * docs(qa): document Buzz live testing * fix(buzz): keep QA relay type private * test(buzz): type QA suite mock * fix(buzz): preserve racing QA reply relations * fix(qa): harden Buzz credential redaction * fix(buzz): ignore retained QA relay events * fix(buzz): clean up observer setup failures * test(buzz): return complete racing messages * fix(buzz): default QA credentials to file * fix(qa): validate pooled Buzz secrets * fix(qa): redact structured Buzz auth tags * fix(buzz): preserve QA credential env override * fix(qa): redact inspected Buzz auth tags * test(buzz): limit live QA to supported scenarios * ci(qa): run Buzz lane in release checks * fix(qa): require secure remote Buzz relays * fix(qa): satisfy Buzz QA validation gates * fix(qa): inject credential leasing into transport adapters
This commit is contained in:
Patrick Erichsen
@@ -81,7 +81,7 @@ on:
|
||||
- qa-parity
|
||||
- qa-live
|
||||
live_suite_filter:
|
||||
description: Optional exact live/E2E suite id, or comma-separated QA live lane ids (qa-live-matrix, qa-live-telegram, qa-live-discord, qa-live-whatsapp, qa-live-slack); blank runs all selected live suites
|
||||
description: Optional exact live/E2E suite id, or comma-separated QA live lane ids (qa-live-matrix, qa-live-buzz, qa-live-telegram, qa-live-discord, qa-live-whatsapp, qa-live-slack); blank runs all selected live suites
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
@@ -146,6 +146,7 @@ jobs:
|
||||
repo_live_suite_filter: ${{ steps.inputs.outputs.repo_live_suite_filter }}
|
||||
cross_os_suite_filter: ${{ steps.inputs.outputs.cross_os_suite_filter }}
|
||||
qa_live_matrix_enabled: ${{ steps.inputs.outputs.qa_live_matrix_enabled }}
|
||||
qa_live_buzz_enabled: ${{ steps.inputs.outputs.qa_live_buzz_enabled }}
|
||||
qa_live_telegram_enabled: ${{ steps.inputs.outputs.qa_live_telegram_enabled }}
|
||||
qa_live_discord_enabled: ${{ steps.inputs.outputs.qa_live_discord_enabled }}
|
||||
qa_live_whatsapp_enabled: ${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}
|
||||
@@ -333,6 +334,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
qa_live_matrix_enabled=true
|
||||
qa_live_buzz_enabled=true
|
||||
qa_live_telegram_enabled=true
|
||||
qa_live_discord_ci_enabled="$(printf '%s' "$RELEASE_QA_DISCORD_LIVE_CI_ENABLED" | tr '[:upper:]' '[:lower:]')"
|
||||
if [[ "$qa_live_discord_ci_enabled" != "true" && "$qa_live_discord_ci_enabled" != "1" && "$qa_live_discord_ci_enabled" != "yes" ]]; then
|
||||
@@ -408,6 +410,7 @@ jobs:
|
||||
qa_filter_seen=false
|
||||
repo_filter_tokens=()
|
||||
matrix_selected=false
|
||||
buzz_selected=false
|
||||
telegram_selected=false
|
||||
discord_selected=false
|
||||
whatsapp_selected=false
|
||||
@@ -424,6 +427,7 @@ jobs:
|
||||
qa-live|qa-live-all|qa-all)
|
||||
qa_filter_seen=true
|
||||
matrix_selected=true
|
||||
buzz_selected=true
|
||||
telegram_selected=true
|
||||
discord_selected="$qa_live_discord_ci_enabled"
|
||||
whatsapp_selected="$qa_live_whatsapp_ci_enabled"
|
||||
@@ -435,6 +439,7 @@ jobs:
|
||||
qa-live-non-slack|qa-non-slack|non-slack|no-slack|without-slack)
|
||||
qa_filter_seen=true
|
||||
matrix_selected=true
|
||||
buzz_selected=true
|
||||
telegram_selected=true
|
||||
discord_selected="$qa_live_discord_ci_enabled"
|
||||
whatsapp_selected="$qa_live_whatsapp_ci_enabled"
|
||||
@@ -445,6 +450,10 @@ jobs:
|
||||
qa_filter_seen=true
|
||||
matrix_selected=true
|
||||
;;
|
||||
qa-live-buzz|qa-buzz|buzz)
|
||||
qa_filter_seen=true
|
||||
buzz_selected=true
|
||||
;;
|
||||
qa-live-telegram|qa-telegram|telegram)
|
||||
qa_filter_seen=true
|
||||
telegram_selected=true
|
||||
@@ -482,6 +491,7 @@ jobs:
|
||||
|
||||
if [[ "$qa_filter_seen" == "true" ]]; then
|
||||
qa_live_matrix_enabled="$matrix_selected"
|
||||
qa_live_buzz_enabled="$buzz_selected"
|
||||
qa_live_telegram_enabled="$telegram_selected"
|
||||
qa_live_discord_enabled="$discord_selected"
|
||||
qa_live_whatsapp_enabled="$whatsapp_selected"
|
||||
@@ -503,6 +513,7 @@ jobs:
|
||||
printf 'repo_live_suite_filter=%s\n' "$repo_live_suite_filter"
|
||||
printf 'cross_os_suite_filter=%s\n' "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT"
|
||||
printf 'qa_live_matrix_enabled=%s\n' "$qa_live_matrix_enabled"
|
||||
printf 'qa_live_buzz_enabled=%s\n' "$qa_live_buzz_enabled"
|
||||
printf 'qa_live_telegram_enabled=%s\n' "$qa_live_telegram_enabled"
|
||||
printf 'qa_live_discord_enabled=%s\n' "$qa_live_discord_enabled"
|
||||
printf 'qa_live_whatsapp_enabled=%s\n' "$qa_live_whatsapp_enabled"
|
||||
@@ -551,7 +562,7 @@ jobs:
|
||||
if [[ -n "${RELEASE_CROSS_OS_SUITE_FILTER// }" ]]; then
|
||||
echo "- Cross-OS suite filter: \`${RELEASE_CROSS_OS_SUITE_FILTER}\`"
|
||||
fi
|
||||
echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`"
|
||||
echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Buzz \`${{ steps.inputs.outputs.qa_live_buzz_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`"
|
||||
if [[ -n "${RELEASE_PACKAGE_SPEC// }" ]]; then
|
||||
echo "- Release package spec: \`${RELEASE_PACKAGE_SPEC}\`"
|
||||
fi
|
||||
@@ -1655,6 +1666,20 @@ jobs:
|
||||
run_matrix: true
|
||||
fail_fast: ${{ fromJSON(needs.resolve_target.outputs.fail_fast) }}
|
||||
|
||||
qa_live_buzz_release_checks:
|
||||
name: Run QA Lab live Buzz lane
|
||||
needs: [resolve_target]
|
||||
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_buzz_enabled == 'true'
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
uses: ./.github/workflows/qa-live-transports-convex.yml
|
||||
with:
|
||||
ref: ${{ needs.resolve_target.outputs.revision }}
|
||||
expected_sha: ${{ needs.resolve_target.outputs.revision }}
|
||||
run_buzz: true
|
||||
buzz_scenario: channel-canary,channel-mention-gating
|
||||
|
||||
# The dispatched child owns Telegram evidence/status artifacts; this blocking job
|
||||
# carries its exact conclusion into the parent summary without copying secrets or artifacts.
|
||||
qa_live_telegram_release_checks:
|
||||
@@ -2108,6 +2133,7 @@ jobs:
|
||||
- qa_lab_runtime_parity_release_checks
|
||||
- runtime_tool_coverage_release_checks
|
||||
- qa_live_release_checks
|
||||
- qa_live_buzz_release_checks
|
||||
- qa_live_telegram_release_checks
|
||||
- qa_live_discord_release_checks
|
||||
- qa_live_whatsapp_release_checks
|
||||
@@ -2143,6 +2169,7 @@ jobs:
|
||||
QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }}
|
||||
RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: ${{ needs.runtime_tool_coverage_release_checks.result }}
|
||||
QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result }}
|
||||
QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_buzz_release_checks.result }}
|
||||
QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }}
|
||||
QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }}
|
||||
QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }}
|
||||
@@ -2169,6 +2196,7 @@ jobs:
|
||||
"qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}"
|
||||
"runtime_tool_coverage_release_checks=${RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT}"
|
||||
"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"
|
||||
"qa_live_buzz_release_checks=${QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT}"
|
||||
"qa_live_telegram_release_checks=${QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT}"
|
||||
"qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}"
|
||||
"qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}"
|
||||
|
||||
@@ -18,6 +18,16 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
run_buzz:
|
||||
description: Run the Buzz live lane
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
buzz_scenario:
|
||||
description: Optional comma-separated Buzz scenario ids
|
||||
required: false
|
||||
default: ""
|
||||
type: string
|
||||
run_matrix:
|
||||
description: Run the Matrix live lane
|
||||
required: false
|
||||
@@ -55,6 +65,15 @@ on:
|
||||
required: true
|
||||
default: main
|
||||
type: string
|
||||
run_buzz:
|
||||
description: Run the Buzz live lane (requires a pooled Buzz credential)
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
buzz_scenario:
|
||||
description: Optional comma-separated Buzz scenario ids
|
||||
required: false
|
||||
type: string
|
||||
scenario:
|
||||
description: Optional comma-separated Telegram scenario ids
|
||||
required: false
|
||||
@@ -427,6 +446,95 @@ jobs:
|
||||
retention-days: 14
|
||||
if-no-files-found: error
|
||||
|
||||
run_live_buzz:
|
||||
name: Run Buzz live QA lane with Convex leases
|
||||
needs: [authorize_actor, validate_selected_ref]
|
||||
if: inputs.run_buzz
|
||||
runs-on: blacksmith-16vcpu-ubuntu-2404
|
||||
timeout-minutes: 60
|
||||
concurrency:
|
||||
group: qa-live-buzz-shared
|
||||
cancel-in-progress: false
|
||||
environment: qa-live-shared
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
install-bun: "true"
|
||||
|
||||
- name: Validate required Buzz QA credential env
|
||||
env:
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
for key in OPENCLAW_QA_CONVEX_SITE_URL OPENCLAW_QA_CONVEX_SECRET_CI; do
|
||||
if [[ -z "${!key:-}" ]]; then
|
||||
echo "Missing required ${key}." >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build private QA runtime
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=12288
|
||||
run: pnpm build
|
||||
|
||||
- name: Run Buzz live lane
|
||||
id: run_lane
|
||||
shell: bash
|
||||
env:
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
|
||||
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
|
||||
OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000"
|
||||
INPUT_SCENARIO: ${{ inputs.buzz_scenario || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
output_dir=".artifacts/qa-e2e/buzz-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
scenario_args=()
|
||||
|
||||
if [[ -n "${INPUT_SCENARIO// }" ]]; then
|
||||
IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}"
|
||||
for raw in "${raw_scenarios[@]}"; do
|
||||
scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
if [[ -n "${scenario}" ]]; then
|
||||
scenario_args+=(--scenario "${scenario}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
pnpm openclaw qa buzz \
|
||||
--repo-root . \
|
||||
--output-dir "${output_dir}" \
|
||||
--provider-mode mock-openai \
|
||||
--credential-source convex \
|
||||
--credential-role ci \
|
||||
"${scenario_args[@]}"
|
||||
|
||||
- name: Upload Buzz QA artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: ${{ inputs.expected_sha != '' && format('release-qa-live-buzz-{0}-{1}', inputs.expected_sha, github.run_attempt) || format('qa-live-buzz-{0}-{1}', github.run_id, github.run_attempt) }}
|
||||
path: ${{ steps.run_lane.outputs.output_dir }}
|
||||
retention-days: 14
|
||||
if-no-files-found: error
|
||||
|
||||
run_live_telegram:
|
||||
name: Run Telegram live QA lane with Convex leases
|
||||
needs: [authorize_actor, validate_selected_ref]
|
||||
|
||||
@@ -338,6 +338,32 @@ openclaw message send \
|
||||
For a full round trip, have an allowed Buzz user mention the bot and confirm that
|
||||
OpenClaw replies in the room.
|
||||
|
||||
### QA Lab round trip
|
||||
|
||||
Source checkouts can exercise the production Buzz channel path with two
|
||||
dedicated test identities:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa buzz \
|
||||
--credential-file /secure/path/buzz-qa-credentials.json \
|
||||
--provider-mode mock-openai
|
||||
```
|
||||
|
||||
The command runs a real relay canary and mention-gating check while using the
|
||||
deterministic mock model. The private JSON credential
|
||||
file contains `relayUrl`, `roomId`, `driverPrivateKey`, and `sutPrivateKey`, plus
|
||||
optional `driverAuthTag` and `sutAuthTag` values for closed relays. Both test
|
||||
public keys must be room members, and the SUT public key must have the **Bot**
|
||||
role. A closed relay may require both public keys to be enrolled separately.
|
||||
Use `--credential-source convex` for pooled QA credentials.
|
||||
|
||||
Use `wss://` for hosted relays. Plaintext `ws://` credential URLs are accepted
|
||||
only for loopback development relays.
|
||||
|
||||
Never use a human owner or admin private key. Private keys and optional
|
||||
authorization values are parent-harness secrets and must not appear in logs,
|
||||
artifacts, screenshots, shell history, or source control.
|
||||
|
||||
## Rotate the bot identity
|
||||
|
||||
Bot identity rotation requires admin approval for the new public key:
|
||||
|
||||
@@ -47,6 +47,7 @@ script aliases; both forms work.
|
||||
| `qa aimock` | Start only the AIMock provider server. |
|
||||
| `qa mock-openai` | Start only the scenario-aware `mock-openai` provider server. |
|
||||
| `qa credentials doctor` / `add` / `list` / `remove` | Manage the shared Convex credential pool. |
|
||||
| `qa buzz` | Live transport lane against a real Buzz relay room with dedicated driver and SUT identities. |
|
||||
| `qa discord` | Live transport lane against a real private Discord guild channel. |
|
||||
| `qa matrix` | QA Lab Matrix catalog scenarios against a disposable Tuwunel homeserver. See [Matrix live lane](#matrix-live-lane). |
|
||||
| `qa slack` | Live transport lane against a real private Slack channel. |
|
||||
@@ -291,6 +292,7 @@ decision still comes from the Discord REST oracle.
|
||||
For the other transport-real smoke lanes:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa buzz
|
||||
pnpm openclaw qa discord
|
||||
pnpm openclaw qa slack
|
||||
pnpm openclaw qa telegram
|
||||
@@ -299,8 +301,8 @@ pnpm openclaw qa whatsapp
|
||||
|
||||
They target a pre-existing real channel with two bots or accounts (driver +
|
||||
SUT). Required env vars, scenario lists, output artifacts, and the Convex
|
||||
credential pool for those four transports are documented in
|
||||
[Discord, Slack, Telegram, and WhatsApp QA reference](#discord-slack-telegram-and-whatsapp-qa-reference)
|
||||
credential pool for those five transports are documented in
|
||||
[Buzz, Discord, Slack, Telegram, and WhatsApp QA reference](#buzz-discord-slack-telegram-and-whatsapp-qa-reference)
|
||||
below.
|
||||
|
||||
### Mantis Slack desktop and visual-task runners
|
||||
@@ -453,16 +455,16 @@ guest: env-based provider keys, the QA live provider config path, and
|
||||
`CODEX_HOME` when present. Keep `--output-dir` under the repo root so the
|
||||
guest can write back through the mounted workspace.
|
||||
|
||||
## Discord, Slack, Telegram, and WhatsApp QA reference
|
||||
## Buzz, Discord, Slack, Telegram, and WhatsApp QA reference
|
||||
|
||||
The Matrix adapter uses the disposable Docker-backed lane documented above.
|
||||
Discord, Slack, Telegram, and WhatsApp run against pre-existing real
|
||||
Buzz, Discord, Slack, Telegram, and WhatsApp run against pre-existing real
|
||||
transports, so their reference lives here.
|
||||
|
||||
### Shared CLI flags
|
||||
|
||||
These lanes register through
|
||||
`extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and
|
||||
These lanes register through the shared QA runner CLI contract. Transport
|
||||
plugins may own the registration while QA Lab remains the suite host. They
|
||||
accept the same flags:
|
||||
|
||||
| Flag | Default | Description |
|
||||
@@ -471,11 +473,12 @@ accept the same flags:
|
||||
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/<transport>-<timestamp>` | Where reports, summaries, evidence, transport-specific artifacts, and the output log are written. Relative paths resolve against `--repo-root`. |
|
||||
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
|
||||
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
|
||||
| `--provider-mode <mode>` | `live-frontier` | `mock-openai`, `aimock`, or `live-frontier`. |
|
||||
| `--provider-mode <mode>` | `live-frontier` (Buzz: `mock-openai`) | `mock-openai`, `aimock`, or `live-frontier`. |
|
||||
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
|
||||
| `--fast` | off | Provider fast mode where supported. |
|
||||
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
|
||||
| `--credential-source <source>` | `env` (Buzz: `file`) | Existing lanes use `env` or `convex`; Buzz uses `file` or `convex`. See [Convex credential pool](#convex-credential-pool). |
|
||||
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
|
||||
| `--credential-file <path>` | - | Buzz-only JSON credential file for local runs. |
|
||||
| `--allow-failures` | off | Write artifacts without returning a failing exit code when scenarios fail. |
|
||||
|
||||
Each lane exits non-zero on any failed scenario. `--allow-failures` writes
|
||||
@@ -483,6 +486,40 @@ artifacts without setting a failing exit code. Telegram also accepts
|
||||
`--list-scenarios` to print available scenario ids and exit; the other lanes
|
||||
do not expose that flag.
|
||||
|
||||
### Buzz QA
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa buzz \
|
||||
--credential-file /secure/path/buzz-qa-credentials.json
|
||||
```
|
||||
|
||||
Targets one real Buzz room with two dedicated Nostr identities. The driver
|
||||
publishes inbound room events; the SUT identity is configured in the child
|
||||
OpenClaw Gateway and its outbound events are observed from the relay. The
|
||||
default `mock-openai` provider proves the real Buzz transport without requiring
|
||||
a model-provider credential.
|
||||
|
||||
Local runs use `--credential-file <path>` with a private JSON file containing
|
||||
`relayUrl`, `roomId`, `driverPrivateKey`, and `sutPrivateKey`. Closed relays may
|
||||
also need `driverAuthTag` and `sutAuthTag`. Relative paths resolve from
|
||||
`--repo-root`. Hosted relays must use `wss://`; plaintext `ws://` is accepted
|
||||
only for loopback development relays.
|
||||
|
||||
Both identities must be members of the dedicated room, and the SUT public key
|
||||
must have the **Bot** role. A hosted closed relay may also require both public
|
||||
keys to be enrolled as relay members. Use dedicated QA identities only; never
|
||||
use a human owner or admin private key. Keep all private keys and authorization
|
||||
values out of logs, command lines, artifacts, screenshots, and source control.
|
||||
|
||||
The default scenarios are:
|
||||
|
||||
- `channel-canary`
|
||||
- `channel-mention-gating`
|
||||
|
||||
Each run writes `qa-suite-report.md`, `qa-suite-summary.json`, and
|
||||
`qa-evidence.json` under the selected output directory. The report identifies
|
||||
the real Buzz relay path but omits credential values.
|
||||
|
||||
### Telegram QA
|
||||
|
||||
```bash
|
||||
@@ -1024,15 +1061,20 @@ Output artifacts:
|
||||
|
||||
### Convex credential pool
|
||||
|
||||
Discord, Slack, Telegram, and WhatsApp lanes can lease credentials from a
|
||||
Buzz, Discord, Slack, Telegram, and WhatsApp lanes can lease credentials from a
|
||||
shared Convex pool instead of reading the env vars above. Pass
|
||||
`--credential-source convex` (or set `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`);
|
||||
QA Lab acquires an exclusive lease, heartbeats it for the duration of the
|
||||
run, and releases it on shutdown. Pool kinds are `"discord"`, `"slack"`,
|
||||
`"telegram"`, and `"whatsapp"`.
|
||||
run, and releases it on shutdown. Pool kinds are `"buzz"`, `"discord"`,
|
||||
`"slack"`, `"telegram"`, and `"whatsapp"`.
|
||||
|
||||
Payload shapes the broker validates on `admin/add`:
|
||||
|
||||
- Buzz (`kind: "buzz"`): `{ relayUrl: string, roomId: string,
|
||||
driverPrivateKey: string, sutPrivateKey: string, driverAuthTag?: string,
|
||||
sutAuthTag?: string }` - `relayUrl` must use `wss://`, with `ws://` allowed only
|
||||
for loopback relays; `roomId` must be a channel UUID, and the identities must
|
||||
be distinct.
|
||||
- Discord (`kind: "discord"`): `{ guildId: string, channelId: string,
|
||||
driverBotToken: string, sutBotToken: string, sutApplicationId: string }`.
|
||||
- Telegram (`kind: "telegram"`): `{ groupId: string, driverToken: string,
|
||||
|
||||
+3
-1
@@ -313,6 +313,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- H2: Manual configuration
|
||||
- H3: Bot key storage
|
||||
- H2: Verify the connection
|
||||
- H3: QA Lab round trip
|
||||
- H2: Rotate the bot identity
|
||||
- H2: Current limits and roadmap
|
||||
- H2: Troubleshooting
|
||||
@@ -2955,8 +2956,9 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- H3: Mantis Slack desktop and visual-task runners
|
||||
- H3: Credential pool health check
|
||||
- H2: Canonical scenario coverage
|
||||
- H2: Discord, Slack, Telegram, and WhatsApp QA reference
|
||||
- H2: Buzz, Discord, Slack, Telegram, and WhatsApp QA reference
|
||||
- H3: Shared CLI flags
|
||||
- H3: Buzz QA
|
||||
- H3: Telegram QA
|
||||
- H3: Discord QA
|
||||
- H3: Slack QA
|
||||
|
||||
+15
-2
@@ -400,6 +400,18 @@ gh workflow run package-acceptance.yml --ref main \
|
||||
- `pnpm openclaw qa aimock`
|
||||
- Starts only the local AIMock provider server for direct protocol smoke
|
||||
testing.
|
||||
- `pnpm openclaw qa buzz`
|
||||
- Runs the Buzz live QA lane against a real relay room using dedicated driver
|
||||
and SUT identities.
|
||||
- Local runs use `--credential-file <path>` with `relayUrl`, `roomId`,
|
||||
`driverPrivateKey`, and `sutPrivateKey`. Closed relays may also need
|
||||
`driverAuthTag` and `sutAuthTag`. Hosted relays require `wss://`; `ws://` is
|
||||
accepted only for loopback development relays.
|
||||
- Defaults to `mock-openai` and runs canary and mention-gating scenarios
|
||||
through the real Buzz plugin path.
|
||||
- Supports `--credential-source convex` with a pooled `kind: "buzz"` row.
|
||||
Both public keys must be relay/room members, and the SUT must have the
|
||||
**Bot** room role. Never use a human owner or admin private key.
|
||||
- `pnpm openclaw qa matrix`
|
||||
- Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel
|
||||
homeserver. Source-checkout only - packaged installs do not ship
|
||||
@@ -488,8 +500,8 @@ drift; the per-lane coverage matrix lives in
|
||||
When `--credential-source convex` (or `OPENCLAW_QA_CREDENTIAL_SOURCE=convex`)
|
||||
is enabled for live transport QA, QA lab acquires an exclusive lease from a
|
||||
Convex-backed pool, heartbeats that lease while the lane is running, and
|
||||
releases the lease on shutdown. The section name predates Discord, Slack, and
|
||||
WhatsApp support; the lease contract is shared across kinds.
|
||||
releases the lease on shutdown. The section name predates Buzz, Discord, Slack,
|
||||
and WhatsApp support; the lease contract is shared across kinds.
|
||||
|
||||
Reference Convex project scaffold: `qa/convex-credential-broker/`
|
||||
|
||||
@@ -575,6 +587,7 @@ Payload shape for Telegram real-user kind:
|
||||
|
||||
Broker-validated multi-channel payloads:
|
||||
|
||||
- Buzz: `{ relayUrl: string, roomId: string, driverPrivateKey: string, sutPrivateKey: string, driverAuthTag?: string, sutAuthTag?: string }`
|
||||
- Discord: `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string, voiceChannelId?: string }`
|
||||
- WhatsApp: `{ driverPhoneE164: string, sutPhoneE164: string, driverAuthArchiveBase64: string, sutAuthArchiveBase64: string, groupJid?: string }`
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ it before dispatching. A narrower `rerun_group` skips this preflight.
|
||||
| Docker assets preflight | **Job:** `Verify Docker runtime image assets`<br />**Child workflow:** none<br />**Proves:** the `runtime-assets` Docker build target still succeeds before any other stage dispatches. Runs only for `rerun_group=all`.<br />**Rerun:** rerun the umbrella with `rerun_group=all`. |
|
||||
| Vitest and normal CI | **Job:** `Run normal full CI`<br />**Child workflow:** `CI`<br />**Proves:** manual full CI graph against the target ref, including Linux Node lanes, bundled plugin shards, plugin and channel contract shards, Node 22 compatibility, `check-*`, `check-additional-*`, built-artifact smoke checks, docs checks, Python skills, Windows, macOS, Control UI i18n, and Android via the umbrella.<br />**Rerun:** `rerun_group=ci`. |
|
||||
| Plugin prerelease | **Job:** `Run plugin prerelease validation`<br />**Child workflow:** `Plugin Prerelease`<br />**Proves:** release-only plugin static checks, agentic plugin coverage, full plugin batch shards, plugin prerelease Docker lanes, and a non-blocking `plugin-inspector-advisory` artifact for compatibility triage.<br />**Rerun:** `rerun_group=plugin-prerelease`. |
|
||||
| Release checks | **Job:** `Run release/live/Docker/QA validation`<br />**Child workflow:** `OpenClaw Release Checks`<br />**Proves:** install smoke, cross-OS package checks, Package Acceptance, QA Lab parity, live Matrix and Telegram, plus gated advisory Discord, WhatsApp, and Slack lanes. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks; beta can opt in with `run_release_soak=true`.<br />**Rerun:** `rerun_group=release-checks` or a narrower release-checks handle. |
|
||||
| Release checks | **Job:** `Run release/live/Docker/QA validation`<br />**Child workflow:** `OpenClaw Release Checks`<br />**Proves:** install smoke, cross-OS package checks, Package Acceptance, QA Lab parity, live Matrix, Buzz, and Telegram, plus gated advisory Discord, WhatsApp, and Slack lanes. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks; beta can opt in with `run_release_soak=true`.<br />**Rerun:** `rerun_group=release-checks` or a narrower release-checks handle. |
|
||||
| Package Telegram | **Job:** `Run package Telegram E2E`<br />**Child workflow:** `NPM Telegram Beta E2E`<br />**Proves:** a focused published-package Telegram E2E when `release_package_spec` or `npm_telegram_package_spec` is set. Full candidate validation uses the canonical Package Acceptance Telegram E2E instead.<br />**Rerun:** `rerun_group=npm-telegram` with `release_package_spec` or `npm_telegram_package_spec`. |
|
||||
| Product performance | **Job:** `Run product performance evidence`<br />**Child workflow:** `OpenClaw Performance`<br />**Proves:** release-profile performance run (`profile=release`, `repeat=3`, `fail_on_regression=true`, `publish_reports=false`) against the target SHA. Kova output stays in workflow artifacts and the child must prove its report publisher was skipped. Required (blocking) only for `rerun_group=all` or `rerun_group=performance`; not required for narrower rerun groups.<br />**Rerun:** `rerun_group=performance`. |
|
||||
| Umbrella verifier | **Job:** `Verify full validation`<br />**Child workflow:** none<br />**Proves:** re-checks recorded child run conclusions and appends slowest-job tables from child workflows.<br />**Rerun:** rerun only this job after rerunning a failed child to green. |
|
||||
@@ -175,6 +175,7 @@ artifact when package or Docker-facing stages need it.
|
||||
| QA runtime parity | **Job:** `Verify QA Lab runtime-pair lanes`<br />**Backing workflow:** direct job<br />**Tests:** the canonical core `openclaw`/`codex` lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex --runtime-pair-lane core`) and, with `run_release_soak=true`, the soak lane. Advisory: individual lane jobs do not block the release-check verifier.<br />**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. |
|
||||
| QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`<br />**Backing workflow:** direct job<br />**Tests:** dynamic tool drift between `openclaw` and `codex` in the canonical core runtime-pair lane (`pnpm openclaw qa coverage --tools`), using that lane's output. Blocking: this job is not advisory-overridable.<br />**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. |
|
||||
| QA live Matrix | **Job:** `Run QA Live Matrix catalog`<br />**Backing workflow:** `QA-Lab - All Lanes` reusable workflow<br />**Tests:** catalog-derived YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment, distributed across deterministic shards.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. |
|
||||
| QA live Buzz | **Job:** `Run QA Lab live Buzz lane`<br />**Backing workflow:** `QA-Lab - All Lanes` reusable workflow<br />**Tests:** signed canary and mention-gating round trips through the real Buzz plugin using dedicated Convex-leased identities and a hosted relay room.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-buzz` for a focused Buzz rerun. |
|
||||
| QA live Telegram | **Job:** `Run QA Lab live Telegram lane`<br />**Backing workflow:** trusted `OpenClaw Release Telegram QA` dispatch<br />**Tests:** live Telegram QA with Convex CI credential leases.<br />**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. |
|
||||
| QA live Discord | **Job:** `Run QA Lab live Discord lane`<br />**Backing workflow:** direct advisory job<br />**Tests:** live Discord QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_DISCORD_LIVE_CI_ENABLED` is enabled.<br />**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-discord`. |
|
||||
| QA live WhatsApp | **Job:** `Run QA Lab live WhatsApp lane`<br />**Backing workflow:** direct advisory job<br />**Tests:** live WhatsApp QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_WHATSAPP_LIVE_CI_ENABLED` is enabled.<br />**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-whatsapp`. |
|
||||
@@ -251,7 +252,7 @@ Use `rerun_group` to avoid repeating unrelated release boxes:
|
||||
| `package` | Package Acceptance. |
|
||||
| `qa` | QA parity plus QA live lanes. |
|
||||
| `qa-parity` | QA parity lanes and report only. |
|
||||
| `qa-live` | QA live Matrix/Telegram plus gated Discord, WhatsApp, and Slack lanes when enabled. |
|
||||
| `qa-live` | QA live Matrix, Buzz, and Telegram plus gated Discord, WhatsApp, and Slack lanes when enabled. |
|
||||
| `npm-telegram` | Published-package Telegram E2E; requires `release_package_spec` or `npm_telegram_package_spec`. |
|
||||
| `performance` | Product performance evidence only. |
|
||||
|
||||
@@ -264,8 +265,8 @@ Valid filter ids are defined in the reusable live/E2E workflow, including
|
||||
`live-codex-harness-docker`.
|
||||
|
||||
For a focused QA transport rerun, set `rerun_group=qa-live` and use the
|
||||
canonical selector `qa-live-matrix`, `qa-live-telegram`, `qa-live-discord`,
|
||||
`qa-live-whatsapp`, or `qa-live-slack`.
|
||||
canonical selector `qa-live-matrix`, `qa-live-buzz`, `qa-live-telegram`,
|
||||
`qa-live-discord`, `qa-live-whatsapp`, or `qa-live-slack`.
|
||||
|
||||
The `live-gateway-advisory-docker` handle is an aggregate rerun handle for its
|
||||
three provider shards, so it still fans out to all advisory Docker gateway jobs.
|
||||
@@ -310,8 +311,8 @@ Useful artifacts:
|
||||
- Docker release-path artifacts under `.artifacts/docker-tests/`
|
||||
- Package Acceptance `package-under-test` and Docker acceptance artifacts
|
||||
- Cross-OS release-check artifacts for each OS and suite
|
||||
- QA parity, runtime parity, and selected Matrix, Telegram, Discord, WhatsApp,
|
||||
or Slack artifacts
|
||||
- QA parity, runtime parity, and selected Matrix, Buzz, Telegram, Discord,
|
||||
WhatsApp, or Slack artifacts
|
||||
|
||||
## Workflow files
|
||||
|
||||
|
||||
@@ -6,6 +6,12 @@
|
||||
"onStartup": false
|
||||
},
|
||||
"channels": ["buzz"],
|
||||
"qaRunners": [
|
||||
{
|
||||
"commandName": "buzz",
|
||||
"description": "Run the Buzz live QA lane against a dedicated relay room"
|
||||
}
|
||||
],
|
||||
"channelConfigs": {
|
||||
"buzz": {
|
||||
"label": "Buzz",
|
||||
|
||||
@@ -1,2 +1,6 @@
|
||||
import { buzzQaCliRegistration } from "./src/qa/cli.js";
|
||||
|
||||
export type { ChannelGatewayContext } from "openclaw/plugin-sdk/channel-contract";
|
||||
export type { ChannelPlugin, PluginRuntime } from "openclaw/plugin-sdk/core";
|
||||
|
||||
export const qaRunnerCliRegistrations = [buzzQaCliRegistration];
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
import type {
|
||||
QaBusInboundMessageInput,
|
||||
QaBusMessage,
|
||||
} from "openclaw/plugin-sdk/qa-channel-protocol";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseBuzzQaCredentialPayload } from "./credentials.js";
|
||||
|
||||
const credentials = parseBuzzQaCredentialPayload({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: "01".repeat(32),
|
||||
sutPrivateKey: "02".repeat(32),
|
||||
});
|
||||
|
||||
const release = vi.hoisted(() => vi.fn(async () => {}));
|
||||
const heartbeatStop = vi.hoisted(() => vi.fn(async () => {}));
|
||||
const acquireQaCredentialLease = vi.hoisted(() =>
|
||||
vi.fn(async (_options: unknown) => ({
|
||||
source: "env" as const,
|
||||
kind: "buzz",
|
||||
payload: credentials,
|
||||
heartbeatIntervalMs: 0,
|
||||
leaseTtlMs: 0,
|
||||
heartbeat: vi.fn(async () => {}),
|
||||
release,
|
||||
})),
|
||||
);
|
||||
const startQaCredentialLeaseHeartbeat = vi.hoisted(() =>
|
||||
vi.fn((_lease: unknown) => ({
|
||||
getFailure: () => null,
|
||||
stop: heartbeatStop,
|
||||
throwIfFailed: vi.fn(),
|
||||
})),
|
||||
);
|
||||
const readBuzzQaCredentialFile = vi.hoisted(() => vi.fn());
|
||||
const sendMessage = vi.hoisted(() =>
|
||||
vi.fn(async () => ({ eventId: "native-inbound", timestamp: 1_750_000_000_000 })),
|
||||
);
|
||||
const closeRelay = vi.hoisted(() => vi.fn(async () => {}));
|
||||
const relayDriverState = vi.hoisted(() => ({
|
||||
onMessage: undefined as
|
||||
| ((message: {
|
||||
id: string;
|
||||
senderPubkey: string;
|
||||
text: string;
|
||||
channelId: string;
|
||||
createdAt: number;
|
||||
threadId?: string;
|
||||
replyToId?: string;
|
||||
mentionedPubkeys: string[];
|
||||
}) => Promise<void>)
|
||||
| undefined,
|
||||
}));
|
||||
const createBuzzQaRelayDriver = vi.hoisted(() =>
|
||||
vi.fn(async (params: { onMessage: NonNullable<typeof relayDriverState.onMessage> }) => {
|
||||
relayDriverState.onMessage = params.onMessage;
|
||||
return {
|
||||
assertHealthy: vi.fn(),
|
||||
close: closeRelay,
|
||||
sendMessage,
|
||||
};
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock("./credentials.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./credentials.js")>()),
|
||||
readBuzzQaCredentialFile,
|
||||
}));
|
||||
vi.mock("./relay-client.js", () => ({ createBuzzQaRelayDriver }));
|
||||
|
||||
import { createBuzzQaTransportAdapter } from "./adapter.runtime.js";
|
||||
|
||||
const credentialHost: Parameters<typeof createBuzzQaTransportAdapter>[0]["credentials"] = {
|
||||
acquire: async (options) => (await acquireQaCredentialLease(options)) as never,
|
||||
startHeartbeat: startQaCredentialLeaseHeartbeat,
|
||||
};
|
||||
|
||||
describe("Buzz QA transport adapter", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
relayDriverState.onMessage = undefined;
|
||||
sendMessage.mockResolvedValue({
|
||||
eventId: "native-inbound",
|
||||
timestamp: 1_750_000_000_000,
|
||||
});
|
||||
readBuzzQaCredentialFile.mockResolvedValue(credentials);
|
||||
});
|
||||
|
||||
it("uses private file credentials by default", async () => {
|
||||
await createBuzzQaTransportAdapter({
|
||||
adapterOptions: {
|
||||
credentialFile: "private/buzz-qa.json",
|
||||
repoRoot: "/repo",
|
||||
},
|
||||
channelId: "buzz",
|
||||
credentials: credentialHost,
|
||||
driver: "live",
|
||||
messages: {
|
||||
addInboundMessage: vi.fn(),
|
||||
addOutboundMessage: vi.fn(),
|
||||
editMessage: vi.fn(),
|
||||
},
|
||||
outputDir: ".artifacts/qa-e2e/buzz",
|
||||
});
|
||||
|
||||
expect(readBuzzQaCredentialFile).toHaveBeenCalledWith({
|
||||
filePath: "private/buzz-qa.json",
|
||||
repoRoot: "/repo",
|
||||
});
|
||||
expect(acquireQaCredentialLease).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ kind: "buzz", source: "env" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("sends a portable mentioned message through the native Buzz relay driver", async () => {
|
||||
const addInboundMessage = vi.fn(async (input) => ({
|
||||
...input,
|
||||
id: "bus-inbound",
|
||||
direction: "inbound" as const,
|
||||
timestamp: input.timestamp ?? 1_750_000_000_000,
|
||||
}));
|
||||
const adapter = await createBuzzQaTransportAdapter({
|
||||
adapterOptions: { credentialSource: "convex", sutAccountId: "sut" },
|
||||
channelId: "buzz",
|
||||
credentials: credentialHost,
|
||||
driver: "live",
|
||||
messages: {
|
||||
addInboundMessage,
|
||||
addOutboundMessage: vi.fn(),
|
||||
editMessage: vi.fn(),
|
||||
},
|
||||
outputDir: ".artifacts/qa-e2e/buzz",
|
||||
});
|
||||
|
||||
const message = await adapter.sendInbound({
|
||||
accountId: "sut",
|
||||
conversation: { id: "qa-routing-primary", kind: "group" },
|
||||
senderId: "driver",
|
||||
senderName: "QA Driver",
|
||||
text: "@openclaw reply exactly: QA-CHANNEL-CANARY-OK",
|
||||
});
|
||||
|
||||
expect(sendMessage).toHaveBeenCalledWith({
|
||||
mentionSut: true,
|
||||
text: "@openclaw reply exactly: QA-CHANNEL-CANARY-OK",
|
||||
});
|
||||
expect(addInboundMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
accountId: "sut",
|
||||
senderId: credentials.driverPublicKey,
|
||||
timestamp: 1_750_000_000_000,
|
||||
}),
|
||||
);
|
||||
expect(message.id).toBe("bus-inbound");
|
||||
|
||||
const gateway = {
|
||||
call: vi.fn(async () => ({
|
||||
channelAccounts: { buzz: [{ accountId: "default", running: true }] },
|
||||
})),
|
||||
};
|
||||
await adapter.waitReady({ gateway });
|
||||
expect(gateway.call).toHaveBeenCalledWith(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
);
|
||||
});
|
||||
|
||||
it("maps native Buzz reply relations back to portable thread ids", async () => {
|
||||
let inboundIndex = 0;
|
||||
let outboundIndex = 0;
|
||||
const addInboundMessage = vi.fn(async (input) => ({
|
||||
...input,
|
||||
id: `bus-inbound-${++inboundIndex}`,
|
||||
direction: "inbound" as const,
|
||||
timestamp: input.timestamp ?? 1_750_000_000_000,
|
||||
}));
|
||||
const addOutboundMessage = vi.fn(async (input) => ({
|
||||
...input,
|
||||
id: `bus-outbound-${++outboundIndex}`,
|
||||
direction: "outbound" as const,
|
||||
conversation: { id: "main", kind: "group" as const },
|
||||
}));
|
||||
sendMessage
|
||||
.mockResolvedValueOnce({ eventId: "native-root", timestamp: 1_750_000_000_000 })
|
||||
.mockResolvedValueOnce({ eventId: "native-follow-up", timestamp: 1_750_000_001_000 });
|
||||
const adapter = await createBuzzQaTransportAdapter({
|
||||
adapterOptions: { credentialSource: "convex", sutAccountId: "sut" },
|
||||
channelId: "buzz",
|
||||
credentials: credentialHost,
|
||||
driver: "live",
|
||||
messages: {
|
||||
addInboundMessage,
|
||||
addOutboundMessage,
|
||||
editMessage: vi.fn(),
|
||||
},
|
||||
outputDir: ".artifacts/qa-e2e/buzz",
|
||||
});
|
||||
|
||||
const root = await adapter.sendInbound({
|
||||
accountId: "sut",
|
||||
conversation: { id: "main", kind: "group" },
|
||||
senderId: "driver",
|
||||
senderName: "QA Driver",
|
||||
text: "@openclaw root",
|
||||
});
|
||||
await relayDriverState.onMessage?.({
|
||||
id: "native-sut-root",
|
||||
senderPubkey: credentials.sutPublicKey,
|
||||
text: "root reply",
|
||||
channelId: credentials.roomId,
|
||||
createdAt: 1_750_000_000,
|
||||
threadId: "native-root",
|
||||
replyToId: "native-root",
|
||||
mentionedPubkeys: [],
|
||||
});
|
||||
const followUp = await adapter.sendInbound({
|
||||
accountId: "sut",
|
||||
conversation: { id: "main", kind: "group" },
|
||||
senderId: "driver",
|
||||
senderName: "QA Driver",
|
||||
text: "@openclaw follow-up",
|
||||
threadId: root.id,
|
||||
});
|
||||
await relayDriverState.onMessage?.({
|
||||
id: "native-sut-follow-up",
|
||||
senderPubkey: credentials.sutPublicKey,
|
||||
text: "thread reply",
|
||||
channelId: credentials.roomId,
|
||||
createdAt: 1_750_000_001,
|
||||
threadId: "native-root",
|
||||
replyToId: "native-follow-up",
|
||||
mentionedPubkeys: [],
|
||||
});
|
||||
|
||||
expect(sendMessage).toHaveBeenLastCalledWith({
|
||||
mentionSut: true,
|
||||
text: "@openclaw follow-up",
|
||||
threadId: "native-root",
|
||||
});
|
||||
expect(addOutboundMessage).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
threadId: root.id,
|
||||
replyToId: followUp.id,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for a racing inbound id mapping before recording reply relations", async () => {
|
||||
let releaseInbound = () => {};
|
||||
const inboundGate = new Promise<void>((resolve) => {
|
||||
releaseInbound = resolve;
|
||||
});
|
||||
const addInboundMessage = vi.fn(async (input: QaBusInboundMessageInput) => {
|
||||
await inboundGate;
|
||||
return {
|
||||
...input,
|
||||
id: "bus-inbound",
|
||||
accountId: input.accountId ?? "sut",
|
||||
direction: "inbound",
|
||||
timestamp: input.timestamp ?? 1_750_000_000_000,
|
||||
reactions: [],
|
||||
} satisfies QaBusMessage;
|
||||
});
|
||||
const addOutboundMessage = vi.fn(async (input) => ({
|
||||
...input,
|
||||
id: "bus-outbound",
|
||||
direction: "outbound" as const,
|
||||
conversation: { id: "main", kind: "group" as const },
|
||||
}));
|
||||
const adapter = await createBuzzQaTransportAdapter({
|
||||
adapterOptions: { credentialSource: "convex", sutAccountId: "sut" },
|
||||
channelId: "buzz",
|
||||
credentials: credentialHost,
|
||||
driver: "live",
|
||||
messages: {
|
||||
addInboundMessage,
|
||||
addOutboundMessage,
|
||||
editMessage: vi.fn(),
|
||||
},
|
||||
outputDir: ".artifacts/qa-e2e/buzz",
|
||||
});
|
||||
|
||||
const inboundPromise = adapter.sendInbound({
|
||||
accountId: "sut",
|
||||
conversation: { id: "main", kind: "group" },
|
||||
senderId: "driver",
|
||||
senderName: "QA Driver",
|
||||
text: "@openclaw root",
|
||||
});
|
||||
await vi.waitFor(() => expect(addInboundMessage).toHaveBeenCalledOnce());
|
||||
const outboundPromise = relayDriverState.onMessage?.({
|
||||
id: "native-sut-reply",
|
||||
senderPubkey: credentials.sutPublicKey,
|
||||
text: "fast reply",
|
||||
channelId: credentials.roomId,
|
||||
createdAt: 1_750_000_000,
|
||||
threadId: "native-inbound",
|
||||
replyToId: "native-inbound",
|
||||
mentionedPubkeys: [],
|
||||
});
|
||||
expect(addOutboundMessage).not.toHaveBeenCalled();
|
||||
|
||||
releaseInbound();
|
||||
await inboundPromise;
|
||||
await outboundPromise;
|
||||
|
||||
expect(addOutboundMessage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
threadId: "bus-inbound",
|
||||
replyToId: "bus-inbound",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("closes relay observation before stopping and releasing the credential lease", async () => {
|
||||
const adapter = await createBuzzQaTransportAdapter({
|
||||
adapterOptions: { credentialSource: "convex" },
|
||||
channelId: "buzz",
|
||||
credentials: credentialHost,
|
||||
driver: "live",
|
||||
messages: {
|
||||
addInboundMessage: vi.fn(),
|
||||
addOutboundMessage: vi.fn(),
|
||||
editMessage: vi.fn(),
|
||||
},
|
||||
outputDir: ".artifacts/qa-e2e/buzz",
|
||||
});
|
||||
|
||||
await adapter.cleanup?.();
|
||||
await adapter.cleanupAfterGatewayStop?.();
|
||||
|
||||
expect(closeRelay).toHaveBeenCalledOnce();
|
||||
expect(heartbeatStop).toHaveBeenCalledOnce();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
expect(closeRelay.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
heartbeatStop.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
expect(heartbeatStop.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
release.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,230 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import type { BuzzInboundMessage } from "../message-event.js";
|
||||
import { buildBuzzTarget } from "../target.js";
|
||||
import { parseBuzzQaCredentialPayload, readBuzzQaCredentialFile } from "./credentials.js";
|
||||
import { createBuzzQaRelayDriver } from "./relay-client.js";
|
||||
|
||||
type AdapterFactory = NonNullable<QaRunnerCliRegistration["adapterFactory"]>;
|
||||
type FactoryContext = Parameters<AdapterFactory["create"]>[0];
|
||||
type AdapterDefinition = Awaited<ReturnType<AdapterFactory["create"]>> & {
|
||||
cleanupAfterGatewayStop?: () => Promise<void>;
|
||||
};
|
||||
|
||||
const BUZZ_GATEWAY_ACCOUNT_ID = "default";
|
||||
const BUZZ_MESSAGE_ID_MAPPING_TIMEOUT_MS = 5_000;
|
||||
|
||||
function isBuzzMention(text: string) {
|
||||
return /(^|\s)@openclaw\b/iu.test(text);
|
||||
}
|
||||
|
||||
async function waitForBuzzChannelRunning(params: {
|
||||
accountId: string;
|
||||
gateway: Parameters<AdapterDefinition["waitReady"]>[0]["gateway"];
|
||||
pollIntervalMs?: number;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const timeoutMs = params.timeoutMs ?? 60_000;
|
||||
const pollIntervalMs = params.pollIntervalMs ?? 500;
|
||||
const startedAt = Date.now();
|
||||
let lastStatus: unknown;
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
const payload = (await params.gateway.call(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
)) as { channelAccounts?: Record<string, Array<Record<string, unknown>>> };
|
||||
const status = payload.channelAccounts?.buzz?.find(
|
||||
(entry) => entry.accountId === params.accountId,
|
||||
);
|
||||
lastStatus = status;
|
||||
if (status?.running === true && status.restartPending !== true) {
|
||||
return;
|
||||
}
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
throw new Error(
|
||||
`buzz account "${params.accountId}" did not become ready; last status: ${JSON.stringify(lastStatus)}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function createBuzzQaTransportAdapter(
|
||||
context: FactoryContext,
|
||||
): Promise<AdapterDefinition> {
|
||||
const options = context.adapterOptions ?? {};
|
||||
const requestedCredentialSource = options.credentialSource?.trim().toLowerCase() || "file";
|
||||
if (requestedCredentialSource !== "file" && requestedCredentialSource !== "convex") {
|
||||
throw new Error('Buzz QA credential source must be "file" or "convex".');
|
||||
}
|
||||
const credentialFile = options.credentialFile?.trim();
|
||||
if (requestedCredentialSource === "file" && !credentialFile) {
|
||||
throw new Error("Buzz QA file credentials require --credential-file <path>.");
|
||||
}
|
||||
if (requestedCredentialSource === "file" && options.credentialRole?.trim()) {
|
||||
throw new Error("Buzz QA --credential-role is only valid with --credential-source convex.");
|
||||
}
|
||||
if (requestedCredentialSource === "convex" && credentialFile) {
|
||||
throw new Error(
|
||||
"Buzz QA --credential-file cannot be combined with --credential-source convex.",
|
||||
);
|
||||
}
|
||||
const fileCredentials =
|
||||
requestedCredentialSource === "file" && credentialFile
|
||||
? await readBuzzQaCredentialFile({ filePath: credentialFile, repoRoot: options.repoRoot })
|
||||
: undefined;
|
||||
const lease = await context.credentials.acquire({
|
||||
kind: "buzz",
|
||||
source: requestedCredentialSource === "convex" ? "convex" : "env",
|
||||
role: options.credentialRole,
|
||||
resolveEnvPayload: () => {
|
||||
if (!fileCredentials) {
|
||||
throw new Error("Buzz QA file credentials are unavailable.");
|
||||
}
|
||||
return fileCredentials;
|
||||
},
|
||||
parsePayload: parseBuzzQaCredentialPayload,
|
||||
});
|
||||
const heartbeat = context.credentials.startHeartbeat(lease);
|
||||
const credentials = lease.payload;
|
||||
const accountId = options.sutAccountId?.trim() || "sut";
|
||||
const nativeMessageIds = new Map<string, string>();
|
||||
const busMessageIds = new Map<string, string>();
|
||||
let logicalConversationId = credentials.roomId;
|
||||
let relayDriver: Awaited<ReturnType<typeof createBuzzQaRelayDriver>>;
|
||||
|
||||
const resolveBusMessageId = async (nativeId: string | undefined) => {
|
||||
if (!nativeId) {
|
||||
return undefined;
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < BUZZ_MESSAGE_ID_MAPPING_TIMEOUT_MS) {
|
||||
const busId = busMessageIds.get(nativeId);
|
||||
if (busId) {
|
||||
return busId;
|
||||
}
|
||||
// A fast relay response can arrive before sendInbound records the
|
||||
// published event's portable id. Preserve native reply relations until
|
||||
// that canonical mapping is available instead of dropping them.
|
||||
await sleep(10);
|
||||
}
|
||||
throw new Error(`Buzz QA could not resolve the portable id for native message ${nativeId}.`);
|
||||
};
|
||||
|
||||
const recordOutbound = async (message: BuzzInboundMessage) => {
|
||||
const [threadId, replyToId] = await Promise.all([
|
||||
resolveBusMessageId(message.threadId),
|
||||
resolveBusMessageId(message.replyToId),
|
||||
]);
|
||||
const outbound = await context.messages.addOutboundMessage({
|
||||
accountId,
|
||||
to: `group:${logicalConversationId}`,
|
||||
senderId: credentials.sutPublicKey,
|
||||
text: message.text,
|
||||
timestamp: message.createdAt * 1_000,
|
||||
threadId,
|
||||
replyToId,
|
||||
});
|
||||
nativeMessageIds.set(outbound.id, message.id);
|
||||
busMessageIds.set(message.id, outbound.id);
|
||||
};
|
||||
|
||||
try {
|
||||
relayDriver = await createBuzzQaRelayDriver({ credentials, onMessage: recordOutbound });
|
||||
} catch (error) {
|
||||
try {
|
||||
await heartbeat.stop();
|
||||
} finally {
|
||||
await lease.release();
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
id: "buzz",
|
||||
label: "Buzz live",
|
||||
accountId,
|
||||
requiredPluginIds: ["buzz"],
|
||||
supportedActions: [],
|
||||
assertTransportHealthy() {
|
||||
heartbeat.throwIfFailed();
|
||||
relayDriver.assertHealthy();
|
||||
},
|
||||
async sendInbound(input) {
|
||||
heartbeat.throwIfFailed();
|
||||
relayDriver.assertHealthy();
|
||||
logicalConversationId = input.conversation.id;
|
||||
const sent = await relayDriver.sendMessage({
|
||||
text: input.text,
|
||||
mentionSut: isBuzzMention(input.text),
|
||||
...(input.threadId ? { threadId: nativeMessageIds.get(input.threadId) } : {}),
|
||||
...(input.replyToId ? { replyToId: nativeMessageIds.get(input.replyToId) } : {}),
|
||||
});
|
||||
const inbound = await context.messages.addInboundMessage({
|
||||
...input,
|
||||
accountId,
|
||||
senderId: credentials.driverPublicKey,
|
||||
timestamp: sent.timestamp,
|
||||
});
|
||||
nativeMessageIds.set(inbound.id, sent.eventId);
|
||||
busMessageIds.set(sent.eventId, inbound.id);
|
||||
return inbound;
|
||||
},
|
||||
resetTransport() {
|
||||
logicalConversationId = credentials.roomId;
|
||||
nativeMessageIds.clear();
|
||||
busMessageIds.clear();
|
||||
},
|
||||
createGatewayConfig: () =>
|
||||
({
|
||||
channels: {
|
||||
buzz: {
|
||||
enabled: true,
|
||||
relayUrl: credentials.relayUrl,
|
||||
privateKey: credentials.sutPrivateKey,
|
||||
...(credentials.sutAuthTag ? { authTag: credentials.sutAuthTag } : {}),
|
||||
groupPolicy: "allowlist",
|
||||
groupAllowFrom: [credentials.driverPublicKey],
|
||||
groups: {
|
||||
[credentials.roomId]: {
|
||||
enabled: true,
|
||||
requireMention: options.transportPolicy?.requireGroupMention ?? true,
|
||||
},
|
||||
},
|
||||
defaultTo: buildBuzzTarget(credentials.roomId),
|
||||
},
|
||||
},
|
||||
}) as Pick<OpenClawConfig, "channels" | "messages">,
|
||||
waitReady: async ({ gateway, timeoutMs, pollIntervalMs }) =>
|
||||
await waitForBuzzChannelRunning({
|
||||
// Buzz is currently single-account; the QA bus keeps its portable SUT
|
||||
// label while Gateway status reports the plugin's canonical account id.
|
||||
accountId: BUZZ_GATEWAY_ACCOUNT_ID,
|
||||
gateway,
|
||||
timeoutMs,
|
||||
pollIntervalMs,
|
||||
}),
|
||||
buildAgentDelivery: () => ({
|
||||
channel: "buzz",
|
||||
to: buildBuzzTarget(credentials.roomId),
|
||||
replyChannel: "buzz",
|
||||
replyTo: buildBuzzTarget(credentials.roomId),
|
||||
}),
|
||||
async handleAction() {
|
||||
throw new Error("Buzz live QA adapter does not implement transport actions");
|
||||
},
|
||||
createReportNotes: () => [
|
||||
"Runs through a real authenticated Buzz relay room; credential values are omitted.",
|
||||
],
|
||||
async cleanup() {
|
||||
await relayDriver.close();
|
||||
},
|
||||
async cleanupAfterGatewayStop() {
|
||||
try {
|
||||
await heartbeat.stop();
|
||||
} finally {
|
||||
await lease.release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { Command } from "commander";
|
||||
import type { LiveTransportQaSuiteCommandOptions } from "openclaw/plugin-sdk/qa-runtime";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runLiveTransportQaSuiteCommand = vi.hoisted(() =>
|
||||
vi.fn<(params: LiveTransportQaSuiteCommandOptions) => Promise<void>>(async () => {}),
|
||||
);
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/qa-runtime", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("openclaw/plugin-sdk/qa-runtime")>()),
|
||||
runLiveTransportQaSuiteCommand,
|
||||
}));
|
||||
|
||||
import { buzzQaCliRegistration } from "./cli.js";
|
||||
|
||||
describe("Buzz QA CLI", () => {
|
||||
beforeEach(() => {
|
||||
runLiveTransportQaSuiteCommand.mockClear();
|
||||
});
|
||||
|
||||
it("runs the portable canary and mention-gating scenarios", async () => {
|
||||
const qa = new Command();
|
||||
buzzQaCliRegistration.register(qa);
|
||||
|
||||
await qa.parseAsync([
|
||||
"node",
|
||||
"openclaw",
|
||||
"buzz",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--credential-file",
|
||||
"/secure/buzz-qa.json",
|
||||
]);
|
||||
|
||||
const params = runLiveTransportQaSuiteCommand.mock.calls[0]?.[0];
|
||||
expect(params).toMatchObject({
|
||||
channelId: "buzz",
|
||||
defaultProviderMode: "mock-openai",
|
||||
options: {
|
||||
providerMode: "mock-openai",
|
||||
credentialFile: "/secure/buzz-qa.json",
|
||||
},
|
||||
});
|
||||
expect(
|
||||
params?.selectScenarioIds({
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
providerMode: "mock-openai",
|
||||
}),
|
||||
).toEqual(["channel-canary", "channel-mention-gating"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
createLiveTransportQaCliRegistration,
|
||||
runLiveTransportQaSuiteCommand,
|
||||
type LiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCommandOptions,
|
||||
} from "openclaw/plugin-sdk/qa-runtime";
|
||||
|
||||
const DEFAULT_BUZZ_QA_SCENARIOS = ["channel-canary", "channel-mention-gating"] as const;
|
||||
|
||||
const loadBuzzQaAdapterRuntime = createLazyCliRuntimeLoader<typeof import("./adapter.runtime.js")>(
|
||||
() => import("./adapter.runtime.js"),
|
||||
);
|
||||
|
||||
async function runQaBuzz(options: LiveTransportQaCommandOptions) {
|
||||
await runLiveTransportQaSuiteCommand({
|
||||
channelId: "buzz",
|
||||
defaultProviderMode: "mock-openai",
|
||||
options,
|
||||
selectScenarioIds: ({ scenarioIds }) =>
|
||||
scenarioIds?.length ? [...scenarioIds] : [...DEFAULT_BUZZ_QA_SCENARIOS],
|
||||
});
|
||||
}
|
||||
|
||||
export const buzzQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
createLiveTransportQaCliRegistration({
|
||||
commandName: "buzz",
|
||||
credentialFileHelp: "JSON credential file for local Buzz QA",
|
||||
adapterFactory: {
|
||||
id: "buzz",
|
||||
matches: ({ channelId, driver }) => channelId === "buzz" && driver === "live",
|
||||
async create(context) {
|
||||
return await (await loadBuzzQaAdapterRuntime()).createBuzzQaTransportAdapter(context);
|
||||
},
|
||||
},
|
||||
credentialOptions: {
|
||||
sourceDescription: "Credential source for Buzz QA: file or convex (default: file)",
|
||||
roleDescription:
|
||||
"Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
|
||||
},
|
||||
defaultProviderMode: "mock-openai",
|
||||
description: "Run the Buzz live QA lane against a dedicated relay room",
|
||||
providerModeHelp: "Provider mode: mock-openai, aimock, or live-frontier",
|
||||
outputDirHelp: "Buzz QA artifact directory",
|
||||
allowFailuresHelp: "Write artifacts without setting a failing exit code when scenarios fail",
|
||||
scenarioHelp: "Run only the named Buzz QA scenario (repeatable)",
|
||||
sutAccountHelp: "Normalized Buzz SUT account id in QA artifacts",
|
||||
run: runQaBuzz,
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { parseBuzzQaCredentialPayload, readBuzzQaCredentialFile } from "./credentials.js";
|
||||
|
||||
const DRIVER_PRIVATE_KEY = "01".repeat(32);
|
||||
const SUT_PRIVATE_KEY = "02".repeat(32);
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempDirs.splice(0).map(async (dir) => await fs.rm(dir, { force: true, recursive: true })),
|
||||
);
|
||||
});
|
||||
|
||||
describe("Buzz QA credentials", () => {
|
||||
it("accepts a strict two-identity room payload", () => {
|
||||
const credentials = parseBuzzQaCredentialPayload({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
driverAuthTag: '["auth","driver","conditions","signature"]',
|
||||
sutAuthTag: '["auth","sut","conditions","signature"]',
|
||||
});
|
||||
|
||||
expect(credentials).toMatchObject({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
});
|
||||
expect(credentials.driverPublicKey).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(credentials.sutPublicKey).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(credentials.driverPublicKey).not.toBe(credentials.sutPublicKey);
|
||||
});
|
||||
|
||||
it.each(["ws://localhost:8080", "ws://127.0.0.1:8080", "ws://[::1]:8080"])(
|
||||
"allows plaintext loopback relay URL %s",
|
||||
(relayUrl) => {
|
||||
expect(
|
||||
parseBuzzQaCredentialPayload({
|
||||
relayUrl,
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
}).relayUrl,
|
||||
).toBe(relayUrl);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects plaintext remote relay URLs", () => {
|
||||
expect(() =>
|
||||
parseBuzzQaCredentialPayload({
|
||||
relayUrl: "ws://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
}),
|
||||
).toThrow("Buzz QA credentials are missing or malformed.");
|
||||
});
|
||||
|
||||
it("never includes credential values in validation errors", () => {
|
||||
const privateKey = "not-a-private-key-value";
|
||||
const authTag = "not-an-auth-tag-value";
|
||||
|
||||
expect(() =>
|
||||
parseBuzzQaCredentialPayload({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: privateKey,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
driverAuthTag: authTag,
|
||||
}),
|
||||
).toThrow("Buzz QA credentials are missing or malformed.");
|
||||
try {
|
||||
parseBuzzQaCredentialPayload({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: privateKey,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
driverAuthTag: authTag,
|
||||
});
|
||||
} catch (error) {
|
||||
expect(String(error)).not.toContain(privateKey);
|
||||
expect(String(error)).not.toContain(authTag);
|
||||
}
|
||||
});
|
||||
|
||||
it("reads a repo-relative private JSON credential file", async () => {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-buzz-qa-"));
|
||||
tempDirs.push(repoRoot);
|
||||
const filePath = path.join("private", "buzz.json");
|
||||
await fs.mkdir(path.join(repoRoot, "private"));
|
||||
await fs.writeFile(
|
||||
path.join(repoRoot, filePath),
|
||||
JSON.stringify({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
|
||||
await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).resolves.toMatchObject({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
driverPrivateKey: DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: SUT_PRIVATE_KEY,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not echo malformed file contents", async () => {
|
||||
const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-buzz-qa-"));
|
||||
tempDirs.push(repoRoot);
|
||||
const filePath = "buzz.json";
|
||||
const secret = "private-key-material-must-not-leak";
|
||||
await fs.writeFile(path.join(repoRoot, filePath), `{${secret}`, { mode: 0o600 });
|
||||
|
||||
await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).rejects.not.toThrow(secret);
|
||||
await expect(readBuzzQaCredentialFile({ filePath, repoRoot })).rejects.toThrow(
|
||||
"is not valid JSON",
|
||||
);
|
||||
try {
|
||||
await readBuzzQaCredentialFile({ filePath, repoRoot });
|
||||
} catch (error) {
|
||||
expect((error as Error & { cause?: unknown }).cause).toBeUndefined();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isLoopbackHost } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { z } from "zod";
|
||||
import { parseBuzzAuthTag } from "../relay-auth.js";
|
||||
import { parseBuzzTarget } from "../target.js";
|
||||
import { resolveBuzzPublicKey } from "../types.js";
|
||||
|
||||
function isSafeBuzzQaRelayUrl(value: string): boolean {
|
||||
try {
|
||||
const relayUrl = new URL(value);
|
||||
return (
|
||||
relayUrl.protocol === "wss:" ||
|
||||
(relayUrl.protocol === "ws:" && isLoopbackHost(relayUrl.hostname))
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const buzzQaCredentialPayloadSchema = z
|
||||
.object({
|
||||
// QA credentials may contain relay auth tags, so plaintext WebSockets must
|
||||
// stay on loopback and never cross a network boundary.
|
||||
relayUrl: z.string().url().refine(isSafeBuzzQaRelayUrl),
|
||||
roomId: z.string().min(1),
|
||||
driverPrivateKey: z.string().min(1),
|
||||
sutPrivateKey: z.string().min(1),
|
||||
driverAuthTag: z.string().optional(),
|
||||
sutAuthTag: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type BuzzQaCredentials = z.output<typeof buzzQaCredentialPayloadSchema> & {
|
||||
driverPublicKey: string;
|
||||
sutPublicKey: string;
|
||||
};
|
||||
|
||||
export function parseBuzzQaCredentialPayload(payload: unknown): BuzzQaCredentials {
|
||||
const parsed = buzzQaCredentialPayloadSchema.safeParse(payload);
|
||||
if (!parsed.success) {
|
||||
throw new Error("Buzz QA credentials are missing or malformed.");
|
||||
}
|
||||
let roomId: string;
|
||||
let driverPublicKey: string;
|
||||
let sutPublicKey: string;
|
||||
try {
|
||||
roomId = parseBuzzTarget(parsed.data.roomId);
|
||||
driverPublicKey = resolveBuzzPublicKey(parsed.data.driverPrivateKey);
|
||||
sutPublicKey = resolveBuzzPublicKey(parsed.data.sutPrivateKey);
|
||||
parseBuzzAuthTag(parsed.data.driverAuthTag ?? "");
|
||||
parseBuzzAuthTag(parsed.data.sutAuthTag ?? "");
|
||||
} catch {
|
||||
throw new Error("Buzz QA credentials are missing or malformed.");
|
||||
}
|
||||
if (driverPublicKey === sutPublicKey) {
|
||||
throw new Error("Buzz QA requires distinct driver and SUT identities.");
|
||||
}
|
||||
return {
|
||||
...parsed.data,
|
||||
roomId,
|
||||
driverPublicKey,
|
||||
sutPublicKey,
|
||||
};
|
||||
}
|
||||
|
||||
export async function readBuzzQaCredentialFile(params: {
|
||||
filePath: string;
|
||||
repoRoot?: string;
|
||||
}): Promise<BuzzQaCredentials> {
|
||||
const resolvedPath = path.resolve(params.repoRoot ?? process.cwd(), params.filePath);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(resolvedPath, "utf8");
|
||||
} catch (error) {
|
||||
throw new Error(`Unable to read Buzz QA credential file ${resolvedPath}.`, { cause: error });
|
||||
}
|
||||
let payload: unknown;
|
||||
try {
|
||||
payload = JSON.parse(raw);
|
||||
} catch {
|
||||
// JSON.parse errors may include source snippets, so never retain the
|
||||
// secret-bearing parser error as a cause.
|
||||
throw new Error(`Buzz QA credential file ${resolvedPath} is not valid JSON.`);
|
||||
}
|
||||
return parseBuzzQaCredentialPayload(payload);
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
import { getPublicKey, type Event, type Filter } from "nostr-tools";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseBuzzQaCredentialPayload } from "./credentials.js";
|
||||
|
||||
const relayMocks = vi.hoisted(() => ({
|
||||
auth: vi.fn(async () => "ok"),
|
||||
close: vi.fn(),
|
||||
connect: vi.fn(async () => {}),
|
||||
publish: vi.fn(async () => "ok"),
|
||||
replayedMessage: undefined as Event | undefined,
|
||||
subscriptions: [] as Array<{
|
||||
filter: Filter;
|
||||
handlers: {
|
||||
onevent?: (event: Event) => void;
|
||||
oneose?: () => void;
|
||||
onclose?: (reason: string) => void;
|
||||
};
|
||||
}>,
|
||||
}));
|
||||
|
||||
vi.mock("nostr-tools", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("nostr-tools")>();
|
||||
return {
|
||||
...actual,
|
||||
Relay: class {
|
||||
onauth?: (template: unknown) => Promise<unknown>;
|
||||
auth = relayMocks.auth;
|
||||
close = relayMocks.close;
|
||||
connect = relayMocks.connect;
|
||||
publish = relayMocks.publish;
|
||||
|
||||
subscribe(
|
||||
filters: Filter[],
|
||||
handlers: (typeof relayMocks.subscriptions)[number]["handlers"],
|
||||
) {
|
||||
const filter = filters[0] ?? {};
|
||||
relayMocks.subscriptions.push({ filter, handlers });
|
||||
if (filter.kinds?.includes(39002)) {
|
||||
handlers.onevent?.({
|
||||
id: "membership",
|
||||
kind: 39002,
|
||||
pubkey: "f".repeat(64),
|
||||
created_at: 1_750_000_000,
|
||||
content: "",
|
||||
sig: "e".repeat(128),
|
||||
tags: [
|
||||
["d", "123e4567-e89b-42d3-a456-426614174000"],
|
||||
[
|
||||
"p",
|
||||
getPublicKey(Uint8Array.from(Buffer.from("01".repeat(32), "hex"))),
|
||||
"",
|
||||
"member",
|
||||
],
|
||||
["p", getPublicKey(Uint8Array.from(Buffer.from("02".repeat(32), "hex"))), "", "bot"],
|
||||
],
|
||||
});
|
||||
handlers.oneose?.();
|
||||
} else {
|
||||
if (relayMocks.replayedMessage) {
|
||||
handlers.onevent?.(relayMocks.replayedMessage);
|
||||
}
|
||||
handlers.oneose?.();
|
||||
}
|
||||
return { close: vi.fn() };
|
||||
}
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { createBuzzQaRelayDriver } from "./relay-client.js";
|
||||
|
||||
const credentials = parseBuzzQaCredentialPayload({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: "01".repeat(32),
|
||||
sutPrivateKey: "02".repeat(32),
|
||||
});
|
||||
|
||||
describe("Buzz QA relay driver", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
relayMocks.subscriptions.length = 0;
|
||||
relayMocks.replayedMessage = undefined;
|
||||
});
|
||||
|
||||
it("authenticates, verifies membership, and publishes a native mentioned thread event", async () => {
|
||||
const driver = await createBuzzQaRelayDriver({
|
||||
credentials,
|
||||
onMessage: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const sent = await driver.sendMessage({
|
||||
text: "@openclaw hello",
|
||||
mentionSut: true,
|
||||
threadId: "root-event",
|
||||
replyToId: "parent-event",
|
||||
});
|
||||
|
||||
expect(sent.eventId).toMatch(/^[a-f0-9]{64}$/u);
|
||||
expect(relayMocks.publish).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
id: sent.eventId,
|
||||
kind: 9,
|
||||
content: "@openclaw hello",
|
||||
tags: [
|
||||
["h", credentials.roomId],
|
||||
["e", "root-event", "", "root"],
|
||||
["e", "parent-event", "", "reply"],
|
||||
["p", credentials.sutPublicKey],
|
||||
],
|
||||
}),
|
||||
);
|
||||
await driver.close();
|
||||
expect(relayMocks.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("ignores retained SUT messages before the observer reaches live events", async () => {
|
||||
const onMessage = vi.fn(async () => {});
|
||||
relayMocks.replayedMessage = {
|
||||
id: "retained-sut-message",
|
||||
kind: 9,
|
||||
pubkey: credentials.sutPublicKey,
|
||||
created_at: 1_750_000_000,
|
||||
content: "old response",
|
||||
sig: "e".repeat(128),
|
||||
tags: [["h", credentials.roomId]],
|
||||
};
|
||||
|
||||
const driver = await createBuzzQaRelayDriver({ credentials, onMessage });
|
||||
|
||||
expect(onMessage).not.toHaveBeenCalled();
|
||||
await driver.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,245 @@
|
||||
import { Relay, finalizeEvent, type Event } from "nostr-tools";
|
||||
import {
|
||||
buildBuzzMessageTags,
|
||||
parseBuzzMessageEvent,
|
||||
type BuzzInboundMessage,
|
||||
} from "../message-event.js";
|
||||
import { authenticateBuzzRelay, createBuzzAuthSigner, parseBuzzAuthTag } from "../relay-auth.js";
|
||||
import {
|
||||
BUZZ_ROOM_MEMBERSHIP_KIND,
|
||||
isNewerBuzzRoomMembership,
|
||||
parseBuzzRoomMembershipEvent,
|
||||
type BuzzRoomMembership,
|
||||
} from "../room-membership.js";
|
||||
import { decodeBuzzPrivateKey } from "../types.js";
|
||||
import type { BuzzQaCredentials } from "./credentials.js";
|
||||
|
||||
const BUZZ_MESSAGE_KIND = 9;
|
||||
const MEMBERSHIP_TIMEOUT_MS = 10_000;
|
||||
const OBSERVER_READY_TIMEOUT_MS = 10_000;
|
||||
|
||||
type BuzzQaRelayDriver = {
|
||||
assertHealthy(): void;
|
||||
close(): Promise<void>;
|
||||
sendMessage(input: {
|
||||
text: string;
|
||||
mentionSut: boolean;
|
||||
threadId?: string;
|
||||
replyToId?: string;
|
||||
}): Promise<{ eventId: string; timestamp: number }>;
|
||||
};
|
||||
|
||||
async function loadBuzzQaRoomMembership(params: {
|
||||
relay: Relay;
|
||||
roomId: string;
|
||||
}): Promise<BuzzRoomMembership> {
|
||||
return await new Promise<BuzzRoomMembership>((resolve, reject) => {
|
||||
let latest: BuzzRoomMembership | undefined;
|
||||
let settled = false;
|
||||
const subscriptionRef: { current?: ReturnType<Relay["subscribe"]> } = {};
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
subscriptionRef.current?.close("membership loaded");
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else if (latest) {
|
||||
resolve(latest);
|
||||
} else {
|
||||
reject(new Error(`Buzz QA room ${params.roomId} has no membership roster.`));
|
||||
}
|
||||
};
|
||||
const timeout = setTimeout(
|
||||
() => finish(new Error(`Timed out loading Buzz QA room ${params.roomId} membership.`)),
|
||||
MEMBERSHIP_TIMEOUT_MS,
|
||||
);
|
||||
subscriptionRef.current = params.relay.subscribe(
|
||||
[{ kinds: [BUZZ_ROOM_MEMBERSHIP_KIND], "#d": [params.roomId], limit: 1 }],
|
||||
{
|
||||
onevent: (event) => {
|
||||
const membership = parseBuzzRoomMembershipEvent(event);
|
||||
if (
|
||||
membership?.roomId === params.roomId &&
|
||||
isNewerBuzzRoomMembership(membership, latest)
|
||||
) {
|
||||
latest = membership;
|
||||
}
|
||||
},
|
||||
oneose: () => finish(),
|
||||
onclose: (reason) => {
|
||||
if (reason !== "membership loaded") {
|
||||
finish(new Error(`Buzz QA membership subscription closed: ${reason}`));
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
if (settled) {
|
||||
subscriptionRef.current.close("membership loaded");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function assertBuzzQaMembership(membership: BuzzRoomMembership, credentials: BuzzQaCredentials) {
|
||||
if (!membership.members.has(credentials.driverPublicKey)) {
|
||||
throw new Error(
|
||||
`Buzz QA driver ${credentials.driverPublicKey} is not a member of room ${credentials.roomId}.`,
|
||||
);
|
||||
}
|
||||
if (
|
||||
!membership.members.has(credentials.sutPublicKey) ||
|
||||
membership.roles.get(credentials.sutPublicKey) !== "bot"
|
||||
) {
|
||||
throw new Error(
|
||||
`Buzz QA SUT ${credentials.sutPublicKey} must have the Bot role in room ${credentials.roomId}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function createBuzzQaRelayDriver(params: {
|
||||
credentials: BuzzQaCredentials;
|
||||
onMessage: (message: BuzzInboundMessage) => Promise<void>;
|
||||
}): Promise<BuzzQaRelayDriver> {
|
||||
const credentials = params.credentials;
|
||||
const secretKey = decodeBuzzPrivateKey(credentials.driverPrivateKey);
|
||||
const relay = new Relay(credentials.relayUrl, { enableReconnect: false });
|
||||
const lifecycleAbort = new AbortController();
|
||||
const signAuth = createBuzzAuthSigner({
|
||||
secretKey,
|
||||
authTag: parseBuzzAuthTag(credentials.driverAuthTag ?? ""),
|
||||
});
|
||||
let transportError: Error | undefined;
|
||||
let messageQueue = Promise.resolve();
|
||||
const observedEventIds = new Set<string>();
|
||||
try {
|
||||
await relay.connect({ abort: lifecycleAbort.signal });
|
||||
await authenticateBuzzRelay({ relay, signAuth, signal: lifecycleAbort.signal });
|
||||
relay.onauth = signAuth;
|
||||
assertBuzzQaMembership(
|
||||
await loadBuzzQaRoomMembership({ relay, roomId: credentials.roomId }),
|
||||
credentials,
|
||||
);
|
||||
} catch (error) {
|
||||
lifecycleAbort.abort(error);
|
||||
relay.close();
|
||||
throw error;
|
||||
}
|
||||
|
||||
let observerReady = false;
|
||||
let resolveObserverReady: (() => void) | undefined;
|
||||
let rejectObserverReady: ((error: Error) => void) | undefined;
|
||||
const observerReadyPromise = new Promise<void>((resolve, reject) => {
|
||||
resolveObserverReady = resolve;
|
||||
rejectObserverReady = reject;
|
||||
});
|
||||
const observerReadyTimeout = setTimeout(() => {
|
||||
rejectObserverReady?.(new Error("Timed out waiting for the Buzz QA message observer."));
|
||||
}, OBSERVER_READY_TIMEOUT_MS);
|
||||
let subscription: ReturnType<Relay["subscribe"]>;
|
||||
try {
|
||||
subscription = relay.subscribe(
|
||||
[
|
||||
{
|
||||
kinds: [BUZZ_MESSAGE_KIND],
|
||||
authors: [credentials.sutPublicKey],
|
||||
"#h": [credentials.roomId],
|
||||
since: Math.floor(Date.now() / 1_000) - 5,
|
||||
},
|
||||
],
|
||||
{
|
||||
onevent: (event: Event) => {
|
||||
if (!observerReady) {
|
||||
return;
|
||||
}
|
||||
if (observedEventIds.has(event.id)) {
|
||||
return;
|
||||
}
|
||||
observedEventIds.add(event.id);
|
||||
const message = parseBuzzMessageEvent(event);
|
||||
if (
|
||||
!message ||
|
||||
message.channelId !== credentials.roomId ||
|
||||
message.senderPubkey !== credentials.sutPublicKey
|
||||
) {
|
||||
return;
|
||||
}
|
||||
messageQueue = messageQueue
|
||||
.then(async () => await params.onMessage(message))
|
||||
.catch((error: unknown) => {
|
||||
transportError = error instanceof Error ? error : new Error(String(error));
|
||||
});
|
||||
},
|
||||
oneose: () => {
|
||||
observerReady = true;
|
||||
clearTimeout(observerReadyTimeout);
|
||||
resolveObserverReady?.();
|
||||
},
|
||||
onclose: (reason) => {
|
||||
if (!observerReady) {
|
||||
clearTimeout(observerReadyTimeout);
|
||||
rejectObserverReady?.(
|
||||
new Error(`Buzz QA message observer closed before it was ready: ${reason}`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (reason !== "shutdown" && reason !== "relay connection closed by us") {
|
||||
transportError = new Error(`Buzz QA message subscription closed: ${reason}`);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
clearTimeout(observerReadyTimeout);
|
||||
lifecycleAbort.abort(error);
|
||||
relay.close();
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
await observerReadyPromise;
|
||||
} catch (error) {
|
||||
lifecycleAbort.abort(error);
|
||||
subscription.close("shutdown");
|
||||
relay.close();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
assertHealthy() {
|
||||
if (transportError) {
|
||||
throw transportError;
|
||||
}
|
||||
},
|
||||
async sendMessage(input) {
|
||||
if (transportError) {
|
||||
throw transportError;
|
||||
}
|
||||
const tags = buildBuzzMessageTags({
|
||||
channelId: credentials.roomId,
|
||||
threadId: input.threadId,
|
||||
replyToId: input.replyToId,
|
||||
});
|
||||
if (input.mentionSut) {
|
||||
tags.push(["p", credentials.sutPublicKey]);
|
||||
}
|
||||
const event = finalizeEvent(
|
||||
{
|
||||
kind: BUZZ_MESSAGE_KIND,
|
||||
content: input.text,
|
||||
created_at: Math.floor(Date.now() / 1_000),
|
||||
tags,
|
||||
},
|
||||
secretKey,
|
||||
);
|
||||
await relay.publish(event);
|
||||
return { eventId: event.id, timestamp: event.created_at * 1_000 };
|
||||
},
|
||||
async close() {
|
||||
lifecycleAbort.abort(new Error("Buzz QA relay driver closed"));
|
||||
subscription.close("shutdown");
|
||||
relay.close();
|
||||
await messageQueue;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -41,6 +41,11 @@ export {
|
||||
setQaChannelRuntime,
|
||||
} from "./src/runtime-api.js";
|
||||
export { startQaLiveLaneGateway } from "./src/live-transports/shared/live-gateway.runtime.js";
|
||||
export { runLiveTransportQaSuiteCommand } from "./src/live-transports/shared/live-transport-suite.runtime.js";
|
||||
export {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
} from "./src/live-transports/shared/credential-lease.runtime.js";
|
||||
export {
|
||||
createQaChannelDriverLifecycle,
|
||||
runQaChannelDriverLifecycleScenarios,
|
||||
|
||||
@@ -160,6 +160,7 @@ export type QaSuiteCommandOptions = QaScenarioRunCommandOptions & {
|
||||
runtimePair?: string;
|
||||
runtimePairLane?: string[];
|
||||
sutAccountId?: string;
|
||||
credentialFile?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
explicitScenarioSelection?: boolean;
|
||||
@@ -1019,6 +1020,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
|
||||
adapterOptions: {
|
||||
repoRoot,
|
||||
sutAccountId: opts.sutAccountId,
|
||||
credentialFile: opts.credentialFile,
|
||||
credentialSource: opts.credentialSource,
|
||||
credentialRole: opts.credentialRole,
|
||||
explicitScenarioSelection:
|
||||
|
||||
@@ -26,6 +26,40 @@ describe("gateway log redaction", () => {
|
||||
expect(formatQaGatewayLogsForError(raw)).not.toContain(token);
|
||||
});
|
||||
|
||||
it("redacts Buzz QA private keys and authorization tags", () => {
|
||||
const privateKey = "01".repeat(32);
|
||||
const authTag = '["auth","pubkey","conditions","signature"]';
|
||||
const raw = [
|
||||
`privateKey: ${privateKey}`,
|
||||
`authTag: ${authTag}`,
|
||||
`{"privateKey":"${privateKey}"}`,
|
||||
JSON.stringify({ authTag }),
|
||||
`channels.buzz.authTag: ${authTag}`,
|
||||
[
|
||||
"channels.buzz.authTag: [",
|
||||
' "auth",',
|
||||
' "pubkey",',
|
||||
' "conditions",',
|
||||
' "signature"',
|
||||
"]",
|
||||
].join("\n"),
|
||||
[
|
||||
"channels.buzz.authTag: [",
|
||||
" 'auth',",
|
||||
" 'pubkey',",
|
||||
" 'conditions',",
|
||||
" 'escaped\\'signature'",
|
||||
"]",
|
||||
].join("\n"),
|
||||
].join("\n");
|
||||
|
||||
const redacted = redactQaGatewayDebugText(raw);
|
||||
expect(redacted).not.toContain(privateKey);
|
||||
expect(redacted).not.toContain(authTag);
|
||||
expect(redacted).not.toContain("signature");
|
||||
expect(redacted.match(/<redacted>/gu)).toHaveLength(7);
|
||||
});
|
||||
|
||||
it("neutralizes GitHub workflow commands at every line boundary", () => {
|
||||
const raw = [
|
||||
"::set-output name=output_dir::/tmp/attacker",
|
||||
|
||||
@@ -20,6 +20,8 @@ const QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS = Object.freeze([
|
||||
"client_secret",
|
||||
"cookie",
|
||||
"driverToken",
|
||||
"privateKey",
|
||||
"authTag",
|
||||
"sutToken",
|
||||
"leaseToken",
|
||||
"refreshToken",
|
||||
@@ -58,21 +60,41 @@ function redactSecretEnvKeyPattern(text: string, pattern: RegExp) {
|
||||
|
||||
function redactSecretValueKey(text: string, key: string) {
|
||||
const escapedKey = escapeRegExp(key);
|
||||
const valuePattern = `[^\\s"';,]+|"(?:\\\\.|[^"\\\\])*"|'[^']*'`;
|
||||
return text
|
||||
.replace(new RegExp(`([?#&]${escapedKey}=)[^&\\s]+`, "gi"), "$1<redacted>")
|
||||
.replace(
|
||||
new RegExp(`(^|\\s)(--${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"),
|
||||
new RegExp(`(^|\\s)(--${escapedKey})(\\s*[=:]\\s*)(${valuePattern})`, "gi"),
|
||||
`$1$2$3<redacted>`,
|
||||
)
|
||||
.replace(
|
||||
new RegExp(`(^|[^\\w?#&-])(${escapedKey})(\\s*[=:]\\s*)([^\\s"';,]+|"[^"]*"|'[^']*')`, "gi"),
|
||||
new RegExp(`(^|[^\\w?#&-])(${escapedKey})(\\s*[=:]\\s*)(${valuePattern})`, "gi"),
|
||||
`$1$2$3<redacted>`,
|
||||
)
|
||||
.replace(new RegExp(`("${escapedKey}"\\s*:\\s*)"[^"]*"`, "gi"), `$1"<redacted>"`);
|
||||
.replace(new RegExp(`("${escapedKey}"\\s*:\\s*)"(?:\\\\.|[^"\\\\])*"`, "gi"), `$1"<redacted>"`);
|
||||
}
|
||||
|
||||
function redactStructuredSecretLine(text: string, key: string) {
|
||||
const escapedKey = escapeRegExp(key);
|
||||
return text.replace(
|
||||
new RegExp(`(^|[\\r\\n])(\\s*"?${escapedKey}"?\\s*[:=]\\s*).*$`, "gim"),
|
||||
`$1$2<redacted>`,
|
||||
);
|
||||
}
|
||||
|
||||
function redactStructuredSecretArray(text: string, key: string) {
|
||||
const escapedKey = escapeRegExp(key);
|
||||
const quotedString = `"(?:\\\\.|[^"\\\\])*"|'(?:\\\\.|[^'\\\\])*'`;
|
||||
const quotedStringArray = `\\[\\s*(?:${quotedString})(?:\\s*,\\s*(?:${quotedString}))*\\s*\\]`;
|
||||
return text.replace(
|
||||
new RegExp(`("?${escapedKey}"?\\s*[:=]\\s*)${quotedStringArray}`, "gi"),
|
||||
`$1<redacted>`,
|
||||
);
|
||||
}
|
||||
|
||||
export function redactQaGatewayDebugText(text: string) {
|
||||
let redacted = redactSensitiveText(redactTelegramBotTokens(text), { mode: "tools" });
|
||||
redacted = redactStructuredSecretArray(redacted, "authTag");
|
||||
for (const key of QA_GATEWAY_DEBUG_SECRET_HEADER_KEYS) {
|
||||
const escapedKey = escapeRegExp(key);
|
||||
redacted = redacted.replace(
|
||||
@@ -97,6 +119,9 @@ export function redactQaGatewayDebugText(text: string) {
|
||||
for (const key of QA_GATEWAY_DEBUG_SECRET_VALUE_KEYS) {
|
||||
redacted = redactSecretValueKey(redacted, key);
|
||||
}
|
||||
for (const key of ["authTag", "privateKey"]) {
|
||||
redacted = redactStructuredSecretLine(redacted, key);
|
||||
}
|
||||
return redacted
|
||||
.replaceAll(/\bsk-ant-oat01-[A-Za-z0-9_-]+\b/g, "<redacted>")
|
||||
.replaceAll(/\bBearer\s+[^\s"'<>]{8,}/gi, "Bearer <redacted>")
|
||||
|
||||
@@ -91,6 +91,10 @@ describe("live transport adapter factories", () => {
|
||||
expect.objectContaining({
|
||||
adapterOptions,
|
||||
channelId,
|
||||
credentials: {
|
||||
acquire: expect.any(Function),
|
||||
startHeartbeat: expect.any(Function),
|
||||
},
|
||||
driver: "live",
|
||||
messages: expect.objectContaining({
|
||||
addInboundMessage: expect.any(Function),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const runQaSuiteCommand = vi.hoisted(() => vi.fn());
|
||||
|
||||
@@ -8,9 +8,14 @@ import { runLiveTransportQaSuiteCommand } from "./live-transport-suite.runtime.j
|
||||
|
||||
describe("live transport suite runtime", () => {
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("OPENCLAW_QA_CREDENTIAL_SOURCE", "");
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("normalizes one live command into the shared suite host", async () => {
|
||||
await runLiveTransportQaSuiteCommand({
|
||||
channelId: "slack",
|
||||
@@ -23,6 +28,7 @@ describe("live transport suite runtime", () => {
|
||||
fastMode: true,
|
||||
allowFailures: true,
|
||||
failFast: true,
|
||||
credentialFile: "/secure/slack-qa.json",
|
||||
credentialSource: " convex ",
|
||||
credentialRole: " ci ",
|
||||
sutAccountId: "slack-sut",
|
||||
@@ -49,6 +55,7 @@ describe("live transport suite runtime", () => {
|
||||
concurrency: 1,
|
||||
scenarioIds: ["slack-canary"],
|
||||
sutAccountId: "slack-sut",
|
||||
credentialFile: "/secure/slack-qa.json",
|
||||
credentialSource: "convex",
|
||||
credentialRole: "ci",
|
||||
explicitScenarioSelection: false,
|
||||
@@ -71,6 +78,21 @@ describe("live transport suite runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes the shared credential source environment override", async () => {
|
||||
vi.stubEnv("OPENCLAW_QA_CREDENTIAL_SOURCE", " convex ");
|
||||
|
||||
await runLiveTransportQaSuiteCommand({
|
||||
channelId: "buzz",
|
||||
defaultProviderMode: "mock-openai",
|
||||
options: {},
|
||||
selectScenarioIds: () => ["channel-canary"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteCommand).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ credentialSource: "convex" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects shared credentials for disposable transports", async () => {
|
||||
await expect(
|
||||
runLiveTransportQaSuiteCommand({
|
||||
|
||||
@@ -20,10 +20,11 @@ export async function runLiveTransportQaSuiteCommand(params: {
|
||||
selectScenarioIds: LiveTransportScenarioSelection;
|
||||
}) {
|
||||
const options = params.options;
|
||||
const credentialSource =
|
||||
options.credentialSource?.trim() || process.env.OPENCLAW_QA_CREDENTIAL_SOURCE?.trim();
|
||||
if (params.credentialMode === "env-only") {
|
||||
const laneLabel = params.laneLabel ?? params.channelId;
|
||||
const credentialSource = options.credentialSource?.trim().toLowerCase();
|
||||
if (credentialSource && credentialSource !== "env") {
|
||||
if (credentialSource && credentialSource.toLowerCase() !== "env") {
|
||||
throw new Error(
|
||||
`QA Lab ${laneLabel} supports only --credential-source env${params.envCredentialReason ? ` because ${params.envCredentialReason}` : "."}`,
|
||||
);
|
||||
@@ -58,10 +59,11 @@ export async function runLiveTransportQaSuiteCommand(params: {
|
||||
concurrency: 1,
|
||||
scenarioIds: selectedScenarioIds,
|
||||
sutAccountId: options.sutAccountId,
|
||||
...(options.credentialFile ? { credentialFile: options.credentialFile } : {}),
|
||||
...(params.credentialMode === "env-only"
|
||||
? {}
|
||||
: {
|
||||
credentialSource: options.credentialSource?.trim(),
|
||||
credentialSource,
|
||||
credentialRole: options.credentialRole?.trim(),
|
||||
}),
|
||||
explicitScenarioSelection: Boolean(options.scenarioIds?.length),
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
// Qa Lab plugin module implements qa transport registry behavior.
|
||||
import type { QaBusState } from "./bus-state.js";
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
} from "./live-transports/shared/credential-lease.runtime.js";
|
||||
import {
|
||||
createQaChannelTransport,
|
||||
QA_CHANNEL_DEFAULT_SUITE_CONCURRENCY,
|
||||
@@ -102,6 +106,10 @@ function createQaTransportAdapterFactoryRegistry(
|
||||
const definition = await factory.create({
|
||||
adapterOptions: context.adapterOptions,
|
||||
channelId: context.channelId,
|
||||
credentials: {
|
||||
acquire: acquireQaCredentialLease,
|
||||
startHeartbeat: startQaCredentialLeaseHeartbeat,
|
||||
},
|
||||
driver: context.driver,
|
||||
messages: {
|
||||
addInboundMessage: (input) => context.state.addInboundMessage(input),
|
||||
|
||||
@@ -59,13 +59,17 @@ Maintainers can manage rows without using the Convex dashboard:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa credentials add \
|
||||
--kind telegram \
|
||||
--payload-file qa/telegram-credential.json
|
||||
--kind buzz \
|
||||
--payload-file qa/buzz-credential.json
|
||||
|
||||
pnpm openclaw qa credentials add \
|
||||
--kind discord \
|
||||
--payload-file qa/discord-credential.json
|
||||
|
||||
pnpm openclaw qa credentials add \
|
||||
--kind telegram \
|
||||
--payload-file qa/telegram-credential.json
|
||||
|
||||
pnpm openclaw qa credentials list --kind telegram
|
||||
|
||||
pnpm openclaw qa credentials remove --credential-id <credential-id>
|
||||
@@ -146,6 +150,18 @@ For `kind: "telegram"`, broker `admin/add` validates that payload includes:
|
||||
- non-empty `driverToken`
|
||||
- non-empty `sutToken`
|
||||
|
||||
For `kind: "buzz"`, broker `admin/add` validates that payload includes:
|
||||
|
||||
- `relayUrl` as a `wss://` URL, or `ws://` only for a loopback relay
|
||||
- `roomId` as a channel UUID
|
||||
- valid, distinct `driverPrivateKey` and `sutPrivateKey` values in nsec or
|
||||
64-character hex form
|
||||
- optional `driverAuthTag` and `sutAuthTag` values matching the four-string
|
||||
Buzz authorization tag JSON shape
|
||||
|
||||
Use dedicated QA identities only. Never add a human owner or admin private key
|
||||
to the shared pool.
|
||||
|
||||
For `kind: "telegram-user"`, broker `admin/add` validates one exclusive real-user
|
||||
credential for both the TDLib CLI driver and the Telegram Desktop visual witness:
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
// Payload Validation module supports OpenClaw QA credential workflows.
|
||||
import { getPublicKey, nip19 } from "nostr-tools";
|
||||
|
||||
class CredentialPayloadValidationError extends Error {
|
||||
code: string;
|
||||
httpStatus: number;
|
||||
@@ -15,6 +17,9 @@ type PayloadValidationFailureFactory = (httpStatus: number, code: string, messag
|
||||
|
||||
const DISCORD_SNOWFLAKE_RE = /^\d{17,20}$/u;
|
||||
const E164_RE = /^\+[1-9]\d{6,14}$/u;
|
||||
const BUZZ_ROOM_ID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
||||
const BUZZ_PRIVATE_KEY_HEX_RE = /^[0-9a-f]{64}$/iu;
|
||||
const SHA256_HEX_RE = /^[a-f0-9]{64}$/u;
|
||||
const TELEGRAM_CHAT_ID_RE = /^-?\d+$/u;
|
||||
const TELEGRAM_USER_ID_RE = /^\d+$/u;
|
||||
@@ -65,6 +70,132 @@ function requireDiscordSnowflakePayloadString(
|
||||
return value;
|
||||
}
|
||||
|
||||
function decodeBuzzPrivateKey(value: string) {
|
||||
if (BUZZ_PRIVATE_KEY_HEX_RE.test(value)) {
|
||||
const bytes = value.match(/.{2}/gu);
|
||||
if (bytes?.length === 32) {
|
||||
return Uint8Array.from(bytes.map((byte) => Number.parseInt(byte, 16)));
|
||||
}
|
||||
}
|
||||
const decoded = nip19.decode(value);
|
||||
if (decoded.type !== "nsec") {
|
||||
throw new Error("not a Buzz private key");
|
||||
}
|
||||
return decoded.data;
|
||||
}
|
||||
|
||||
function requireBuzzPrivateKey(
|
||||
payload: Record<string, unknown>,
|
||||
key: "driverPrivateKey" | "sutPrivateKey",
|
||||
createFailure: PayloadValidationFailureFactory,
|
||||
) {
|
||||
const value = requirePayloadString(payload, key, "buzz", createFailure);
|
||||
try {
|
||||
return { value, publicKey: getPublicKey(decodeBuzzPrivateKey(value)) };
|
||||
} catch {
|
||||
throwPayloadError(
|
||||
createFailure,
|
||||
`Credential payload for kind "buzz" must include "${key}" as an nsec or 64-character hex private key.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function requireBuzzAuthTag(
|
||||
payload: Record<string, unknown>,
|
||||
key: "driverAuthTag" | "sutAuthTag",
|
||||
createFailure: PayloadValidationFailureFactory,
|
||||
) {
|
||||
const value = requirePayloadString(payload, key, "buzz", createFailure);
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(value);
|
||||
} catch {
|
||||
parsed = undefined;
|
||||
}
|
||||
if (
|
||||
!Array.isArray(parsed) ||
|
||||
parsed.length !== 4 ||
|
||||
parsed[0] !== "auth" ||
|
||||
parsed.some((entry) => typeof entry !== "string")
|
||||
) {
|
||||
throwPayloadError(
|
||||
createFailure,
|
||||
`Credential payload for kind "buzz" must include "${key}" as an auth tag JSON array.`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function normalizeBuzzCredentialPayload(
|
||||
payload: Record<string, unknown>,
|
||||
createFailure: PayloadValidationFailureFactory,
|
||||
) {
|
||||
const kind = "buzz";
|
||||
const relayUrl = requirePayloadString(payload, "relayUrl", kind, createFailure);
|
||||
let parsedRelayUrl: URL | undefined;
|
||||
try {
|
||||
parsedRelayUrl = new URL(relayUrl);
|
||||
} catch {
|
||||
parsedRelayUrl = undefined;
|
||||
}
|
||||
const relayProtocol = parsedRelayUrl?.protocol;
|
||||
const relayUsesSafeTransport =
|
||||
relayProtocol === "wss:" ||
|
||||
(relayProtocol === "ws:" && isBuzzLoopbackHostname(parsedRelayUrl?.hostname ?? ""));
|
||||
if (!relayUsesSafeTransport) {
|
||||
throwPayloadError(
|
||||
createFailure,
|
||||
'Credential payload for kind "buzz" must include "relayUrl" using wss:// (ws:// is allowed only for loopback).',
|
||||
);
|
||||
}
|
||||
const roomId = requirePayloadString(payload, "roomId", kind, createFailure).toLowerCase();
|
||||
if (!BUZZ_ROOM_ID_RE.test(roomId)) {
|
||||
throwPayloadError(
|
||||
createFailure,
|
||||
'Credential payload for kind "buzz" must include "roomId" as a channel UUID.',
|
||||
);
|
||||
}
|
||||
const driverIdentity = requireBuzzPrivateKey(payload, "driverPrivateKey", createFailure);
|
||||
const sutIdentity = requireBuzzPrivateKey(payload, "sutPrivateKey", createFailure);
|
||||
if (driverIdentity.publicKey === sutIdentity.publicKey) {
|
||||
throwPayloadError(
|
||||
createFailure,
|
||||
'Credential payload for kind "buzz" must use distinct driver and SUT identities.',
|
||||
);
|
||||
}
|
||||
const optionalString = (key: "driverAuthTag" | "sutAuthTag") => {
|
||||
if (payload[key] === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
return requireBuzzAuthTag(payload, key, createFailure);
|
||||
};
|
||||
const driverAuthTag = optionalString("driverAuthTag");
|
||||
const sutAuthTag = optionalString("sutAuthTag");
|
||||
|
||||
return {
|
||||
relayUrl,
|
||||
roomId,
|
||||
driverPrivateKey: driverIdentity.value,
|
||||
sutPrivateKey: sutIdentity.value,
|
||||
...(driverAuthTag ? { driverAuthTag } : {}),
|
||||
...(sutAuthTag ? { sutAuthTag } : {}),
|
||||
} satisfies Record<string, unknown>;
|
||||
}
|
||||
|
||||
function isBuzzLoopbackHostname(hostname: string): boolean {
|
||||
const normalized = hostname.toLowerCase().replace(/^\[|\]$/gu, "");
|
||||
if (normalized === "localhost" || normalized === "::1") {
|
||||
return true;
|
||||
}
|
||||
const ipv4 = normalized.startsWith("::ffff:") ? normalized.slice("::ffff:".length) : normalized;
|
||||
const octets = ipv4.split(".");
|
||||
return (
|
||||
octets.length === 4 &&
|
||||
octets[0] === "127" &&
|
||||
octets.every((octet) => /^\d{1,3}$/u.test(octet) && Number(octet) <= 255)
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeTelegramCredentialPayload(
|
||||
payload: Record<string, unknown>,
|
||||
createFailure: PayloadValidationFailureFactory,
|
||||
@@ -263,6 +394,7 @@ const credentialPayloadNormalizers: Record<
|
||||
createFailure: PayloadValidationFailureFactory,
|
||||
) => Record<string, unknown>
|
||||
> = {
|
||||
buzz: normalizeBuzzCredentialPayload,
|
||||
discord: normalizeDiscordCredentialPayload,
|
||||
telegram: normalizeTelegramCredentialPayload,
|
||||
"telegram-user": normalizeTelegramUserCredentialPayload,
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
"dev": "convex dev"
|
||||
},
|
||||
"dependencies": {
|
||||
"convex": "1.42.3"
|
||||
"convex": "1.42.3",
|
||||
"nostr-tools": "2.24.1"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ type QaRunnerAdapterOptions = {
|
||||
repoRoot?: string;
|
||||
scenarioIds?: readonly string[];
|
||||
sutAccountId?: string;
|
||||
credentialFile?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
transportPolicy?: QaRunnerTransportPolicy;
|
||||
@@ -40,6 +41,46 @@ type QaRunnerMessageRecorder = {
|
||||
editMessage: (input: QaBusEditMessageInput) => QaBusMessage | Promise<QaBusMessage>;
|
||||
};
|
||||
|
||||
type QaRunnerCredentialLease<TPayload> = {
|
||||
credentialId?: string;
|
||||
heartbeat(): Promise<void>;
|
||||
heartbeatIntervalMs: number;
|
||||
kind: string;
|
||||
leaseToken?: string;
|
||||
leaseTtlMs: number;
|
||||
ownerId?: string;
|
||||
payload: TPayload;
|
||||
release(): Promise<void>;
|
||||
role?: "ci" | "maintainer";
|
||||
source: "convex" | "env";
|
||||
};
|
||||
|
||||
type QaRunnerCredentialLeaseOptions<TPayload> = {
|
||||
kind: string;
|
||||
parsePayload: (payload: unknown) => TPayload;
|
||||
resolveEnvPayload: () => TPayload;
|
||||
role?: string;
|
||||
source?: string;
|
||||
};
|
||||
|
||||
type QaRunnerCredentialHeartbeat = {
|
||||
getFailure(): Error | null;
|
||||
stop(): Promise<void>;
|
||||
throwIfFailed(): void;
|
||||
};
|
||||
|
||||
type QaRunnerCredentialHost = {
|
||||
acquire<TPayload>(
|
||||
options: QaRunnerCredentialLeaseOptions<TPayload>,
|
||||
): Promise<QaRunnerCredentialLease<TPayload>>;
|
||||
startHeartbeat(
|
||||
lease: Pick<
|
||||
QaRunnerCredentialLease<unknown>,
|
||||
"heartbeat" | "heartbeatIntervalMs" | "kind" | "source"
|
||||
>,
|
||||
): QaRunnerCredentialHeartbeat;
|
||||
};
|
||||
|
||||
type QaRunnerTransportFlowPreparationInput = {
|
||||
config: Record<string, unknown>;
|
||||
scenarioId: string;
|
||||
@@ -147,6 +188,7 @@ type QaRunnerTransportFactory = {
|
||||
create: (context: {
|
||||
adapterOptions?: QaRunnerAdapterOptions;
|
||||
channelId: string;
|
||||
credentials: QaRunnerCredentialHost;
|
||||
driver: string;
|
||||
messages: QaRunnerMessageRecorder;
|
||||
outputDir: string;
|
||||
|
||||
@@ -118,6 +118,30 @@ describe("plugin-sdk qa-runtime", () => {
|
||||
expect(module.isQaRuntimeAvailable()).toBe(false);
|
||||
});
|
||||
|
||||
it("runs a plugin-owned transport through the private QA suite host", async () => {
|
||||
const runLiveTransportQaSuiteCommand = vi.fn(async () => {});
|
||||
loadBundledPluginPublicSurfaceModuleSync.mockReturnValue({
|
||||
runLiveTransportQaSuiteCommand,
|
||||
});
|
||||
const module = await import("./qa-runtime.js");
|
||||
const options = { providerMode: "mock-openai" };
|
||||
const selectScenarioIds = vi.fn(() => ["channel-canary"]);
|
||||
|
||||
await module.runLiveTransportQaSuiteCommand({
|
||||
channelId: "buzz",
|
||||
defaultProviderMode: "mock-openai",
|
||||
options,
|
||||
selectScenarioIds,
|
||||
});
|
||||
|
||||
expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith({
|
||||
channelId: "buzz",
|
||||
defaultProviderMode: "mock-openai",
|
||||
options,
|
||||
selectScenarioIds,
|
||||
});
|
||||
});
|
||||
|
||||
it("registers shared live transport QA CLI options", async () => {
|
||||
const module = await import("./qa-runtime.js");
|
||||
const run = vi.fn(async () => {});
|
||||
@@ -126,6 +150,7 @@ describe("plugin-sdk qa-runtime", () => {
|
||||
module
|
||||
.createLiveTransportQaCliRegistration({
|
||||
commandName: "telegram",
|
||||
credentialFileHelp: "Private JSON credential file",
|
||||
credentialOptions: {
|
||||
sourceDescription: "Credential source for Telegram QA",
|
||||
roleDescription: "Credential role for Telegram QA",
|
||||
@@ -172,6 +197,8 @@ describe("plugin-sdk qa-runtime", () => {
|
||||
"--fail-fast",
|
||||
"--sut-account",
|
||||
"sut-2",
|
||||
"--credential-file",
|
||||
"/secure/telegram-qa.json",
|
||||
"--credential-source",
|
||||
"convex",
|
||||
"--credential-role",
|
||||
@@ -191,6 +218,7 @@ describe("plugin-sdk qa-runtime", () => {
|
||||
scenarioIds: ["alpha", "beta"],
|
||||
listScenarios: true,
|
||||
sutAccountId: "sut-2",
|
||||
credentialFile: "/secure/telegram-qa.json",
|
||||
credentialSource: "convex",
|
||||
credentialRole: "maintainer",
|
||||
});
|
||||
|
||||
@@ -24,6 +24,7 @@ type QaRuntimeSurface = {
|
||||
},
|
||||
) => string;
|
||||
startQaLiveLaneGateway: (...args: unknown[]) => Promise<unknown>;
|
||||
runLiveTransportQaSuiteCommand: (params: LiveTransportQaSuiteCommandOptions) => Promise<unknown>;
|
||||
};
|
||||
|
||||
function isMissingQaRuntimeError(error: unknown) {
|
||||
@@ -71,10 +72,31 @@ export type LiveTransportQaCommandOptions = {
|
||||
scenarioIds?: string[];
|
||||
listScenarios?: boolean;
|
||||
sutAccountId?: string;
|
||||
credentialFile?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
};
|
||||
|
||||
export type LiveTransportQaSuiteCommandOptions = {
|
||||
channelId: string;
|
||||
credentialMode?: "env-only" | "shared-lease";
|
||||
defaultProviderMode: string;
|
||||
envCredentialReason?: string;
|
||||
laneLabel?: string;
|
||||
options: LiveTransportQaCommandOptions;
|
||||
selectScenarioIds: (params: {
|
||||
profile?: string;
|
||||
primaryModel: string;
|
||||
providerMode: string;
|
||||
scenarioIds?: readonly string[];
|
||||
}) => string[];
|
||||
};
|
||||
|
||||
/** Run a plugin-owned transport adapter through QA Lab's shared suite host. */
|
||||
export async function runLiveTransportQaSuiteCommand(params: LiveTransportQaSuiteCommandOptions) {
|
||||
return await loadQaRuntimeModule().runLiveTransportQaSuiteCommand(params);
|
||||
}
|
||||
|
||||
type LiveTransportQaCommanderOptions = {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
@@ -88,6 +110,7 @@ type LiveTransportQaCommanderOptions = {
|
||||
failFast?: boolean;
|
||||
profile?: string;
|
||||
sutAccount?: string;
|
||||
credentialFile?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
};
|
||||
@@ -104,6 +127,7 @@ export type LiveTransportQaCredentialCliOptions = {
|
||||
/** Declarative command metadata and runner used to install a live-transport QA CLI. */
|
||||
export type LiveTransportQaCliRegistrationOptions = {
|
||||
commandName: string;
|
||||
credentialFileHelp?: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
defaultProviderMode: string;
|
||||
description: string;
|
||||
@@ -149,6 +173,7 @@ function mapLiveTransportQaCommanderOptions(
|
||||
scenarioIds: opts.scenario,
|
||||
listScenarios: opts.listScenarios,
|
||||
sutAccountId: opts.sutAccount,
|
||||
credentialFile: opts.credentialFile,
|
||||
credentialSource: opts.credentialSource,
|
||||
credentialRole: opts.credentialRole,
|
||||
};
|
||||
@@ -177,6 +202,10 @@ function registerLiveTransportQaCli(
|
||||
|
||||
command.option("--sut-account <id>", params.sutAccountHelp, "sut");
|
||||
|
||||
if (params.credentialFileHelp) {
|
||||
command.option("--credential-file <path>", params.credentialFileHelp);
|
||||
}
|
||||
|
||||
if (params.listScenariosHelp) {
|
||||
command.option("--list-scenarios", params.listScenariosHelp, false);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,106 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeCredentialPayloadForKind } from "../qa/convex-credential-broker/convex/payload_validation.js";
|
||||
|
||||
const BUZZ_DRIVER_PRIVATE_KEY = "01".repeat(32);
|
||||
const BUZZ_SUT_PRIVATE_KEY = "02".repeat(32);
|
||||
const BUZZ_DRIVER_NSEC = "nsec1qyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqszqgpqyqstywftw";
|
||||
|
||||
describe("QA Convex credential payload validation", () => {
|
||||
it("normalizes Buzz credential payloads", () => {
|
||||
expect(
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: " wss://relay.qa.example ",
|
||||
roomId: " 123E4567-E89B-42D3-A456-426614174000 ",
|
||||
driverPrivateKey: ` ${BUZZ_DRIVER_PRIVATE_KEY} `,
|
||||
sutPrivateKey: ` ${BUZZ_SUT_PRIVATE_KEY} `,
|
||||
driverAuthTag: ' ["auth","driver","conditions","signature"] ',
|
||||
ignored: true,
|
||||
}),
|
||||
).toEqual({
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: BUZZ_SUT_PRIVATE_KEY,
|
||||
driverAuthTag: '["auth","driver","conditions","signature"]',
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["ws://localhost:8080", "ws://127.0.0.1:8080", "ws://[::1]:8080"])(
|
||||
"allows plaintext loopback Buzz relay URL %s",
|
||||
(relayUrl) => {
|
||||
expect(
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl,
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: BUZZ_SUT_PRIVATE_KEY,
|
||||
}),
|
||||
).toMatchObject({ relayUrl });
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects plaintext remote Buzz relay URLs", () => {
|
||||
expect(() =>
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: "ws://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: BUZZ_SUT_PRIVATE_KEY,
|
||||
}),
|
||||
).toThrow(/wss:\/\//u);
|
||||
});
|
||||
|
||||
it("rejects malformed Buzz credential payloads without echoing values", () => {
|
||||
const privateKey = BUZZ_DRIVER_PRIVATE_KEY;
|
||||
const invalidRelay = "https://relay.qa.example/private-path";
|
||||
expect(() =>
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: invalidRelay,
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: privateKey,
|
||||
sutPrivateKey: privateKey,
|
||||
}),
|
||||
).toThrow(/wss:\/\//u);
|
||||
try {
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: privateKey,
|
||||
sutPrivateKey: privateKey,
|
||||
});
|
||||
} catch (error) {
|
||||
expect(String(error)).not.toContain(privateKey);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects runtime-invalid Buzz secrets and equivalent key encodings", () => {
|
||||
expect(() =>
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: BUZZ_DRIVER_NSEC,
|
||||
}),
|
||||
).toThrow(/distinct driver and SUT identities/u);
|
||||
expect(() =>
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: "not-a-private-key",
|
||||
sutPrivateKey: BUZZ_SUT_PRIVATE_KEY,
|
||||
}),
|
||||
).toThrow(/nsec or 64-character hex private key/u);
|
||||
expect(() =>
|
||||
normalizeCredentialPayloadForKind("buzz", {
|
||||
relayUrl: "wss://relay.qa.example",
|
||||
roomId: "123e4567-e89b-42d3-a456-426614174000",
|
||||
driverPrivateKey: BUZZ_DRIVER_PRIVATE_KEY,
|
||||
sutPrivateKey: BUZZ_SUT_PRIVATE_KEY,
|
||||
driverAuthTag: "not-an-auth-tag",
|
||||
}),
|
||||
).toThrow(/auth tag JSON array/u);
|
||||
});
|
||||
|
||||
it("normalizes Discord credential payloads", () => {
|
||||
expect(
|
||||
normalizeCredentialPayloadForKind("discord", {
|
||||
|
||||
@@ -394,6 +394,7 @@ function runReleaseChecksSummary(params: {
|
||||
QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LIVE_RELEASE_CHECKS_RESULT: "skipped",
|
||||
QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: "skipped",
|
||||
@@ -2590,7 +2591,7 @@ describe("package artifact reuse", () => {
|
||||
fail_fast: "${{ fromJSON(needs.resolve_target.outputs.fail_fast) }}",
|
||||
run_matrix: true,
|
||||
});
|
||||
for (const lane of ["mock_parity", "telegram", "discord", "whatsapp", "slack"]) {
|
||||
for (const lane of ["mock_parity", "buzz", "telegram", "discord", "whatsapp", "slack"]) {
|
||||
expect(releaseJob.with?.[`run_${lane}`]).toBeUndefined();
|
||||
}
|
||||
expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_mock_parity").if).toBe(
|
||||
@@ -2645,6 +2646,30 @@ describe("package artifact reuse", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("routes release Buzz through the QA Lab selector", () => {
|
||||
const releaseJob = workflowJob(RELEASE_CHECKS_WORKFLOW, "qa_live_buzz_release_checks");
|
||||
|
||||
expect(releaseJob.uses).toBe("./.github/workflows/qa-live-transports-convex.yml");
|
||||
expect(releaseJob.secrets).toBeUndefined();
|
||||
expect(releaseJob.permissions).toEqual({ contents: "read", "pull-requests": "read" });
|
||||
expect(releaseJob.if).toContain('contains(fromJSON(\'["all","qa","qa-live"]\')');
|
||||
expect(releaseJob.if).toContain("needs.resolve_target.outputs.qa_live_buzz_enabled == 'true'");
|
||||
expect(releaseJob.with).toMatchObject({
|
||||
buzz_scenario: "channel-canary,channel-mention-gating",
|
||||
expected_sha: "${{ needs.resolve_target.outputs.revision }}",
|
||||
run_buzz: true,
|
||||
});
|
||||
expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_buzz").if).toBe("inputs.run_buzz");
|
||||
expect(
|
||||
workflowStep(
|
||||
workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_buzz"),
|
||||
"Upload Buzz QA artifacts",
|
||||
).with?.name,
|
||||
).toBe(
|
||||
"${{ inputs.expected_sha != '' && format('release-qa-live-buzz-{0}-{1}', inputs.expected_sha, github.run_attempt) || format('qa-live-buzz-{0}-{1}', github.run_id, github.run_attempt) }}",
|
||||
);
|
||||
});
|
||||
|
||||
it("runs live transport lanes nightly while release checks stay gated", () => {
|
||||
const releaseWorkflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8");
|
||||
const qaWorkflow = readFileSync(QA_LIVE_TRANSPORTS_WORKFLOW, "utf8");
|
||||
@@ -2674,6 +2699,7 @@ describe("package artifact reuse", () => {
|
||||
"always() && steps.run_lane.outputs.output_dir != ''",
|
||||
],
|
||||
["run_live_matrix", "Upload Matrix QA artifacts", "always()"],
|
||||
["run_live_buzz", "Upload Buzz QA artifacts", "always()"],
|
||||
["run_live_telegram", "Upload Telegram QA artifacts", "always()"],
|
||||
["run_live_discord", "Upload Discord QA artifacts", "always()"],
|
||||
["run_live_whatsapp", "Upload WhatsApp QA artifacts", "always()"],
|
||||
@@ -3431,6 +3457,7 @@ describe("package artifact reuse", () => {
|
||||
|
||||
const verifyStep = workflowStep(summary, "Verify release check results");
|
||||
expect(verifyStep.env).toMatchObject({
|
||||
QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: "${{ needs.qa_live_buzz_release_checks.result }}",
|
||||
QA_LIVE_RELEASE_CHECKS_RESULT: "${{ needs.qa_live_release_checks.result }}",
|
||||
RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}",
|
||||
RELEASE_CHECK_RUN_ID: "${{ github.run_id }}",
|
||||
@@ -3454,6 +3481,7 @@ describe("package artifact reuse", () => {
|
||||
"::warning::${name} ended with ${result}; Tideclaw alpha treats non-package-safety release-check lanes as advisory.",
|
||||
"::error::${name} ended with ${result}",
|
||||
'"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"',
|
||||
'"qa_live_buzz_release_checks=${QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT}"',
|
||||
]);
|
||||
expect(verifyStep.run).not.toContain("qa_live_matrix_release_checks");
|
||||
expect(verifyStep.run).not.toContain(
|
||||
|
||||
Reference in New Issue
Block a user