diff --git a/.agents/skills/openclaw-qa-testing/SKILL.md b/.agents/skills/openclaw-qa-testing/SKILL.md index eb3c955af7aa..28eae8a499ac 100644 --- a/.agents/skills/openclaw-qa-testing/SKILL.md +++ b/.agents/skills/openclaw-qa-testing/SKILL.md @@ -62,20 +62,16 @@ scenario through qa-channel, decodes the emitted protobuf spans, and verifies the exported trace names and privacy contract. It does not require Opik, Langfuse, or external collector credentials. -## Matrix live profiles +## Matrix live selection -`pnpm openclaw qa matrix` defaults to the full `all` profile. Use explicit -profiles for faster CI/release proof: +`pnpm openclaw qa matrix` derives its default set from flow scenarios that +explicitly declare Matrix channel eligibility. It has no named profiles or +runner-owned scenario-id list. Repeat `--scenario ` for a focused run, or +use `--shard /` to reproduce one deterministic CI partition. -```bash -pnpm openclaw qa matrix --profile release -``` - -- `fast` and `release`: focused release-critical scenarios. -- `transport`: transport and routing coverage. -- `all`: the full Matrix profile. -- `QA-Lab - All Lanes` fans manual `all` runs across the transport, media, and - E2EE profiles. Release checks use the focused reusable Matrix job. +`QA-Lab - All Lanes` and release checks fan the same catalog-derived selection +across five balanced shards. Shards distribute work only; they do not define +semantic membership or priority. ## QA credentials and 1Password diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index eb4e8fa684df..6e9fbcb08b9d 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -452,19 +452,17 @@ jobs, followed by a report job that downloads both artifacts and runs first; inspect the report job when both lane summaries exist but the comparison fails. -### QA Lab Matrix Profiles +### QA Lab Matrix selection -`pnpm openclaw qa matrix` defaults to `--profile all`. Do not assume the CLI -default is the fast release path. Use explicit profiles: +`pnpm openclaw qa matrix` runs every flow scenario that explicitly declares +Matrix channel eligibility. There are no Matrix named profiles or runner-owned +scenario-id lists. Repeat `--scenario ` for an explicit subset, or use +`--shard /` to reproduce one deterministic CI partition. -- `--profile fast|release`: focused release-critical scenarios -- `--profile transport|all`: broad Matrix proof -- repeated `--scenario ` flags: explicit scenario selection - -`QA-Lab - All Lanes` and `OpenClaw Release Checks` use the same QA Lab selector -and standard artifacts. Manual dispatch keeps `matrix_profile=all` as the -default and fans it across the transport, media, and E2EE profiles; focused -dispatches select `fast`, `release`, or `transport`. +`QA-Lab - All Lanes` and `OpenClaw Release Checks` use the same catalog-derived +selection and standard artifacts. CI fans the selection across five balanced +shards; shard placement is operational only and does not define membership or +priority. ### Reusable Live/E2E Checks diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 29eaf106d95b..71b90fd961ad 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1625,7 +1625,7 @@ jobs: if-no-files-found: error qa_live_release_checks: - name: Run QA Live Matrix profile + name: Run QA Live Matrix catalog needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true' permissions: @@ -1635,13 +1635,7 @@ jobs: with: ref: ${{ needs.resolve_target.outputs.revision }} expected_sha: ${{ needs.resolve_target.outputs.revision }} - matrix_profile: release - matrix_provider_mode: mock-openai - matrix_primary_model: mock-openai/gpt-5.6-luna - matrix_alternate_model: mock-openai/gpt-5.6-luna-alt - matrix_attempts: 2 run_matrix: true - matrix_advisory: true # The dispatched child owns Telegram evidence/status artifacts; this blocking job # carries its exact conclusion into the parent summary without copying secrets or artifacts. @@ -2130,7 +2124,7 @@ jobs: QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_parity_report_release_checks.result }} QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }} RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: ${{ needs.runtime_tool_coverage_release_checks.result }} - QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }} + QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result }} QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }} QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }} QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }} diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index b4a8df17fde8..bf8ada7b46d5 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -4,10 +4,6 @@ on: schedule: - cron: "41 4 * * *" workflow_call: - outputs: - matrix_status: - description: Matrix release profile status - value: ${{ jobs.run_live_matrix.outputs.status }} inputs: ref: description: Ref, tag, or SHA to run @@ -17,31 +13,6 @@ on: description: Exact resolved SHA expected from the trusted caller required: true type: string - matrix_profile: - description: Matrix QA profile - required: false - default: release - type: string - matrix_provider_mode: - description: Matrix provider mode - required: false - default: mock-openai - type: string - matrix_primary_model: - description: Matrix primary model - required: false - default: mock-openai/gpt-5.6-luna - type: string - matrix_alternate_model: - description: Matrix alternate model - required: false - default: mock-openai/gpt-5.6-luna-alt - type: string - matrix_attempts: - description: Matrix lane attempts - required: false - default: 2 - type: number run_mock_parity: description: Run the mock parity lane required: false @@ -72,11 +43,6 @@ on: required: false default: false type: boolean - matrix_advisory: - description: Report Matrix failure through the status output instead of failing the reusable job - required: false - default: false - type: boolean workflow_dispatch: inputs: ref: @@ -100,17 +66,6 @@ on: description: Optional comma-separated Slack scenario ids required: false type: string - matrix_profile: - description: Matrix QA profile for the live Matrix lane - required: false - default: all - type: choice - options: - - fast - - release - - all - - transport - permissions: contents: read pull-requests: read @@ -407,15 +362,16 @@ jobs: if-no-files-found: error run_live_matrix: - name: Run Matrix live QA lane + name: Run Matrix live QA lane (shard ${{ matrix.shard }}/5) needs: [authorize_actor, validate_selected_ref] - if: (github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all') - continue-on-error: ${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }} + if: github.event_name != 'workflow_call' || inputs.run_matrix runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 environment: qa-live-shared - outputs: - status: ${{ steps.record_status.outputs.status }} + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5] steps: - name: Checkout selected ref uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -430,19 +386,6 @@ jobs: node-version: ${{ env.NODE_VERSION }} install-bun: "true" - - name: Validate required QA credential env - env: - MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }} - OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} - shell: bash - run: | - set -euo pipefail - - if [[ "${MATRIX_PROVIDER_MODE}" == "live-frontier" && -z "${OPENAI_API_KEY:-}" ]]; then - echo "Missing required OPENAI_API_KEY." >&2 - exit 1 - fi - - name: Build private QA runtime env: NODE_OPTIONS: --max-old-space-size=12288 @@ -450,80 +393,40 @@ jobs: - name: Run Matrix live lane id: run_lane + continue-on-error: true shell: bash env: - OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} - INPUT_MATRIX_PROFILE: ${{ inputs.expected_sha != '' && inputs.matrix_profile || github.event_name == 'workflow_dispatch' && inputs.matrix_profile || 'release' }} - MATRIX_ALTERNATE_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }} - MATRIX_ATTEMPTS: ${{ inputs.expected_sha != '' && inputs.matrix_attempts || 1 }} - MATRIX_PRIMARY_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }} - MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" run: | set -euo pipefail - if [[ ! "${MATRIX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then - echo "matrix_attempts must be a positive integer; got: ${MATRIX_ATTEMPTS}" >&2 - exit 1 - fi - case "${MATRIX_PROVIDER_MODE}" in - mock-openai|live-frontier) ;; - *) - echo "matrix_provider_mode must be mock-openai or live-frontier; got: ${MATRIX_PROVIDER_MODE}" >&2 - exit 1 - ;; - esac - - resolve_matrix_profile() { - local requested="$1" - local help_text="$2" - if [[ "${requested}" == "release" ]] && ! grep -Fq "fast, release, transport" <<<"${help_text}"; then - printf 'fast\n' - return - fi - printf '%s\n' "${requested}" - } + output_dir=".artifacts/qa-e2e/matrix-live-shard-${{ matrix.shard }}-of-5-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -p "${output_dir}" matrix_help="$(pnpm openclaw qa matrix --help 2>&1)" - matrix_profile="$(resolve_matrix_profile "${INPUT_MATRIX_PROFILE}" "${matrix_help}")" - if [[ "${matrix_profile}" != "${INPUT_MATRIX_PROFILE}" ]]; then - echo "Selected target does not support Matrix profile '${INPUT_MATRIX_PROFILE}'; using '${matrix_profile}'." + if ! grep -Fq -- "--shard " <<<"${matrix_help}"; then + echo "Selected target predates profile-free Matrix catalog sharding; update the target revision." >&2 + exit 1 fi - output_dir=".artifacts/qa-e2e/matrix-live-${matrix_profile}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - for attempt in $(seq 1 "${MATRIX_ATTEMPTS}"); do - attempt_output_dir="${output_dir}/attempt-${attempt}" - if pnpm openclaw qa matrix \ - --repo-root . \ - --output-dir "${attempt_output_dir}" \ - --provider-mode "${MATRIX_PROVIDER_MODE}" \ - --model "${MATRIX_PRIMARY_MODEL}" \ - --alt-model "${MATRIX_ALTERNATE_MODEL}" \ - --profile "${matrix_profile}" \ - --fast; then - exit 0 - fi - if [[ "${attempt}" == "${MATRIX_ATTEMPTS}" ]]; then - exit 1 - fi - echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2 - sleep 10 - done + pnpm openclaw qa matrix \ + --repo-root . \ + --output-dir "${output_dir}" \ + --provider-mode mock-openai \ + --shard "${{ matrix.shard }}/5" \ + --fast - name: Upload Matrix QA artifacts id: upload_artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }} - path: ${{ steps.run_lane.outputs.output_dir }} + name: ${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}-shard-{1}-of-5', inputs.expected_sha, matrix.shard) || format('qa-live-matrix-{0}-{1}-shard-{2}-of-5', github.run_id, github.run_attempt, matrix.shard) }} + path: .artifacts/qa-e2e/matrix-live-shard-${{ matrix.shard }}-of-5-${{ github.run_id }}-${{ github.run_attempt }} retention-days: 14 if-no-files-found: error - - name: Record Matrix QA status - id: record_status + - name: Enforce Matrix QA shard result if: always() shell: bash env: @@ -532,98 +435,11 @@ jobs: run: | set -euo pipefail - status="success" - for outcome in "${RUN_OUTCOME:-}" "${UPLOAD_OUTCOME:-}"; do - case "${outcome}" in - failure) status="failure" ;; - cancelled) - if [[ "${status}" != "failure" ]]; then - status="cancelled" - fi - ;; - success|skipped|"") ;; - *) status="failure" ;; - esac - done - echo "status=${status}" >> "$GITHUB_OUTPUT" - - run_live_matrix_sharded: - name: Run Matrix live QA lane (${{ matrix.profile }}) - needs: [authorize_actor, validate_selected_ref] - if: ${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all' }} - runs-on: blacksmith-16vcpu-ubuntu-2404 - timeout-minutes: 60 - environment: qa-live-shared - strategy: - fail-fast: false - matrix: - profile: - - transport - - media - - e2ee-smoke - - e2ee-deep - - e2ee-cli - steps: - - name: Checkout selected ref - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} - fetch-depth: 1 - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - node-version: ${{ env.NODE_VERSION }} - install-bun: "true" - - - name: Validate required QA credential env - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - shell: bash - run: | - set -euo pipefail - - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "Missing required OPENAI_API_KEY." >&2 + if [[ "${RUN_OUTCOME:-failure}" != "success" || "${UPLOAD_OUTCOME:-failure}" != "success" ]]; then + echo "Matrix QA shard failed: run=${RUN_OUTCOME:-missing}, upload=${UPLOAD_OUTCOME:-missing}." >&2 exit 1 fi - - name: Build private QA runtime - env: - NODE_OPTIONS: --max-old-space-size=12288 - run: pnpm build - - - name: Run Matrix live lane shard - id: run_lane - shell: bash - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - run: | - set -euo pipefail - - output_dir=".artifacts/qa-e2e/matrix-live-${{ matrix.profile }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - pnpm openclaw qa matrix \ - --repo-root . \ - --output-dir "${output_dir}" \ - --provider-mode live-frontier \ - --model "${OPENCLAW_CI_OPENAI_MODEL}" \ - --alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \ - --profile "${{ matrix.profile }}" \ - --fast - - - name: Upload Matrix QA shard artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: qa-live-matrix-${{ matrix.profile }}-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ steps.run_lane.outputs.output_dir }} - retention-days: 14 - if-no-files-found: error - run_live_telegram: name: Run Telegram live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] diff --git a/docs/channels/qa-channel.md b/docs/channels/qa-channel.md index 24ab5a0ebb2b..537c8f4b8099 100644 --- a/docs/channels/qa-channel.md +++ b/docs/channels/qa-channel.md @@ -90,7 +90,7 @@ Builds the QA site, starts the Docker-backed gateway + QA Lab stack, and prints ## Related -- [QA overview](/concepts/qa-e2e-automation) - overall stack, transport adapters, Matrix profiles, and scenario authoring +- [QA overview](/concepts/qa-e2e-automation) - overall stack, transport adapters, the Matrix live lane, and scenario authoring - [Pairing](/channels/pairing) - [Groups](/channels/groups) - [Channels overview](/channels) diff --git a/docs/ci.md b/docs/ci.md index cd10ad5487e9..4c6738ab2207 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -34,7 +34,7 @@ dispatch. | `build-artifacts` | Build `dist/`, Control UI, built-CLI smoke checks, startup memory, and embedded built-artifact checks | Node-relevant changes | | `control-ui-i18n` | Verify generated Control UI locale bundles, metadata, and translation memory; advisory on automatic runs, blocking on manual release CI | Control UI i18n-relevant changes and manual CI | | `checks-fast-core` | Fast Linux correctness lanes: suppression-baseline max-lines ratchet, bundled + protocol, Bun launcher, and the CI-routing fast task | Node-relevant changes | -| `qa-smoke-ci-profile` | Two self-contained balanced parts of the bounded automatic QA Smoke representative set; full taxonomy coverage remains available through explicit QA profiles | Node-relevant changes | +| `qa-smoke-ci-profile` | Self-contained balanced parts of the automatic QA Smoke coverage set; full taxonomy coverage remains available through explicit QA profiles | Node-relevant changes | | `checks-fast-contracts-plugins-*` | Two weighted plugin contract shards | Node-relevant changes | | `checks-fast-contracts-channels-*` | Two weighted channel contract shards | Node-relevant changes | | `checks-node-*` | Changed-target Node tests on pull requests; full core shards on `main`, manual, release, and broad-fallback runs | Node-relevant changes | @@ -566,9 +566,9 @@ QA Lab has dedicated CI lanes outside the main smart-scoped workflow. Agentic pa - The `QA-Lab - All Lanes` workflow runs nightly on `main` and on manual dispatch; it fans out mock parity plus live Matrix, Telegram, Discord, WhatsApp, and Slack jobs. Live jobs use the `qa-live-shared` environment; Telegram, Discord, WhatsApp, and Slack use Convex leases, while Matrix provisions disposable local credentials. -Release checks run Matrix and Telegram live transport lanes with the deterministic mock provider and mock-qualified models (`mock-openai/gpt-5.6-luna` and `mock-openai/gpt-5.6-luna-alt`) so the channel contract is isolated from live model latency and normal provider-plugin startup. The live transport gateway disables memory search because QA parity covers memory behavior separately; provider connectivity is covered by the separate live model, native provider, and Docker provider suites. +Scheduled, manual, and release Matrix checks use the deterministic mock provider so the live transport contract is isolated from model latency and normal provider-plugin startup. Telegram release checks use the same deterministic model boundary. The live transport gateway disables memory search because QA parity covers memory behavior separately; provider connectivity is covered by the separate live model, native provider, and Docker provider suites. -Scheduled and release Matrix gates use the shared QA Lab suite host and live adapter with the release scenarios. The CLI default and manual workflow input remain `all`; manual `all` dispatches fan out the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles so the 93-scenario proof stays within per-job timeouts. Focused manual dispatches select `fast`, `release`, or `transport` in one job. +Scheduled, manual, and release Matrix gates use the shared QA Lab suite host and live adapter. Default membership comes from flow scenarios that explicitly declare Matrix channel eligibility; the runner and workflow keep no curated profile or scenario-id list. CI distributes that catalog-derived selection across five deterministic balanced shards so membership is order-independent and each job stays within its timeout. Focused local runs repeat `--scenario `. `OpenClaw Release Checks` also runs the release-critical QA Lab lanes before release approval; its QA parity gate runs the candidate and baseline packs as parallel lane jobs, then downloads both artifacts into a small report job for the final parity comparison. diff --git a/docs/concepts/personal-agent-benchmark-pack.md b/docs/concepts/personal-agent-benchmark-pack.md index 78f5476131a6..7e5327787a1e 100644 --- a/docs/concepts/personal-agent-benchmark-pack.md +++ b/docs/concepts/personal-agent-benchmark-pack.md @@ -30,20 +30,19 @@ Ten scenarios, defined in `qa/scenarios/personal/*.yaml`: | `personal-no-fake-progress` | Proof-backed completion claims that avoid fake progress before local evidence exists | | `personal-failure-recovery` | Failure recovery that reports partial status and keeps retry boundaries clear | -The machine-readable pack metadata (id list, title, description) lives in -`extensions/qa-lab/src/scenario-packs.ts` as `QA_PERSONAL_AGENT_SCENARIO_IDS`. -Run the pack with `--pack personal-agent`: +The machine-readable `personal-agent` profile lives in root `taxonomy.yaml` as +semantic coverage IDs. QA Lab resolves every primary owner from the catalog; +there is no second scenario-ID list. Run it with: ```bash -OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 pnpm openclaw qa suite \ +OPENCLAW_ENABLE_PRIVATE_QA_CLI=1 pnpm openclaw qa run \ + --qa-profile personal-agent \ --provider-mode mock-openai \ - --pack personal-agent \ --concurrency 1 ``` -`--pack` is additive with repeated `--scenario` flags. Explicit scenarios run -first, then the pack scenarios run in `QA_PERSONAL_AGENT_SCENARIO_IDS` order -with duplicates removed. +Use repeated `--scenario` flags to narrow the profile. Scenario file and +taxonomy order do not affect membership or execution order. The pack targets `qa-channel` with `mock-openai` or another local QA provider lane. Do not point it at live chat services or real personal accounts. @@ -61,9 +60,10 @@ inspect and file in issues. ## Extending the pack -Add new `.yaml` cases under `qa/scenarios/personal/`, then add the scenario id -to `QA_PERSONAL_AGENT_SCENARIO_IDS`. Keep each case small, local, deterministic -in `mock-openai`, and focused on one personal assistant behavior. +Add new `.yaml` cases under `qa/scenarios/personal/`, declare the exact primary +coverage ID they prove, and add that semantic ID to the taxonomy profile when +it belongs in this benchmark. Keep each case small, local, deterministic in +`mock-openai`, and focused on one personal assistant behavior. Good follow-up candidates: redacted trajectory export checks, local-only plugin workflow checks. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index aa2a9e5ab6ae..045bd0f4c41b 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -16,7 +16,7 @@ Pieces: - `extensions/qa-channel`: synthetic message channel with DM, channel, thread, reaction, edit, and delete surfaces. -- `extensions/qa-lab`: debugger UI, QA bus, scenario profiles, and live +- `extensions/qa-lab`: debugger UI, QA bus, scenario runners, and live transport adapters for observing the transcript, injecting inbound messages, and exporting a Markdown report. - `qa/`: repo-backed seed assets for the kickoff task and baseline QA @@ -48,7 +48,7 @@ script aliases; both forms work. | `qa mock-openai` | Start only the scenario-aware `mock-openai` provider server. | | `qa credentials doctor` / `add` / `list` / `remove` | Manage the shared Convex credential pool. | | `qa discord` | Live transport lane against a real private Discord guild channel. | -| `qa matrix` | QA Lab Matrix profiles against a disposable Tuwunel homeserver. See [Matrix smoke lanes](#matrix-smoke-lanes). | +| `qa matrix` | QA Lab Matrix catalog scenarios against a disposable Tuwunel homeserver. See [Matrix live lane](#matrix-live-lane). | | `qa slack` | Live transport lane against a real private Slack channel. | | `qa telegram` | Live transport lane against a real private Telegram group. | | `qa whatsapp` | Live transport lane against real WhatsApp Web accounts. | @@ -67,7 +67,9 @@ fulfills matching IDs, while secondary coverage stays advisory. Every coverage ID is exactly `taxonomy-surface.feature`, using the short surface ID from `taxonomy.yaml`. A scenario's separate `surface` field is an execution/reporting label (for example, `channel` or `runtime-tool`); it does not define taxonomy -ownership. +ownership. An explicit profile coverage ID selects every eligible primary owner +for that ID, deduplicated by scenario. Scenario file and taxonomy order do not +affect membership or execution order. Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`; `smoke-ci` defaults to slim, and `--evidence-mode full` restores full entries: @@ -179,13 +181,13 @@ checks the authenticated scrape includes release-critical metric families without prompt content, response content, raw diagnostic identifiers, auth tokens, or local paths. -### Matrix smoke lanes +### Matrix live lane -For a transport-real Matrix smoke lane that does not require model-provider -credentials, run the release profile with the deterministic mock OpenAI provider: +For a transport-real Matrix lane that does not require model-provider +credentials, use the deterministic mock OpenAI provider: ```bash -pnpm openclaw qa matrix --provider-mode mock-openai --profile release +pnpm openclaw qa matrix --provider-mode mock-openai ``` For the live-frontier provider lane, supply OpenAI-compatible credentials @@ -193,27 +195,20 @@ explicitly: ```bash OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" \ - pnpm openclaw qa matrix --provider-mode live-frontier --profile release + pnpm openclaw qa matrix --provider-mode live-frontier ``` -Plain `pnpm openclaw qa matrix` runs the full `all` profile and continues after -scenario failures. Use `--fail-fast` for a shorter feedback loop or repeat -`--scenario ` to select individual scenarios; explicit scenario ids take -precedence over `--profile`. +Plain `pnpm openclaw qa matrix` runs every flow scenario that explicitly +declares Matrix eligibility through `execution.channel` or +`execution.channels`, and it continues after scenario failures. Use +`--fail-fast` for a shorter feedback loop or repeat `--scenario ` for an +explicit subset, including portable scenarios with no channel restriction. -| Profile | Scenarios | Purpose | -| ------------ | --------- | ---------------------------------------------------------------------------------------------------------------------------------------- | -| `all` | 93 | Complete catalog (default). | -| `release` | 2 | Release-critical channel baseline and live allowlist reload. | -| `fast` | 12 | Focused threading, reactions, approvals, policy, bot-gating, and encrypted-reply coverage. | -| `transport` | 50 | Threading, DM/room routing, autojoin, approvals, reactions, restarts, mention/allowlist policy, edits, and multi-actor ordering. | -| `media` | 7 | Image, generated-image, voice, attachment, unsupported-media, and encrypted-media coverage. | -| `e2ee-smoke` | 8 | Minimum encrypted reply, threading, bootstrap, recovery, restart, redaction, and failure coverage. | -| `e2ee-deep` | 18 | State-loss, backup, key recovery, device hygiene, and SAS/QR/DM verification. | -| `e2ee-cli` | 9 | `openclaw matrix encryption setup`, recovery-key, multi-account, gateway round-trip, and self-verification commands through the harness. | - -Profile membership and channel requirements live with the declarative Matrix -scenarios under `qa/scenarios/channels/`. The run chooses the channel driver. +Declarative scenario metadata is the only default-membership source. The +Matrix runner has no named profiles or scenario-id allowlists. The run chooses +the channel driver; deterministic `--shard /` partitioning only +distributes the selected catalog and does not define semantic membership or +execution priority. Their live implementations live under `extensions/qa-lab/src/live-transports/matrix/scenarios/`. @@ -228,8 +223,8 @@ Common options: | Flag | Default | Purpose | | ------------------------ | ----------------- | ------------------------------------------------------------------------------------ | -| `--profile ` | `all` | Select one of the profiles above. | | `--scenario ` | - | Select one scenario; repeatable. | +| `--shard ` | - | Run one deterministic, balanced partition of the selected Matrix catalog. | | `--fail-fast` | off | Stop after the first failed check or scenario. | | `--allow-failures` | off | Write artifacts without returning a failing exit code for scenario failures. | | `--provider-mode ` | `live-frontier` | Use `mock-openai` for deterministic dispatch or `live-frontier` for a live provider. | @@ -257,23 +252,19 @@ directory: `qa-suite-report.md`, `qa-suite-summary.json`, and increase the no-reply window; on fast CI, a smaller window can shorten negative assertions. -The scenarios cover transport behavior that unit tests cannot prove end to +The catalog covers transport behavior that unit tests cannot prove end to end: mention gating, allow-bot policies, allowlists, top-level and threaded replies, DM routing, reaction handling, inbound edit suppression, restart replay dedupe, homeserver interruption recovery, approval metadata delivery, media handling, and Matrix E2EE bootstrap/recovery/verification flows. The -E2EE CLI profile also drives `openclaw matrix encryption setup` and +E2EE CLI scenarios also drive `openclaw matrix encryption setup` and verification commands through the same disposable homeserver before checking gateway replies. -`matrix-room-block-streaming` and `subagent-thread-spawn` remain available by -explicit `--scenario` selection but stay outside the default `all` profile. - CI uses the same command surface in -`.github/workflows/qa-live-transports-convex.yml`. Scheduled and release runs -execute the release scenarios. Manual `matrix_profile=all` dispatches fan out -the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles; -focused dispatches select `fast`, `release`, or `transport` in one job. +`.github/workflows/qa-live-transports-convex.yml`. Scheduled, release, and +manual runs fan the catalog-derived selection across five deterministic shards +so membership stays scenario-owned while each job remains within its timeout. ### Discord Mantis scenarios @@ -414,11 +405,11 @@ when the maintainer secret is present. The root `taxonomy.yaml` defines semantic coverage IDs. Scenario YAML files under `qa/scenarios/` map each scenario to those IDs and own execution -metadata: `channel` is the only channel requirement, and `profiles` declare -named run membership. The channel driver is an interchangeable run-level -implementation choice. TypeScript -runners query that catalog; they do not maintain parallel scenario or coverage -inventories. +metadata; `channel` is the only channel requirement. Taxonomy profiles select +coverage IDs or whole categories, and the catalog resolves their primary +scenario owners. Transport runners apply channel and provider eligibility to +that result instead of keeping scenario-ID allowlists. The channel driver is +an interchangeable run-level implementation choice. Static `qa coverage` output reports the taxonomy-to-scenario mapping. Actual proof comes from `qa-evidence.json`, which records the executed scenario, @@ -441,12 +432,10 @@ Host and Multipass suite runs execute multiple selected scenarios in parallel with isolated gateway workers by default. `qa-channel` defaults to concurrency 4, capped by the selected scenario count. Use `--concurrency ` to tune the worker count, or `--concurrency 1` for serial execution. -Use `--pack personal-agent` to run the personal assistant benchmark pack (10 -scenarios). The pack selector is additive with repeated `--scenario` flags: -explicit scenarios run first, then pack scenarios run in pack order with -duplicates removed. Use `--pack observability` to select the -`otel-trace-smoke` and `docker-prometheus-smoke` scenarios together when a -custom QA runner already supplies the OpenTelemetry collector setup. +Use `qa run --qa-profile personal-agent --provider-mode mock-openai` for the +personal assistant benchmark, or `--qa-profile observability` for the source +checkout telemetry checks. CI uses the same profile resolver for `smoke-ci`; +none of these selectors maintains a second scenario-ID list. The command exits non-zero when any scenario fails. Use `--allow-failures` when you want artifacts without a failing exit code. diff --git a/docs/docs_map.md b/docs/docs_map.md index d80e9b33d0e5..eaf88535e00a 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -2925,7 +2925,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H3: Profile-backed qa run - H2: Operator flow - H3: Observability smokes - - H3: Matrix smoke lanes + - H3: Matrix live lane - H3: Discord Mantis scenarios - H3: Mantis Slack desktop and visual-task runners - H3: Credential pool health check diff --git a/docs/help/testing.md b/docs/help/testing.md index 8933191b8727..7d5b658416dd 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -15,7 +15,7 @@ regressions for real-world provider/model bugs. **QA stack (qa-lab, qa-channel, live transport lanes)** is documented separately: -- [QA overview](/concepts/qa-e2e-automation) - architecture, command surface, scenario authoring, and Matrix profiles. +- [QA overview](/concepts/qa-e2e-automation) - architecture, command surface, scenario authoring, and the Matrix live lane. - [Maturity scorecard](/maturity/scorecard) - how release QA evidence supports stability and LTS decisions. - [QA channel](/channels/qa-channel) - the synthetic transport plugin used by repo-backed scenarios. @@ -173,12 +173,12 @@ checks keep exhaustive live/Docker soak behind `run_release_soak=true`; the `full` profile forces soak on. `QA-Lab - All Lanes` runs nightly on `main` and from manual dispatch with the mock parity lane, live Matrix lane, Convex-managed live Telegram lane, and Convex-managed live Discord lane as -parallel jobs. Scheduled QA and release checks run the Matrix release profile -through the shared live adapter. The Matrix CLI and manual workflow input -default remain `all`; manual `all` dispatches fan out the transport, media, and -E2EE profiles, while focused dispatches can select `fast`, `release`, or -`transport`. `OpenClaw Release Checks` runs parity plus the reusable Matrix -live-adapter profile and Telegram lane before release approval. Release +parallel jobs. Scheduled QA and release checks run the catalog-derived Matrix +selection through the shared live adapter. The Matrix CLI and workflow have no +curated profiles: scenario channel eligibility defines default membership, +explicit `--scenario` flags narrow local runs, and CI fans the same selection +across deterministic shards. `OpenClaw Release Checks` runs parity plus the +reusable Matrix live-adapter lane and Telegram lane before release approval. Release transport checks use `mock-openai/gpt-5.6-luna` so they stay deterministic and avoid normal provider-plugin startup. These live transport gateways disable memory search; memory behavior stays covered by the QA parity suites. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index b146ec2f46e0..b9d0d23cc0e5 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -279,7 +279,7 @@ A legacy fallback correction tag may reuse base-package evidence only when the c Manually dispatch `Windows Node Release` only for recovery, and always pass an exact tag, never `latest`, plus the explicit `expected_installer_digests` JSON map from the approved source release. Website download links should target exact OpenClaw release asset URLs for the current stable release, or `releases/latest/download/...` only after verifying GitHub's latest redirect points at that same release; do not link only to the companion repo release page. -- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix release profile and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. Run the manual `QA-Lab - All Lanes` workflow with `matrix_profile=all` when you want every maintained Matrix scenario; the workflow fans that selection across the transport, media, and E2EE profiles to keep full proof within per-job timeouts. +- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix catalog and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. The `QA-Lab - All Lanes` workflow derives Matrix membership from scenario channel eligibility and fans it across deterministic balanced shards to keep proof within per-job timeouts; there is no separate Matrix selector input. - Cross-OS install and upgrade runtime validation is part of public `OpenClaw Release Checks` and `Full Release Validation`, which call the reusable workflow `.github/workflows/openclaw-cross-os-release-checks-reusable.yml` directly. This split is intentional: keep the real npm release path short, deterministic, and artifact-focused, while slower live checks stay in their own lane so they do not stall or block publish. - Secret-bearing release checks should be dispatched through `Full Release Validation` or from the `main`/release workflow ref so workflow logic and secrets stay controlled. - `OpenClaw Release Checks` accepts a branch, tag, or full commit SHA as long as the resolved commit is reachable from an OpenClaw branch or release tag. @@ -433,11 +433,11 @@ The QA Lab box is also part of `OpenClaw Release Checks`. It is the agentic beha Release QA Lab coverage includes: - mock parity lane comparing the OpenAI candidate lane against the `anthropic/claude-opus-4-8` baseline using the agentic parity pack -- Matrix live-adapter release profile using the `qa-live-shared` environment +- Matrix live-adapter catalog lane using the `qa-live-shared` environment - live Telegram QA lane using Convex CI credential leases - `pnpm qa:otel:smoke`, `pnpm qa:otel:collector-smoke`, `pnpm qa:prometheus:smoke`, or `pnpm qa:observability:smoke` when release telemetry needs explicit local proof -Use this box to answer "does the release behave correctly in QA scenarios and live channel flows?" Keep the artifact URLs for parity, Matrix, and Telegram lanes when approving the release. Full Matrix coverage remains available as a manual sharded QA-Lab run rather than the default release-critical lane. +Use this box to answer "does the release behave correctly in QA scenarios and live channel flows?" Keep the artifact URLs for parity, Matrix, and Telegram lanes when approving the release. Matrix runs use the same catalog-derived sharded selection in scheduled, manual, and release workflows. ### Package diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index aca40be8d817..fc62d6f520e6 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -171,7 +171,7 @@ artifact when package or Docker-facing stages need it. | QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`
**Backing workflow:** direct jobs
**Tests:** candidate and baseline agentic parity packs, then the parity report.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA runtime parity | **Job:** `Verify QA Lab runtime-pair lanes`
**Backing workflow:** direct job
**Tests:** the canonical core `openclaw`/`codex` lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex --runtime-pair-lane core`) and, with `run_release_soak=true`, the soak lane. Advisory: individual lane jobs do not block the release-check verifier.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`
**Backing workflow:** direct job
**Tests:** dynamic tool drift between `openclaw` and `codex` in the canonical core runtime-pair lane (`pnpm openclaw qa coverage --tools`), using that lane's output. Blocking: this job is not advisory-overridable.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | -| QA live Matrix | **Job:** `Run QA Live Matrix profile`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** parity-proven YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. | +| QA live Matrix | **Job:** `Run QA Live Matrix catalog`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** catalog-derived YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment, distributed across deterministic shards.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. | | QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** trusted `OpenClaw Release Telegram QA` dispatch
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | | QA live Discord | **Job:** `Run QA Lab live Discord lane`
**Backing workflow:** direct advisory job
**Tests:** live Discord QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_DISCORD_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-discord`. | | QA live WhatsApp | **Job:** `Run QA Lab live WhatsApp lane`
**Backing workflow:** direct advisory job
**Tests:** live WhatsApp QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_WHATSAPP_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-whatsapp`. | diff --git a/extensions/qa-lab/src/ci-smoke-plan.test.ts b/extensions/qa-lab/src/ci-smoke-plan.test.ts index 844f34698569..004bde17dfc9 100644 --- a/extensions/qa-lab/src/ci-smoke-plan.test.ts +++ b/extensions/qa-lab/src/ci-smoke-plan.test.ts @@ -1,11 +1,55 @@ -// Qa Lab tests cover bounded CI smoke profile planning. -import { OPENCLAW_CRABLINE_DEFAULT_CHANNEL } from "@openclaw/crabline"; +// Qa Lab tests cover bounded CI smoke pack planning. import { expectDefined } from "@openclaw/normalization-core"; -import { describe, expect, it } from "vitest"; -import { createQaSmokeCiPart } from "./ci-smoke-plan.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createQaSmokeCiPart, selectQaSmokeCiEligibilityChannel } from "./ci-smoke-plan.js"; +import { resolveQaProfileScenarios } from "./profile-planning.js"; import { readQaScenarioPack } from "./scenario-catalog.js"; import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; +const smokeProfileMock = vi.hoisted(() => ({ + mode: "actual" as "actual" | "empty" | "ineligible" | "missing-coverage" | "unsupported", +})); + +vi.mock("./profile-planning.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveQaProfileScenarios(params: Parameters[0]) { + const selection = actual.resolveQaProfileScenarios(params); + if (params.profile !== "smoke-ci") { + return selection; + } + if (smokeProfileMock.mode === "empty") { + return { ...selection, scenarios: [] }; + } + const scenarioPack = readQaScenarioPack(); + if (smokeProfileMock.mode === "ineligible") { + const replacement = expectDefined( + scenarioPack.scenarios.find((scenario) => scenario.id === "otel-trace-smoke"), + "ineligible smoke replacement", + ); + return { ...selection, scenarios: [replacement, ...selection.scenarios.slice(1)] }; + } + if (smokeProfileMock.mode === "missing-coverage") { + return { + ...selection, + scenarios: selection.scenarios.filter( + (scenario) => !scenario.coverage?.primary.includes("gateway.health-apis"), + ), + }; + } + if (smokeProfileMock.mode === "unsupported") { + const replacement = expectDefined( + scenarioPack.scenarios.find((scenario) => scenario.id === "discord-canary"), + "unsupported smoke replacement", + ); + return { ...selection, scenarios: [replacement, ...selection.scenarios.slice(1)] }; + } + return selection; + }, + }; +}); + type QaScenario = ReturnType["scenarios"][number]; function estimateScenarioCost(scenario: QaScenario | undefined): number { @@ -22,59 +66,65 @@ function estimateScenarioCost(scenario: QaScenario | undefined): number { } describe("createQaSmokeCiPart", () => { - it("balances the bounded automatic smoke set across four profile parts", () => { + afterEach(() => { + smokeProfileMock.mode = "actual"; + }); + + it("balances the bounded smoke pack across four profile parts", () => { const parts = ["profile-1", "profile-2", "profile-3", "profile-4"].map((partId) => createQaSmokeCiPart(partId), ); const repeatedLast = createQaSmokeCiPart("profile-4"); expect(repeatedLast).toEqual(parts[3]); - for (const part of parts) { - expect(part.runs[0]?.channel).toBe(OPENCLAW_CRABLINE_DEFAULT_CHANNEL); - } - // The matrix channel run rides only on the last part. - expect( - parts.slice(0, 3).some((part) => part.runs.some((run) => run.channel === "matrix")), - ).toBe(false); - expect(parts[3]?.runs.some((run) => run.channel === "matrix")).toBe(true); + expect(parts.slice(0, 3).some((part) => part.runs.some((run) => run.slug === "matrix"))).toBe( + false, + ); + expect(parts[3]?.runs.some((run) => run.slug === "matrix")).toBe(true); const scenarioIds = parts.flatMap((part) => part.runs.flatMap((run) => run.scenario_ids)); expect(new Set(scenarioIds).size).toBe(scenarioIds.length); + const scenarioPack = readQaScenarioPack(); const scenarioById = new Map( - readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario] as const), + scenarioPack.scenarios.map((scenario) => [scenario.id, scenario] as const), ); + const smokeSelection = resolveQaProfileScenarios({ + profile: "smoke-ci", + providerMode: "mock-openai", + eligibleChannels: ["telegram", "matrix"], + }); + const smokeProfileScenarioIds = smokeSelection.scenarios.map((scenario) => scenario.id); + expect(new Set(scenarioIds)).toEqual(new Set(smokeProfileScenarioIds)); expect( new Set(scenarioIds.map((scenarioId) => scenarioById.get(scenarioId)?.execution.kind)), ).toEqual(new Set(["flow", "playwright", "script"])); - expect(scenarioIds).toHaveLength(11); - expect(scenarioIds).toContain("control-ui-chat-flow-playwright"); - expect(scenarioIds).toContain("gateway-smoke"); - expect(scenarioIds).toContain("matrix-restart-resume"); const selectedScenarioPaths = new Set( scenarioIds.map((scenarioId) => scenarioById.get(scenarioId)?.sourcePath), ); const scorecardReport = readQaScorecardTaxonomyReport([...scenarioById.values()]); - const smokeScenarioRefs = new Set( - scorecardReport.categories - .filter((category) => category.profiles.includes("smoke-ci")) - .flatMap((category) => category.scenarioRefs), + const taxonomyProfile = expectDefined( + scorecardReport.profiles.find((profile) => profile.id === "smoke-ci"), + "smoke-ci taxonomy profile", ); + const smokeScenarioRefs = new Set(taxonomyProfile.scenarioRefs); expect( [...selectedScenarioPaths].every( (scenarioPath) => scenarioPath !== undefined && smokeScenarioRefs.has(scenarioPath), ), ).toBe(true); - const uncoveredCategoryIds = scorecardReport.categories - .filter((category) => category.profiles.includes("smoke-ci")) - .filter((category) => !category.scenarioRefs.some((ref) => selectedScenarioPaths.has(ref))) - .map((category) => category.id); - expect(uncoveredCategoryIds).toEqual([]); + const selectedCoverageIds = new Set( + smokeSelection.scenarios.flatMap((scenario) => + (scenario.coverage?.primary ?? []).filter((coverageId) => + taxonomyProfile.coverageIds.includes(coverageId), + ), + ), + ); + expect(selectedCoverageIds).toEqual(new Set(taxonomyProfile.coverageIds)); const primaryScenarioIds = parts.map( (part) => part.runs.find((run) => run.slug === "primary")?.scenario_ids ?? [], ); - expect(primaryScenarioIds[1]).toContain("system-agent-ring-zero-setup"); const primaryRunCosts = primaryScenarioIds.map((ids) => ids.reduce( (cost, scenarioId) => cost + estimateScenarioCost(scenarioById.get(scenarioId)), @@ -94,8 +144,6 @@ describe("createQaSmokeCiPart", () => { primaryRunCosts.toSorted((left, right) => left - right)[0], "lightest QA smoke run cost", ); - // Greedy balance: no part carries more than one heaviest-scenario cost - // beyond the lightest, and every part runs at least one scenario. expect(heaviestRunCost - lightestRunCost).toBeLessThanOrEqual(largestScenarioCost); expect(primaryScenarioIds.every((ids) => ids.length > 0)).toBe(true); }); @@ -105,4 +153,42 @@ describe("createQaSmokeCiPart", () => { "unknown QA smoke CI profile part: profile-5", ); }); + + it("accepts a portable multi-channel scenario through a supported CI channel", () => { + const scenario = expectDefined( + readQaScenarioPack().scenarios.find((candidate) => candidate.id === "channel-message-flows"), + "channel-message-flows scenario", + ); + + expect(scenario.execution).toMatchObject({ channels: ["qa-channel", "telegram"] }); + expect(selectQaSmokeCiEligibilityChannel(scenario)).toBe("telegram"); + }); + + it("fails when the smoke pack resolves empty", () => { + smokeProfileMock.mode = "empty"; + expect(() => createQaSmokeCiPart("profile-1")).toThrow( + "smoke-ci taxonomy profile did not resolve any CI scenarios", + ); + }); + + it("fails when the smoke pack contains a taxonomy-ineligible scenario", () => { + smokeProfileMock.mode = "ineligible"; + expect(() => createQaSmokeCiPart("profile-1")).toThrow( + "smoke-ci taxonomy profile resolved ineligible CI scenarios", + ); + }); + + it("fails when the smoke pack contains an unsupported channel", () => { + smokeProfileMock.mode = "unsupported"; + expect(() => createQaSmokeCiPart("profile-1")).toThrow( + "smoke-ci taxonomy profile resolved unsupported CI channels: discord", + ); + }); + + it("fails when an exact profile coverage ID has no eligible primary owner", () => { + smokeProfileMock.mode = "missing-coverage"; + expect(() => createQaSmokeCiPart("profile-1")).toThrow( + "smoke-ci taxonomy profile leaves coverage IDs without eligible CI scenarios: gateway.health-apis", + ); + }); }); diff --git a/extensions/qa-lab/src/ci-smoke-plan.ts b/extensions/qa-lab/src/ci-smoke-plan.ts index 26e2f37daf5a..e4d7bc14ce33 100644 --- a/extensions/qa-lab/src/ci-smoke-plan.ts +++ b/extensions/qa-lab/src/ci-smoke-plan.ts @@ -1,33 +1,22 @@ -// Qa Lab plugin module plans the bounded CI smoke profile parts. -import { OPENCLAW_CRABLINE_DEFAULT_CHANNEL } from "@openclaw/crabline"; +// Qa Lab plugin module plans the bounded CI smoke pack parts. import { defaultQaModelForMode, normalizeQaProviderMode } from "./model-selection.js"; +import { resolveQaProfileScenarios } from "./profile-planning.js"; import { readQaScenarioPack } from "./scenario-catalog.js"; -import { scenarioMatchesQaProviderLane } from "./scenario-lane.js"; +import { describeQaProviderLaneMismatches } from "./scenario-lane.js"; import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; const QA_SMOKE_PROFILE = "smoke-ci"; // Four parts keep each smoke job near the fixed setup cost (~1min) instead of // serializing ~4min of scenarios into one job that owns the PR wall clock. const QA_SMOKE_CI_PARTS = ["profile-1", "profile-2", "profile-3", "profile-4"] as const; -const QA_SMOKE_CI_CHANNELS = ["matrix", OPENCLAW_CRABLINE_DEFAULT_CHANNEL] as const; -const QA_SMOKE_CI_SCENARIO_IDS = new Set([ - "control-ui-chat-flow-playwright", - "system-agent-ring-zero-setup", - "gateway-smoke", - "model-switch-follow-up", - "group-visible-reply-tool", - "long-running-release-audit", - "matrix-restart-resume", - "personal-task-followthrough-status", - "plugin-lifecycle-hot-reload", - "subagent-completion-direct-fallback", - "telegram-commands-command", -]); +const QA_SMOKE_CI_CHANNELS = ["telegram", "matrix"] as const; type QaSmokeCiPartId = (typeof QA_SMOKE_CI_PARTS)[number]; +type QaSmokeCiScenario = ReturnType["scenarios"][number]; +// CI consumes only the run slug and ids. `qa run` resolves the taxonomy-owned +// channel driver so this planner does not encode driver-specific channel policy. type QaSmokeCiRun = { - channel: string; slug: string; scenario_ids: string[]; }; @@ -41,9 +30,7 @@ function isQaSmokeCiPartId(value: string): value is QaSmokeCiPartId { return QA_SMOKE_CI_PARTS.includes(value as QaSmokeCiPartId); } -function estimateScenarioCost( - scenario: ReturnType["scenarios"][number], -) { +function estimateScenarioCost(scenario: QaSmokeCiScenario) { if (scenario.execution.kind === "script") { return 8; } @@ -53,6 +40,18 @@ function estimateScenarioCost( return scenario.execution.kind === "flow" && scenario.execution.isolationReason ? 4 : 1; } +function listQaSmokeCiDeclaredChannels(scenario: QaSmokeCiScenario): readonly string[] { + if (scenario.execution.channel) { + return [scenario.execution.channel]; + } + return scenario.execution.kind === "flow" ? (scenario.execution.channels ?? []) : []; +} + +export function selectQaSmokeCiEligibilityChannel(scenario: QaSmokeCiScenario): string | undefined { + const declaredChannels = listQaSmokeCiDeclaredChannels(scenario); + return QA_SMOKE_CI_CHANNELS.find((channel) => declaredChannels.includes(channel)); +} + export function createQaSmokeCiPart(partId: string): QaSmokeCiPart { if (!isQaSmokeCiPartId(partId)) { throw new Error(`unknown QA smoke CI profile part: ${partId}`); @@ -64,42 +63,94 @@ export function createQaSmokeCiPart(partId: string): QaSmokeCiPart { if (!profile) { throw new Error(`taxonomy.yaml does not define QA run profile ${QA_SMOKE_PROFILE}.`); } - const providerMode = normalizeQaProviderMode("mock-openai"); - const primaryModel = defaultQaModelForMode(providerMode); - const scenarios = scenarioPack.scenarios.filter( - (scenario) => - QA_SMOKE_CI_SCENARIO_IDS.has(scenario.id) && - scenarioMatchesQaProviderLane({ - scenario, - providerMode, - primaryModel, - channelDriver: profile.channelDriver, - channel: scenario.execution.channel ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL, - }), - ); + + let scenarios: QaSmokeCiScenario[]; + let excludedScenarios: ReturnType["excludedScenarios"]; + try { + const selection = resolveQaProfileScenarios({ + profile: QA_SMOKE_PROFILE, + providerMode: "mock-openai", + eligibleChannels: QA_SMOKE_CI_CHANNELS, + }); + scenarios = selection.scenarios; + excludedScenarios = selection.excludedScenarios; + } catch (error) { + throw new Error(`${QA_SMOKE_PROFILE} taxonomy profile did not resolve any CI scenarios.`, { + cause: error, + }); + } if (scenarios.length === 0) { - throw new Error(`${QA_SMOKE_PROFILE} did not resolve any executable QA scenarios.`); + throw new Error(`${QA_SMOKE_PROFILE} taxonomy profile did not resolve any CI scenarios.`); } const supportedChannels = new Set(QA_SMOKE_CI_CHANNELS); const unsupportedChannels = new Set( - scenarios - .map((scenario) => scenario.execution.channel ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL) - .filter((channel) => !supportedChannels.has(channel)), + scenarios.flatMap((scenario) => { + const declaredChannels = listQaSmokeCiDeclaredChannels(scenario); + return declaredChannels.length > 0 && !selectQaSmokeCiEligibilityChannel(scenario) + ? declaredChannels.filter((channel) => !supportedChannels.has(channel)) + : []; + }), ); if (unsupportedChannels.size > 0) { throw new Error( - `${QA_SMOKE_PROFILE} resolved unsupported CI channels: ${[...unsupportedChannels].toSorted().join(", ")}.`, + `${QA_SMOKE_PROFILE} taxonomy profile resolved unsupported CI channels: ${[...unsupportedChannels].toSorted().join(", ")}.`, ); } - const matrixScenarios = scenarios.filter((scenario) => scenario.execution.channel === "matrix"); - const defaultChannelScenarios = scenarios - .filter( - (scenario) => - (scenario.execution.channel ?? OPENCLAW_CRABLINE_DEFAULT_CHANNEL) === - OPENCLAW_CRABLINE_DEFAULT_CHANNEL, - ) + const providerMode = normalizeQaProviderMode("mock-openai"); + const primaryModel = defaultQaModelForMode(providerMode); + const smokeScenarioRefs = new Set(profile.scenarioRefs); + const ineligibleScenarios = scenarios.flatMap((scenario) => { + const reasons = describeQaProviderLaneMismatches({ + scenario, + providerMode, + primaryModel, + channelDriver: profile.channelDriver, + channel: selectQaSmokeCiEligibilityChannel(scenario), + }); + if (!smokeScenarioRefs.has(scenario.sourcePath)) { + reasons.unshift(`not a primary owner selected by ${QA_SMOKE_PROFILE}`); + } + return reasons.length > 0 ? [`${scenario.id} (${reasons.join(", ")})`] : []; + }); + if (ineligibleScenarios.length > 0) { + throw new Error( + `${QA_SMOKE_PROFILE} taxonomy profile resolved ineligible CI scenarios: ${ineligibleScenarios.toSorted().join("; ")}.`, + ); + } + + const selectedCoverageIds = new Set( + scenarios.flatMap((scenario) => + (scenario.coverage?.primary ?? []).filter((coverageId) => + profile.coverageIds.includes(coverageId), + ), + ), + ); + const uncoveredCoverageIds = profile.coverageIds.filter( + (coverageId) => !selectedCoverageIds.has(coverageId), + ); + if (uncoveredCoverageIds.length > 0) { + const excludedOwners = excludedScenarios + .filter(({ scenario }) => + (scenario.coverage?.primary ?? []).some((coverageId) => + uncoveredCoverageIds.includes(coverageId), + ), + ) + .map(({ scenario, reasons }) => `${scenario.id} (${reasons.join(", ")})`) + .toSorted(); + const exclusionDetails = + excludedOwners.length > 0 ? ` Excluded owners: ${excludedOwners.join("; ")}.` : ""; + throw new Error( + `${QA_SMOKE_PROFILE} taxonomy profile leaves coverage IDs without eligible CI scenarios: ${uncoveredCoverageIds.join(", ")}.${exclusionDetails}`, + ); + } + + const matrixScenarios = scenarios.filter( + (scenario) => selectQaSmokeCiEligibilityChannel(scenario) === "matrix", + ); + const primaryScenarios = scenarios + .filter((scenario) => selectQaSmokeCiEligibilityChannel(scenario) !== "matrix") .toSorted( (left, right) => estimateScenarioCost(right) - estimateScenarioCost(left) || left.id.localeCompare(right.id), @@ -112,7 +163,7 @@ export function createQaSmokeCiPart(partId: string): QaSmokeCiPart { if (!firstPartition) { throw new Error(`${QA_SMOKE_PROFILE} declares no CI profile parts.`); } - for (const scenario of defaultChannelScenarios) { + for (const scenario of primaryScenarios) { const partition = partitions.reduce( (lightest, candidate) => (candidate.cost < lightest.cost ? candidate : lightest), firstPartition, @@ -121,8 +172,8 @@ export function createQaSmokeCiPart(partId: string): QaSmokeCiPart { partition.cost += estimateScenarioCost(scenario); } - // The matrix channel run rides on the last part so the greedy cost balance - // above stays undisturbed for the shared default-channel scenarios. + // The Matrix run rides on the last part so the greedy cost balance above + // stays undisturbed for scenarios that use the run-level channel driver. const matrixPartIndex = QA_SMOKE_CI_PARTS.length - 1; const partIndex = QA_SMOKE_CI_PARTS.indexOf(partId); const selectedPartition = partitions[partIndex]; @@ -131,14 +182,12 @@ export function createQaSmokeCiPart(partId: string): QaSmokeCiPart { } const runs: QaSmokeCiRun[] = [ { - channel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL, slug: "primary", scenario_ids: selectedPartition.scenarios.map((scenario) => scenario.id).toSorted(), }, ]; if (partIndex === matrixPartIndex) { runs.push({ - channel: "matrix", slug: "matrix", scenario_ids: matrixScenarios.map((scenario) => scenario.id).toSorted(), }); diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 0762a9d72461..c4c960bde8eb 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -385,15 +385,15 @@ describe("qa cli runtime", () => { { test: { kind: "qa-scenario", - id: "channel-top-level-reply-shape", - title: "Channel top-level reply shape", + id: "telegram-commands-command", + title: "Telegram commands list reply", source: { - path: "qa/scenarios/channels/channel-top-level-reply-shape.yaml", + path: "qa/scenarios/channels/telegram-commands-command.yaml", }, }, coverage: [ { - id: "channels.group-final-reply", + id: "telegram.built-in-commands", role: "primary", }, ], @@ -440,9 +440,9 @@ describe("qa cli runtime", () => { repoRoot: "/tmp/openclaw-repo", outputDir: ".artifacts/qa-e2e/smoke-ci", profile: "smoke-ci", - surface: "channels", - category: "channels.outbound-delivery-and-reply-pipeline", - scenarioIds: ["channel-top-level-reply-shape"], + surface: "telegram", + category: "telegram.native-controls-and-approvals", + scenarioIds: ["telegram-commands-command"], transportId: "qa-channel", fastMode: true, concurrency: 2, @@ -463,7 +463,7 @@ describe("qa cli runtime", () => { channel: "telegram", channelDriver: "crabline", }); - expect(suiteArgs.scenarioIds).toEqual(["channel-top-level-reply-shape"]); + expect(suiteArgs.scenarioIds).toEqual(["telegram-commands-command"]); expect(process.env.OPENCLAW_QA_PROFILE).toBe("release"); const evidence = JSON.parse(await fs.readFile(suiteEvidencePath, "utf8")) as { evidenceMode?: unknown; @@ -489,15 +489,11 @@ describe("qa cli runtime", () => { expect(evidence.scorecard).not.toHaveProperty("kind"); expect(evidence.scorecard).not.toHaveProperty("taxonomy"); expect(evidence.scorecard).not.toHaveProperty("profile"); - expect(evidence.scorecard?.coverageIds?.fulfilled).toBe(1); expect(evidence.scorecard?.categoryReports?.[0]).toMatchObject({ - id: "channels.outbound-delivery-and-reply-pipeline", - coverageIds: { - fulfilled: 1, - }, + id: "telegram.native-controls-and-approvals", }); expect(evidence.entries?.[0]).not.toHaveProperty("execution"); - expect(JSON.stringify(evidence.scorecard)).not.toContain("channel-top-level-reply-shape"); + expect(JSON.stringify(evidence.scorecard)).not.toContain("telegram-commands-command"); expectWriteContains(stdoutWrite, "QA run profile: smoke-ci; categories: 1; scenarios:"); expectWriteContains(stdoutWrite, `QA profile scorecard: ${suiteEvidencePath}`); } finally { @@ -529,12 +525,12 @@ describe("qa cli runtime", () => { profile: "release", surface: "channels", providerMode: "mock-openai", - scenarioIds: ["channel-chat-baseline", "telegram-help-command"], + scenarioIds: ["channel-chat-baseline", "thread-follow-up"], }); const suiteArgs = mockFirstObjectArg(runQaSuite); expect(suiteArgs.scenarioIds).toContain("channel-chat-baseline"); - expect(suiteArgs.scenarioIds).toContain("telegram-help-command"); + expect(suiteArgs.scenarioIds).toContain("thread-follow-up"); expect(suiteArgs.adapterFactories).toBe( listLiveTransportQaAdapterFactories.mock.results[0]?.value, ); @@ -574,7 +570,7 @@ describe("qa cli runtime", () => { const suiteArgs = mockFirstObjectArg(runQaSuite); expect(suiteArgs.channelDriver).toBe("crabline"); - expect(suiteArgs.scenarioIds).toContain("channel-top-level-reply-shape"); + expect(suiteArgs.scenarioIds).toContain("telegram-commands-command"); const scenarioById = new Map( readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]), ); @@ -587,33 +583,7 @@ describe("qa cli runtime", () => { ); }), ).toBe(true); - expect(suiteArgs.scenarioIds).not.toEqual( - expect.arrayContaining([ - "instruction-followthrough-repo-contract", - "subagent-forked-context", - "subagent-handoff", - "group-message-tool-unavailable-fallback", - "qa-channel-reconnect-dedupe", - "reaction-edit-delete", - "claude-cli-provider-capabilities", - "claude-cli-provider-capabilities-subscription", - "image-generation-roundtrip", - "image-understanding-attachment", - "native-image-generation", - "active-memory-preprompt-recall", - "memory-recall", - "session-memory-ranking", - "thread-memory-isolation", - "personal-channel-thread-reply", - "personal-memory-preference-recall", - "personal-reminder-roundtrip", - "cron-natural-fire-no-duplicate", - "cron-one-minute-ping", - "cron-single-run-no-duplicate", - "control-ui-qa-channel-image-roundtrip", - "config-apply-restart-wakeup", - ]), - ); + expect(suiteArgs.scenarioIds).not.toContain("control-ui-qa-channel-image-roundtrip"); }); it("rejects explicit profile selections with an incompatible scenario", async () => { @@ -621,7 +591,7 @@ describe("qa cli runtime", () => { runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", profile: "smoke-ci", - scenarioIds: ["channel-top-level-reply-shape", "control-ui-qa-channel-image-roundtrip"], + scenarioIds: ["control-ui-qa-channel-image-roundtrip"], }), ).rejects.toThrow( "qa run --qa-profile smoke-ci cannot run explicitly selected scenario(s): control-ui-qa-channel-image-roundtrip (channelDriver=qa-channel).", @@ -678,7 +648,9 @@ describe("qa cli runtime", () => { repoRoot: "/tmp/openclaw-repo", profile: "nightly", }), - ).rejects.toThrow('--qa-profile must be one of smoke-ci, release, all, got "nightly".'); + ).rejects.toThrow( + '--qa-profile must be one of smoke-ci, personal-agent, observability, release, all, got "nightly".', + ); expect(runQaSuite).not.toHaveBeenCalled(); }); @@ -1113,8 +1085,8 @@ describe("qa cli runtime", () => { expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( expect.objectContaining({ scenarioIds: expect.arrayContaining([ - "channel-canary", - "channel-mention-gating", + "telegram-commands-command", + "telegram-help-command", "telegram-other-bot-command-gating", ]), }), @@ -1145,7 +1117,7 @@ describe("qa cli runtime", () => { vi.stubEnv("OPENCLAW_QA_TELEGRAM_SUT_UID", "1001"); await runQaTelegramCommand({ repoRoot: candidateRoot, - scenarioIds: ["telegram-help-command", "telegram-stream-final-single-message"], + scenarioIds: ["telegram-help-command", "telegram-commands-command"], }); const sutOpenClawCommand = { @@ -1266,7 +1238,7 @@ describe("qa cli runtime", () => { repoRoot: "/tmp/openclaw-repo", scenarioIds: ["telegram-help-command", "missing-telegram-scenario"], }), - ).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario"); + ).rejects.toThrow("unknown QA scenario id(s): missing-telegram-scenario"); expect(runQaFlowSuiteFromRuntime).not.toHaveBeenCalled(); }); @@ -1729,31 +1701,6 @@ describe("qa cli runtime", () => { }); }); - it("expands the personal-agent pack onto the suite scenario list", async () => { - await runQaSuiteCommand({ - repoRoot: "/tmp/openclaw-repo", - pack: "personal-agent", - scenarioIds: ["channel-chat-baseline"], - }); - - expectFields(mockFirstObjectArg(runQaSuite), { - repoRoot: path.resolve("/tmp/openclaw-repo"), - scenarioIds: [ - "channel-chat-baseline", - "personal-reminder-roundtrip", - "personal-channel-thread-reply", - "personal-memory-preference-recall", - "personal-redaction-no-secret-leak", - "personal-tool-safety-followthrough", - "personal-approval-denial-stop", - "personal-task-followthrough-status", - "personal-share-safe-diagnostics-artifact", - "personal-no-fake-progress", - "personal-failure-recovery", - ], - }); - }); - it("expands runtime-pair lane selections onto the suite scenario list", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", @@ -1873,15 +1820,6 @@ describe("qa cli runtime", () => { ).rejects.toThrow('--runtime-pair-lane must be one of core, extended, soak, got "coreish".'); }); - it("rejects unknown suite packs", async () => { - await expect( - runQaSuiteCommand({ - repoRoot: "/tmp/openclaw-repo", - pack: "personal-admin", - }), - ).rejects.toThrow('--pack must be one of personal-agent, observability, got "personal-admin"'); - }); - it("rejects unknown suite CLI auth modes", async () => { await expect( runQaSuiteCommand({ diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 894866d97e65..a4c9462f1cc4 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -84,7 +84,6 @@ import { type QaRuntimePairLane, } from "./scenario-catalog.js"; import { scenarioMatchesQaProviderLane } from "./scenario-lane.js"; -import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js"; import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js"; import { qaScorecardChannelDriverSchema, @@ -150,7 +149,6 @@ export type QaSuiteCommandOptions = QaScenarioRunCommandOptions & { thinking?: string; cliAuthMode?: string; parityPack?: string; - pack?: string; scenarioIds?: string[]; enabledPluginIds?: string[]; image?: string; @@ -812,12 +810,9 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) { const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel); const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel); const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver); - const explicitScenarioIds = resolveQaScenarioPackScenarioIds({ - pack: opts.pack, - scenarioIds: resolveQaParityPackScenarioIds({ - parityPack: opts.parityPack, - scenarioIds: opts.scenarioIds, - }), + const explicitScenarioIds = resolveQaParityPackScenarioIds({ + parityPack: opts.parityPack, + scenarioIds: opts.scenarioIds, }); const runtimePairLanes = parseQaRuntimePairLaneFilters(opts.runtimePairLane); const runtimePairLaneSelection = resolveQaRuntimePairLaneScenarioIds({ diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 22d10fa7fe15..4a6b7846bb26 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -926,13 +926,6 @@ describe("qa cli registration", () => { expect(requireQaSuiteOptions()).toEqual(expect.objectContaining({ failFast: true, runner })); }); - it("forwards --pack for suite runs", async () => { - await program.parseAsync(["node", "openclaw", "qa", "suite", "--pack", "personal-agent"]); - - const options = requireQaSuiteOptions(); - expect(options.pack).toBe("personal-agent"); - }); - it("forwards --runtime-pair-lane for suite runs", async () => { await program.parseAsync([ "node", diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index a3114a4132bd..597c5a73e1bf 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -72,7 +72,6 @@ type QaSuiteCliOptions = QaScenarioRunCliOptions & { thinking?: QaSuiteCommandOptions["thinking"]; cliAuthMode?: QaSuiteCommandOptions["cliAuthMode"]; parityPack?: QaSuiteCommandOptions["parityPack"]; - pack?: QaSuiteCommandOptions["pack"]; scenario?: QaSuiteCommandOptions["scenarioIds"]; enablePlugin?: QaSuiteCommandOptions["enabledPluginIds"]; image?: QaSuiteCommandOptions["image"]; @@ -481,10 +480,6 @@ export function registerQaLabCli(program: Command) { "CLI backend auth mode for live Claude CLI runs: auto, api-key, or subscription", ) .option("--parity-pack ", 'Preset scenario pack; currently only "agentic" is supported') - .option( - "--pack ", - 'Scenario pack id; currently "personal-agent" and "observability" are supported', - ) .option("--scenario ", "Run only the named QA scenario (repeatable)", collectString, []) .option( "--enable-plugin ", @@ -535,7 +530,6 @@ export function registerQaLabCli(program: Command) { thinking: opts.thinking, cliAuthMode: opts.cliAuthMode, parityPack: opts.parityPack, - pack: opts.pack, scenarioIds: opts.scenario, enabledPluginIds: opts.enablePlugin, concurrency: opts.concurrency, diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index ab937db7c039..a62c110db0dd 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -53,6 +53,7 @@ function testMaturityTaxonomy(params?: { includeAllCategories: false, channelDriver: "crabline" as const, categoryIds: [categoryId], + coverageIds: [], }, { id: "release", @@ -60,6 +61,7 @@ function testMaturityTaxonomy(params?: { includeAllCategories: params?.includeAllCategories ?? false, channelDriver: "qa-channel" as const, categoryIds: params?.includeAllCategories ? [] : [categoryId], + coverageIds: [], }, ], surfaces: [ @@ -164,7 +166,7 @@ describe("qa coverage report", () => { expect(inventory.secondaryCoverageIdCount).toBeGreaterThan(0); expect(inventory.overlappingCoverage.length).toBeGreaterThan(0); expect(inventory.missingCoverage).toStrictEqual([]); - expect(inventory.scorecardTaxonomy.profileCount).toBe(3); + expect(inventory.scorecardTaxonomy.profileCount).toBe(5); expect( inventory.scorecardTaxonomy.profiles.find((profile) => profile.id === "smoke-ci"), ).toMatchObject({ @@ -236,18 +238,6 @@ describe("qa coverage report", () => { role: "primary", scenarioRefs: ["qa/scenarios/ui/control-ui-chat-flow-playwright.yaml"], }); - expect(inventory.scenarioPacks.map((pack) => pack.id)).toEqual([ - "observability", - "personal-agent", - ]); - const personalPack = inventory.scenarioPacks.find((pack) => pack.id === "personal-agent"); - const observabilityPack = inventory.scenarioPacks.find((pack) => pack.id === "observability"); - expect(personalPack?.missingScenarioIds).toStrictEqual([]); - expect(personalPack?.scenarioIds).toContain("personal-share-safe-diagnostics-artifact"); - expect(personalPack?.coverageIds).toContain("security.redaction-personal-redaction"); - expect(observabilityPack?.missingScenarioIds).toStrictEqual([]); - expect(observabilityPack?.scenarioIds).toEqual(["otel-trace-smoke", "docker-prometheus-smoke"]); - expect(observabilityPack?.coverageIds).toContain("observability.prometheus"); expect( expectDefined(inventory.byTheme.memory, "memory QA theme").map((coverage) => coverage.id), ).toContain("session-memory.memory-recall"); @@ -370,14 +360,6 @@ describe("qa coverage report", () => { expect(report).toContain("session-memory.embedding-search-recall"); expect(report).toContain("primary: memory-recall (qa/scenarios/memory/memory-recall.yaml)"); expect(report).toContain("secondary: active-memory-preprompt-recall"); - expect(report).toContain("## Scenario Packs"); - expect(report).toContain( - "- personal-agent (Personal Agent Benchmark Pack): 10 scenarios; coverage IDs:", - ); - expect(report).toContain( - "- observability (Observability Smoke Pack): 2 scenarios; coverage IDs:", - ); - expect(report).toContain("otel-trace-smoke, docker-prometheus-smoke"); expect(report).toContain("personal-share-safe-diagnostics-artifact"); expect(report).toContain("## Scorecard Taxonomy"); expect(report).toContain("- Taxonomy: taxonomy.yaml"); @@ -621,6 +603,7 @@ describe("qa coverage report", () => { includeAllCategories: false, channelDriver: "qa-channel", categoryIds: ["agent-runtime.agent-turn-execution"], + coverageIds: [], }, ], surfaces: [ diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index 610f6b4865c7..2fd69c836017 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -1,6 +1,6 @@ // Qa Lab plugin module implements coverage report behavior. import { normalizeStringEntriesLower } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { QA_SCENARIO_PACKS, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { readQaScorecardTaxonomyReport, type QaScorecardTaxonomyReport, @@ -39,14 +39,6 @@ type QaCoverageIdSummary = { scenarios: QaCoverageScenarioReference[]; }; -type QaCoverageScenarioPackSummary = { - id: string; - title: string; - scenarioIds: string[]; - coverageIds: string[]; - missingScenarioIds: string[]; -}; - type QaCoverageInventory = { scenarioCount: number; coverageIdCount: number; @@ -57,7 +49,6 @@ type QaCoverageInventory = { missingCoverage: QaCoverageScenarioSummary[]; byTheme: Record; bySurface: Record; - scenarioPacks: QaCoverageScenarioPackSummary[]; scorecardTaxonomy: QaScorecardTaxonomyReport; }; @@ -188,36 +179,6 @@ function sortCoverageIds(coverageIds: readonly QaCoverageIdSummary[]) { return coverageIds.toSorted((left, right) => left.id.localeCompare(right.id)); } -function buildScenarioPackSummaries( - scenarios: readonly QaSeedScenarioWithSource[], -): QaCoverageScenarioPackSummary[] { - const scenariosById = new Map(scenarios.map((scenario) => [scenario.id, scenario])); - return QA_SCENARIO_PACKS.map((pack) => { - const coverageIds = new Set(); - const missingScenarioIds: string[] = []; - for (const scenarioId of pack.scenarioIds) { - const scenario = scenariosById.get(scenarioId); - if (!scenario) { - missingScenarioIds.push(scenarioId); - continue; - } - for (const coverageId of [ - ...(scenario.coverage?.primary ?? []), - ...(scenario.coverage?.secondary ?? []), - ]) { - coverageIds.add(coverageId); - } - } - return { - id: pack.id, - title: pack.title, - scenarioIds: [...pack.scenarioIds], - coverageIds: [...coverageIds].toSorted(), - missingScenarioIds, - }; - }).toSorted((left, right) => left.id.localeCompare(right.id)); -} - export function buildQaCoverageInventory( scenarios: readonly QaSeedScenarioWithSource[], params?: { nonYamlScenarios?: readonly { id: string; sourcePath: string }[] }, @@ -292,7 +253,6 @@ export function buildQaCoverageInventory( missingCoverage, byTheme, bySurface, - scenarioPacks: buildScenarioPackSummaries(scenarios), scorecardTaxonomy: readQaScorecardTaxonomyReport(scenarios), }; } @@ -306,17 +266,6 @@ function pushCoverageIdLines(lines: string[], coverageIds: readonly QaCoverageId } } -function pushScenarioPackLines(lines: string[], packs: readonly QaCoverageScenarioPackSummary[]) { - for (const pack of packs) { - const missing = - pack.missingScenarioIds.length > 0 ? pack.missingScenarioIds.join(", ") : "none"; - lines.push( - `- ${pack.id} (${pack.title}): ${pack.scenarioIds.length} scenarios; coverage IDs: ${pack.coverageIds.join(", ")}; missing scenarios: ${missing}`, - ); - lines.push(` - scenarios: ${pack.scenarioIds.join(", ")}`); - } -} - function pushScorecardTaxonomyLines(lines: string[], report: QaScorecardTaxonomyReport) { lines.push("## Scorecard Taxonomy", ""); lines.push(`- Taxonomy: ${report.taxonomyPath ?? "missing"}`); @@ -393,12 +342,6 @@ export function renderQaCoverageMarkdownReport(inventory: QaCoverageInventory): "", ]; - if (inventory.scenarioPacks.length > 0) { - lines.push("## Scenario Packs", ""); - pushScenarioPackLines(lines, inventory.scenarioPacks); - lines.push(""); - } - lines.push("## By Theme", ""); for (const theme of Object.keys(inventory.byTheme).toSorted()) { lines.push(`### ${theme}`, ""); diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts index e6fd461ce409..10c7b16b9b10 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { afterEach, describe, expect, it, vi } from "vitest"; import { discordQaScenarioSupport } from "./discord-live.runtime.js"; +import { resolveDiscordQaScenarioIds } from "./scenario-selection.js"; const { testing } = discordQaScenarioSupport; @@ -285,11 +286,9 @@ describe("discord live qa runtime", () => { }); it("includes the Discord live scenarios", () => { - expect(testing.findScenario().map((scenario) => scenario.id)).toEqual([ - "discord-canary", - "discord-mention-gating", - "discord-native-help-command-registration", - ]); + expect(testing.findScenario().map((scenario) => scenario.id)).toEqual( + resolveDiscordQaScenarioIds({}), + ); expect( testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id), ).toEqual(["discord-status-reactions-tool-only"]); @@ -465,7 +464,7 @@ describe("discord live qa runtime", () => { it("fails when any requested Discord scenario id is unknown", () => { expect(() => testing.findScenario(["discord-canary", "typo-scenario"])).toThrow( - "unknown Discord QA scenario id(s): typo-scenario", + "unknown QA scenario id(s): typo-scenario", ); }); diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts index 31999c0c556a..98a96a1587fa 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -18,8 +18,8 @@ import { chromium } from "playwright-core"; import { z } from "zod"; import { startQaGatewayChild } from "../../gateway-child.js"; import { isTruthyOptIn } from "../../mantis-options.runtime.js"; -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; import { assertLiveScenarioReply as assertDiscordScenarioReply } from "../shared/live-scenario-reply.js"; +import { resolveDiscordQaScenarioIds } from "./scenario-selection.js"; type DiscordQaRuntimeEnv = { guildId: string; @@ -1345,10 +1345,7 @@ function buildObservedMessagesArtifact(params: { } function findScenario(ids?: string[]) { - const requestedIds = - ids && ids.length > 0 - ? ids - : listQaScenariosForExecutionProfile("discord:adapter").map((scenario) => scenario.id); + const requestedIds = resolveDiscordQaScenarioIds({ scenarioIds: ids }); const scenariosById = new Map(DISCORD_QA_SCENARIOS.map((scenario) => [scenario.id, scenario])); const missingIds = requestedIds.filter((id) => !scenariosById.has(id as DiscordQaScenarioId)); if (missingIds.length > 0) { diff --git a/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts b/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts index 1ec43b320f37..bc3996803cf2 100644 --- a/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts +++ b/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts @@ -1,7 +1,14 @@ -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; +import type { QaProviderModeInput } from "../../model-selection.js"; +import { resolveLiveTransportQaScenarioIds } from "../shared/scenario-selection.js"; -export function resolveDiscordQaScenarioIds({ scenarioIds }: { scenarioIds?: readonly string[] }) { - return scenarioIds?.length - ? [...scenarioIds] - : listQaScenariosForExecutionProfile("discord:adapter").map((scenario) => scenario.id); +export function resolveDiscordQaScenarioIds(params: { + profile?: string; + providerMode?: QaProviderModeInput; + scenarioIds?: readonly string[]; +}) { + return resolveLiveTransportQaScenarioIds({ + channelId: "discord", + ...params, + providerMode: params.providerMode ?? "live-frontier", + }); } diff --git a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts index c15f43f14486..327d63410603 100644 --- a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts +++ b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it, vi } from "vitest"; import { createQaBusState } from "../bus-state.js"; import { createQaChannelTransport } from "../qa-channel-transport.js"; import { createQaTransportAdapter } from "../qa-transport-registry.js"; -import { listQaScenariosForExecutionProfile } from "../scenario-catalog.js"; const { createDiscord, createMatrix, createSlack, createTelegram, createWhatsApp } = vi.hoisted( () => ({ @@ -63,32 +62,6 @@ describe("live transport adapter factories", () => { expect(whatsappQaAdapterFactory.isolatesInstances).toBeUndefined(); }); - it("selects Slack generic defaults from the YAML adapter profile", () => { - expect( - listQaScenariosForExecutionProfile("slack:adapter").map((scenario) => scenario.id), - ).toEqual([ - "channel-chat-baseline", - "channel-canary", - "channel-mention-gating", - "channel-top-level-reply-shape", - "thread-follow-up", - "thread-isolation", - ]); - }); - - it("selects WhatsApp DM-safe defaults from the YAML adapter profile", () => { - expect( - listQaScenariosForExecutionProfile("whatsapp:adapter").map((scenario) => scenario.id), - ).toEqual([ - "dm-chat-baseline", - "channel-canary", - "channel-dm-group-routing", - "channel-mention-gating", - "channel-top-level-reply-shape", - "whatsapp-help-command", - ]); - }); - it.each([ ["discord", createDiscord], ["matrix", createMatrix], diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts index a962347ccced..10a1a390f37e 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts @@ -63,15 +63,16 @@ describe("QA Lab Matrix CLI registration", () => { "--model", "--alt-model", "--scenario", + "--shard", "--fast", "--fail-fast", - "--profile", "--sut-account", ]) { expect(optionNames).toContain(optionName); } - expect(optionNames).not.toContain("--credential-source"); - expect(optionNames).not.toContain("--credential-role"); + for (const optionName of ["--profile", "--credential-source", "--credential-role"]) { + expect(optionNames).not.toContain(optionName); + } }); it("delegates command options to the Matrix runtime", async () => { @@ -83,10 +84,10 @@ describe("QA Lab Matrix CLI registration", () => { "node", "openclaw", "matrix", - "--profile", - "release", "--scenario", "matrix-allowlist-hot-reload", + "--shard", + "1/1", ]); expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith( @@ -94,9 +95,9 @@ describe("QA Lab Matrix CLI registration", () => { channelId: "matrix", credentialMode: "env-only", options: expect.objectContaining({ - profile: "release", providerMode: "live-frontier", scenarioIds: ["matrix-allowlist-hot-reload"], + shard: "1/1", }), }), ); diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.ts b/extensions/qa-lab/src/live-transports/matrix/cli.ts index 0e410fd50481..cb9a784460d6 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.ts @@ -8,7 +8,7 @@ import { type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -import { resolveMatrixQaScenarioIds } from "./profiles.js"; +import { resolveMatrixQaScenarioIds } from "./scenario-selection.js"; const DISABLE_MATRIX_QA_FORCE_EXIT_ENV = "OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT"; @@ -46,7 +46,11 @@ async function runQaMatrix(opts: LiveTransportQaCommandOptions) { envCredentialReason: "its homeserver is disposable and local.", laneLabel: "Matrix", options: opts, - selectScenarioIds: resolveMatrixQaScenarioIds, + selectScenarioIds: (selection) => + resolveMatrixQaScenarioIds({ + scenarioIds: selection.scenarioIds, + shard: opts.shard, + }), }); }; if (process.env[DISABLE_MATRIX_QA_FORCE_EXIT_ENV] === "1") { @@ -79,9 +83,8 @@ export const matrixQaCliRegistration: LiveTransportQaCliRegistration = defaultProviderMode: "live-frontier", description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver", outputDirHelp: "Matrix QA artifact directory", - profileHelp: - "QA Lab Matrix profile: all, fast, release, transport, media, e2ee-smoke, e2ee-deep, or e2ee-cli (default: all)", scenarioHelp: "Run only the named Matrix QA scenario (repeatable)", + shardHelp: "Run one deterministic Matrix catalog shard as /", failFastHelp: "Stop after the first failed Matrix QA scenario", sutAccountHelp: "Temporary Matrix account id inside the QA gateway config", run: runQaMatrix, diff --git a/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts b/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts deleted file mode 100644 index ffbe006ed215..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { readQaScenarioById } from "../../scenario-catalog.js"; -import { resolveMatrixQaScenarioIds } from "./profiles.js"; - -const MATRIX_QA_PROFILE_NAMES = [ - "all", - "fast", - "release", - "transport", - "media", - "e2ee-smoke", - "e2ee-deep", - "e2ee-cli", -] as const; - -describe("QA Lab Matrix profiles", () => { - it("preserves the profile sizes and default selection", () => { - const allScenarioIds = resolveMatrixQaScenarioIds({ profile: "all" }); - expect(allScenarioIds).toHaveLength(93); - expect(resolveMatrixQaScenarioIds({ profile: "fast" })).toHaveLength(12); - expect(resolveMatrixQaScenarioIds({ profile: "release" })).toEqual([ - "channel-chat-baseline", - "matrix-allowlist-hot-reload", - ]); - expect(resolveMatrixQaScenarioIds({ profile: "transport" })).toHaveLength(50); - expect(resolveMatrixQaScenarioIds({ profile: "media" })).toHaveLength(7); - expect(resolveMatrixQaScenarioIds({ profile: "e2ee-smoke" })).toHaveLength(8); - expect(resolveMatrixQaScenarioIds({ profile: "e2ee-deep" })).toHaveLength(18); - expect(resolveMatrixQaScenarioIds({ profile: "e2ee-cli" })).toHaveLength(9); - expect(resolveMatrixQaScenarioIds({})).toEqual(allScenarioIds); - expect(resolveMatrixQaScenarioIds({ profile: "fast" })).toContain("channel-chat-baseline"); - expect(allScenarioIds).toContain("channel-chat-baseline"); - }); - - it("keeps every profile unique, catalog-backed, and contained by all", () => { - const allScenarioIds = resolveMatrixQaScenarioIds({ profile: "all" }); - const allScenarioIdSet = new Set(allScenarioIds); - for (const profile of MATRIX_QA_PROFILE_NAMES) { - const scenarioIds = resolveMatrixQaScenarioIds({ profile }); - expect(new Set(scenarioIds).size).toBe(scenarioIds.length); - for (const scenarioId of scenarioIds) { - expect(readQaScenarioById(scenarioId).id).toBe(scenarioId); - if (profile !== "all") { - expect(allScenarioIdSet.has(scenarioId), `${profile}:${scenarioId}`).toBe(true); - } - } - } - }); - - it("keeps explicit-only scenarios out of every named profile", () => { - const profileScenarioIds = MATRIX_QA_PROFILE_NAMES.flatMap((profile) => - resolveMatrixQaScenarioIds({ profile }), - ); - - expect(profileScenarioIds).not.toContain("matrix-room-block-streaming"); - expect(profileScenarioIds).not.toContain("subagent-thread-spawn"); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/matrix/profiles.ts b/extensions/qa-lab/src/live-transports/matrix/profiles.ts deleted file mode 100644 index 80844c052349..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/profiles.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; - -const MATRIX_QA_PROFILES = [ - "all", - "fast", - "release", - "transport", - "media", - "e2ee-smoke", - "e2ee-deep", - "e2ee-cli", -] as const; - -export function resolveMatrixQaScenarioIds(params: { - profile?: string; - scenarioIds?: readonly string[]; -}): string[] { - if (params.scenarioIds?.length) { - return [...params.scenarioIds]; - } - const profile = params.profile?.trim() || "all"; - if (!MATRIX_QA_PROFILES.some((candidate) => candidate === profile)) { - throw new Error( - `Unknown QA Lab Matrix profile "${profile}". Expected one of: ${MATRIX_QA_PROFILES.join(", ")}.`, - ); - } - return listQaScenariosForExecutionProfile(`matrix:${profile}`).map((scenario) => scenario.id); -} diff --git a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts new file mode 100644 index 000000000000..a67605cbd79d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { readQaScenarioPack } from "../../scenario-catalog.js"; +import { resolveMatrixQaScenarioIds } from "./scenario-selection.js"; + +describe("QA Lab Matrix scenario selection", () => { + it("derives the default set from explicit Matrix channel eligibility", () => { + const catalog = readQaScenarioPack().scenarios; + const scenarioById = new Map(catalog.map((scenario) => [scenario.id, scenario] as const)); + const scenarioIds = resolveMatrixQaScenarioIds({}); + + expect(scenarioIds.length).toBeGreaterThan(0); + expect(new Set(scenarioIds).size).toBe(scenarioIds.length); + for (const scenarioId of scenarioIds) { + const scenario = scenarioById.get(scenarioId); + expect(scenario?.execution.kind, scenarioId).toBe("flow"); + expect( + scenario?.execution.channel === "matrix" || + (scenario?.execution.kind === "flow" && scenario.execution.channels?.includes("matrix")), + scenarioId, + ).toBe(true); + } + + const unrelatedScenario = catalog.find( + (scenario) => + scenario.execution.kind === "flow" && + scenario.execution.channel !== "matrix" && + !scenario.execution.channels?.includes("matrix"), + ); + expect(unrelatedScenario).toBeDefined(); + expect(scenarioIds).not.toContain(unrelatedScenario?.id); + }); + + it("preserves an explicit scenario subset without a named profile", () => { + const explicitScenarioIds = resolveMatrixQaScenarioIds({}).slice(0, 2).toReversed(); + + expect(resolveMatrixQaScenarioIds({ scenarioIds: explicitScenarioIds })).toEqual( + explicitScenarioIds, + ); + }); + + it("balances deterministic shards independently of input order", () => { + const scenarioIds = resolveMatrixQaScenarioIds({}); + const shardValues = ["1/5", "2/5", "3/5", "4/5", "5/5"] as const; + const shards = shardValues.map((shard) => resolveMatrixQaScenarioIds({ scenarioIds, shard })); + const sizes = shards.map((shard) => shard.length); + + expect(Math.max(...sizes) - Math.min(...sizes)).toBeLessThanOrEqual(1); + expect(shards.flat().toSorted()).toEqual(scenarioIds.toSorted()); + expect(new Set(shards.flat()).size).toBe(scenarioIds.length); + expect( + resolveMatrixQaScenarioIds({ scenarioIds: scenarioIds.toReversed(), shard: shardValues[0] }), + ).toEqual(shards[0]); + }); + + it("rejects invalid and empty shard selections honestly", () => { + expect(() => resolveMatrixQaScenarioIds({ scenarioIds: ["scenario"], shard: "0/5" })).toThrow( + "Expected /", + ); + expect(() => resolveMatrixQaScenarioIds({ scenarioIds: ["scenario"], shard: "2/5" })).toThrow( + "resolved no scenarios", + ); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts new file mode 100644 index 000000000000..ecbaa3689d1d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/matrix/scenario-selection.ts @@ -0,0 +1,82 @@ +import { readQaScenarioPack } from "../../scenario-catalog.js"; + +const MATRIX_QA_CHANNEL_ID = "matrix"; + +type QaScenarioShard = { + index: number; + total: number; +}; + +function compareStrings(left: string, right: string) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function hashScenarioId(id: string) { + let hash = 0x811c9dc5; + for (let index = 0; index < id.length; index += 1) { + hash ^= id.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +function parseQaScenarioShard(value: string): QaScenarioShard { + const match = /^(\d+)\/(\d+)$/.exec(value.trim()); + const index = Number.parseInt(match?.[1] ?? "", 10); + const total = Number.parseInt(match?.[2] ?? "", 10); + if ( + !match || + !Number.isSafeInteger(index) || + !Number.isSafeInteger(total) || + index < 1 || + total < 1 || + index > total + ) { + throw new Error( + `Invalid Matrix QA shard "${value}". Expected / with 1 <= index <= total.`, + ); + } + return { index, total }; +} + +function shardMatrixQaScenarioIds(scenarioIds: readonly string[], shardValue: string): string[] { + const shard = parseQaScenarioShard(shardValue); + // Hash ordering makes shard membership independent of catalog/YAML order while + // round-robin assignment keeps every shard within one scenario of the others. + const orderedIds = [...new Set(scenarioIds)].toSorted((left, right) => { + const hashDelta = hashScenarioId(left) - hashScenarioId(right); + return hashDelta || compareStrings(left, right); + }); + const selectedIds = orderedIds.filter((_, index) => index % shard.total === shard.index - 1); + if (selectedIds.length === 0) { + throw new Error( + `Matrix QA shard ${shardValue} resolved no scenarios from ${orderedIds.length} selected scenario(s).`, + ); + } + return selectedIds; +} + +function listMatrixQaScenarioIds(): string[] { + return readQaScenarioPack() + .scenarios.filter( + (scenario) => + scenario.execution.kind === "flow" && + (scenario.execution.channel === MATRIX_QA_CHANNEL_ID || + scenario.execution.channels?.includes(MATRIX_QA_CHANNEL_ID)), + ) + .map((scenario) => scenario.id) + .toSorted(compareStrings); +} + +export function resolveMatrixQaScenarioIds(params: { + scenarioIds?: readonly string[]; + shard?: string; +}): string[] { + const scenarioIds = params.scenarioIds?.length + ? [...new Set(params.scenarioIds)] + : listMatrixQaScenarioIds(); + if (scenarioIds.length === 0) { + throw new Error("Matrix QA catalog selection resolved no scenarios."); + } + return params.shard ? shardMatrixQaScenarioIds(scenarioIds, params.shard) : scenarioIds; +} diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index 100a6920976b..807d0c05f857 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -13,6 +13,7 @@ export type LiveTransportQaCommandOptions = { allowFailures?: boolean; failFast?: boolean; profile?: string; + shard?: string; scenarioIds?: string[]; listScenarios?: boolean; sutAccountId?: string; @@ -32,6 +33,7 @@ type LiveTransportQaCommanderOptions = { allowFailures?: boolean; failFast?: boolean; profile?: string; + shard?: string; sutAccount?: string; credentialSource?: string; credentialRole?: string; @@ -51,6 +53,7 @@ type LiveTransportQaCliRegistrationOptions = { listScenariosHelp?: string; outputDirHelp: string; profileHelp?: string; + shardHelp?: string; failFastHelp?: string; allowFailuresHelp?: string; scenarioHelp: string; @@ -83,6 +86,7 @@ function mapCommanderOptions(opts: LiveTransportQaCommanderOptions): LiveTranspo allowFailures: opts.allowFailures, failFast: opts.failFast, profile: opts.profile, + shard: opts.shard, scenarioIds: opts.scenario, listScenarios: opts.listScenarios, sutAccountId: opts.sutAccount, @@ -119,6 +123,9 @@ function createSharedLiveTransportQaCliRegistration( if (params.profileHelp) { command.option("--profile ", params.profileHelp); } + if (params.shardHelp) { + command.option("--shard ", params.shardHelp); + } if (params.failFastHelp) { command.option("--fail-fast", params.failFastHelp, false); } diff --git a/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts b/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts new file mode 100644 index 000000000000..008e370b5c89 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/scenario-selection.ts @@ -0,0 +1,54 @@ +import { + defaultQaModelForMode, + normalizeQaProviderMode, + type QaProviderModeInput, +} from "../../model-selection.js"; +import { + resolveQaProfileScenarios, + resolveQaRunProfileExecutionSelection, + scenarioDeclaresQaChannel, +} from "../../profile-planning.js"; +import { readQaScenarioPack } from "../../scenario-catalog.js"; + +export function resolveLiveTransportQaScenarioIds(params: { + channelId: string; + profile?: string; + providerMode: QaProviderModeInput; + scenarioIds?: readonly string[]; +}) { + return resolveQaProfileScenarios({ + profile: params.profile?.trim() || "release", + providerMode: params.providerMode, + channelDriver: "live", + channel: params.channelId, + requireDeclaredChannel: true, + scenarioIds: params.scenarioIds, + }).scenarios.map((scenario) => scenario.id); +} + +export function listLiveTransportQaScenarios(params: { + channelId: string; + providerMode: QaProviderModeInput; +}) { + const defaultIds = new Set(resolveLiveTransportQaScenarioIds(params)); + const providerMode = normalizeQaProviderMode(params.providerMode); + const eligibleScenarios = readQaScenarioPack().scenarios.filter((scenario) => + scenarioDeclaresQaChannel(scenario, params.channelId), + ); + const scenarios = resolveQaRunProfileExecutionSelection({ + scenarios: eligibleScenarios, + providerMode, + primaryModel: defaultQaModelForMode(providerMode), + channelDriver: "live", + channel: params.channelId, + }).selectedScenarios; + return scenarios.map((scenario) => { + return { + id: scenario.id, + title: scenario.title, + rationale: scenario.objective, + regressionRefs: scenario.regressionRefs ?? [], + defaultEnabled: defaultIds.has(scenario.id), + }; + }); +} diff --git a/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts b/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts index 24f88b07638d..d999d3dd487a 100644 --- a/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts +++ b/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts @@ -1,7 +1,14 @@ -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; +import type { QaProviderModeInput } from "../../model-selection.js"; +import { resolveLiveTransportQaScenarioIds } from "../shared/scenario-selection.js"; -export function resolveSlackQaScenarioIds({ scenarioIds }: { scenarioIds?: readonly string[] }) { - return scenarioIds?.length - ? [...scenarioIds] - : listQaScenariosForExecutionProfile("slack:default").map((scenario) => scenario.id); +export function resolveSlackQaScenarioIds(params: { + profile?: string; + providerMode?: QaProviderModeInput; + scenarioIds?: readonly string[]; +}) { + return resolveLiveTransportQaScenarioIds({ + channelId: "slack", + ...params, + providerMode: params.providerMode ?? "live-frontier", + }); } diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 09e26347c46e..c5c47a864652 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -139,7 +139,7 @@ describe("Slack live QA runtime helpers", () => { ]); }); - it("selects opt-in native scenarios by id without changing standard scenario coverage", () => { + it("selects native scenarios by explicit id", () => { expect( testing .findScenario([ @@ -173,15 +173,6 @@ describe("Slack live QA runtime helpers", () => { "slack-codex-approval-exec-native", "slack-codex-approval-plugin-native", ]); - expect(testing.findScenario().map((scenario) => scenario.id)).not.toContain( - "slack-table-invalid-blocks-fallback", - ); - expect(testing.findScenario().map((scenario) => scenario.id)).not.toContain( - "slack-progress-commentary-true", - ); - expect(testing.findScenario().map((scenario) => scenario.id)).not.toContain( - "slack-channel-disabled-warning", - ); expect(testing.findScenario(["slack-codex-approval-exec-native"])[0]?.forcedRuntime).toBe( "codex", ); diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index 8305f61c6bd2..7d07881254c3 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -10,8 +10,8 @@ import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import { createTelegramQaTransportAdapter } from "./adapter.runtime.js"; -import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js"; import { resolveTelegramQaRunOptions } from "./run-options.runtime.js"; +import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./scenario-selection.js"; const TELEGRAM_QA_SUT_OPENCLAW_COMMAND_ENV = "OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND"; const TELEGRAM_QA_SUT_UID_ENV = "OPENCLAW_QA_TELEGRAM_SUT_UID"; diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index b5e83e64b6fc..a815c40e2a72 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -31,7 +31,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration = description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness", listScenariosHelp: "Print available Telegram scenario ids and exit", outputDirHelp: "Telegram QA artifact directory", - profileHelp: "QA Lab Telegram profile: release or all (default: release)", + profileHelp: "Taxonomy profile for Telegram scenario selection (default: release)", async run(opts: LiveTransportQaCommandOptions) { await (await loadTelegramQaCliRuntime()).runQaTelegramCommand(opts); }, diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts index 6fc1ebcb84fe..8be051cf3baa 100644 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts @@ -1,21 +1,25 @@ import { describe, expect, it } from "vitest"; -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; -import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js"; +import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./scenario-selection.js"; describe("Telegram QA profiles", () => { - it("keeps release focused and adds the scripted long-final check for mock runs", () => { + it("derives release membership from taxonomy and provider eligibility", () => { const live = resolveTelegramQaScenarioIds({ providerMode: "live-frontier" }); const mock = resolveTelegramQaScenarioIds({ providerMode: "mock-openai" }); expect(live).toContain("telegram-other-bot-command-gating"); expect(live).not.toContain("telegram-long-final-reuses-preview"); - expect(mock).toEqual([...live, "telegram-long-final-reuses-preview"]); + expect(mock).toContain("telegram-long-final-reuses-preview"); + expect(mock).toContain("telegram-assistant-transcript-role-boundary"); }); - it("selects every migrated Telegram scenario through all", () => { - expect(resolveTelegramQaScenarioIds({ providerMode: "mock-openai", profile: "all" })).toEqual( - listQaScenariosForExecutionProfile("telegram:all").map((scenario) => scenario.id), - ); + it("selects every taxonomy-owned executable Telegram scenario through all", () => { + const scenarioIds = resolveTelegramQaScenarioIds({ + providerMode: "mock-openai", + profile: "all", + }); + + expect(scenarioIds).toContain("channel-message-flows"); + expect(scenarioIds).toContain("native-command-session-target"); }); it("lets explicit scenarios override profile selection", () => { @@ -23,36 +27,35 @@ describe("Telegram QA profiles", () => { resolveTelegramQaScenarioIds({ profile: "release", providerMode: "live-frontier", - scenarioIds: ["thread-follow-up"], + scenarioIds: ["telegram-help-command"], }), - ).toEqual(["thread-follow-up"]); + ).toEqual(["telegram-help-command"]); }); - it("rejects unknown profiles and leaves explicit scenario validation to the suite catalog", () => { + it("rejects unknown profiles and channel-ineligible explicit scenarios", () => { expect(() => resolveTelegramQaScenarioIds({ providerMode: "live-frontier", profile: "transport" }), - ).toThrow('Unknown QA Lab Telegram profile "transport"'); - expect( + ).toThrow("QA run profile must be one of"); + expect(() => resolveTelegramQaScenarioIds({ providerMode: "live-frontier", scenarioIds: ["channel-chat-baseline"], }), - ).toEqual(["channel-chat-baseline"]); + ).toThrow("cannot run ineligible scenario(s)"); }); - it("lists the YAML catalog with provider-specific release defaults", () => { + it("lists catalog-eligible scenarios with provider-specific release defaults", () => { const scenarios = listTelegramQaScenarios("mock-openai"); + const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode: "mock-openai" })); - expect(scenarios.map(({ id }) => id).toSorted()).toEqual( - listQaScenariosForExecutionProfile("telegram:all") - .map((scenario) => scenario.id) - .toSorted(), - ); + expect( + new Set(scenarios.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)), + ).toEqual(defaultIds); expect( scenarios.find(({ id }) => id === "telegram-long-final-reuses-preview")?.defaultEnabled, ).toBe(true); expect( scenarios.find(({ id }) => id === "telegram-long-final-three-chunks")?.defaultEnabled, - ).toBe(false); + ).toBe(true); }); }); diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.ts deleted file mode 100644 index d1838cb6d9dc..000000000000 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { listQaScenariosForExecutionProfile, readQaScenarioPack } from "../../scenario-catalog.js"; - -type TelegramQaProfile = "all" | "release"; - -function resolveTelegramQaProfile(profile: string | undefined): TelegramQaProfile { - const normalized = profile?.trim() || "release"; - if (normalized === "all" || normalized === "release") { - return normalized; - } - throw new Error( - `Unknown QA Lab Telegram profile "${normalized}". Expected one of: all, release.`, - ); -} - -function listTelegramQaProfileScenarios(profile: string) { - return listQaScenariosForExecutionProfile(`telegram:${profile}`); -} - -export function resolveTelegramQaScenarioIds(params: { - profile?: string; - providerMode: string; - scenarioIds?: readonly string[]; -}): string[] { - if (params.scenarioIds?.length) { - const knownIds = new Set(readQaScenarioPack().scenarios.map((scenario) => scenario.id)); - const unknownIds = params.scenarioIds.filter((id) => !knownIds.has(id)); - if (unknownIds.length > 0) { - throw new Error(`unknown Telegram QA scenario id(s): ${unknownIds.join(", ")}`); - } - return [...params.scenarioIds]; - } - const profile = resolveTelegramQaProfile(params.profile); - const executionProfile = - profile === "release" && params.providerMode === "mock-openai" ? "mock-release" : profile; - return listTelegramQaProfileScenarios(executionProfile).map((scenario) => scenario.id); -} - -export function listTelegramQaScenarios(providerMode: string) { - const defaultIds = new Set(resolveTelegramQaScenarioIds({ providerMode, profile: "release" })); - return listTelegramQaProfileScenarios("all").map((scenario) => ({ - id: scenario.id, - title: scenario.title, - rationale: scenario.objective, - regressionRefs: scenario.regressionRefs ?? [], - defaultEnabled: defaultIds.has(scenario.id), - })); -} diff --git a/extensions/qa-lab/src/live-transports/telegram/scenario-selection.ts b/extensions/qa-lab/src/live-transports/telegram/scenario-selection.ts new file mode 100644 index 000000000000..83cdc552f2bc --- /dev/null +++ b/extensions/qa-lab/src/live-transports/telegram/scenario-selection.ts @@ -0,0 +1,22 @@ +import type { QaProviderModeInput } from "../../model-selection.js"; +import { + listLiveTransportQaScenarios, + resolveLiveTransportQaScenarioIds, +} from "../shared/scenario-selection.js"; + +const TELEGRAM_QA_CHANNEL_ID = "telegram"; + +export function resolveTelegramQaScenarioIds(params: { + profile?: string; + providerMode: QaProviderModeInput; + scenarioIds?: readonly string[]; +}): string[] { + return resolveLiveTransportQaScenarioIds({ channelId: TELEGRAM_QA_CHANNEL_ID, ...params }); +} + +export function listTelegramQaScenarios(providerMode: QaProviderModeInput) { + return listLiveTransportQaScenarios({ + channelId: TELEGRAM_QA_CHANNEL_ID, + providerMode, + }); +} diff --git a/extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts b/extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts index 88a78e2a3cb3..7a15c0cd0e4e 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts @@ -1,12 +1,10 @@ -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; +import type { QaProviderModeInput } from "../../model-selection.js"; +import { resolveLiveTransportQaScenarioIds } from "../shared/scenario-selection.js"; export function resolveWhatsAppQaScenarioIds(params: { - providerMode: string; + profile?: string; + providerMode: QaProviderModeInput; scenarioIds?: readonly string[]; }) { - if (params.scenarioIds?.length) { - return [...params.scenarioIds]; - } - const profile = params.providerMode === "mock-openai" ? "mock-default" : "default"; - return listQaScenariosForExecutionProfile(`whatsapp:${profile}`).map((scenario) => scenario.id); + return resolveLiveTransportQaScenarioIds({ channelId: "whatsapp", ...params }); } diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index f40136f205f1..07002f4053b8 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -1011,64 +1011,20 @@ describe("WhatsApp QA live runtime", () => { }); }); - it("keeps mock-backed and native approval scenarios out of default live-frontier selection", () => { - const expectedDefaultIds = [ - "whatsapp-canary", - "whatsapp-mention-gating", - "whatsapp-top-level-reply-shape", - "whatsapp-reply-to-message", - "whatsapp-group-reply-to-message", - "whatsapp-status-reactions", - "whatsapp-group-allowlist-block", - ]; + it("derives live-frontier selection from taxonomy and provider eligibility", () => { + const scenarioIds = testing.resolveWhatsAppQaScenarioIds({ providerMode: "live-frontier" }); - expect( - testing.resolveWhatsAppQaScenarioIds({ providerMode: "live-frontier" }).slice(0, -1), - ).toEqual(expectedDefaultIds); + expect(scenarioIds).toContain("whatsapp-canary"); + expect(scenarioIds).toContain("whatsapp-mention-gating"); + expect(scenarioIds).not.toContain("whatsapp-native-new-command"); }); - it("adds deterministic audio preflight to the default mock-openai WhatsApp selection", () => { - expect(testing.resolveWhatsAppQaScenarioIds({ providerMode: "mock-openai" })).toEqual([ - "whatsapp-canary", - "whatsapp-mention-gating", - "whatsapp-group-pending-history-context", - "whatsapp-broadcast-group-fanout", - "whatsapp-group-activation-always", - "whatsapp-group-reply-to-bot-triggers", - "whatsapp-top-level-reply-shape", - "whatsapp-reply-to-message", - "whatsapp-group-reply-to-message", - "whatsapp-reply-to-mode-batched", - "whatsapp-agent-message-action-react", - "whatsapp-agent-message-action-upload-file", - "whatsapp-group-agent-message-action-react", - "whatsapp-group-agent-message-action-upload-file", - "whatsapp-inbound-reaction-no-trigger", - "whatsapp-reply-context-isolation", - "whatsapp-inbound-image-caption", - "whatsapp-audio-preflight", - "whatsapp-outbound-media-matrix", - "whatsapp-outbound-document-preserves-filename", - "whatsapp-outbound-poll", - "whatsapp-group-outbound-media", - "whatsapp-group-outbound-audio", - "whatsapp-group-outbound-poll", - "whatsapp-message-actions", - "whatsapp-inbound-structured-messages", - "whatsapp-group-audio-gating", - "whatsapp-reply-delivery-shape", - "whatsapp-stream-final-message-accounting", - "whatsapp-status-reactions", - "whatsapp-status-reaction-lifecycle", - "whatsapp-group-allowlist-block", - "whatsapp-help-command", - "whatsapp-commands-command", - "whatsapp-tools-compact-command", - "whatsapp-whoami-command", - "whatsapp-context-command", - "whatsapp-tool-only-usage-footer", - "whatsapp-native-new-command", - ]); + it("includes mock-only scenarios when the provider lane supports them", () => { + const scenarioIds = testing.resolveWhatsAppQaScenarioIds({ providerMode: "mock-openai" }); + + expect(scenarioIds).toContain("whatsapp-audio-preflight"); + expect(scenarioIds).toContain("whatsapp-native-new-command"); + expect(scenarioIds).toContain("whatsapp-tool-only-usage-footer"); }); it("defines Phase 2 WhatsApp group scenarios as mock-backed user-path scenarios", () => { diff --git a/extensions/qa-lab/src/profile-planning.ts b/extensions/qa-lab/src/profile-planning.ts index bb0b4ff67660..c417f3b04d83 100644 --- a/extensions/qa-lab/src/profile-planning.ts +++ b/extensions/qa-lab/src/profile-planning.ts @@ -1,7 +1,12 @@ // Qa Lab plugin module owns canonical taxonomy profile membership planning. import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { QaCliBackendAuthMode } from "./gateway-child.js"; -import type { QaProviderMode } from "./model-selection.js"; +import { + defaultQaModelForMode, + normalizeQaProviderMode, + type QaProviderMode, + type QaProviderModeInput, +} from "./model-selection.js"; import { readQaScenarioPack, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { describeQaProviderLaneMismatches } from "./scenario-lane.js"; import { @@ -75,7 +80,9 @@ export function resolveQaRunProfileMembership( const scenarioBySourcePath = new Map( scenarios.map((scenario) => [scenario.sourcePath, scenario] as const), ); - const profileScenarios = uniqueStrings(categories.flatMap((category) => category.scenarioRefs)) + const categoryScenarioRefs = new Set(categories.flatMap((category) => category.scenarioRefs)); + const profileScenarios = profile.scenarioRefs + .filter((scenarioRef) => categoryScenarioRefs.has(scenarioRef)) .map((scenarioRef) => scenarioBySourcePath.get(scenarioRef)) .filter((scenario): scenario is QaSeedScenarioWithSource => scenario !== undefined); const requestedScenarioIds = uniqueStrings( @@ -156,3 +163,91 @@ export function resolveQaRunProfileExecutionSelection(params: { } return { excludedScenarios, selectedScenarios }; } + +export function scenarioDeclaresQaChannel(scenario: QaSeedScenarioWithSource, channel: string) { + const normalizedChannel = channel.trim().toLowerCase(); + if (scenario.execution.channel === normalizedChannel) { + return true; + } + return ( + scenario.execution.kind === "flow" && + scenario.execution.channels?.includes(normalizedChannel) === true + ); +} + +export function resolveQaProfileScenarios(params: { + profile: string; + providerMode: QaProviderModeInput; + primaryModel?: string; + channelDriver?: QaScorecardChannelDriver; + channel?: string; + eligibleChannels?: readonly string[]; + requireDeclaredChannel?: boolean; + scenarioIds?: readonly string[]; +}) { + const membership = resolveQaRunProfileMembership({ + profile: params.profile, + scenarioIds: params.scenarioIds, + }); + if (membership.excludedScenarioIds.length > 0) { + throw new Error(`unknown QA scenario id(s): ${membership.excludedScenarioIds.join(", ")}`); + } + + const providerMode = normalizeQaProviderMode(params.providerMode); + const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const channelDriver = params.channelDriver ?? membership.profile.channelDriver; + const channel = params.channel?.trim().toLowerCase(); + const eligibleChannels = new Set( + params.eligibleChannels?.map((candidate) => candidate.trim().toLowerCase()), + ); + const evaluatedCandidates = membership.selectedScenarios.map((scenario) => { + const reasons: string[] = []; + if (params.requireDeclaredChannel && channel && !scenarioDeclaresQaChannel(scenario, channel)) { + reasons.push(`does not declare channel ${channel}`); + } + if (eligibleChannels.size > 0) { + const declaredChannels = scenario.execution.channel + ? [scenario.execution.channel] + : scenario.execution.kind === "flow" + ? (scenario.execution.channels ?? []) + : []; + if ( + declaredChannels.length > 0 && + !declaredChannels.some((declaredChannel) => eligibleChannels.has(declaredChannel)) + ) { + reasons.push( + `declared channels ${declaredChannels.join(", ")} do not match eligible channels ${[...eligibleChannels].join(", ")}`, + ); + } + } + reasons.push( + ...resolveQaRunProfileExecutionSelection({ + scenarios: [scenario], + providerMode, + primaryModel, + channelDriver, + channel: channel ?? scenario.execution.channel, + }).excludedScenarios.flatMap((entry) => entry.reasons), + ); + return { scenario, reasons: uniqueStrings(reasons) }; + }); + const scenarios = evaluatedCandidates + .filter(({ reasons }) => reasons.length === 0) + .map(({ scenario }) => scenario); + const excludedScenarios = evaluatedCandidates.filter(({ reasons }) => reasons.length > 0); + + if (params.scenarioIds?.length && excludedScenarios.length > 0) { + const ineligible = excludedScenarios + .map(({ scenario, reasons }) => `${scenario.id} (${reasons.join(", ")})`) + .toSorted(); + throw new Error( + `QA profile ${membership.profile.id} cannot run ineligible scenario(s) for the selected lane: ${ineligible.join("; ")}.`, + ); + } + if (scenarios.length === 0) { + throw new Error( + `QA taxonomy profile ${membership.profile.id} resolved no executable scenarios.`, + ); + } + return { profile: membership.profile, scenarios, excludedScenarios }; +} diff --git a/extensions/qa-lab/src/profile-selection.test.ts b/extensions/qa-lab/src/profile-selection.test.ts new file mode 100644 index 000000000000..175d9b77e70c --- /dev/null +++ b/extensions/qa-lab/src/profile-selection.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { resolveLiveTransportQaScenarioIds } from "./live-transports/shared/scenario-selection.js"; +import { resolveQaProfileScenarios } from "./profile-planning.js"; +import { readQaScenarioPack } from "./scenario-catalog.js"; +import { readQaScorecardTaxonomyReport } from "./scorecard-taxonomy.js"; + +describe("taxonomy profile scenario selection", () => { + it("resolves smoke membership from every primary coverage owner", () => { + const catalog = readQaScenarioPack().scenarios; + const report = readQaScorecardTaxonomyReport(catalog); + const profile = report.profiles.find((candidate) => candidate.id === "smoke-ci"); + expect(profile).toBeDefined(); + const selectedCoverageIds = new Set(profile?.coverageIds ?? []); + const selectedScenarioRefs = new Set(profile?.scenarioRefs ?? []); + + expect(selectedScenarioRefs.size).toBeGreaterThan(0); + for (const scenario of catalog.filter((candidate) => + selectedScenarioRefs.has(candidate.sourcePath), + )) { + expect(scenario.coverage?.primary.some((id) => selectedCoverageIds.has(id))).toBe(true); + } + for (const coverageId of selectedCoverageIds) { + const primaryOwners = catalog.filter((scenario) => + scenario.coverage?.primary.includes(coverageId), + ); + expect(primaryOwners.length).toBeGreaterThan(0); + expect(primaryOwners.every((scenario) => selectedScenarioRefs.has(scenario.sourcePath))).toBe( + true, + ); + } + }); + + it("does not pull unrelated scenarios from a selected coverage category", () => { + const scenarioPack = readQaScenarioPack(); + const report = readQaScorecardTaxonomyReport(scenarioPack.scenarios); + const profile = report.profiles.find((candidate) => candidate.id === "smoke-ci"); + expect(profile).toBeDefined(); + const categoryScenarioRefs = new Set( + report.categories + .filter((category) => category.profiles.includes("smoke-ci")) + .flatMap((category) => category.scenarioRefs), + ); + const profileScenarioRefs = new Set(profile?.scenarioRefs ?? []); + const unrelatedRefs = [...categoryScenarioRefs].filter((ref) => !profileScenarioRefs.has(ref)); + + expect(unrelatedRefs.length).toBeGreaterThan(0); + expect( + resolveQaProfileScenarios({ profile: "smoke-ci", providerMode: "mock-openai" }).scenarios, + ).not.toContainEqual(expect.objectContaining({ sourcePath: unrelatedRefs[0] })); + }); + + it("derives channel defaults from catalog metadata and lane constraints", () => { + const liveTelegram = resolveLiveTransportQaScenarioIds({ + channelId: "telegram", + providerMode: "live-frontier", + }); + const mockTelegram = resolveLiveTransportQaScenarioIds({ + channelId: "telegram", + providerMode: "mock-openai", + }); + + expect(liveTelegram).toContain("telegram-help-command"); + expect(liveTelegram).not.toContain("telegram-assistant-transcript-role-boundary"); + expect(mockTelegram).toContain("telegram-assistant-transcript-role-boundary"); + expect(mockTelegram).not.toContain("discord-canary"); + }); +}); diff --git a/extensions/qa-lab/src/run-config.test.ts b/extensions/qa-lab/src/run-config.test.ts index 343b596280da..7a554554a803 100644 --- a/extensions/qa-lab/src/run-config.test.ts +++ b/extensions/qa-lab/src/run-config.test.ts @@ -30,9 +30,30 @@ import { const DEFAULT_LIVE_FRONTIER_MODEL = defaultQaProviderModelForMode("live-frontier"); const profiles: QaScorecardTaxonomyReport["profiles"] = [ - { id: "smoke-ci", evidenceMode: "slim", channelDriver: "crabline", categoryIds: [] }, - { id: "release", evidenceMode: "full", channelDriver: "live", categoryIds: [] }, - { id: "all", evidenceMode: "full", channelDriver: "live", categoryIds: [] }, + { + id: "smoke-ci", + evidenceMode: "slim", + channelDriver: "crabline", + categoryIds: [], + coverageIds: [], + scenarioRefs: [], + }, + { + id: "release", + evidenceMode: "full", + channelDriver: "live", + categoryIds: [], + coverageIds: [], + scenarioRefs: [], + }, + { + id: "all", + evidenceMode: "full", + channelDriver: "live", + categoryIds: [], + coverageIds: [], + scenarioRefs: [], + }, ]; const scenarios = [ diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 5bc4f54225e8..3f3900be1f4d 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -901,9 +901,6 @@ describe("qa scenario catalog", () => { const scenario = readQaScenarioById(scenarioId); expect(scenario.execution.channel, scenarioId).toBeUndefined(); - expect(Object.keys(scenario.execution.profiles ?? {}), scenarioId).toEqual( - expect.arrayContaining(["matrix:adapter", "slack:adapter"]), - ); } }); diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index c1d6b075e147..2f4aa0343c09 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -67,13 +67,6 @@ const qaScenarioChannelSchema = z message: "scenario execution channel ids must use lowercase dotted or dashed tokens", }); -const qaScenarioProfileSchema = z - .string() - .trim() - .regex(/^[a-z0-9]+(?:[.:/-][a-z0-9]+)*$/, { - message: "scenario execution profiles must use lowercase namespaced tokens", - }); - const qaScenarioTransportPolicySchema = z.object({ requireGroupMention: z.literal(true).optional(), senderAllowlist: z.array(z.string().trim().min(1)).min(1).optional(), @@ -92,7 +85,6 @@ const qaFlowScenarioExecutionSchema = z message: "scenario execution channel ids must be unique", }) .optional(), - profiles: z.record(qaScenarioProfileSchema, z.number().int().nonnegative()).optional(), suiteIsolation: z.literal("isolated").optional(), isolationReason: z.string().trim().min(1).optional(), transportPolicy: qaScenarioTransportPolicySchema.optional(), @@ -103,7 +95,6 @@ const qaFlowScenarioExecutionSchema = z const qaTestFileScenarioExecutionBaseSchema = z.object({ summary: z.string().trim().min(1).optional(), channel: qaScenarioChannelSchema.optional(), - profiles: z.record(qaScenarioProfileSchema, z.number().int().nonnegative()).optional(), path: qaScenarioRepoRefSchema, config: qaScenarioConfigSchema.optional(), }); @@ -370,8 +361,6 @@ export type QaBootstrapScenarioCatalog = { scenarios: QaSeedScenarioWithSource[]; }; -export { QA_SCENARIO_PACKS } from "./scenario-packs.js"; - const QA_SCENARIO_PACK_INDEX_PATH = "qa/scenarios/index.yaml"; const QA_SCENARIO_LEGACY_OVERVIEW_PATH = "qa/scenarios.md"; const QA_SCENARIO_DIR_PATH = "qa/scenarios"; @@ -556,21 +545,6 @@ export function readQaScenarioExecutionConfig(id: string): Record candidate.id === id)?.execution?.config; } -export function listQaScenariosForExecutionProfile(profile: string): QaSeedScenarioWithSource[] { - const normalized = qaScenarioProfileSchema.parse(profile); - const scenarios = readQaScenarioPack() - .scenarios.filter((scenario) => scenario.execution.profiles?.[normalized] !== undefined) - .toSorted((left, right) => { - const orderDelta = - left.execution.profiles![normalized]! - right.execution.profiles![normalized]!; - return orderDelta || left.sourcePath.localeCompare(right.sourcePath); - }); - if (scenarios.length === 0) { - throw new Error(`unknown QA scenario execution profile: ${normalized}`); - } - return scenarios; -} - export function validateQaScenarioExecutionConfig(config: Record) { return qaScenarioConfigSchema.parse(config); } diff --git a/extensions/qa-lab/src/scenario-packs.test.ts b/extensions/qa-lab/src/scenario-packs.test.ts deleted file mode 100644 index cfdb607bff09..000000000000 --- a/extensions/qa-lab/src/scenario-packs.test.ts +++ /dev/null @@ -1,176 +0,0 @@ -// Qa Lab tests cover scenario packs plugin behavior. -import { describe, expect, it } from "vitest"; -import { QA_SCENARIO_PACKS, readQaScenarioById } from "./scenario-catalog.js"; -import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js"; - -function scenarioIdsForPack(packId: string): readonly string[] { - const pack = QA_SCENARIO_PACKS.find((candidate) => candidate.id === packId); - if (!pack) { - throw new Error(`missing QA scenario pack: ${packId}`); - } - return pack.scenarioIds; -} - -describe("qa scenario packs", () => { - it("points every pack scenario id at a loadable YAML scenario", () => { - expect(QA_SCENARIO_PACKS.length).toBeGreaterThan(0); - - for (const pack of QA_SCENARIO_PACKS) { - expect(pack.id).toMatch(/^[a-z0-9]+(?:-[a-z0-9]+)*$/); - expect(pack.title.trim()).toBe(pack.title); - expect(pack.description.trim()).toBe(pack.description); - expect(pack.scenarioIds.length).toBeGreaterThan(0); - expect(new Set(pack.scenarioIds).size).toBe(pack.scenarioIds.length); - - for (const scenarioId of pack.scenarioIds) { - const scenario = readQaScenarioById(scenarioId); - - expect(scenario.id).toBe(scenarioId); - expect(scenario.execution.kind).toBe("flow"); - expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0); - } - } - }); - - it("keeps the personal-agent pack scoped to the personal scenarios directory", () => { - const personalPack = QA_SCENARIO_PACKS.find((pack) => pack.id === "personal-agent"); - - expect(personalPack?.scenarioIds).toEqual([ - "personal-reminder-roundtrip", - "personal-channel-thread-reply", - "personal-memory-preference-recall", - "personal-redaction-no-secret-leak", - "personal-tool-safety-followthrough", - "personal-approval-denial-stop", - "personal-task-followthrough-status", - "personal-share-safe-diagnostics-artifact", - "personal-no-fake-progress", - "personal-failure-recovery", - ]); - - for (const scenarioId of personalPack?.scenarioIds ?? []) { - const scenario = readQaScenarioById(scenarioId); - - expect(scenario.sourcePath).toMatch(/^qa\/scenarios\/personal\//); - expect(scenario.coverage?.primary.length).toBeGreaterThan(0); - expect( - scenario.coverage?.primary.every((id) => /^[a-z0-9]+(?:[.-][a-z0-9]+)*$/.test(id)), - ).toBe(true); - } - }); - - it("expands the personal-agent pack in pack order", () => { - expect(resolveQaScenarioPackScenarioIds({ pack: "personal-agent" })).toEqual([ - ...scenarioIdsForPack("personal-agent"), - ]); - }); - - it("expands the observability pack in pack order", () => { - expect(resolveQaScenarioPackScenarioIds({ pack: "observability" })).toEqual([ - ...scenarioIdsForPack("observability"), - ]); - }); - - it("combines explicit scenarios with pack scenarios", () => { - expect( - resolveQaScenarioPackScenarioIds({ - pack: "personal-agent", - scenarioIds: ["channel-chat-baseline", "personal-reminder-roundtrip"], - }), - ).toEqual(["channel-chat-baseline", ...scenarioIdsForPack("personal-agent")]); - }); - - it("rejects unknown scenario packs", () => { - expect(() => resolveQaScenarioPackScenarioIds({ pack: "personal-admin" })).toThrow( - '--pack must be one of personal-agent, observability, got "personal-admin"', - ); - }); - - it("keeps personal pack mock debug assertions scoped to each reviewed scenario", () => { - const redactionFlow = JSON.stringify( - readQaScenarioById("personal-redaction-no-secret-leak").execution.flow, - ); - const toolSafetyFlow = JSON.stringify( - readQaScenarioById("personal-tool-safety-followthrough").execution.flow, - ); - const approvalDenialFlow = JSON.stringify( - readQaScenarioById("personal-approval-denial-stop").execution.flow, - ); - const taskFollowthroughScenario = readQaScenarioById("personal-task-followthrough-status"); - const taskFollowthroughFlow = JSON.stringify(taskFollowthroughScenario.execution.flow); - const diagnosticsScenario = readQaScenarioById("personal-share-safe-diagnostics-artifact"); - const diagnosticsFlow = JSON.stringify(diagnosticsScenario.execution.flow); - const noFakeProgressScenario = readQaScenarioById("personal-no-fake-progress"); - const noFakeProgressFlow = JSON.stringify(noFakeProgressScenario.execution.flow); - const failureRecoveryScenario = readQaScenarioById("personal-failure-recovery"); - const failureRecoveryFlow = JSON.stringify(failureRecoveryScenario.execution.flow); - const memoryScenario = readQaScenarioById("personal-memory-preference-recall"); - const memoryFlow = JSON.stringify(memoryScenario.execution.flow); - - expect(redactionFlow).toContain("config.promptSnippet"); - expect(redactionFlow).toContain("plannedToolName === 'read'"); - expect(redactionFlow).toContain("!newOutbounds.some"); - - expect(toolSafetyFlow).toContain("config.preActionPrompt"); - expect(toolSafetyFlow).toContain("preActionOutbound"); - expect(toolSafetyFlow).toContain("request.plannedToolName"); - expect(toolSafetyFlow).toContain("plannedToolName === 'read'"); - - expect(approvalDenialFlow).toContain("config.denialPromptSnippet"); - expect(approvalDenialFlow).toContain("request.plannedToolName"); - expect(approvalDenialFlow).toContain("config.deniedReadMarker"); - expect(approvalDenialFlow).toContain("beforeDenialOutboundCursor"); - - expect(taskFollowthroughScenario.execution.config?.prompt).toContain( - "Personal task followthrough check", - ); - expect(taskFollowthroughFlow).toContain("personal-task-status.txt"); - expect(taskFollowthroughFlow).toContain("plannedToolName === 'write'"); - expect(taskFollowthroughFlow).toContain("readIndices[1] < firstWrite"); - expect(taskFollowthroughScenario.successCriteria.join("\n").toLowerCase()).toContain("blocked"); - - expect(diagnosticsScenario.execution.config?.prompt).toContain( - "Personal share-safe diagnostics check", - ); - expect(diagnosticsScenario.execution.config?.artifactName).toBe( - "personal-diagnostics-summary.txt", - ); - expect(diagnosticsFlow).toContain("plannedToolName === 'write'"); - expect(diagnosticsFlow).toContain("readIndices[1] < firstWrite"); - expect(diagnosticsFlow).toContain("forbiddenNeedles"); - expect(diagnosticsScenario.successCriteria.join("\n").toLowerCase()).toContain("share-safe"); - - expect(noFakeProgressScenario.execution.config?.prompt).toContain( - "Personal no-fake-progress check", - ); - expect(noFakeProgressScenario.execution.config?.artifactName).toBe( - "personal-progress-proof.txt", - ); - expect(noFakeProgressFlow).toContain("plannedToolName === 'write'"); - expect(noFakeProgressFlow).toContain("readIndices[1] < firstWrite"); - expect(noFakeProgressFlow).toContain("forbiddenNeedles"); - expect(noFakeProgressScenario.successCriteria.join("\n").toLowerCase()).toContain( - "local evidence", - ); - - expect(failureRecoveryScenario.execution.config?.prompt).toContain( - "Personal failure recovery check", - ); - expect(failureRecoveryScenario.execution.config?.artifactName).toBe( - "personal-failure-recovery.txt", - ); - expect(failureRecoveryFlow).toContain("plannedToolName === 'write'"); - expect(failureRecoveryFlow).toContain("readIndices[1] < firstWrite"); - expect(failureRecoveryFlow).toContain("length === 1"); - expect(failureRecoveryScenario.successCriteria.join("\n").toLowerCase()).toContain( - "retry boundary", - ); - - expect(memoryFlow).toContain("config.rememberPrompt"); - expect(memoryFlow).toContain("config.recallPrompt"); - expect(memoryScenario.execution.config?.recallPrompt).toContain("Memory tools check"); - expect(memoryFlow).toContain("recallStartIndex"); - expect(memoryFlow).toContain("slice(recallStartIndex)"); - expect(memoryFlow).toContain("recallExpectedAny"); - }); -}); diff --git a/extensions/qa-lab/src/scenario-packs.ts b/extensions/qa-lab/src/scenario-packs.ts deleted file mode 100644 index a506e010a915..000000000000 --- a/extensions/qa-lab/src/scenario-packs.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Qa Lab plugin module implements scenario packs behavior. -import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; - -type QaScenarioPackDefinition = { - id: string; - title: string; - description: string; - scenarioIds: readonly string[]; -}; - -const QA_PERSONAL_AGENT_SCENARIO_IDS = [ - "personal-reminder-roundtrip", - "personal-channel-thread-reply", - "personal-memory-preference-recall", - "personal-redaction-no-secret-leak", - "personal-tool-safety-followthrough", - "personal-approval-denial-stop", - "personal-task-followthrough-status", - "personal-share-safe-diagnostics-artifact", - "personal-no-fake-progress", - "personal-failure-recovery", -] as const; - -const QA_OBSERVABILITY_SCENARIO_IDS = ["otel-trace-smoke", "docker-prometheus-smoke"] as const; - -export const QA_SCENARIO_PACKS = [ - { - id: "personal-agent", - title: "Personal Agent Benchmark Pack", - description: - "Local-only personal assistant workflow scenarios for reminders, channel replies, memory recall, redaction, safe tool followthrough, approval denial, task status honesty, share-safe diagnostics, proof-backed completion claims, and failure recovery.", - scenarioIds: QA_PERSONAL_AGENT_SCENARIO_IDS, - }, - { - id: "observability", - title: "Observability Smoke Pack", - description: - "Source-checkout diagnostics smoke scenarios for OpenTelemetry signal export and protected Prometheus scraping.", - scenarioIds: QA_OBSERVABILITY_SCENARIO_IDS, - }, -] as const satisfies readonly QaScenarioPackDefinition[]; - -export function resolveQaScenarioPackScenarioIds(params: { - pack?: string; - scenarioIds?: string[]; -}): string[] { - const normalizedPack = params.pack?.trim().toLowerCase(); - const explicitScenarioIds = uniqueStrings(params.scenarioIds ?? []); - if (!normalizedPack) { - return explicitScenarioIds; - } - const pack = QA_SCENARIO_PACKS.find((candidate) => candidate.id === normalizedPack); - if (!pack) { - throw new Error( - `--pack must be one of ${QA_SCENARIO_PACKS.map((candidate) => candidate.id).join(", ")}, got "${params.pack}"`, - ); - } - return uniqueStrings([...explicitScenarioIds, ...pack.scenarioIds]); -} diff --git a/extensions/qa-lab/src/scorecard-taxonomy.ts b/extensions/qa-lab/src/scorecard-taxonomy.ts index 6d27e6410a24..2f143e655ac5 100644 --- a/extensions/qa-lab/src/scorecard-taxonomy.ts +++ b/extensions/qa-lab/src/scorecard-taxonomy.ts @@ -41,6 +41,7 @@ const qaScorecardProfileSchema = z.object({ includeAllCategories: z.boolean().default(false), channelDriver: qaScorecardChannelDriverSchema.default("qa-channel"), categoryIds: z.array(qaScorecardIdSchema).default([]), + coverageIds: z.array(qaCoverageIdSchema).default([]), }); function maturityScoreLabelForScore(score: number) { @@ -222,6 +223,20 @@ const qaMaturityTaxonomySchema = z message: `profile ${profile.id} cannot set categoryIds when includeAllCategories is true`, }); } + if (profile.includeAllCategories && profile.coverageIds.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["profiles", profileIndex, "coverageIds"], + message: `profile ${profile.id} cannot set coverageIds when includeAllCategories is true`, + }); + } + if (profile.categoryIds.length > 0 && profile.coverageIds.length > 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["profiles", profileIndex, "coverageIds"], + message: `profile ${profile.id} must select categories or coverage IDs, not both`, + }); + } if (profile.channelDriver === "crabline" && profile.includeAllCategories) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -229,11 +244,15 @@ const qaMaturityTaxonomySchema = z message: `profile ${profile.id} cannot set includeAllCategories when channelDriver is crabline`, }); } - if (profile.channelDriver === "crabline" && !profile.categoryIds.length) { + if ( + profile.channelDriver === "crabline" && + !profile.categoryIds.length && + !profile.coverageIds.length + ) { ctx.addIssue({ code: z.ZodIssueCode.custom, - path: ["profiles", profileIndex, "categoryIds"], - message: `profile ${profile.id} requires categoryIds when channelDriver is crabline`, + path: ["profiles", profileIndex], + message: `profile ${profile.id} requires categoryIds or coverageIds when channelDriver is crabline`, }); } @@ -248,6 +267,18 @@ const qaMaturityTaxonomySchema = z } seenProfileCategoryIds.add(categoryId); } + + const seenProfileCoverageIds = new Set(); + for (const [coverageIndex, coverageId] of profile.coverageIds.entries()) { + if (seenProfileCoverageIds.has(coverageId)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["profiles", profileIndex, "coverageIds", coverageIndex], + message: `duplicate coverage ID in profile ${profile.id}: ${coverageId}`, + }); + } + seenProfileCoverageIds.add(coverageId); + } } const categoryIds = new Set(); @@ -331,6 +362,15 @@ const qaMaturityTaxonomySchema = z }); } } + for (const [coverageIndex, coverageId] of profile.coverageIds.entries()) { + if (!coverageIdOwners.has(coverageId)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ["profiles", profileIndex, "coverageIds", coverageIndex], + message: `profile ${profile.id} references missing coverage ID ${coverageId}`, + }); + } + } } }); @@ -361,6 +401,7 @@ type QaScorecardValidationIssueCode = | "taxonomy-ref-not-found" | "taxonomy-category-ref-not-found" | "profile-category-ref-not-found" + | "profile-coverage-ref-not-found" | "profile-category-missing-inventory"; type QaScorecardValidationIssue = { @@ -404,6 +445,8 @@ type QaScorecardProfileReport = { evidenceMode: QaScorecardEvidenceMode; channelDriver: QaScorecardChannelDriver; categoryIds: string[]; + coverageIds: string[]; + scenarioRefs: string[]; }; export type QaScorecardTaxonomyReport = { @@ -538,6 +581,33 @@ function scenarioCoverageIds(scenario: QaSeedScenarioWithSource) { return [...(scenario.coverage?.primary ?? []), ...(scenario.coverage?.secondary ?? [])]; } +function selectQaScorecardProfileScenarios(params: { + coverageIds: readonly string[]; + profileId: string; + scenarios: readonly QaSeedScenarioWithSource[]; +}) { + if (params.coverageIds.length === 0) { + return [...params.scenarios]; + } + const selected: QaSeedScenarioWithSource[] = []; + const selectedIds = new Set(); + for (const coverageId of params.coverageIds) { + const candidates = params.scenarios.filter((scenario) => + scenario.coverage?.primary.includes(coverageId), + ); + if (candidates.length === 0) { + throw new Error(`${params.profileId} profile coverage ${coverageId} has no primary owner.`); + } + for (const candidate of candidates) { + if (!selectedIds.has(candidate.id)) { + selectedIds.add(candidate.id); + selected.push(candidate); + } + } + } + return selected; +} + type ScenarioInventoryRef = { sourcePath: string; kind: QaScorecardEvidenceKind; @@ -846,11 +916,15 @@ export function readQaScorecardProfileOptions(profileId: string | undefined, rep function pushMissingPrimaryInventoryIssues(params: { issues: QaScorecardValidationIssue[]; category: MaturityCategoryRef; + requiredCoverageIds: ReadonlySet; coverageIdsWithPrimaryInventory: ReadonlySet; coverageIdsWithSecondaryInventory: ReadonlySet; }) { for (const feature of params.category.features) { for (const coverageId of feature.coverageIds) { + if (!params.requiredCoverageIds.has(coverageId)) { + continue; + } if (params.coverageIdsWithPrimaryInventory.has(coverageId)) { continue; } @@ -955,9 +1029,10 @@ function buildQaScorecardTaxonomyReport(params: { } const profileCategoryIdsByCategoryId = new Map>(); + const requiredCoverageIdsByCategoryId = new Map>(); const profiles = params.taxonomy?.profiles.map((profile) => { - const validCategoryIds: string[] = []; + const selectedCoverageIds = new Set(); const selectedCategoryIds = profile.includeAllCategories ? [...maturityRefs.categories.keys()] : profile.categoryIds; @@ -971,16 +1046,61 @@ function buildQaScorecardTaxonomyReport(params: { }); continue; } - const profileIds = profileCategoryIdsByCategoryId.get(categoryId) ?? new Set(); - profileIds.add(profile.id); - profileCategoryIdsByCategoryId.set(categoryId, profileIds); - validCategoryIds.push(categoryId); + for (const coverageId of maturityRefs.categories.get(categoryId)?.coverageIds ?? []) { + selectedCoverageIds.add(coverageId); + } } + for (const coverageId of profile.coverageIds) { + if (!maturityRefs.coverageIds.has(coverageId)) { + issues.push({ + code: "profile-coverage-ref-not-found", + severity: "warning", + ref: coverageId, + message: `${profile.id} profile references missing taxonomy coverage ID ${coverageId}`, + }); + continue; + } + selectedCoverageIds.add(coverageId); + } + + const validCategoryIds = new Set(); + for (const coverageId of selectedCoverageIds) { + for (const coverageRef of maturityRefs.coverageIds.get(coverageId) ?? []) { + const categoryId = coverageRef.categoryId; + validCategoryIds.add(categoryId); + const profileIds = profileCategoryIdsByCategoryId.get(categoryId) ?? new Set(); + profileIds.add(profile.id); + profileCategoryIdsByCategoryId.set(categoryId, profileIds); + const requiredCoverageIds = + requiredCoverageIdsByCategoryId.get(categoryId) ?? new Set(); + requiredCoverageIds.add(coverageId); + requiredCoverageIdsByCategoryId.set(categoryId, requiredCoverageIds); + } + } + const validCoverageIds = uniqueSorted(selectedCoverageIds); + const scenarioRefs = + profile.coverageIds.length > 0 + ? selectQaScorecardProfileScenarios({ + coverageIds: validCoverageIds, + profileId: profile.id, + scenarios: params.scenarios, + }) + .map((scenario) => scenario.sourcePath) + .toSorted() + : uniqueSorted( + validCoverageIds.flatMap((coverageId) => + (primaryInventoryRefsByCoverageId.get(coverageId) ?? []).map( + (ref) => ref.sourcePath, + ), + ), + ); return { id: profile.id, evidenceMode: profile.evidenceMode ?? "full", channelDriver: profile.channelDriver, - categoryIds: validCategoryIds, + categoryIds: uniqueSorted(validCategoryIds), + coverageIds: validCoverageIds, + scenarioRefs, }; }) ?? []; @@ -1014,7 +1134,9 @@ function buildQaScorecardTaxonomyReport(params: { } const profileIds = uniqueSorted(profileCategoryIdsByCategoryId.get(categoryId) ?? []); - const required = profileIds.length > 0; + const requiredCoverageIdsForCategory = + requiredCoverageIdsByCategoryId.get(categoryId) ?? new Set(); + const required = requiredCoverageIdsForCategory.size > 0; const inventoryRefs: QaScorecardInventoryRef[] = []; const categoryScenarioRefs = new Set(); const inventoriedCoverageIds = new Set(); @@ -1068,7 +1190,7 @@ function buildQaScorecardTaxonomyReport(params: { inventoriedCoverageIds.has(coverageId), ).length; if (required) { - for (const coverageId of category.coverageIds) { + for (const coverageId of requiredCoverageIdsForCategory) { requiredCoverageIds.add(coverageId); if (inventoriedCoverageIds.has(coverageId)) { inventoriedRequiredCoverageIds.add(coverageId); @@ -1077,6 +1199,7 @@ function buildQaScorecardTaxonomyReport(params: { pushMissingPrimaryInventoryIssues({ issues, category, + requiredCoverageIds: requiredCoverageIdsForCategory, coverageIdsWithPrimaryInventory: inventoriedCoverageIds, coverageIdsWithSecondaryInventory: secondaryOnlyCoverageIds, }); @@ -1091,12 +1214,16 @@ function buildQaScorecardTaxonomyReport(params: { } const missingCoverageIds = required - ? category.coverageIds.filter((coverageId) => !coverageIdsWithAnyInventory.has(coverageId)) + ? [...requiredCoverageIdsForCategory].filter( + (coverageId) => !coverageIdsWithAnyInventory.has(coverageId), + ) : []; const inventoryStatus = required && - category.coverageIds.length > 0 && - inventoriedCoverageIdCountForCategory === category.coverageIds.length + requiredCoverageIdsForCategory.size > 0 && + [...requiredCoverageIdsForCategory].every((coverageId) => + inventoriedCoverageIds.has(coverageId), + ) ? "complete" : inventoryRefs.length > 0 ? "partial" diff --git a/qa/scenarios/channels/channel-canary.yaml b/qa/scenarios/channels/channel-canary.yaml index 3af6e6f0812c..1f98ebb3c5b1 100644 --- a/qa/scenarios/channels/channel-canary.yaml +++ b/qa/scenarios/channels/channel-canary.yaml @@ -20,16 +20,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/suite.ts execution: - profiles: - { - "matrix:adapter": 1, - "slack:adapter": 1, - "telegram:adapter": 1, - "telegram:all": 0, - "telegram:mock-release": 0, - "telegram:release": 0, - "whatsapp:adapter": 1, - } kind: flow summary: Run the shared channel canary through QA Channel, Crabline, or a live adapter. transportPolicy: diff --git a/qa/scenarios/channels/channel-chat-baseline.yaml b/qa/scenarios/channels/channel-chat-baseline.yaml index 353aa4cc72dd..c08d548a3f4d 100644 --- a/qa/scenarios/channels/channel-chat-baseline.yaml +++ b/qa/scenarios/channels/channel-chat-baseline.yaml @@ -21,15 +21,6 @@ scenario: - extensions/qa-channel/src/inbound.ts - extensions/qa-lab/src/bus-state.ts execution: - profiles: - { - "matrix:adapter": 0, - "matrix:all": 0, - "matrix:fast": 0, - "matrix:release": 0, - "slack:adapter": 0, - "telegram:adapter": 0, - } kind: flow suiteIsolation: isolated transportPolicy: diff --git a/qa/scenarios/channels/channel-dm-group-routing.yaml b/qa/scenarios/channels/channel-dm-group-routing.yaml index 6978a85ad559..82993129eb98 100644 --- a/qa/scenarios/channels/channel-dm-group-routing.yaml +++ b/qa/scenarios/channels/channel-dm-group-routing.yaml @@ -20,7 +20,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - src/routing/resolve-route.ts execution: - profiles: { "matrix:adapter": 2, "whatsapp:adapter": 2 } kind: flow summary: Send sequential DM and group turns and assert route separation. transportPolicy: diff --git a/qa/scenarios/channels/channel-mention-gating.yaml b/qa/scenarios/channels/channel-mention-gating.yaml index 664397369397..729050ec8a2c 100644 --- a/qa/scenarios/channels/channel-mention-gating.yaml +++ b/qa/scenarios/channels/channel-mention-gating.yaml @@ -19,19 +19,6 @@ scenario: - extensions/qa-channel/src/inbound.ts - extensions/qa-lab/src/qa-transport.ts execution: - profiles: - { - "matrix:adapter": 3, - "matrix:all": 35, - "matrix:fast": 7, - "matrix:transport": 34, - "slack:adapter": 2, - "telegram:adapter": 2, - "telegram:all": 1, - "telegram:mock-release": 1, - "telegram:release": 1, - "whatsapp:adapter": 3, - } kind: flow summary: Prove the shared mention gate with a quiet turn followed by a mentioned turn. transportPolicy: diff --git a/qa/scenarios/channels/channel-multi-actor-ordering.yaml b/qa/scenarios/channels/channel-multi-actor-ordering.yaml index 1a972605e1ab..0ab6e97b6849 100644 --- a/qa/scenarios/channels/channel-multi-actor-ordering.yaml +++ b/qa/scenarios/channels/channel-multi-actor-ordering.yaml @@ -18,7 +18,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:adapter": 7, "matrix:all": 48, "matrix:transport": 47 } kind: flow summary: Preserve actor order across a blocked observer turn and a successful driver turn. transportPolicy: diff --git a/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml b/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml index d9f39ee73e2e..d27e6bf7ce15 100644 --- a/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml +++ b/qa/scenarios/channels/channel-secondary-conversation-isolation.yaml @@ -17,7 +17,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:adapter": 6, "matrix:all": 19, "matrix:transport": 18 } kind: flow summary: Send sequential turns to primary and secondary channel conversations and assert isolation. transportPolicy: diff --git a/qa/scenarios/channels/channel-sender-allowlist.yaml b/qa/scenarios/channels/channel-sender-allowlist.yaml index e26ad3e80ff1..dcef8723ae3a 100644 --- a/qa/scenarios/channels/channel-sender-allowlist.yaml +++ b/qa/scenarios/channels/channel-sender-allowlist.yaml @@ -19,7 +19,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - src/channels/turn/kernel.ts execution: - profiles: { "matrix:adapter": 4, "matrix:all": 47, "matrix:fast": 10, "matrix:transport": 46 } kind: flow summary: Exercise sender allowlist rejection and the configured driver override in one ordered flow. transportPolicy: diff --git a/qa/scenarios/channels/channel-top-level-reply-shape.yaml b/qa/scenarios/channels/channel-top-level-reply-shape.yaml index e702a9105728..f4dbd068c503 100644 --- a/qa/scenarios/channels/channel-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/channel-top-level-reply-shape.yaml @@ -19,15 +19,6 @@ scenario: - extensions/qa-lab/src/qa-transport.ts - extensions/qa-lab/src/runtime-api.ts execution: - profiles: - { - "matrix:adapter": 5, - "matrix:all": 5, - "matrix:fast": 3, - "matrix:transport": 4, - "slack:adapter": 3, - "whatsapp:adapter": 4, - } kind: flow summary: Assert portable top-level reply shape while native adapters normalize transport relations. transportPolicy: diff --git a/qa/scenarios/channels/discord-canary.yaml b/qa/scenarios/channels/discord-canary.yaml index e915143a8f47..29d289a8c64f 100644 --- a/qa/scenarios/channels/discord-canary.yaml +++ b/qa/scenarios/channels/discord-canary.yaml @@ -6,7 +6,6 @@ scenario: primary: - discord.configured-and-runtime-routing execution: - profiles: { "discord:adapter": 0, "discord:default": 0 } kind: flow channel: discord timeoutMs: 45000 diff --git a/qa/scenarios/channels/discord-mention-gating.yaml b/qa/scenarios/channels/discord-mention-gating.yaml index edee3823f95b..f0d85f6233a2 100644 --- a/qa/scenarios/channels/discord-mention-gating.yaml +++ b/qa/scenarios/channels/discord-mention-gating.yaml @@ -6,7 +6,6 @@ scenario: primary: - discord.mention-gating execution: - profiles: { "discord:adapter": 1, "discord:default": 1 } kind: flow channel: discord timeoutMs: 8000 diff --git a/qa/scenarios/channels/discord-native-help-command-registration.yaml b/qa/scenarios/channels/discord-native-help-command-registration.yaml index 61d3ecdc6677..68acac3a51d7 100644 --- a/qa/scenarios/channels/discord-native-help-command-registration.yaml +++ b/qa/scenarios/channels/discord-native-help-command-registration.yaml @@ -5,7 +5,6 @@ scenario: coverage: primary: [discord.native-slash-command-registration] execution: - profiles: { "discord:adapter": 2, "discord:default": 2 } kind: flow channel: discord timeoutMs: 45000 diff --git a/qa/scenarios/channels/dm-chat-baseline.yaml b/qa/scenarios/channels/dm-chat-baseline.yaml index 799390f3152a..26286db318fb 100644 --- a/qa/scenarios/channels/dm-chat-baseline.yaml +++ b/qa/scenarios/channels/dm-chat-baseline.yaml @@ -21,7 +21,6 @@ scenario: - extensions/qa-channel/src/gateway.ts - extensions/qa-lab/src/lab-server.ts execution: - profiles: { "matrix:all": 14, "matrix:transport": 13, "whatsapp:adapter": 0 } kind: flow summary: Verify the QA agent can chat coherently in a DM, explain the QA setup, and stay in character. config: diff --git a/qa/scenarios/channels/dm-per-room-session.yaml b/qa/scenarios/channels/dm-per-room-session.yaml index bdf9ed1805ff..a506a5ed81c9 100644 --- a/qa/scenarios/channels/dm-per-room-session.yaml +++ b/qa/scenarios/channels/dm-per-room-session.yaml @@ -25,7 +25,6 @@ scenario: - extensions/matrix/src/matrix/monitor/handler.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:adapter": 12, "matrix:all": 17, "matrix:transport": 16 } kind: flow channel: matrix summary: Send consecutive DMs in two Matrix rooms and require per-room isolation. diff --git a/qa/scenarios/channels/dm-shared-session.yaml b/qa/scenarios/channels/dm-shared-session.yaml index 19c707138899..bd7bffb28c60 100644 --- a/qa/scenarios/channels/dm-shared-session.yaml +++ b/qa/scenarios/channels/dm-shared-session.yaml @@ -17,7 +17,6 @@ scenario: - extensions/matrix/src/matrix/monitor/handler.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:adapter": 11, "matrix:all": 15, "matrix:transport": 14 } kind: flow channel: matrix summary: Send consecutive DMs from one Matrix user in two rooms and require the shared-session notice. diff --git a/qa/scenarios/channels/matrix-allowbots-default-block.yaml b/qa/scenarios/channels/matrix-allowbots-default-block.yaml index 61d3a9880fb9..d042c2705952 100644 --- a/qa/scenarios/channels/matrix-allowbots-default-block.yaml +++ b/qa/scenarios/channels/matrix-allowbots-default-block.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 36, "matrix:fast": 8, "matrix:transport": 35 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml index 5b263c1e02a8..f9b6724d9459 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-dm-unmentioned.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.dm-policy execution: - profiles: { "matrix:all": 40, "matrix:transport": 39 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml index b3b4114c40cf..c16eb5bae246 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-mentioned-room.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 38, "matrix:fast": 9, "matrix:transport": 37 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml b/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml index 6837b833448b..9392f56561bd 100644 --- a/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml +++ b/qa/scenarios/channels/matrix-allowbots-mentions-unmentioned-open-room-block.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 39, "matrix:transport": 38 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml b/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml index 68f0e50cab38..028697b5c1ca 100644 --- a/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml +++ b/qa/scenarios/channels/matrix-allowbots-room-override-blocks-account-true.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 41, "matrix:transport": 40 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml b/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml index eb29e5ec8da0..77eb97794c50 100644 --- a/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml +++ b/qa/scenarios/channels/matrix-allowbots-room-override-enables-account-off.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 42, "matrix:transport": 41 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml b/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml index aaf4f33adbff..e91343a91c20 100644 --- a/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml +++ b/qa/scenarios/channels/matrix-allowbots-self-sender-ignored.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 43, "matrix:transport": 42 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml b/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml index d9241b16d213..b1b2e5656a47 100644 --- a/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml +++ b/qa/scenarios/channels/matrix-allowbots-true-unmentioned-open-room.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 37, "matrix:transport": 36 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml index 235cbafb3d5a..673dcd7307cd 100644 --- a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml +++ b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml @@ -15,7 +15,6 @@ scenario: codeRefs: - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:all": 46, "matrix:release": 1, "matrix:transport": 45 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-approval-channel-target-both.yaml b/qa/scenarios/channels/matrix-approval-channel-target-both.yaml index d0f49b5a8528..7a2057372a8d 100644 --- a/qa/scenarios/channels/matrix-approval-channel-target-both.yaml +++ b/qa/scenarios/channels/matrix-approval-channel-target-both.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 30, "matrix:transport": 29 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-approval-deny-reaction.yaml b/qa/scenarios/channels/matrix-approval-deny-reaction.yaml index c14414abfed1..b9bec58bc349 100644 --- a/qa/scenarios/channels/matrix-approval-deny-reaction.yaml +++ b/qa/scenarios/channels/matrix-approval-deny-reaction.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 28, "matrix:transport": 27 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml b/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml index e20b3a8d9330..f80047ff2902 100644 --- a/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml +++ b/qa/scenarios/channels/matrix-approval-exec-metadata-chunked.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 26, "matrix:fast": 6, "matrix:transport": 25 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml b/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml index ff718906c8b9..33bb9985ff35 100644 --- a/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml +++ b/qa/scenarios/channels/matrix-approval-exec-metadata-single-event.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 25, "matrix:fast": 5, "matrix:transport": 24 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml b/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml index be815fb24589..ba142f3e1e2f 100644 --- a/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml +++ b/qa/scenarios/channels/matrix-approval-plugin-metadata-single-event.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 27, "matrix:transport": 26 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-approval-thread-target.yaml b/qa/scenarios/channels/matrix-approval-thread-target.yaml index 3ff873014694..876405c034d1 100644 --- a/qa/scenarios/channels/matrix-approval-thread-target.yaml +++ b/qa/scenarios/channels/matrix-approval-thread-target.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.channel-native-approval-prompts execution: - profiles: { "matrix:all": 29, "matrix:transport": 28 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-attachment-only-ignored.yaml b/qa/scenarios/channels/matrix-attachment-only-ignored.yaml index 5447bfe49c80..203f8391d6b8 100644 --- a/qa/scenarios/channels/matrix-attachment-only-ignored.yaml +++ b/qa/scenarios/channels/matrix-attachment-only-ignored.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.inbound-media-normalization execution: - profiles: { "matrix:all": 55, "matrix:media": 4 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml b/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml index 8c95cd2dfac2..869a35e2c265 100644 --- a/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml +++ b/qa/scenarios/channels/matrix-dm-thread-reply-override.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.matrix-thread-reply-routing execution: - profiles: { "matrix:all": 16, "matrix:transport": 15 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml b/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml index 703dfab3db56..028c84d5a725 100644 --- a/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml +++ b/qa/scenarios/channels/matrix-e2ee-artifact-redaction.yaml @@ -6,7 +6,6 @@ scenario: primary: - security.redaction-artifact-safety execution: - profiles: { "matrix:all": 64, "matrix:e2ee-smoke": 6 } kind: flow channel: matrix timeoutMs: 150000 diff --git a/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml b/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml index 3b9ca6777adc..5371eb258a07 100644 --- a/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml +++ b/qa/scenarios/channels/matrix-e2ee-basic-reply.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 58, "matrix:e2ee-smoke": 0, "matrix:fast": 11 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml b/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml index 30e1aaf03389..c07be34c2f64 100644 --- a/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml +++ b/qa/scenarios/channels/matrix-e2ee-bootstrap-success.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 60, "matrix:e2ee-smoke": 2 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml b/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml index e8c23de08550..e076e3e6419c 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-account-add-enable-e2ee.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 84, "matrix:e2ee-cli": 0 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml index 6b77bf175797..712c16871ce3 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-bootstrap-failure.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 87, "matrix:e2ee-cli": 3 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml index eac3ba7fefd7..9a86ecde1181 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-idempotent.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 86, "matrix:e2ee-cli": 2 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml index 463fb0484c9a..f5e42abda43c 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup-multi-account.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 90, "matrix:e2ee-cli": 6 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml index b98f1f6d5aa7..4cdb8cc6f2ca 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-encryption-setup.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 85, "matrix:e2ee-cli": 1 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml index f39ae915ac05..c535561bd1a2 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-invalid.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 89, "matrix:e2ee-cli": 5 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml index bc98163410ad..92c4065c1bf5 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-recovery-key-setup.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 88, "matrix:e2ee-cli": 4 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml b/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml index f8a9c96a6d79..1cb729037d30 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-self-verification.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 92, "matrix:e2ee-cli": 8 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml b/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml index 63eae338bab8..ec9025cfab71 100644 --- a/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml +++ b/qa/scenarios/channels/matrix-e2ee-cli-setup-then-gateway-reply.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 91, "matrix:e2ee-cli": 7 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml b/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml index c7529a6a12bb..63975f945a99 100644 --- a/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml +++ b/qa/scenarios/channels/matrix-e2ee-corrupt-crypto-idb-snapshot.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 73, "matrix:e2ee-deep": 7 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml b/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml index 039ac9ee6f52..eb12c19728d9 100644 --- a/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-device-sas-verification.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 78, "matrix:e2ee-deep": 12 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml b/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml index 9ea90820ec08..418ef10e6924 100644 --- a/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-dm-sas-verification.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 81, "matrix:e2ee-deep": 15 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml b/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml index 1adc3539deed..cdaf58ec9f77 100644 --- a/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml +++ b/qa/scenarios/channels/matrix-e2ee-history-exists-backup-empty.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 77, "matrix:e2ee-deep": 11 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml b/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml index 1c4396c8c92b..951199b0525e 100644 --- a/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml +++ b/qa/scenarios/channels/matrix-e2ee-key-bootstrap-failure.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 65, "matrix:e2ee-smoke": 7 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-media-image.yaml b/qa/scenarios/channels/matrix-e2ee-media-image.yaml index 3ce1130e4b66..40975623e45d 100644 --- a/qa/scenarios/channels/matrix-e2ee-media-image.yaml +++ b/qa/scenarios/channels/matrix-e2ee-media-image.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encrypted-media-upload-download execution: - profiles: { "matrix:all": 57, "matrix:media": 6 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml b/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml index 83a310095c7b..5b07b78533b6 100644 --- a/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml +++ b/qa/scenarios/channels/matrix-e2ee-qr-verification.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 79, "matrix:e2ee-deep": 13 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml b/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml index 39e1e286fad9..f3d703dcee56 100644 --- a/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml +++ b/qa/scenarios/channels/matrix-e2ee-recovery-key-lifecycle.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 61, "matrix:e2ee-smoke": 3 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml b/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml index 555e36a8c64b..6ea65bb62ba1 100644 --- a/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml +++ b/qa/scenarios/channels/matrix-e2ee-recovery-owner-verification-required.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 62, "matrix:e2ee-smoke": 4 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml b/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml index 369116fc4046..c31ab88a6cea 100644 --- a/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml +++ b/qa/scenarios/channels/matrix-e2ee-restart-resume.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 63, "matrix:e2ee-smoke": 5 } kind: flow channel: matrix timeoutMs: 150000 diff --git a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml index 26ae523a5f5d..578b3fced04e 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-reupload-restores.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 72, "matrix:e2ee-deep": 6 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml index d5c2a7841505..f86149bd415b 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-backup-deleted-local-state-intact.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 71, "matrix:e2ee-deep": 5 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml index 1acb8d2cd207..bb1bdb2eb15b 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-local-state-intact.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 74, "matrix:e2ee-deep": 8 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml index b5584816adba..190e42c30d23 100644 --- a/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml +++ b/qa/scenarios/channels/matrix-e2ee-server-device-deleted-relogin-recovers.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 75, "matrix:e2ee-deep": 9 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml b/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml index 0e7e52fcabf4..459cb3083574 100644 --- a/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml +++ b/qa/scenarios/channels/matrix-e2ee-stale-device-hygiene.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 80, "matrix:e2ee-deep": 14 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml b/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml index 43c230830ece..41a1bab93b1f 100644 --- a/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml +++ b/qa/scenarios/channels/matrix-e2ee-stale-recovery-key-after-backup-reset.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 70, "matrix:e2ee-deep": 4 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml b/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml index 5a8aa5bb1baf..eb09ff54802e 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-after-missing-encryption.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 66, "matrix:e2ee-deep": 0 } kind: flow channel: matrix timeoutMs: 150000 diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml index 75258e0ff0ae..3c01eb9f4b55 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-external-recovery-key.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 67, "matrix:e2ee-deep": 1 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml index eac726fea6f1..34636ba56afd 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-no-recovery-key.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 69, "matrix:e2ee-deep": 3 } kind: flow channel: matrix timeoutMs: 120000 diff --git a/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml index 93951c4ce233..6728fa685de2 100644 --- a/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-state-loss-stored-recovery-key.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 68, "matrix:e2ee-deep": 2 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml b/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml index cc3c80a44570..a3ef10d2cd34 100644 --- a/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml +++ b/qa/scenarios/channels/matrix-e2ee-sync-state-loss-crypto-intact.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.legacy-state execution: - profiles: { "matrix:all": 76, "matrix:e2ee-deep": 10 } kind: flow channel: matrix timeoutMs: 150000 diff --git a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml index e46e3a530d69..b43ce747b072 100644 --- a/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml +++ b/qa/scenarios/channels/matrix-e2ee-thread-follow-up.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.matrix-thread-reply-routing execution: - profiles: { "matrix:all": 59, "matrix:e2ee-smoke": 1 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml b/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml index 4898d1971943..d6cf206a86d0 100644 --- a/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml +++ b/qa/scenarios/channels/matrix-e2ee-verification-notice-no-trigger.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encrypted-media-upload-download execution: - profiles: { "matrix:all": 82, "matrix:e2ee-deep": 16 } kind: flow channel: matrix timeoutMs: 30000 diff --git a/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml b/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml index 9297eb8c75e6..58572a2f6b9d 100644 --- a/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml +++ b/qa/scenarios/channels/matrix-e2ee-wrong-account-recovery-key.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.encryption-setup execution: - profiles: { "matrix:all": 83, "matrix:e2ee-deep": 17 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml b/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml index 452175c567dd..72596b96ccf1 100644 --- a/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml +++ b/qa/scenarios/channels/matrix-homeserver-restart-resume.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.conversation-routing-and-delivery execution: - profiles: { "matrix:all": 34, "matrix:transport": 33 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml b/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml index 0d61ff4ba361..38ab0db6eed3 100644 --- a/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml +++ b/qa/scenarios/channels/matrix-inbound-edit-ignored.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 49, "matrix:transport": 48 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml index 5960178ce81b..bd6aeecd439c 100644 --- a/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml +++ b/qa/scenarios/channels/matrix-inbound-edit-no-duplicate-trigger.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 50, "matrix:transport": 49 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml b/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml index 03f8598e8a29..2937411b05ab 100644 --- a/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml +++ b/qa/scenarios/channels/matrix-initial-catchup-then-incremental.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.conversation-routing-and-delivery execution: - profiles: { "matrix:all": 31, "matrix:transport": 30 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-media-type-coverage.yaml b/qa/scenarios/channels/matrix-media-type-coverage.yaml index 76303facb6d3..3a861e41c7c8 100644 --- a/qa/scenarios/channels/matrix-media-type-coverage.yaml +++ b/qa/scenarios/channels/matrix-media-type-coverage.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.media-and-rich-content execution: - profiles: { "matrix:all": 53, "matrix:media": 2 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml b/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml index 47167cafa36b..322a7dd4b91b 100644 --- a/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml +++ b/qa/scenarios/channels/matrix-mention-metadata-spoof-block.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 45, "matrix:transport": 44 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml b/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml index c2a1ff7ab88d..e6c734a39e18 100644 --- a/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml +++ b/qa/scenarios/channels/matrix-mxid-prefixed-command-block.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.channel-native-commands execution: - profiles: { "matrix:all": 44, "matrix:transport": 43 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml b/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml index c704e6b2ff6f..4dec91844287 100644 --- a/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml +++ b/qa/scenarios/channels/matrix-reaction-not-a-reply.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 23, "matrix:transport": 22 } kind: flow channel: matrix timeoutMs: 8000 diff --git a/qa/scenarios/channels/matrix-reaction-notification.yaml b/qa/scenarios/channels/matrix-reaction-notification.yaml index 12771165e5ac..44a432bcb835 100644 --- a/qa/scenarios/channels/matrix-reaction-notification.yaml +++ b/qa/scenarios/channels/matrix-reaction-notification.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 21, "matrix:fast": 4, "matrix:transport": 20 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml b/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml index 04a4b8e56b31..f0b6078eaa70 100644 --- a/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml +++ b/qa/scenarios/channels/matrix-reaction-redaction-observed.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 24, "matrix:transport": 23 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-reaction-threaded.yaml b/qa/scenarios/channels/matrix-reaction-threaded.yaml index 7e7a0a72e790..0570a89aa358 100644 --- a/qa/scenarios/channels/matrix-reaction-threaded.yaml +++ b/qa/scenarios/channels/matrix-reaction-threaded.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-send-read-edit-delete execution: - profiles: { "matrix:all": 22, "matrix:transport": 21 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-room-autojoin-invite.yaml b/qa/scenarios/channels/matrix-room-autojoin-invite.yaml index 32422e9334c5..bc2ad2ee612a 100644 --- a/qa/scenarios/channels/matrix-room-autojoin-invite.yaml +++ b/qa/scenarios/channels/matrix-room-autojoin-invite.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.conversation-routing-and-delivery execution: - profiles: { "matrix:all": 18, "matrix:transport": 17 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml index 29bc4b4990d1..61f177d14fa0 100644 --- a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml +++ b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.media-and-rich-content execution: - profiles: { "matrix:all": 52, "matrix:media": 1 } kind: flow channel: matrix timeoutMs: 180000 diff --git a/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml b/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml index cae452397cde..fbabfeb801ac 100644 --- a/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml +++ b/qa/scenarios/channels/matrix-room-image-understanding-attachment.yaml @@ -10,7 +10,6 @@ scenario: docsRefs: - docs/tools/media-overview.md execution: - profiles: { "matrix:all": 51, "matrix:media": 0 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-membership-loss.yaml b/qa/scenarios/channels/matrix-room-membership-loss.yaml index d723514b9bd9..b35f8dd55129 100644 --- a/qa/scenarios/channels/matrix-room-membership-loss.yaml +++ b/qa/scenarios/channels/matrix-room-membership-loss.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.conversation-routing-and-delivery execution: - profiles: { "matrix:all": 33, "matrix:transport": 32 } kind: flow channel: matrix timeoutMs: 75000 diff --git a/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml b/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml index 232ebe84e919..6dd38af0ab79 100644 --- a/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml +++ b/qa/scenarios/channels/matrix-room-partial-streaming-preview.yaml @@ -8,7 +8,6 @@ scenario: docsRefs: - docs/concepts/streaming.md execution: - profiles: { "matrix:all": 7, "matrix:transport": 6 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml b/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml index 594de0cff81e..be94f7b4f990 100644 --- a/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml +++ b/qa/scenarios/channels/matrix-room-quiet-streaming-preview.yaml @@ -8,7 +8,6 @@ scenario: docsRefs: - docs/concepts/streaming.md execution: - profiles: { "matrix:all": 8, "matrix:transport": 7 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml b/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml index 6c5741daf87d..f3b2ec5e4000 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-command-preview.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-presentation-metadata execution: - profiles: { "matrix:all": 10, "matrix:transport": 9 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-tool-progress-error.yaml b/qa/scenarios/channels/matrix-room-tool-progress-error.yaml index ef241b9f0c8d..14da8cbeedda 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-error.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-error.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-presentation-metadata execution: - profiles: { "matrix:all": 12, "matrix:transport": 11 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml b/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml index 2e268b324c03..219fe5db1d6b 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-mention-safety.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-presentation-metadata execution: - profiles: { "matrix:all": 13, "matrix:transport": 12 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml b/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml index 4cb053e5e939..1cb98c952ce1 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-preview-opt-out.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-presentation-metadata execution: - profiles: { "matrix:all": 11, "matrix:transport": 10 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml b/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml index c4b4a982ed80..47c3258167ac 100644 --- a/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml +++ b/qa/scenarios/channels/matrix-room-tool-progress-preview.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.message-presentation-metadata execution: - profiles: { "matrix:all": 9, "matrix:transport": 8 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml b/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml index c58326d07330..2fee3fa3865e 100644 --- a/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml +++ b/qa/scenarios/channels/matrix-secondary-room-open-trigger.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.mention-gates execution: - profiles: { "matrix:all": 20, "matrix:transport": 19 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml b/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml index f2bee94d3d6d..bd76ac56fcda 100644 --- a/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml +++ b/qa/scenarios/channels/matrix-stale-sync-replay-dedupe.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.conversation-routing-and-delivery execution: - profiles: { "matrix:all": 32, "matrix:transport": 31 } kind: flow channel: matrix timeoutMs: 90000 diff --git a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml index 0a3ea3565b7f..7362b3b0fa45 100644 --- a/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml +++ b/qa/scenarios/channels/matrix-thread-nested-reply-shape.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.matrix-thread-reply-routing execution: - profiles: { "matrix:all": 3, "matrix:transport": 2 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-thread-root-preservation.yaml b/qa/scenarios/channels/matrix-thread-root-preservation.yaml index 048147c94c29..7c355f9562bd 100644 --- a/qa/scenarios/channels/matrix-thread-root-preservation.yaml +++ b/qa/scenarios/channels/matrix-thread-root-preservation.yaml @@ -6,7 +6,6 @@ scenario: primary: - matrix.matrix-thread-reply-routing execution: - profiles: { "matrix:all": 2, "matrix:transport": 1 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/matrix-unsupported-media-safe.yaml b/qa/scenarios/channels/matrix-unsupported-media-safe.yaml index 7d6c2aae6a27..95ee25aaab48 100644 --- a/qa/scenarios/channels/matrix-unsupported-media-safe.yaml +++ b/qa/scenarios/channels/matrix-unsupported-media-safe.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.inbound-media-normalization execution: - profiles: { "matrix:all": 56, "matrix:media": 5 } kind: flow channel: matrix timeoutMs: 45000 diff --git a/qa/scenarios/channels/matrix-voice-preflight-mention.yaml b/qa/scenarios/channels/matrix-voice-preflight-mention.yaml index 5d870c0213c2..ffe55120f723 100644 --- a/qa/scenarios/channels/matrix-voice-preflight-mention.yaml +++ b/qa/scenarios/channels/matrix-voice-preflight-mention.yaml @@ -21,7 +21,6 @@ scenario: mentionPatterns: - 'matrix\W+qa\W+voice\W+pre[ -]?flight\W+ok(?:ay)?' execution: - profiles: { "matrix:all": 54, "matrix:media": 3 } kind: flow channel: matrix providerMode: mock-openai diff --git a/qa/scenarios/channels/slack-allowlist-block.yaml b/qa/scenarios/channels/slack-allowlist-block.yaml index efdf8cc74057..cd8a8b011a88 100644 --- a/qa/scenarios/channels/slack-allowlist-block.yaml +++ b/qa/scenarios/channels/slack-allowlist-block.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.channel-allowlists execution: - profiles: { "slack:default": 2 } kind: flow channel: slack timeoutMs: 8000 diff --git a/qa/scenarios/channels/slack-approval-exec-native.yaml b/qa/scenarios/channels/slack-approval-exec-native.yaml index b036cfdd37a7..1efa21acda66 100644 --- a/qa/scenarios/channels/slack-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-approval-exec-native.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.channel-native-approval-prompts execution: - profiles: { "slack:default": 7 } kind: flow channel: slack timeoutMs: 60000 diff --git a/qa/scenarios/channels/slack-approval-plugin-native.yaml b/qa/scenarios/channels/slack-approval-plugin-native.yaml index 100daa116bc5..426d0cdf15c5 100644 --- a/qa/scenarios/channels/slack-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-approval-plugin-native.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.channel-native-approval-prompts execution: - profiles: { "slack:default": 8 } kind: flow channel: slack timeoutMs: 60000 diff --git a/qa/scenarios/channels/slack-canary.yaml b/qa/scenarios/channels/slack-canary.yaml index cd189de50e15..8a63de0552a8 100644 --- a/qa/scenarios/channels/slack-canary.yaml +++ b/qa/scenarios/channels/slack-canary.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.socket execution: - profiles: { "slack:default": 0 } kind: flow channel: slack timeoutMs: 45000 diff --git a/qa/scenarios/channels/slack-chart-presentation-native.yaml b/qa/scenarios/channels/slack-chart-presentation-native.yaml index e7a54da2b6fe..b2324d0962f2 100644 --- a/qa/scenarios/channels/slack-chart-presentation-native.yaml +++ b/qa/scenarios/channels/slack-chart-presentation-native.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.media-and-rich-content execution: - profiles: { "slack:default": 4 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml index b4de21e5b462..c9283db3541b 100644 --- a/qa/scenarios/channels/slack-codex-approval-exec-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-exec-native.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.channel-native-approval-prompts execution: - profiles: { "slack:default": 9 } kind: flow channel: slack runtime: codex diff --git a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml index bf290b4830ac..4a426413385f 100644 --- a/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml +++ b/qa/scenarios/channels/slack-codex-approval-plugin-native.yaml @@ -8,7 +8,6 @@ scenario: secondary: - channels.channel-native-approval-prompts execution: - profiles: { "slack:default": 10 } kind: flow channel: slack runtime: codex diff --git a/qa/scenarios/channels/slack-mention-gating.yaml b/qa/scenarios/channels/slack-mention-gating.yaml index e8d71354c8c2..f37cb5147bbe 100644 --- a/qa/scenarios/channels/slack-mention-gating.yaml +++ b/qa/scenarios/channels/slack-mention-gating.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.channel-allowlists execution: - profiles: { "slack:default": 1 } kind: flow channel: slack timeoutMs: 8000 diff --git a/qa/scenarios/channels/slack-reaction-glyph-native.yaml b/qa/scenarios/channels/slack-reaction-glyph-native.yaml index 9e9415e212eb..bffa256f63d1 100644 --- a/qa/scenarios/channels/slack-reaction-glyph-native.yaml +++ b/qa/scenarios/channels/slack-reaction-glyph-native.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.actions execution: - profiles: { "slack:default": 6 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-table-presentation-native.yaml b/qa/scenarios/channels/slack-table-presentation-native.yaml index 370b7683a26b..da6659c81a4d 100644 --- a/qa/scenarios/channels/slack-table-presentation-native.yaml +++ b/qa/scenarios/channels/slack-table-presentation-native.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.media-and-rich-content execution: - profiles: { "slack:default": 5 } kind: flow channel: slack timeoutMs: 90000 diff --git a/qa/scenarios/channels/slack-top-level-reply-shape.yaml b/qa/scenarios/channels/slack-top-level-reply-shape.yaml index d9f1fc958529..cdfba1c9a0d7 100644 --- a/qa/scenarios/channels/slack-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/slack-top-level-reply-shape.yaml @@ -6,7 +6,6 @@ scenario: primary: - slack.thread-routing execution: - profiles: { "slack:default": 3 } kind: flow channel: slack timeoutMs: 45000 diff --git a/qa/scenarios/channels/telegram-commands-command.yaml b/qa/scenarios/channels/telegram-commands-command.yaml index 7bbb919f46f0..c8634da1f927 100644 --- a/qa/scenarios/channels/telegram-commands-command.yaml +++ b/qa/scenarios/channels/telegram-commands-command.yaml @@ -7,6 +7,7 @@ scenario: coverage: primary: - channels.channel-native-commands + - telegram.built-in-commands objective: Verify Telegram renders the full native command catalog. successCriteria: - The command list is returned through the Telegram group command path. @@ -15,13 +16,6 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 4, - "telegram:all": 3, - "telegram:mock-release": 3, - "telegram:release": 3, - } kind: flow channel: telegram summary: Request the Telegram command catalog and verify representative entries. diff --git a/qa/scenarios/channels/telegram-context-command.yaml b/qa/scenarios/channels/telegram-context-command.yaml index 5175d47f9b00..083879aa53ef 100644 --- a/qa/scenarios/channels/telegram-context-command.yaml +++ b/qa/scenarios/channels/telegram-context-command.yaml @@ -15,13 +15,6 @@ scenario: - src/auto-reply/reply/commands-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 9, - "telegram:all": 8, - "telegram:mock-release": 8, - "telegram:release": 8, - } kind: flow channel: telegram summary: Run Telegram context help and verify the list shortcut. diff --git a/qa/scenarios/channels/telegram-current-session-status-tool.yaml b/qa/scenarios/channels/telegram-current-session-status-tool.yaml index 0f77ea3620d0..e054abca4889 100644 --- a/qa/scenarios/channels/telegram-current-session-status-tool.yaml +++ b/qa/scenarios/channels/telegram-current-session-status-tool.yaml @@ -17,7 +17,6 @@ scenario: - src/agents/tools/session-status-tool.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 11, "telegram:all": 10 } kind: flow channel: telegram summary: Ask the model to resolve the current Telegram group session through session_status. diff --git a/qa/scenarios/channels/telegram-help-command.yaml b/qa/scenarios/channels/telegram-help-command.yaml index 49408342785a..c5b475ff9a58 100644 --- a/qa/scenarios/channels/telegram-help-command.yaml +++ b/qa/scenarios/channels/telegram-help-command.yaml @@ -15,13 +15,6 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 3, - "telegram:all": 2, - "telegram:mock-release": 2, - "telegram:release": 2, - } kind: flow channel: telegram summary: Send Telegram help and verify the concise native command guide. diff --git a/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml index 0cfdfc997d25..56df7cf44c38 100644 --- a/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml +++ b/qa/scenarios/channels/telegram-long-final-reuses-preview.yaml @@ -17,7 +17,6 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 15, "telegram:all": 14, "telegram:mock-release": 10 } kind: flow channel: telegram summary: Run the scripted long final and verify preview reuse and chunk count. diff --git a/qa/scenarios/channels/telegram-long-final-three-chunks.yaml b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml index 68c7a4d166ab..836d9286915c 100644 --- a/qa/scenarios/channels/telegram-long-final-three-chunks.yaml +++ b/qa/scenarios/channels/telegram-long-final-three-chunks.yaml @@ -17,7 +17,6 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 16, "telegram:all": 15 } kind: flow channel: telegram summary: Run the scripted three-chunk final and verify final chunk accounting. diff --git a/qa/scenarios/channels/telegram-other-bot-command-gating.yaml b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml index d52fbffecbaf..bea68e4e7441 100644 --- a/qa/scenarios/channels/telegram-other-bot-command-gating.yaml +++ b/qa/scenarios/channels/telegram-other-bot-command-gating.yaml @@ -15,13 +15,6 @@ scenario: - extensions/telegram/src/bot-message-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 10, - "telegram:all": 9, - "telegram:mock-release": 9, - "telegram:release": 9, - } kind: flow channel: telegram summary: Address a command to another bot and verify OpenClaw ignores it. diff --git a/qa/scenarios/channels/telegram-repeated-command-authorization.yaml b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml index acbe47bcbaf9..98bae19461b9 100644 --- a/qa/scenarios/channels/telegram-repeated-command-authorization.yaml +++ b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml @@ -15,13 +15,6 @@ scenario: - extensions/telegram/src/bot-message-context.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 8, - "telegram:all": 7, - "telegram:mock-release": 7, - "telegram:release": 7, - } kind: flow channel: telegram suiteIsolation: isolated diff --git a/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml index 6a6f125bc924..15313944ed2f 100644 --- a/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml +++ b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply-payload.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 13, "telegram:all": 12 } kind: flow channel: telegram summary: Request one exact Telegram marker and verify single-message delivery. diff --git a/qa/scenarios/channels/telegram-status-command.yaml b/qa/scenarios/channels/telegram-status-command.yaml index dc55bae2f8f5..400d63c98898 100644 --- a/qa/scenarios/channels/telegram-status-command.yaml +++ b/qa/scenarios/channels/telegram-status-command.yaml @@ -17,13 +17,6 @@ scenario: - src/auto-reply/reply/commands-status.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 7, - "telegram:all": 6, - "telegram:mock-release": 6, - "telegram:release": 6, - } kind: flow channel: telegram summary: Run Telegram status and verify representative status fields. diff --git a/qa/scenarios/channels/telegram-stream-final-single-message.yaml b/qa/scenarios/channels/telegram-stream-final-single-message.yaml index d319d9e80df9..08a0ae9570c9 100644 --- a/qa/scenarios/channels/telegram-stream-final-single-message.yaml +++ b/qa/scenarios/channels/telegram-stream-final-single-message.yaml @@ -17,7 +17,6 @@ scenario: - extensions/telegram/src/send.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 14, "telegram:all": 13 } kind: flow channel: telegram summary: Run the scripted quiet-streaming reply and verify one final message. diff --git a/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml index 7a1764ac5ada..30c4b56cfd8e 100644 --- a/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml +++ b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml @@ -19,7 +19,6 @@ scenario: - src/auto-reply/reply/commands-session.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: { "telegram:adapter": 12, "telegram:all": 11 } kind: flow channel: telegram summary: Enable token usage mode, then verify the next Telegram reply footer. diff --git a/qa/scenarios/channels/telegram-tools-compact-command.yaml b/qa/scenarios/channels/telegram-tools-compact-command.yaml index 7002b2d7a338..be574422aa92 100644 --- a/qa/scenarios/channels/telegram-tools-compact-command.yaml +++ b/qa/scenarios/channels/telegram-tools-compact-command.yaml @@ -15,13 +15,6 @@ scenario: - src/auto-reply/reply/commands-info.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 5, - "telegram:all": 4, - "telegram:mock-release": 4, - "telegram:release": 4, - } kind: flow channel: telegram summary: Request compact tools through Telegram and verify inventory text. diff --git a/qa/scenarios/channels/telegram-whoami-command.yaml b/qa/scenarios/channels/telegram-whoami-command.yaml index f6a2f5c017ac..985c9d6312a3 100644 --- a/qa/scenarios/channels/telegram-whoami-command.yaml +++ b/qa/scenarios/channels/telegram-whoami-command.yaml @@ -15,13 +15,6 @@ scenario: - src/auto-reply/reply/commands-whoami.ts - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts execution: - profiles: - { - "telegram:adapter": 6, - "telegram:all": 5, - "telegram:mock-release": 5, - "telegram:release": 5, - } kind: flow channel: telegram summary: Run whoami and verify channel identity context. diff --git a/qa/scenarios/channels/thread-follow-up.yaml b/qa/scenarios/channels/thread-follow-up.yaml index e6c200b14625..d6a414e87ac6 100644 --- a/qa/scenarios/channels/thread-follow-up.yaml +++ b/qa/scenarios/channels/thread-follow-up.yaml @@ -22,14 +22,6 @@ scenario: - extensions/qa-lab/src/live-transports/slack/adapter.runtime.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: - { - "matrix:adapter": 8, - "matrix:all": 1, - "matrix:fast": 1, - "matrix:transport": 0, - "slack:adapter": 4, - } kind: flow summary: Send a deterministic follow-up through the shared host and require native thread relation evidence. config: diff --git a/qa/scenarios/channels/thread-isolation.yaml b/qa/scenarios/channels/thread-isolation.yaml index b614e8074f7e..13bf7fef73a4 100644 --- a/qa/scenarios/channels/thread-isolation.yaml +++ b/qa/scenarios/channels/thread-isolation.yaml @@ -20,14 +20,6 @@ scenario: - extensions/qa-lab/src/live-transports/slack/adapter.runtime.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: - { - "matrix:adapter": 9, - "matrix:all": 4, - "matrix:fast": 2, - "matrix:transport": 3, - "slack:adapter": 5, - } kind: flow summary: Establish a thread, then require a fresh top-level reply without stale relation metadata. config: diff --git a/qa/scenarios/channels/thread-reply-override.yaml b/qa/scenarios/channels/thread-reply-override.yaml index bc77be716c07..31004288cdfb 100644 --- a/qa/scenarios/channels/thread-reply-override.yaml +++ b/qa/scenarios/channels/thread-reply-override.yaml @@ -17,7 +17,6 @@ scenario: - extensions/matrix/src/matrix/monitor/threads.ts - extensions/qa-lab/src/live-transports/matrix/adapter.runtime.ts execution: - profiles: { "matrix:adapter": 10, "matrix:all": 6, "matrix:transport": 5 } kind: flow channel: matrix timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml index 5a59eb4d524f..82733ae573d1 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-react.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.message-actions execution: - profiles: { "whatsapp:mock-default": 10 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml index e6d6ad3a1173..1a5d307c5971 100644 --- a/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-agent-message-action-upload-file.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-image execution: - profiles: { "whatsapp:mock-default": 11 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-audio-preflight.yaml b/qa/scenarios/channels/whatsapp-audio-preflight.yaml index 1c247b4f6d8c..b863ff0ba5ab 100644 --- a/qa/scenarios/channels/whatsapp-audio-preflight.yaml +++ b/qa/scenarios/channels/whatsapp-audio-preflight.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.inbound-media-download execution: - profiles: { "whatsapp:mock-default": 17 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml index 973d83b4dcd7..ed268b7229f3 100644 --- a/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml +++ b/qa/scenarios/channels/whatsapp-broadcast-group-fanout.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-session-keys execution: - profiles: { "whatsapp:mock-default": 3 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-canary.yaml b/qa/scenarios/channels/whatsapp-canary.yaml index f776d55bcfc4..5f59b267d97d 100644 --- a/qa/scenarios/channels/whatsapp-canary.yaml +++ b/qa/scenarios/channels/whatsapp-canary.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:default": 0, "whatsapp:mock-default": 0 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-commands-command.yaml b/qa/scenarios/channels/whatsapp-commands-command.yaml index eb26278e0487..0fb8974bde51 100644 --- a/qa/scenarios/channels/whatsapp-commands-command.yaml +++ b/qa/scenarios/channels/whatsapp-commands-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 33 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/channels/whatsapp-context-command.yaml b/qa/scenarios/channels/whatsapp-context-command.yaml index aece59ca493a..6a33ab9f6951 100644 --- a/qa/scenarios/channels/whatsapp-context-command.yaml +++ b/qa/scenarios/channels/whatsapp-context-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-context.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 36 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/channels/whatsapp-group-activation-always.yaml b/qa/scenarios/channels/whatsapp-group-activation-always.yaml index cfc36fa7aaa7..ca853a128c41 100644 --- a/qa/scenarios/channels/whatsapp-group-activation-always.yaml +++ b/qa/scenarios/channels/whatsapp-group-activation-always.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-allowlists execution: - profiles: { "whatsapp:mock-default": 4 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml index 80182c107a24..3a22b9034737 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-react.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.message-actions execution: - profiles: { "whatsapp:mock-default": 12 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml index f8d47e34bdad..f4601c4b250e 100644 --- a/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml +++ b/qa/scenarios/channels/whatsapp-group-agent-message-action-upload-file.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-image execution: - profiles: { "whatsapp:mock-default": 13 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml index 73da02f41717..c2d519a634b8 100644 --- a/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml +++ b/qa/scenarios/channels/whatsapp-group-allowlist-block.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-allowlists execution: - profiles: { "whatsapp:default": 6, "whatsapp:mock-default": 31 } kind: flow channel: whatsapp timeoutMs: 8000 diff --git a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml index 244de8d37dbe..655cc69b8b00 100644 --- a/qa/scenarios/channels/whatsapp-group-audio-gating.yaml +++ b/qa/scenarios/channels/whatsapp-group-audio-gating.yaml @@ -7,7 +7,6 @@ scenario: - whatsapp.group-allowlists - whatsapp.inbound-media-download execution: - profiles: { "whatsapp:mock-default": 26 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml index 35ec89a5f06d..ee82159c40f6 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-audio.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-audio execution: - profiles: { "whatsapp:mock-default": 22 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml index 0e30570b6e5a..a2380d7d4b5c 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-media.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-media.yaml @@ -7,7 +7,6 @@ scenario: - whatsapp.outbound-image - whatsapp.outbound-document execution: - profiles: { "whatsapp:mock-default": 21 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml index acb6a153c706..ec9d03d0341d 100644 --- a/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-group-outbound-poll.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.provider-specific-channeldata execution: - profiles: { "whatsapp:mock-default": 23 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml index 7031871ae3ed..ba2cd9b47c73 100644 --- a/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml +++ b/qa/scenarios/channels/whatsapp-group-pending-history-context.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-session-keys execution: - profiles: { "whatsapp:mock-default": 2 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml index 489aba9d87e9..483abe9ab7b5 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-bot-triggers.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-allowlists execution: - profiles: { "whatsapp:mock-default": 5 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml index 894b87bec6c2..e1784521e6c7 100644 --- a/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-group-reply-to-message.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:default": 4, "whatsapp:mock-default": 8 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-help-command.yaml b/qa/scenarios/channels/whatsapp-help-command.yaml index 85a0592d127c..45e46aaed213 100644 --- a/qa/scenarios/channels/whatsapp-help-command.yaml +++ b/qa/scenarios/channels/whatsapp-help-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:adapter": 5, "whatsapp:default": 7, "whatsapp:mock-default": 32 } kind: flow channel: whatsapp summary: Send WhatsApp help and verify command guidance. diff --git a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml index 8aa9448ca3e4..4702fc390583 100644 --- a/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-image-caption.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.inbound-media-download execution: - profiles: { "whatsapp:mock-default": 16 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml index 1f67d156f476..9571c21c91ac 100644 --- a/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-reaction-no-trigger.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.message-actions execution: - profiles: { "whatsapp:mock-default": 14 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml index 14492207cdfe..0ab9545bdfa5 100644 --- a/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml +++ b/qa/scenarios/channels/whatsapp-inbound-structured-messages.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.provider-specific-channeldata execution: - profiles: { "whatsapp:mock-default": 25 } kind: flow channel: whatsapp timeoutMs: 240000 diff --git a/qa/scenarios/channels/whatsapp-mention-gating.yaml b/qa/scenarios/channels/whatsapp-mention-gating.yaml index f9e0711f6328..b199b30fcefd 100644 --- a/qa/scenarios/channels/whatsapp-mention-gating.yaml +++ b/qa/scenarios/channels/whatsapp-mention-gating.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.group-allowlists execution: - profiles: { "whatsapp:default": 1, "whatsapp:mock-default": 1 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-message-actions.yaml b/qa/scenarios/channels/whatsapp-message-actions.yaml index 4d269e7f6360..6811660ebfa7 100644 --- a/qa/scenarios/channels/whatsapp-message-actions.yaml +++ b/qa/scenarios/channels/whatsapp-message-actions.yaml @@ -7,7 +7,6 @@ scenario: - channels.message-actions - whatsapp.outbound-image execution: - profiles: { "whatsapp:mock-default": 24 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-native-new-command.yaml b/qa/scenarios/channels/whatsapp-native-new-command.yaml index 27ad0cb9ca03..4847ef16fdb6 100644 --- a/qa/scenarios/channels/whatsapp-native-new-command.yaml +++ b/qa/scenarios/channels/whatsapp-native-new-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-core.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 38 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml index b2c46298784e..dd4c6d6682d4 100644 --- a/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-document-preserves-filename.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-document execution: - profiles: { "whatsapp:mock-default": 19 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml index 30052bdffa33..d435d51e4d3b 100644 --- a/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-media-matrix.yaml @@ -8,7 +8,6 @@ scenario: - whatsapp.outbound-document - whatsapp.outbound-audio execution: - profiles: { "whatsapp:mock-default": 18 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-outbound-poll.yaml b/qa/scenarios/channels/whatsapp-outbound-poll.yaml index ad699ce9043b..42978964296c 100644 --- a/qa/scenarios/channels/whatsapp-outbound-poll.yaml +++ b/qa/scenarios/channels/whatsapp-outbound-poll.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.provider-specific-channeldata execution: - profiles: { "whatsapp:mock-default": 20 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml index eea4df2bf67a..85b7e4311948 100644 --- a/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml +++ b/qa/scenarios/channels/whatsapp-reply-context-isolation.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:mock-default": 15 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml index b01192d49da3..ccc7f4558850 100644 --- a/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml +++ b/qa/scenarios/channels/whatsapp-reply-delivery-shape.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:mock-default": 27 } kind: flow channel: whatsapp timeoutMs: 120000 diff --git a/qa/scenarios/channels/whatsapp-reply-to-message.yaml b/qa/scenarios/channels/whatsapp-reply-to-message.yaml index 6cc66487c86b..63cd1389d93b 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-message.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-message.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:default": 3, "whatsapp:mock-default": 7 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml index bf36c7c4eeb6..0816e433dcbd 100644 --- a/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml +++ b/qa/scenarios/channels/whatsapp-reply-to-mode-batched.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:mock-default": 9 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml index 8d1101199316..1e5572bf00a0 100644 --- a/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml +++ b/qa/scenarios/channels/whatsapp-status-reaction-lifecycle.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.reply-message-actions execution: - profiles: { "whatsapp:mock-default": 30 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-status-reactions.yaml b/qa/scenarios/channels/whatsapp-status-reactions.yaml index bc1027783ac7..bbb6fafe545e 100644 --- a/qa/scenarios/channels/whatsapp-status-reactions.yaml +++ b/qa/scenarios/channels/whatsapp-status-reactions.yaml @@ -6,7 +6,6 @@ scenario: primary: - channels.reply-message-actions execution: - profiles: { "whatsapp:default": 5, "whatsapp:mock-default": 29 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml index 875a54ab01e5..597ba05f6b28 100644 --- a/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml +++ b/qa/scenarios/channels/whatsapp-stream-final-message-accounting.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:mock-default": 28 } kind: flow channel: whatsapp timeoutMs: 90000 diff --git a/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml index 1fd6f03b6013..0482011ab98e 100644 --- a/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml +++ b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml @@ -17,7 +17,6 @@ scenario: - src/auto-reply/reply/commands-session.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 37 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/channels/whatsapp-tools-compact-command.yaml b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml index 7fc0d8095f34..b0d1affb9edc 100644 --- a/qa/scenarios/channels/whatsapp-tools-compact-command.yaml +++ b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-info.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 34 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml index e2db4ef4d4e7..54f2023913a7 100644 --- a/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml +++ b/qa/scenarios/channels/whatsapp-top-level-reply-shape.yaml @@ -6,7 +6,6 @@ scenario: primary: - whatsapp.outbound-text-sends execution: - profiles: { "whatsapp:default": 2, "whatsapp:mock-default": 6 } kind: flow channel: whatsapp timeoutMs: 60000 diff --git a/qa/scenarios/channels/whatsapp-whoami-command.yaml b/qa/scenarios/channels/whatsapp-whoami-command.yaml index 5542a88faf74..c507319e290b 100644 --- a/qa/scenarios/channels/whatsapp-whoami-command.yaml +++ b/qa/scenarios/channels/whatsapp-whoami-command.yaml @@ -15,7 +15,6 @@ scenario: - src/auto-reply/reply/commands-whoami.ts - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts execution: - profiles: { "whatsapp:mock-default": 35 } kind: flow channel: whatsapp config: diff --git a/qa/scenarios/index.yaml b/qa/scenarios/index.yaml index 9caf398ce668..a2be4f193409 100644 --- a/qa/scenarios/index.yaml +++ b/qa/scenarios/index.yaml @@ -23,6 +23,8 @@ title: OpenClaw QA Scenario Pack # - taxonomy feature coverage IDs are exact proof targets, not aliases # - scenario primary can list multiple IDs only when this scenario is primary # proof for each +# - taxonomy profiles select every primary owner of each requested coverage ID; +# scenario file and taxonomy order do not affect membership or execution order # - prefer reusing an existing coverage ID over minting a scenario-shaped ID # - avoid copying the scenario title into coverage IDs # - use `pnpm openclaw qa coverage` to render the current inventory diff --git a/taxonomy.yaml b/taxonomy.yaml index 94f05f665904..1538ce12754f 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -8,21 +8,43 @@ snapshot: profiles: - id: smoke-ci description: Deterministic PR and merge proof with mock model providers and the - Crabline-backed channel driver; categories are limited to exact owners exercised by the - fixed bounded CI plan. + Crabline-backed channel driver; coverage IDs resolve to every eligible primary owner. evidenceMode: slim channelDriver: crabline - categoryIds: - - agent-runtime.model-and-runtime-selection - - agent-runtime.tool-calls-and-response-handling - - session-memory.cross-client-history-and-session-parity - - gateway.gateway-rpc-apis-and-events - - gateway.websocket-connection - - channels.channel-actions-commands-and-approvals - - channels.channel-setup - - channels.outbound-delivery-and-reply-pipeline - - plugins.installing-and-running-plugins - - control-ui.browser-ui + coverageIds: + - agent-runtime.runtime-override-switching + - agent-runtime.subagent-turns-delivery + - agent-runtime.tool-fs-read + - agent-runtime.tool-fs-write + - agent-runtime.tool-task-followthrough + - channels.outbound-group-reply + - channels.resumed-final-reply + - control-ui.gateway-hosted-ui-control + - gateway.health-apis + - gateway.hello-ok-snapshot + - gateway.websocket-transport + - plugins.activation-lifecycle + - session-memory.memory-files-dreaming + - telegram.built-in-commands + - id: personal-agent + description: Local personal-assistant reliability proof selected by semantic coverage targets. + coverageIds: + - agent-runtime.failure-recovery + - agent-runtime.progress-visibility-no-fake-progress + - agent-runtime.tool-safety-controls + - agent-runtime.tool-task-followthrough + - automation.agent-cron-tool-reminders + - channels.native-threads-channel-replies + - observability.support-zip-diagnostics + - security.approval-policy-approval-denial + - security.redaction-personal-redaction + - session-memory.personal-memory-recall + - id: observability + description: Source-checkout telemetry proof selected by semantic coverage targets. + coverageIds: + - observability.otlp-http-traces-otel + - observability.prometheus + - observability.prometheus-api-auth - id: release description: Stable/LTS proof selector for live providers, live channels, package artifacts, upgrade paths, and platform proof where the claim depends on real upstreams or release diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index ded38ab7f288..b079ed6de0a4 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -106,6 +106,7 @@ type WorkflowJob = { include?: WorkflowMatrixEntry[]; lane?: string; profile?: string[]; + shard?: number[]; }; }; secrets?: string | Record; @@ -118,6 +119,11 @@ type WorkflowJob = { type Workflow = { env?: Record; jobs?: Record; + on?: { + workflow_call?: { + inputs?: Record; + }; + }; }; function readWorkflow(path: string): Workflow { @@ -2581,13 +2587,7 @@ describe("package artifact reuse", () => { expect(releaseJob.if).toContain('contains(fromJSON(\'["all","qa","qa-live"]\')'); expect(releaseJob.with).toMatchObject({ expected_sha: "${{ needs.resolve_target.outputs.revision }}", - matrix_profile: "release", - matrix_provider_mode: "mock-openai", - matrix_primary_model: "mock-openai/gpt-5.6-luna", - matrix_alternate_model: "mock-openai/gpt-5.6-luna-alt", - matrix_attempts: 2, run_matrix: true, - matrix_advisory: true, }); for (const lane of ["mock_parity", "telegram", "discord", "whatsapp", "slack"]) { expect(releaseJob.with?.[`run_${lane}`]).toBeUndefined(); @@ -2596,7 +2596,7 @@ describe("package artifact reuse", () => { "inputs.expected_sha == '' || inputs.run_mock_parity", ); expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix").if).toBe( - "(github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all')", + "github.event_name != 'workflow_call' || inputs.run_matrix", ); for (const channel of ["telegram", "discord", "whatsapp", "slack"]) { expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, `run_live_${channel}`).if).toBe( @@ -2607,9 +2607,6 @@ describe("package artifact reuse", () => { expect(releaseWorkflow).not.toContain("Run QA Lab live Matrix lane"); expect(releaseWorkflow).not.toContain("pnpm openclaw qa matrix"); expect(qaWorkflow).toContain("pnpm openclaw qa matrix"); - expect(qaWorkflow).toContain('for attempt in $(seq 1 "${MATRIX_ATTEMPTS}")'); - expect(qaWorkflow).toContain("matrix_status:"); - expect(qaWorkflow).toContain("value: ${{ jobs.run_live_matrix.outputs.status }}"); expect(qaWorkflow).toContain('trusted_reason="repository-branch"'); expect(qaWorkflow).toContain('"${selected_revision}" != "${EXPECTED_SHA}"'); expect(qaWorkflow).toContain("EXPECTED_SHA: ${{ inputs.expected_sha }}"); @@ -2623,71 +2620,33 @@ describe("package artifact reuse", () => { expect(qaWorkflow).not.toContain('"${{ inputs.expected_sha }}" !== ""'); expect(qaWorkflow).toContain('if [[ -n "${EXPECTED_SHA}" ]]; then'); const matrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix"); - const conditionalOpenAiSecret = - "${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}"; - expect(workflowStep(matrixJob, "Validate required QA credential env").env?.OPENAI_API_KEY).toBe( - conditionalOpenAiSecret, + expect(workflowStep(matrixJob, "Run Matrix live lane").run).toContain( + "--provider-mode mock-openai", ); - expect(workflowStep(matrixJob, "Run Matrix live lane").env?.OPENAI_API_KEY).toBe( - conditionalOpenAiSecret, - ); - expect(workflowStep(matrixJob, "Run Matrix live lane").env).toMatchObject({ - MATRIX_PROVIDER_MODE: - "${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }}", - MATRIX_PRIMARY_MODEL: - "${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }}", - MATRIX_ALTERNATE_MODEL: - "${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }}", - }); + const matrixSpecificInputs = Object.keys( + readWorkflow(QA_LIVE_TRANSPORTS_WORKFLOW).on?.workflow_call?.inputs ?? {}, + ).filter((input) => input.startsWith("matrix_")); + expect(matrixSpecificInputs).toEqual([]); expect(workflowStep(matrixJob, "Upload Matrix QA artifacts").with?.name).toBe( - "${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }}", + "${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}-shard-{1}-of-5', inputs.expected_sha, matrix.shard) || format('qa-live-matrix-{0}-{1}-shard-{2}-of-5', github.run_id, github.run_attempt, matrix.shard) }}", ); - expect(matrixJob["continue-on-error"]).toBe( - "${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }}", + expect(matrixJob["continue-on-error"]).toBeUndefined(); + expect(matrixJob.strategy?.matrix?.shard).toEqual([1, 2, 3, 4, 5]); + expect(workflowStep(matrixJob, "Run Matrix live lane").run).toContain( + '--shard "${{ matrix.shard }}/5"', ); - expect(qaWorkflow).toContain("status: ${{ steps.record_status.outputs.status }}"); - expect(qaWorkflow).not.toContain('matrix_runner="legacy"'); - const shardedMatrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix_sharded"); - expect(shardedMatrixJob.if).toBe( - "${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all' }}", - ); - expect(shardedMatrixJob.strategy?.matrix?.profile).toEqual([ - "transport", - "media", - "e2ee-smoke", - "e2ee-deep", - "e2ee-cli", - ]); - expect(workflowStep(shardedMatrixJob, "Run Matrix live lane shard").run).toContain( - '--profile "${{ matrix.profile }}"', + expect(workflowStep(matrixJob, "Run Matrix live lane").run).toContain( + "Selected target predates profile-free Matrix catalog sharding", ); + expect(readWorkflow(QA_LIVE_TRANSPORTS_WORKFLOW).jobs?.run_live_matrix_sharded).toBeUndefined(); expect(releaseTelegramWorkflow).toContain( 'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2', ); - expect(qaWorkflow).toContain( - 'echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2', - ); + expect(workflowStep(matrixJob, "Run Matrix live lane").run).not.toContain("for attempt in"); + expect(qaWorkflow).not.toContain("Matrix live lane failed on attempt"); expect(qaWorkflow).not.toContain("OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS"); - expect(qaWorkflow).toContain('--profile "${matrix_profile}"'); + expect(qaWorkflow).not.toContain("--profile"); expect(qaWorkflow).not.toContain("--fail-fast"); - - const matrixRunScript = workflowStep(matrixJob, "Run Matrix live lane").run ?? ""; - const resolveMatrixProfile = shellFunctionSource(matrixRunScript, "resolve_matrix_profile"); - const resolveProfile = (requested: string, helpText: string) => - execFileSync( - "bash", - [ - "-c", - `${resolveMatrixProfile}\nresolve_matrix_profile "$1" "$2"`, - "resolve-matrix-profile", - requested, - helpText, - ], - { encoding: "utf8" }, - ).trim(); - expect(resolveProfile("release", "profiles: all, fast, release, transport")).toBe("release"); - expect(resolveProfile("release", "profiles: all, fast, transport")).toBe("fast"); - expect(resolveProfile("e2ee-smoke", "profiles: all, fast, transport")).toBe("e2ee-smoke"); }); it("runs live transport lanes nightly while release checks stay gated", () => { @@ -2719,7 +2678,6 @@ describe("package artifact reuse", () => { "always() && steps.run_lane.outputs.output_dir != ''", ], ["run_live_matrix", "Upload Matrix QA artifacts", "always()"], - ["run_live_matrix_sharded", "Upload Matrix QA shard artifacts", "always()"], ["run_live_telegram", "Upload Telegram QA artifacts", "always()"], ["run_live_discord", "Upload Discord QA artifacts", "always()"], ["run_live_whatsapp", "Upload WhatsApp QA artifacts", "always()"], @@ -3370,7 +3328,6 @@ describe("package artifact reuse", () => { for (const jobName of [ "run_mock_parity", "run_live_matrix", - "run_live_matrix_sharded", "run_live_telegram", "run_live_discord", "run_live_whatsapp", @@ -3485,8 +3442,7 @@ describe("package artifact reuse", () => { const verifyStep = workflowStep(summary, "Verify release check results"); expect(verifyStep.env).toMatchObject({ - QA_LIVE_RELEASE_CHECKS_RESULT: - "${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }}", + QA_LIVE_RELEASE_CHECKS_RESULT: "${{ needs.qa_live_release_checks.result }}", RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}", RELEASE_CHECK_RUN_ID: "${{ github.run_id }}", RELEASE_CHECK_TARGET_SHA: "${{ needs.resolve_target.outputs.revision }}",