From 6c4e76ea8b349384a15ade3ced81464ba9335f71 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 15 Jul 2026 01:48:59 -0700 Subject: [PATCH] ci(qa): reuse Matrix live workflow for releases (#103604) * ci(qa): reuse Matrix live workflow for releases * docs(testing): restore Tailscale isolation flag * fix(ci): make Matrix artifact names ref-safe * test(ci): type Matrix workflow profiles * style(testing): format Tailscale isolation guidance --- .agents/skills/openclaw-qa-testing/SKILL.md | 15 +- .agents/skills/openclaw-testing/SKILL.md | 18 +- .github/workflows/openclaw-release-checks.yml | 115 ++-------- .../workflows/qa-live-transports-convex.yml | 214 ++++++++++++++---- docs/ci.md | 2 +- docs/concepts/qa-e2e-automation.md | 17 +- docs/help/testing.md | 15 +- docs/reference/RELEASING.md | 4 +- docs/reference/full-release-validation.md | 2 +- .../package-acceptance-workflow.test.ts | 112 +++++++-- 10 files changed, 320 insertions(+), 194 deletions(-) diff --git a/.agents/skills/openclaw-qa-testing/SKILL.md b/.agents/skills/openclaw-qa-testing/SKILL.md index 01f1d5ee3dc3..eb3c955af7aa 100644 --- a/.agents/skills/openclaw-qa-testing/SKILL.md +++ b/.agents/skills/openclaw-qa-testing/SKILL.md @@ -68,17 +68,14 @@ Langfuse, or external collector credentials. profiles for faster CI/release proof: ```bash -OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \ -pnpm openclaw qa matrix --profile fast --fail-fast +pnpm openclaw qa matrix --profile release ``` -- `fast`: release-critical transport contract, excluding generated image and - deep E2EE recovery inventory. -- `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, `e2ee-cli`: sharded full - Matrix coverage. -- `QA-Lab - All Lanes` uses explicit `fast` Matrix on scheduled runs. Manual - dispatch keeps `matrix_profile=all` as the default and always shards that full - Matrix selection. +- `fast` and `release`: focused release-critical scenarios. +- `transport`: transport and routing coverage. +- `all`: the full Matrix profile. +- `QA-Lab - All Lanes` fans manual `all` runs across the transport, media, and + E2EE profiles. Release checks use the focused reusable Matrix job. ## QA credentials and 1Password diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index 117da38d4689..ae11f6cb8345 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -450,18 +450,14 @@ fails. `pnpm openclaw qa matrix` defaults to `--profile all`. Do not assume the CLI default is the fast release path. Use explicit profiles: -- `--profile fast`: release-critical Matrix transport contract; add - `--fail-fast` only when the target CLI supports it -- `--profile transport|media|e2ee-smoke|e2ee-deep|e2ee-cli`: sharded full - Matrix proof -- `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`: CI-friendly no-reply quiet - window when paired with fast or sharded gates +- `--profile fast|release`: focused release-critical scenarios +- `--profile transport|all`: broad Matrix proof +- repeated `--scenario ` flags: explicit scenario selection -`QA-Lab - All Lanes` uses explicit fast Matrix on scheduled runs; manual -dispatch keeps `matrix_profile=all` as the default and always shards that full -Matrix selection. `OpenClaw Release Checks` uses explicit fast Matrix; run the -all-lanes workflow when release investigation needs full Matrix media/E2EE -inventory. +`QA-Lab - All Lanes` and `OpenClaw Release Checks` use the same QA Lab selector +and standard artifacts. Manual dispatch keeps `matrix_profile=all` as the +default and fans it across the transport, media, and E2EE profiles; focused +dispatches select `fast`, `release`, or `transport`. ### Reusable Live/E2E Checks diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index e68cd8ab3881..1df586604976 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1468,104 +1468,24 @@ jobs: retention-days: 14 if-no-files-found: error - qa_live_matrix_release_checks: - name: Run QA Lab live Matrix lane + qa_live_release_checks: + name: Run QA Live Matrix profile needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true' - continue-on-error: true - runs-on: blacksmith-16vcpu-ubuntu-2404 - timeout-minutes: 60 permissions: contents: read pull-requests: read - environment: qa-live-shared - env: - OPENCLAW_BUILD_PRIVATE_QA: "1" - OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" - steps: - - name: Checkout selected ref - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - persist-credentials: false - ref: ${{ needs.resolve_target.outputs.revision }} - fetch-depth: 1 - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - node-version: ${{ env.NODE_VERSION }} - install-bun: "true" - - - name: Build private QA runtime - env: - NODE_OPTIONS: --max-old-space-size=8192 - run: node scripts/build-all.mjs qaRuntime - - - name: Run Matrix live lane - id: run_lane - shell: bash - env: - OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS: "90000" - OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000" - run: | - set -euo pipefail - - output_dir=".artifacts/qa-e2e/matrix-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - matrix_args=( - --repo-root . \ - --provider-mode mock-openai \ - --model mock-openai/gpt-5.6-luna \ - --alt-model mock-openai/gpt-5.6-luna-alt \ - --profile fast \ - --fast - ) - if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then - matrix_args+=(--fail-fast) - fi - - for attempt in 1 2; do - attempt_output_dir="${output_dir}/attempt-${attempt}" - if pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"; then - exit 0 - fi - if [[ "${attempt}" == "2" ]]; then - exit 1 - fi - echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2 - sleep 10 - done - - - name: Upload Matrix QA artifacts - id: upload_matrix_qa_artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: release-qa-live-matrix-${{ needs.resolve_target.outputs.revision }} - path: .artifacts/qa-e2e/ - retention-days: 14 - if-no-files-found: error - - - name: Record advisory status - if: always() - shell: bash - env: - RELEASE_CHECK_JOB: qa_live_matrix_release_checks - RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} - JOB_STATUS: ${{ job.status }} - RELEASE_CHECK_STEP_OUTCOMES: ${{ steps.run_lane.outcome }} ${{ steps.upload_matrix_qa_artifacts.outcome }} - run: *record_release_check_advisory_status - - - name: Upload advisory status - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: release-check-status-qa-live-matrix-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} - path: .artifacts/release-check-status/qa_live_matrix_release_checks-${{ github.run_id }}-${{ github.run_attempt }}.env - retention-days: 14 - if-no-files-found: error + uses: ./.github/workflows/qa-live-transports-convex.yml + with: + ref: ${{ needs.resolve_target.outputs.revision }} + expected_sha: ${{ needs.resolve_target.outputs.revision }} + matrix_profile: release + matrix_provider_mode: mock-openai + matrix_primary_model: mock-openai/gpt-5.6-luna + matrix_alternate_model: mock-openai/gpt-5.6-luna-alt + matrix_attempts: 2 + run_matrix: true + matrix_advisory: true # The dispatched child owns Telegram evidence/status artifacts; this blocking job # carries its exact conclusion into the parent summary without copying secrets or artifacts. @@ -2022,7 +1942,7 @@ jobs: - qa_lab_parity_report_release_checks - qa_lab_runtime_parity_release_checks - runtime_tool_coverage_release_checks - - qa_live_matrix_release_checks + - qa_live_release_checks - qa_live_telegram_release_checks - qa_live_discord_release_checks - qa_live_whatsapp_release_checks @@ -2057,7 +1977,7 @@ jobs: QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_parity_report_release_checks.result }} QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }} RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: ${{ needs.runtime_tool_coverage_release_checks.result }} - QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_matrix_release_checks.result }} + QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }} QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }} QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }} QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }} @@ -2083,7 +2003,7 @@ jobs: "qa_lab_parity_report_release_checks=${QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT}" "qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}" "runtime_tool_coverage_release_checks=${RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT}" - "qa_live_matrix_release_checks=${QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT}" + "qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}" "qa_live_telegram_release_checks=${QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT}" "qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}" "qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}" @@ -2093,7 +2013,6 @@ jobs: "qa_lab_parity_lane_release_checks=${QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT}" "qa_lab_parity_report_release_checks=${QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT}" "qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}" - "qa_live_matrix_release_checks=${QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT}" "qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}" "qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}" "qa_live_slack_release_checks=${QA_LIVE_SLACK_RELEASE_CHECKS_RESULT}" @@ -2247,7 +2166,7 @@ jobs: } advisory_status_override_allowed() { case "$1" in - qa_lab_parity_lane_release_checks|qa_lab_parity_report_release_checks|qa_lab_runtime_parity_release_checks|qa_live_matrix_release_checks|qa_live_discord_release_checks|qa_live_whatsapp_release_checks|qa_live_slack_release_checks) + qa_lab_parity_lane_release_checks|qa_lab_parity_report_release_checks|qa_lab_runtime_parity_release_checks|qa_live_discord_release_checks|qa_live_whatsapp_release_checks|qa_live_slack_release_checks) return 0 ;; *) diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index 82c0ca7a65f0..3c3d7945d3f4 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -3,6 +3,80 @@ name: QA-Lab - All Lanes on: schedule: - cron: "41 4 * * *" + workflow_call: + outputs: + matrix_status: + description: Matrix release profile status + value: ${{ jobs.run_live_matrix.outputs.status }} + inputs: + ref: + description: Ref, tag, or SHA to run + required: true + type: string + expected_sha: + description: Exact resolved SHA expected from the trusted caller + required: true + type: string + matrix_profile: + description: Matrix QA profile + required: false + default: release + type: string + matrix_provider_mode: + description: Matrix provider mode + required: false + default: mock-openai + type: string + matrix_primary_model: + description: Matrix primary model + required: false + default: mock-openai/gpt-5.6-luna + type: string + matrix_alternate_model: + description: Matrix alternate model + required: false + default: mock-openai/gpt-5.6-luna-alt + type: string + matrix_attempts: + description: Matrix lane attempts + required: false + default: 2 + type: number + run_mock_parity: + description: Run the mock parity lane + required: false + default: false + type: boolean + run_matrix: + description: Run the Matrix live lane + required: false + default: false + type: boolean + run_telegram: + description: Run the Telegram live lane + required: false + default: false + type: boolean + run_discord: + description: Run the Discord live lane + required: false + default: false + type: boolean + run_whatsapp: + description: Run the WhatsApp live lane + required: false + default: false + type: boolean + run_slack: + description: Run the Slack live lane + required: false + default: false + type: boolean + matrix_advisory: + description: Report Matrix failure through the status output instead of failing the reusable job + required: false + default: false + type: boolean workflow_dispatch: inputs: ref: @@ -33,19 +107,16 @@ on: type: choice options: - fast + - release - all - transport - - media - - e2ee-smoke - - e2ee-deep - - e2ee-cli permissions: contents: read pull-requests: read concurrency: - group: qa-lab-all-lanes-${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + group: qa-lab-all-lanes-${{ github.event_name != 'schedule' && inputs.ref || github.sha }} cancel-in-progress: false env: @@ -66,10 +137,12 @@ jobs: - name: Require maintainer-level repository access id: permission uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} with: script: | - if (context.eventName === "schedule") { - core.info("Scheduled default-branch QA run; actor permission check is only required for manual dispatch."); + if (context.eventName === "schedule" || (process.env.EXPECTED_SHA ?? "") !== "") { + core.info("Trusted scheduled or reusable QA run; actor permission check is only required for manual dispatch."); core.setOutput("authorized", "true"); return; } @@ -104,23 +177,37 @@ jobs: uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + ref: ${{ github.event_name != 'schedule' && inputs.ref || github.sha }} fetch-depth: 0 - name: Validate selected ref id: validate env: + EXPECTED_SHA: ${{ inputs.expected_sha }} GH_TOKEN: ${{ github.token }} - INPUT_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} + INPUT_REF: ${{ github.event_name != 'schedule' && inputs.ref || github.sha }} shell: bash run: | set -euo pipefail selected_revision="$(git rev-parse HEAD)" trusted_reason="" + if [[ -n "${EXPECTED_SHA}" && "${selected_revision}" != "${EXPECTED_SHA}" ]]; then + echo "Ref '${INPUT_REF}' resolved to ${selected_revision}, expected ${EXPECTED_SHA}." >&2 + exit 1 + fi + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main - if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then + if [[ -n "${EXPECTED_SHA}" ]]; then + git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' + git fetch --tags origin '+refs/tags/*:refs/tags/*' + if git tag --points-at "$selected_revision" | grep -Eq '^v'; then + trusted_reason="release-tag" + elif git for-each-ref --format='%(refname:short)' --contains "$selected_revision" refs/remotes/origin | grep -Eq '^origin/'; then + trusted_reason="repository-branch" + fi + elif git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then trusted_reason="main-ancestor" elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then trusted_reason="release-tag" @@ -159,6 +246,7 @@ jobs: run_mock_parity: name: Run QA Lab mock parity lane needs: [validate_selected_ref] + if: inputs.expected_sha == '' || inputs.run_mock_parity runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 30 env: @@ -322,10 +410,13 @@ jobs: run_live_matrix: name: Run Matrix live QA lane needs: [authorize_actor, validate_selected_ref] - if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all') }} + if: (github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all') + continue-on-error: ${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }} runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 environment: qa-live-shared + outputs: + status: ${{ steps.record_status.outputs.status }} steps: - name: Checkout selected ref uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -342,12 +433,13 @@ jobs: - name: Validate required QA credential env env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }} + OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} shell: bash run: | set -euo pipefail - if [[ -z "${OPENAI_API_KEY:-}" ]]; then + if [[ "${MATRIX_PROVIDER_MODE}" == "live-frontier" && -z "${OPENAI_API_KEY:-}" ]]; then echo "Missing required OPENAI_API_KEY." >&2 exit 1 fi @@ -361,40 +453,85 @@ jobs: id: run_lane shell: bash env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_MATRIX_PROFILE: ${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile || 'fast' }} + OPENAI_API_KEY: ${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }} + INPUT_MATRIX_PROFILE: ${{ inputs.expected_sha != '' && inputs.matrix_profile || github.event_name == 'workflow_dispatch' && inputs.matrix_profile || 'release' }} + MATRIX_ALTERNATE_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }} + MATRIX_ATTEMPTS: ${{ inputs.expected_sha != '' && inputs.matrix_attempts || 1 }} + MATRIX_PRIMARY_MODEL: ${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }} + MATRIX_PROVIDER_MODE: ${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000" run: | set -euo pipefail - output_dir=".artifacts/qa-e2e/matrix-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + if [[ ! "${MATRIX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then + echo "matrix_attempts must be a positive integer; got: ${MATRIX_ATTEMPTS}" >&2 + exit 1 + fi + case "${MATRIX_PROVIDER_MODE}" in + mock-openai|live-frontier) ;; + *) + echo "matrix_provider_mode must be mock-openai or live-frontier; got: ${MATRIX_PROVIDER_MODE}" >&2 + exit 1 + ;; + esac + + output_dir=".artifacts/qa-e2e/matrix-live-${INPUT_MATRIX_PROFILE}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - matrix_args=( - --repo-root . \ - --output-dir "${output_dir}" \ - --provider-mode live-frontier \ - --model "${OPENCLAW_CI_OPENAI_MODEL}" \ - --alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \ - --profile "${INPUT_MATRIX_PROFILE}" \ - --fast - ) - if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then - matrix_args+=(--fail-fast) - fi - - pnpm openclaw qa matrix "${matrix_args[@]}" + for attempt in $(seq 1 "${MATRIX_ATTEMPTS}"); do + attempt_output_dir="${output_dir}/attempt-${attempt}" + if pnpm openclaw qa matrix \ + --repo-root . \ + --output-dir "${attempt_output_dir}" \ + --provider-mode "${MATRIX_PROVIDER_MODE}" \ + --model "${MATRIX_PRIMARY_MODEL}" \ + --alt-model "${MATRIX_ALTERNATE_MODEL}" \ + --profile "${INPUT_MATRIX_PROFILE}" \ + --fast; then + exit 0 + fi + if [[ "${attempt}" == "${MATRIX_ATTEMPTS}" ]]; then + exit 1 + fi + echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2 + sleep 10 + done - name: Upload Matrix QA artifacts + id: upload_artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: qa-live-matrix-${{ github.run_id }}-${{ github.run_attempt }} + name: ${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }} path: ${{ steps.run_lane.outputs.output_dir }} retention-days: 14 if-no-files-found: error + - name: Record Matrix QA status + id: record_status + if: always() + shell: bash + env: + RUN_OUTCOME: ${{ steps.run_lane.outcome }} + UPLOAD_OUTCOME: ${{ steps.upload_artifacts.outcome }} + run: | + set -euo pipefail + + status="success" + for outcome in "${RUN_OUTCOME:-}" "${UPLOAD_OUTCOME:-}"; do + case "${outcome}" in + failure) status="failure" ;; + cancelled) + if [[ "${status}" != "failure" ]]; then + status="cancelled" + fi + ;; + success|skipped|"") ;; + *) status="failure" ;; + esac + done + echo "status=${status}" >> "$GITHUB_OUTPUT" + run_live_matrix_sharded: name: Run Matrix live QA lane (${{ matrix.profile }}) needs: [authorize_actor, validate_selected_ref] @@ -448,14 +585,13 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000" run: | set -euo pipefail output_dir=".artifacts/qa-e2e/matrix-live-${{ matrix.profile }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - matrix_args=( + pnpm openclaw qa matrix \ --repo-root . \ --output-dir "${output_dir}" \ --provider-mode live-frontier \ @@ -463,12 +599,6 @@ jobs: --alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \ --profile "${{ matrix.profile }}" \ --fast - ) - if pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"; then - matrix_args+=(--fail-fast) - fi - - pnpm openclaw qa matrix "${matrix_args[@]}" - name: Upload Matrix QA shard artifacts if: always() @@ -482,6 +612,7 @@ jobs: run_live_telegram: name: Run Telegram live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] + if: inputs.expected_sha == '' || inputs.run_telegram runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 environment: qa-live-shared @@ -578,6 +709,7 @@ jobs: run_live_discord: name: Run Discord live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] + if: inputs.expected_sha == '' || inputs.run_discord runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 environment: qa-live-shared @@ -653,6 +785,7 @@ jobs: run_live_whatsapp: name: Run WhatsApp live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] + if: inputs.expected_sha == '' || inputs.run_whatsapp runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 concurrency: @@ -731,6 +864,7 @@ jobs: run_live_slack: name: Run Slack live QA lane with Convex leases needs: [authorize_actor, validate_selected_ref] + if: inputs.expected_sha == '' || inputs.run_slack runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 60 environment: qa-live-shared diff --git a/docs/ci.md b/docs/ci.md index aa1a2a3afa1b..96d20511d232 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -538,7 +538,7 @@ QA Lab has dedicated CI lanes outside the main smart-scoped workflow. Agentic pa Release checks run Matrix and Telegram live transport lanes with the deterministic mock provider and mock-qualified models (`mock-openai/gpt-5.6-luna` and `mock-openai/gpt-5.6-luna-alt`) so the channel contract is isolated from live model latency and normal provider-plugin startup. The live transport gateway disables memory search because QA parity covers memory behavior separately; provider connectivity is covered by the separate live model, native provider, and Docker provider suites. -Matrix uses `--profile fast` for scheduled and release gates, adding `--fail-fast` only when the checked-out CLI supports it. The CLI default and manual workflow input remain `all`; manual `matrix_profile=all` dispatch always shards full Matrix coverage into `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs. +Scheduled and release Matrix gates use the shared QA Lab suite host and live adapter with the release scenarios. The CLI default and manual workflow input remain `all`; manual `all` dispatches fan out the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles so the 92-scenario proof stays within per-job timeouts. Focused manual dispatches select `fast`, `release`, or `transport` in one job. `OpenClaw Release Checks` also runs the release-critical QA Lab lanes before release approval; its QA parity gate runs the candidate and baseline packs as parallel lane jobs, then downloads both artifacts into a small report job for the final parity comparison. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 214e86cd4a42..13b8618dc24a 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -164,11 +164,10 @@ tokens, or local paths. ### Matrix smoke lanes For a transport-real Matrix smoke lane that does not require model-provider -credentials, run the fast profile with the deterministic mock OpenAI provider: +credentials, run the release profile with the deterministic mock OpenAI provider: ```bash -OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \ - pnpm openclaw qa matrix --provider-mode mock-openai --profile fast --fail-fast +pnpm openclaw qa matrix --provider-mode mock-openai --profile release ``` For the live-frontier provider lane, supply OpenAI-compatible credentials @@ -176,8 +175,7 @@ explicitly: ```bash OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" \ -OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000 \ - pnpm openclaw qa matrix --provider-mode live-frontier --profile fast --fail-fast + pnpm openclaw qa matrix --provider-mode live-frontier --profile release ``` Plain `pnpm openclaw qa matrix` runs the full `all` profile and continues after @@ -255,11 +253,10 @@ gateway replies. explicit `--scenario` selection but stay outside the default `all` profile. CI uses the same command surface in -`.github/workflows/qa-live-transports-convex.yml`. Scheduled and default -manual runs execute the fast Matrix profile with QA-provided live-frontier -credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`. -Manual `matrix_profile=all` fans out into five profile shards: `transport`, -`media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli`. +`.github/workflows/qa-live-transports-convex.yml`. Scheduled and release runs +execute the release scenarios. Manual `matrix_profile=all` dispatches fan out +the `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` profiles; +focused dispatches select `fast`, `release`, or `transport` in one job. ### Discord Mantis scenarios diff --git a/docs/help/testing.md b/docs/help/testing.md index 4c3b57ce1a87..64b028444549 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -167,13 +167,14 @@ checks keep exhaustive live/Docker soak behind `run_release_soak=true`; the `full` profile forces soak on. `QA-Lab - All Lanes` runs nightly on `main` and from manual dispatch with the mock parity lane, live Matrix lane, Convex-managed live Telegram lane, and Convex-managed live Discord lane as -parallel jobs. Scheduled QA and release checks pass Matrix `--profile fast` -explicitly, while the Matrix CLI and manual workflow input default remains -`all`; manual dispatch can shard `all` into `transport`, `media`, -`e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs. `OpenClaw Release Checks` runs -parity plus the fast Matrix and Telegram lanes before release approval, using -`mock-openai/gpt-5.6-luna` for release transport checks so they stay deterministic -and avoid normal provider-plugin startup. These live transport gateways +parallel jobs. Scheduled QA and release checks run the Matrix release profile +through the shared live adapter. The Matrix CLI and manual workflow input +default remain `all`; manual `all` dispatches fan out the transport, media, and +E2EE profiles, while focused dispatches can select `fast`, `release`, or +`transport`. `OpenClaw Release Checks` runs parity plus the reusable Matrix +live-adapter profile and Telegram lane before release approval. Release +transport checks use `mock-openai/gpt-5.6-luna` so they stay deterministic and +avoid normal provider-plugin startup. These live transport gateways disable memory search; memory behavior stays covered by the QA parity suites. Full release live media shards use diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 87392032e4fe..c5ce47b9f273 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -258,7 +258,7 @@ A legacy fallback correction tag may reuse base-package evidence only when the c Manually dispatch `Windows Node Release` only for recovery, and always pass an exact tag, never `latest`, plus the explicit `expected_installer_digests` JSON map from the approved source release. Website download links should target exact OpenClaw release asset URLs for the current stable release, or `releases/latest/download/...` only after verifying GitHub's latest redirect points at that same release; do not link only to the companion repo release page. -- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the fast live Matrix profile and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. Run the manual `QA-Lab - All Lanes` workflow with `matrix_profile=all` and `matrix_shards=true` when you want full Matrix transport, media, and E2EE inventory in parallel. +- Release checks now run in a separate manual workflow: `OpenClaw Release Checks`. It also runs the QA Lab mock parity lane plus the Matrix release profile and Telegram QA lane before release approval. The live lanes use the `qa-live-shared` environment; Telegram also uses Convex CI credential leases. Run the manual `QA-Lab - All Lanes` workflow with `matrix_profile=all` when you want every maintained Matrix scenario; the workflow fans that selection across the transport, media, and E2EE profiles to keep full proof within per-job timeouts. - Cross-OS install and upgrade runtime validation is part of public `OpenClaw Release Checks` and `Full Release Validation`, which call the reusable workflow `.github/workflows/openclaw-cross-os-release-checks-reusable.yml` directly. This split is intentional: keep the real npm release path short, deterministic, and artifact-focused, while slower live checks stay in their own lane so they do not stall or block publish. - Secret-bearing release checks should be dispatched through `Full Release Validation` or from the `main`/release workflow ref so workflow logic and secrets stay controlled. - `OpenClaw Release Checks` accepts a branch, tag, or full commit SHA as long as the resolved commit is reachable from an OpenClaw branch or release tag. @@ -412,7 +412,7 @@ The QA Lab box is also part of `OpenClaw Release Checks`. It is the agentic beha Release QA Lab coverage includes: - mock parity lane comparing the OpenAI candidate lane against the `anthropic/claude-opus-4-8` baseline using the agentic parity pack -- fast live Matrix QA profile using the `qa-live-shared` environment +- Matrix live-adapter release profile using the `qa-live-shared` environment - live Telegram QA lane using Convex CI credential leases - `pnpm qa:otel:smoke`, `pnpm qa:otel:collector-smoke`, `pnpm qa:prometheus:smoke`, or `pnpm qa:observability:smoke` when release telemetry needs explicit local proof diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index d27bb4b2e068..4184a3ac49ce 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -132,7 +132,7 @@ or Docker-facing stages need it. | QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`
**Backing workflow:** direct jobs
**Tests:** candidate and baseline agentic parity packs, then the parity report.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA runtime parity | **Job:** `Run QA Lab runtime parity lane`
**Backing workflow:** direct job
**Tests:** an `openclaw`/`codex` runtime-pair agentic parity lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex`), including a standard tier and, with `run_release_soak=true`, a soak tier. Advisory: individual failures do not block the release-check verifier.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`
**Backing workflow:** direct job
**Tests:** dynamic tool drift between `openclaw` and `codex` in the standard runtime-parity tier (`pnpm openclaw qa coverage --tools`), using the QA runtime parity lane's output. Blocking: this job is not advisory-overridable.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | -| QA live Matrix | **Job:** `Run QA Lab live Matrix lane`
**Backing workflow:** direct job
**Tests:** fast live Matrix QA profile in the `qa-live-shared` environment.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | +| QA live Matrix | **Job:** `Run QA Live Matrix profile`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** parity-proven YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. | | QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** direct job
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | | Release verifier | **Job:** `Verify release checks`
**Backing workflow:** none
**Tests:** required release-check jobs for the selected rerun group.
**Rerun:** rerun after focused child jobs pass. | diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 3edd116a8381..12b4ba3f39c2 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -89,13 +89,15 @@ type WorkflowJob = { "fail-fast"?: boolean; matrix?: { include?: WorkflowMatrixEntry[]; + profile?: string[]; tier?: string; }; }; + secrets?: string | Record; "timeout-minutes"?: number | string; steps?: WorkflowStep[]; uses?: string; - with?: Record; + with?: Record; }; type Workflow = { @@ -281,7 +283,7 @@ function runReleaseChecksSummary(params: { QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: "skipped", QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: "skipped", - QA_LIVE_MATRIX_RELEASE_CHECKS_RESULT: "skipped", + QA_LIVE_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: params.currentResult, QA_LIVE_TELEGRAM_SELECTED: String(params.telegramSelected ?? true), @@ -2139,28 +2141,107 @@ describe("package artifact reuse", () => { } }); - it("detects Matrix fail-fast support for older release refs", () => { + it("routes release Matrix through the QA Lab selector", () => { const releaseWorkflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8"); const releaseTelegramWorkflow = readFileSync(RELEASE_TELEGRAM_QA_WORKFLOW, "utf8"); const qaWorkflow = readFileSync(".github/workflows/qa-live-transports-convex.yml", "utf8"); + const releaseJob = workflowJob(RELEASE_CHECKS_WORKFLOW, "qa_live_release_checks"); - expect(releaseWorkflow).toContain("matrix_args=("); - expect(releaseWorkflow).toContain( - 'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"', + expect(releaseJob.uses).toBe("./.github/workflows/qa-live-transports-convex.yml"); + expect(releaseJob.secrets).toBeUndefined(); + expect(releaseJob.permissions).toEqual({ contents: "read", "pull-requests": "read" }); + expect(releaseJob.if).toContain('contains(fromJSON(\'["all","qa","qa-live"]\')'); + expect(releaseJob.with).toMatchObject({ + expected_sha: "${{ needs.resolve_target.outputs.revision }}", + matrix_profile: "release", + matrix_provider_mode: "mock-openai", + matrix_primary_model: "mock-openai/gpt-5.6-luna", + matrix_alternate_model: "mock-openai/gpt-5.6-luna-alt", + matrix_attempts: 2, + run_matrix: true, + matrix_advisory: true, + }); + for (const lane of ["mock_parity", "telegram", "discord", "whatsapp", "slack"]) { + expect(releaseJob.with?.[`run_${lane}`]).toBeUndefined(); + } + expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_mock_parity").if).toBe( + "inputs.expected_sha == '' || inputs.run_mock_parity", ); - expect(releaseWorkflow).toContain("matrix_args+=(--fail-fast)"); - expect(releaseWorkflow).toContain( - 'pnpm openclaw qa matrix --output-dir "${attempt_output_dir}" "${matrix_args[@]}"', + expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix").if).toBe( + "(github.event_name != 'workflow_call' || inputs.run_matrix) && !(github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all')", ); - expect(releaseWorkflow).toContain( - 'echo "Matrix live lane failed on attempt ${attempt}; retrying once..." >&2', + for (const channel of ["telegram", "discord", "whatsapp", "slack"]) { + expect(workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, `run_live_${channel}`).if).toBe( + `inputs.expected_sha == '' || inputs.run_${channel}`, + ); + } + expect(releaseWorkflow).not.toContain("qa_live_matrix_release_checks"); + expect(releaseWorkflow).not.toContain("Run QA Lab live Matrix lane"); + expect(releaseWorkflow).not.toContain("pnpm openclaw qa matrix"); + expect(qaWorkflow).toContain("pnpm openclaw qa matrix"); + expect(qaWorkflow).toContain('for attempt in $(seq 1 "${MATRIX_ATTEMPTS}")'); + expect(qaWorkflow).toContain("matrix_status:"); + expect(qaWorkflow).toContain("value: ${{ jobs.run_live_matrix.outputs.status }}"); + expect(qaWorkflow).toContain('trusted_reason="repository-branch"'); + expect(qaWorkflow).toContain('"${selected_revision}" != "${EXPECTED_SHA}"'); + expect(qaWorkflow).toContain("EXPECTED_SHA: ${{ inputs.expected_sha }}"); + expect( + workflowStep( + workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "authorize_actor"), + "Require maintainer-level repository access", + ).env?.EXPECTED_SHA, + ).toBe("${{ inputs.expected_sha }}"); + expect(qaWorkflow).toContain('(process.env.EXPECTED_SHA ?? "") !== ""'); + expect(qaWorkflow).not.toContain('"${{ inputs.expected_sha }}" !== ""'); + expect(qaWorkflow).toContain('if [[ -n "${EXPECTED_SHA}" ]]; then'); + const matrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix"); + const conditionalOpenAiSecret = + "${{ (inputs.expected_sha != '' && inputs.matrix_provider_mode == 'live-frontier' || inputs.expected_sha == '' && github.event_name != 'workflow_dispatch') && secrets.OPENAI_API_KEY || '' }}"; + expect(workflowStep(matrixJob, "Validate required QA credential env").env?.OPENAI_API_KEY).toBe( + conditionalOpenAiSecret, + ); + expect(workflowStep(matrixJob, "Run Matrix live lane").env?.OPENAI_API_KEY).toBe( + conditionalOpenAiSecret, + ); + expect(workflowStep(matrixJob, "Run Matrix live lane").env).toMatchObject({ + MATRIX_PROVIDER_MODE: + "${{ inputs.expected_sha != '' && inputs.matrix_provider_mode || github.event_name == 'workflow_dispatch' && 'mock-openai' || 'live-frontier' }}", + MATRIX_PRIMARY_MODEL: + "${{ inputs.expected_sha != '' && inputs.matrix_primary_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna' || env.OPENCLAW_CI_OPENAI_MODEL }}", + MATRIX_ALTERNATE_MODEL: + "${{ inputs.expected_sha != '' && inputs.matrix_alternate_model || github.event_name == 'workflow_dispatch' && 'mock-openai/gpt-5.6-luna-alt' || env.OPENCLAW_CI_OPENAI_FALLBACK_MODEL }}", + }); + expect(workflowStep(matrixJob, "Upload Matrix QA artifacts").with?.name).toBe( + "${{ inputs.expected_sha != '' && format('release-qa-live-matrix-{0}', inputs.expected_sha) || format('qa-live-matrix-{0}-{1}', github.run_id, github.run_attempt) }}", + ); + expect(matrixJob["continue-on-error"]).toBe( + "${{ github.event_name == 'workflow_call' && inputs.matrix_advisory }}", + ); + expect(qaWorkflow).toContain("status: ${{ steps.record_status.outputs.status }}"); + expect(qaWorkflow).not.toContain('matrix_runner="legacy"'); + const shardedMatrixJob = workflowJob(QA_LIVE_TRANSPORTS_WORKFLOW, "run_live_matrix_sharded"); + expect(shardedMatrixJob.if).toBe( + "${{ github.event_name == 'workflow_dispatch' && inputs.matrix_profile == 'all' }}", + ); + expect(shardedMatrixJob.strategy?.matrix?.profile).toEqual([ + "transport", + "media", + "e2ee-smoke", + "e2ee-deep", + "e2ee-cli", + ]); + expect(workflowStep(shardedMatrixJob, "Run Matrix live lane shard").run).toContain( + '--profile "${{ matrix.profile }}"', ); expect(releaseTelegramWorkflow).toContain( 'echo "Telegram live lane failed on attempt ${attempt}; retrying once..." >&2', ); expect(qaWorkflow).toContain( - 'pnpm openclaw qa matrix --help 2>/dev/null | grep -F -q -- "--fail-fast"', + 'echo "Matrix live lane failed on attempt ${attempt}; retrying..." >&2', ); + expect(qaWorkflow).not.toContain("OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS"); + expect(qaWorkflow).toContain('--profile "${INPUT_MATRIX_PROFILE}"'); + expect(qaWorkflow).not.toContain("--fail-fast"); }); it("runs live transport lanes nightly while release checks stay gated", () => { @@ -2224,7 +2305,6 @@ describe("package artifact reuse", () => { ["qa_lab_parity_lane_release_checks", "Upload parity lane artifacts"], ["qa_lab_parity_report_release_checks", "Upload parity artifacts"], ["qa_lab_runtime_parity_release_checks", "Upload runtime parity artifacts"], - ["qa_live_matrix_release_checks", "Upload Matrix QA artifacts"], ["qa_live_discord_release_checks", "Upload Discord QA artifacts"], ["qa_live_whatsapp_release_checks", "Upload WhatsApp QA artifacts"], ["qa_live_slack_release_checks", "Upload Slack QA artifacts"], @@ -2684,7 +2764,6 @@ describe("package artifact reuse", () => { for (const jobName of [ "qa_lab_parity_lane_release_checks", "qa_lab_parity_report_release_checks", - "qa_live_matrix_release_checks", ]) { expect(releaseChecksWorkflow).toMatch( new RegExp(`${jobName}:[\\s\\S]*?runs-on: ubuntu-24\\.04`, "u"), @@ -2725,7 +2804,6 @@ describe("package artifact reuse", () => { "qa_lab_parity_lane_release_checks", "qa_lab_parity_report_release_checks", "qa_lab_runtime_parity_release_checks", - "qa_live_matrix_release_checks", "qa_live_discord_release_checks", "qa_live_whatsapp_release_checks", "qa_live_slack_release_checks", @@ -2819,6 +2897,8 @@ describe("package artifact reuse", () => { const verifyStep = workflowStep(summary, "Verify release check results"); expect(verifyStep.env).toMatchObject({ + QA_LIVE_RELEASE_CHECKS_RESULT: + "${{ needs.qa_live_release_checks.result == 'skipped' && 'skipped' || needs.qa_live_release_checks.outputs.matrix_status || 'failure' }}", RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}", RELEASE_CHECK_RUN_ID: "${{ github.run_id }}", RELEASE_CHECK_TARGET_SHA: "${{ needs.resolve_target.outputs.revision }}", @@ -2840,7 +2920,9 @@ describe("package artifact reuse", () => { 'if advisory_status_override_allowed "$name"; then', "::warning::${name} ended with ${result}; Tideclaw alpha treats non-package-safety release-check lanes as advisory.", "::error::${name} ended with ${result}", + '"qa_live_release_checks=${QA_LIVE_RELEASE_CHECKS_RESULT}"', ]); + expect(verifyStep.run).not.toContain("qa_live_matrix_release_checks"); expect(verifyStep.run).not.toContain( "QA release-check lanes are advisory and do not block release validation.", );