From 750f2f37629d75599a4be7152879caa8dda16baa Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 20 Aug 2026 23:35:16 -0700 Subject: [PATCH] fix(release): require concrete validation retry groups (#127012) * fix(release): require concrete validation retry groups * fix(release): reject mismatched retry filters * fix(release): align retry controller vocabulary * fix(release): preserve historical validation evidence * fix(release): validate retry filters before scheduling * test(release): follow shared filter validator * docs(testing): clarify release QA retry groups --- .agents/skills/openclaw-testing/SKILL.md | 21 +- .agents/skills/release-openclaw-ci/SKILL.md | 6 + .../references/release-ci-notes.md | 5 + .github/workflows/full-release-validation.yml | 62 ++- .github/workflows/openclaw-release-checks.yml | 141 ++++-- docs/help/testing.md | 10 +- docs/reference/RELEASING.md | 2 +- docs/reference/full-release-validation.md | 60 +-- scripts/full-release-validation-at-sha.mts | 20 +- .../github/validate-release-suite-filters.sh | 107 ++++ scripts/release-ci-summary.mjs | 29 +- .../full-release-validation-at-sha.test.ts | 8 + ...openclaw-cross-os-release-workflow.test.ts | 21 +- .../package-acceptance-workflow.test.ts | 55 ++- test/scripts/release-ci-summary.test.ts | 23 + test/scripts/release-no-push-workflow.test.ts | 459 ++++++++++++++++++ 16 files changed, 879 insertions(+), 150 deletions(-) create mode 100755 scripts/github/validate-release-suite-filters.sh diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index ac32b9168266..6875624d919f 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -366,10 +366,12 @@ editing. Only a confirmed product failure changes the Code SHA. Use one diagnosis, one fix when needed, and one narrow retry with `-f rerun_group=`, then reassess. Supported umbrella groups are `all`, `ci`, `plugin-prerelease`, -`release-checks`, `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, -`qa-parity`, `qa-live`, and `npm-telegram`. Use the narrowest group that covers -the failed box. Do not automatically dispatch `all` after a narrow retry. For a -single failed live/E2E shard, use +`install-smoke`, `cross-os`, `live-e2e`, `package`, `qa-parity`, `qa-live`, +`npm-telegram`, and `performance`. The old `release-checks` aggregate retry +handle is invalid because it silently selected every release-check lane. `qa` +is a direct-child manual aggregate, not an umbrella/controller retry API. Use +the narrowest concrete group that covers the failed box. Do not automatically +dispatch `all` after a narrow retry. For a single failed live/E2E shard, use `-f rerun_group=live-e2e -f live_suite_filter=` so the Blacksmith workflow only spends setup and queue time on that suite. @@ -426,11 +428,16 @@ gh workflow run openclaw-release-checks.yml \ -f provider=openai \ -f mode=both \ -f release_profile=stable \ - -f rerun_group=all + -f rerun_group= ``` -Release-check rerun groups are `all`, `install-smoke`, `cross-os`, `live-e2e`, -`package`, `qa`, `qa-parity`, and `qa-live`. +Concrete release-check rerun groups are `install-smoke`, `cross-os`, +`live-e2e`, `package`, `qa-parity`, and `qa-live`. Direct manual dispatch may +use `qa` to aggregate parity and live QA, but controllers must select one of +those two concrete groups. Reserve `all` for an intentional whole-child +validation, never automatic recovery. Non-empty live or cross-OS filters must +match their owning group; mismatches fail before scheduling and never widen to +an unfiltered run. `OpenClaw Release Checks` uses the trusted workflow ref to resolve the selected ref once as `release-package-under-test` and passes that artifact into cross-OS release checks, release-path Docker live/E2E checks, and Package Acceptance. diff --git a/.agents/skills/release-openclaw-ci/SKILL.md b/.agents/skills/release-openclaw-ci/SKILL.md index 60ac35a595b1..17d27993542a 100644 --- a/.agents/skills/release-openclaw-ci/SKILL.md +++ b/.agents/skills/release-openclaw-ci/SKILL.md @@ -96,6 +96,12 @@ and Release SHA separately in the lifecycle ledger. - Recover one failed surface with one diagnosis, one fix when needed, and one narrow retry. Then reassess the release decision. Do not automatically dispatch `rerun_group=all`. +- Controller retries are `ci`, `plugin-prerelease`, `install-smoke`, + `cross-os`, `live-e2e`, `package`, `qa-parity`, `qa-live`, `npm-telegram`, + or `performance`. Never use the removed `release-checks` handle. `qa` is + only a direct-child manual aggregate, not a controller retry API. +- Filtered retries fail closed unless the filter belongs to the selected group. + Never turn an empty derived filter into an unfiltered broad run. - A new all-group parent is justified only when shared orchestration changed, earlier evidence is invalid for the selected tuple, or the operator explicitly requests it. Record the invalidating event. diff --git a/.agents/skills/release-openclaw-ci/references/release-ci-notes.md b/.agents/skills/release-openclaw-ci/references/release-ci-notes.md index 0f2838aaf001..36c5c340607a 100644 --- a/.agents/skills/release-openclaw-ci/references/release-ci-notes.md +++ b/.agents/skills/release-openclaw-ci/references/release-ci-notes.md @@ -25,6 +25,11 @@ - Classify one failed surface, make one fix when needed, and retry the narrowest failed group once. Then reassess whether to ship, explicitly waive, or block instead of creating another verification loop. +- Release-check recovery uses one concrete group. The removed `release-checks` + aggregate handle must never be substituted with `all`. +- Controller recovery uses `qa-parity` or `qa-live`; `qa` is reserved for a + deliberate direct-child manual aggregate. Filters that do not belong to the + selected group fail closed. - Preserve successful exact-tuple evidence when the documented finalization rules allow reuse. Narrow evidence does not become publish authorization by itself, and there is no standalone rerunnable finalizer today. diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 91c88d9e28e5..4a56092e8127 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -74,12 +74,10 @@ on: - all - ci - plugin-prerelease - - release-checks - install-smoke - cross-os - live-e2e - package - - qa - qa-parity - qa-live - npm-telegram @@ -171,6 +169,8 @@ jobs: timeout-minutes: 10 outputs: sha: ${{ steps.resolve.outputs.sha }} + live_suite_filter: ${{ steps.filters.outputs.live_suite_filter }} + cross_os_suite_filter: ${{ steps.filters.outputs.cross_os_suite_filter }} steps: - name: Checkout trusted workflow helper uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -193,6 +193,26 @@ jobs: --expected-sha "$EXPECTED_SHA" \ --github-output "$GITHUB_OUTPUT" + - name: Validate suite filters + id: filters + env: + RERUN_GROUP: ${{ inputs.rerun_group }} + RAW_LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + RAW_CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + RELEASE_FILTER_VALIDATOR: workflow/scripts/github/validate-release-suite-filters.sh + run: | + set -euo pipefail + source "$RELEASE_FILTER_VALIDATOR" + validate_release_suite_filters \ + "$RERUN_GROUP" \ + "$RAW_LIVE_SUITE_FILTER" \ + "$RAW_CROSS_OS_SUITE_FILTER" \ + controller + { + printf 'live_suite_filter=%s\n' "$RELEASE_FILTER_LIVE_SUITE_FILTER" + printf 'cross_os_suite_filter=%s\n' "$RELEASE_FILTER_CROSS_OS_SUITE_FILTER" + } >> "$GITHUB_OUTPUT" + - name: Checkout target package manifest uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -304,8 +324,8 @@ jobs: SKIP_PACKAGE_TELEGRAM_E2E: ${{ inputs.skip_package_telegram_e2e }} ALLOW_UNRELEASED_CHANGELOG: ${{ inputs.allow_unreleased_changelog || (inputs.target_context_ref == '' && (inputs.ref == 'main' || inputs.ref == 'refs/heads/main')) }} RERUN_GROUP: ${{ inputs.rerun_group }} - LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} - CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + LIVE_SUITE_FILTER: ${{ steps.filters.outputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ steps.filters.outputs.cross_os_suite_filter }} PLUGIN_PRERELEASE_NODE_EXCLUDE_PATTERNS_JSON: ${{ inputs.plugin_prerelease_node_exclude_patterns_json }} run: | plugin_prerelease_node_exclusions="$(jq -c . <<< "$PLUGIN_PRERELEASE_NODE_EXCLUDE_PATTERNS_JSON")" @@ -343,7 +363,7 @@ jobs: else echo "- Plugin prerelease: skipped by rerun group" fi - if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "release-checks" || "$RERUN_GROUP" == "install-smoke" || "$RERUN_GROUP" == "cross-os" || "$RERUN_GROUP" == "live-e2e" || "$RERUN_GROUP" == "package" || "$RERUN_GROUP" == "qa" || "$RERUN_GROUP" == "qa-parity" || "$RERUN_GROUP" == "qa-live" ]]; then + if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "install-smoke" || "$RERUN_GROUP" == "cross-os" || "$RERUN_GROUP" == "live-e2e" || "$RERUN_GROUP" == "package" || "$RERUN_GROUP" == "qa-parity" || "$RERUN_GROUP" == "qa-live" ]]; then echo "- Release/live/Docker/package/QA: \`OpenClaw Release Checks\`" else echo "- Release/live/Docker/package/QA: skipped by rerun group" @@ -357,7 +377,7 @@ jobs: echo "- Published-package Telegram E2E: \`${RELEASE_PACKAGE_SPEC}\`" elif [[ "$RERUN_GROUP" == "npm-telegram" ]]; then echo "- Package Telegram E2E: focused rerun requires \`release_package_spec\` or \`npm_telegram_package_spec\`" - elif [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "release-checks" || "$RERUN_GROUP" == "package" ]]; then + elif [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "package" ]]; then if [[ "$SKIP_PACKAGE_TELEGRAM_E2E" == "true" ]]; then echo "- Package Telegram E2E: deferred by \`skip_package_telegram_e2e\`" else @@ -426,8 +446,8 @@ jobs: PROVIDER: ${{ inputs.provider }} MODE: ${{ inputs.mode }} TARGET_CONTEXT_REF: ${{ inputs.target_context_ref }} - LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} - CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + LIVE_SUITE_FILTER: ${{ needs.resolve_target.outputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ needs.resolve_target.outputs.cross_os_suite_filter }} RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }} @@ -533,7 +553,7 @@ jobs: prepare_release_candidate: name: Prepare shared release candidate needs: [resolve_target, evidence_reuse] - if: ${{ always() && needs.resolve_target.result == 'success' && needs.evidence_reuse.outputs.reuse != 'true' && inputs.release_package_spec == '' && inputs.package_acceptance_package_spec == '' && contains(fromJSON('["all","plugin-prerelease","release-checks","cross-os","live-e2e","package"]'), inputs.rerun_group) }} + if: ${{ always() && needs.resolve_target.result == 'success' && needs.evidence_reuse.outputs.reuse != 'true' && inputs.release_package_spec == '' && inputs.package_acceptance_package_spec == '' && (contains(fromJSON('["all","plugin-prerelease","cross-os","package"]'), inputs.rerun_group) || (inputs.rerun_group == 'live-e2e' && needs.resolve_target.outputs.live_suite_filter == '')) }} permissions: actions: read contents: read @@ -992,10 +1012,6 @@ jobs: fi echo "- Package Telegram E2E deferred: \`${SKIP_PACKAGE_TELEGRAM_E2E}\`" } >> "$GITHUB_STEP_SUMMARY" - child_rerun_group="$RERUN_GROUP" - if [[ "$child_rerun_group" == "release-checks" ]]; then - child_rerun_group=all - fi args=( -f ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" @@ -1006,7 +1022,7 @@ jobs: -f fail_fast="$FAIL_FAST" -f allow_unreleased_changelog="$ALLOW_UNRELEASED_CHANGELOG" -f skip_package_telegram_e2e="$SKIP_PACKAGE_TELEGRAM_E2E" - -f rerun_group="$child_rerun_group" + -f rerun_group="$RERUN_GROUP" ) if [[ -n "${TARGET_CONTEXT_REF// }" ]]; then args+=(-f target_context_ref="$TARGET_CONTEXT_REF") @@ -1115,7 +1131,7 @@ jobs: release_checks: name: Run release/live/Docker/QA validation needs: [resolve_target, evidence_reuse, prepare_release_candidate] - if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","release-checks","install-smoke","cross-os","live-e2e","package","qa","qa-parity","qa-live"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }} + if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","install-smoke","cross-os","live-e2e","package","qa-parity","qa-live"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }} runs-on: blacksmith-4vcpu-ubuntu-2404 # The bounded package critical path tops out at 310 minutes; 420 leaves # queue/API margin. Parent timeout preserves the adopted child for exact cancellation. @@ -1142,8 +1158,8 @@ jobs: FAIL_FAST: ${{ inputs.fail_fast }} ALLOW_UNRELEASED_CHANGELOG: ${{ inputs.allow_unreleased_changelog || (inputs.target_context_ref == '' && (inputs.ref == 'main' || inputs.ref == 'refs/heads/main')) }} RERUN_GROUP: ${{ inputs.rerun_group }} - LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} - CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + LIVE_SUITE_FILTER: ${{ needs.resolve_target.outputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ needs.resolve_target.outputs.cross_os_suite_filter }} RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }} @@ -1240,6 +1256,7 @@ jobs: RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} NPM_TELEGRAM_PACKAGE_SPEC: ${{ inputs.npm_telegram_package_spec }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} + LIVE_SUITE_FILTER: ${{ needs.resolve_target.outputs.live_suite_filter }} SKIP_PACKAGE_TELEGRAM_E2E: ${{ inputs.skip_package_telegram_e2e }} EVIDENCE_REUSE: ${{ needs.evidence_reuse.outputs.reuse }} EVIDENCE_ROOT_RUN_ID: ${{ needs.evidence_reuse.outputs.evidence_root_run_id }} @@ -1519,9 +1536,12 @@ jobs: echo "- Package Telegram E2E deferred: \`${SKIP_PACKAGE_TELEGRAM_E2E}\`" >> "$GITHUB_STEP_SUMMARY" if [[ -z "${RELEASE_PACKAGE_SPEC// }" && -z "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then case "$RERUN_GROUP" in - all|plugin-prerelease|release-checks|cross-os|live-e2e|package) + all|plugin-prerelease|cross-os|package) candidate_required=1 ;; + live-e2e) + [[ -n "${LIVE_SUITE_FILTER// }" ]] || candidate_required=1 + ;; esac fi if [[ "$candidate_required" == "1" && "$EVIDENCE_REUSE" != "true" && "$PREPARE_RELEASE_CANDIDATE_RESULT" != "success" ]]; then @@ -1588,7 +1608,7 @@ jobs: plugin-prerelease) plugin_prerelease_required=1 ;; - release-checks|install-smoke|cross-os|live-e2e|package|qa|qa-parity|qa-live) + install-smoke|cross-os|live-e2e|package|qa-parity|qa-live) release_checks_required=1 ;; performance) @@ -1760,8 +1780,8 @@ jobs: PROVIDER: ${{ inputs.provider }} MODE: ${{ inputs.mode }} TARGET_CONTEXT_REF: ${{ inputs.target_context_ref }} - LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} - CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + LIVE_SUITE_FILTER: ${{ needs.resolve_target.outputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ needs.resolve_target.outputs.cross_os_suite_filter }} RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }} diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index a5f29088b730..2fb796db70ae 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -149,6 +149,10 @@ jobs: release_profile: ${{ steps.inputs.outputs.release_profile }} run_release_soak: ${{ steps.inputs.outputs.run_release_soak }} qa_live_scheduled: ${{ steps.inputs.outputs.qa_live_scheduled }} + release_check_groups_json: ${{ steps.inputs.outputs.release_check_groups_json }} + package_required: ${{ steps.inputs.outputs.package_required }} + docker_required: ${{ steps.inputs.outputs.docker_required }} + install_smoke_scheduled: ${{ steps.inputs.outputs.install_smoke_scheduled }} fail_fast: ${{ steps.inputs.outputs.fail_fast }} run_maturity_scorecard: ${{ steps.inputs.outputs.run_maturity_scorecard }} allow_unreleased_changelog: ${{ steps.inputs.outputs.allow_unreleased_changelog }} @@ -168,7 +172,9 @@ jobs: package_acceptance_package_spec: ${{ steps.inputs.outputs.package_acceptance_package_spec }} codex_plugin_spec: ${{ steps.inputs.outputs.codex_plugin_spec }} cross_os_scheduled: ${{ steps.inputs.outputs.cross_os_scheduled }} - docker_release_scheduled: ${{ steps.inputs.outputs.docker_release_scheduled }} + live_e2e_scheduled: ${{ steps.inputs.outputs.live_e2e_scheduled }} + package_acceptance_scheduled: ${{ steps.inputs.outputs.package_acceptance_scheduled }} + qa_parity_scheduled: ${{ steps.inputs.outputs.qa_parity_scheduled }} steps: - name: Require trusted workflow ref for release checks env: @@ -423,6 +429,7 @@ jobs: RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT: ${{ inputs.package_acceptance_package_spec }} RELEASE_CODEX_PLUGIN_SPEC_INPUT: ${{ inputs.codex_plugin_spec }} CANDIDATE_ARTIFACT_JSON_INPUT: ${{ inputs.candidate_artifact_json }} + RELEASE_FILTER_VALIDATOR: workflow/scripts/github/validate-release-suite-filters.sh run: | set -euo pipefail if [[ -n "${CANDIDATE_ARTIFACT_JSON_INPUT// }" ]] && @@ -511,12 +518,19 @@ jobs: if [[ -z "${codex_plugin_spec// }" && "$RELEASE_PACKAGE_SPEC_INPUT" =~ ^openclaw@(.+)$ ]]; then codex_plugin_spec="npm:@openclaw/codex@${BASH_REMATCH[1]}" fi + source "$RELEASE_FILTER_VALIDATOR" + validate_release_suite_filters \ + "$RELEASE_RERUN_GROUP_INPUT" \ + "$RELEASE_LIVE_SUITE_FILTER_INPUT" \ + "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT" \ + release-checks + live_suite_filter="$RELEASE_FILTER_LIVE_SUITE_FILTER" + cross_os_suite_filter="$RELEASE_FILTER_CROSS_OS_SUITE_FILTER" - qa_filter_seen=false - filter="$(printf '%s' "$RELEASE_LIVE_SUITE_FILTER_INPUT" | tr '[:upper:]' '[:lower:]')" - repo_live_suite_filter="$filter" - if [[ -n "${filter// }" ]]; then - repo_filter_tokens=() + qa_filter_seen="$RELEASE_FILTER_QA_FILTER_SEEN" + filter="$live_suite_filter" + repo_live_suite_filter="$RELEASE_FILTER_REPO_LIVE_SUITE_FILTER" + if [[ -n "$filter" ]]; then matrix_selected=false buzz_selected=false telegram_selected=false @@ -525,12 +539,8 @@ jobs: slack_selected=false disabled_required_lanes=() - IFS=', ' read -r -a filter_tokens <<< "$filter" + IFS=',' read -r -a filter_tokens <<< "$filter" for token in "${filter_tokens[@]}"; do - token="${token//$'\t'/}" - token="${token//$'\r'/}" - token="${token//$'\n'/}" - [[ -z "$token" ]] && continue case "$token" in qa-live|qa-live-all|qa-all) qa_filter_seen=true @@ -577,20 +587,13 @@ jobs: [[ "$qa_live_whatsapp_ci_enabled" == "true" ]] || disabled_required_lanes+=("qa-live-whatsapp") ;; qa-live-slack|qa-slack|slack) - qa_filter_seen=true slack_selected="$qa_live_slack_ci_enabled" [[ "$qa_live_slack_ci_enabled" == "true" ]] || disabled_required_lanes+=("qa-live-slack") ;; - *) - repo_filter_tokens+=("$token") - ;; + *) ;; esac done - if [[ "$qa_filter_seen" == "true" ]]; then - repo_live_suite_filter="$(IFS=,; printf '%s' "${repo_filter_tokens[*]-}")" - fi - if [[ "${#disabled_required_lanes[@]}" -gt 0 ]]; then echo "live_suite_filter explicitly requested disabled QA live lane(s): ${disabled_required_lanes[*]}" >&2 echo "Enable the matching OPENCLAW_RELEASE_QA_*_LIVE_CI_ENABLED repo variable or remove the lane from live_suite_filter." >&2 @@ -607,21 +610,55 @@ jobs: fi fi - qa_live_scheduled=false - if [[ "$RELEASE_RERUN_GROUP_INPUT" == "qa" || "$RELEASE_RERUN_GROUP_INPUT" == "qa-live" ]] || - [[ "$RELEASE_RERUN_GROUP_INPUT" == "all" && ( "$run_release_soak" == "true" || "$qa_filter_seen" == "true" ) ]]; then - qa_live_scheduled=true - fi + release_check_groups=() + case "$RELEASE_RERUN_GROUP_INPUT" in + all) + release_check_groups=(install-smoke cross-os package qa-parity) + if [[ "$run_release_soak" == "true" ]]; then + release_check_groups+=(live-e2e) + fi + if [[ "$run_release_soak" == "true" || "$qa_filter_seen" == "true" ]]; then + release_check_groups+=(qa-live) + fi + ;; + qa) + release_check_groups=(qa-parity qa-live) + ;; + install-smoke|cross-os|live-e2e|package|qa-parity|qa-live) + release_check_groups=("$RELEASE_RERUN_GROUP_INPUT") + ;; + *) + echo "rerun_group must be one of: all, install-smoke, cross-os, live-e2e, package, qa, qa-parity, qa-live" >&2 + exit 1 + ;; + esac + release_check_groups_json="$(printf '%s\n' "${release_check_groups[@]}" | jq -Rsc 'split("\n") | map(select(length > 0))')" + group_selected() { + jq -e --arg group "$1" 'index($group) != null' <<< "$release_check_groups_json" >/dev/null + } + install_smoke_scheduled=false cross_os_scheduled=false - if [[ "$RELEASE_RERUN_GROUP_INPUT" == "all" || "$RELEASE_RERUN_GROUP_INPUT" == "cross-os" ]]; then - cross_os_scheduled=true + live_e2e_scheduled=false + package_acceptance_scheduled=false + qa_parity_scheduled=false + qa_live_scheduled=false + group_selected install-smoke && install_smoke_scheduled=true + group_selected cross-os && cross_os_scheduled=true + group_selected live-e2e && live_e2e_scheduled=true + group_selected package && package_acceptance_scheduled=true + group_selected qa-parity && qa_parity_scheduled=true + group_selected qa-live && qa_live_scheduled=true + + docker_required=false + if [[ "$live_e2e_scheduled" == "true" && -z "$repo_live_suite_filter" ]]; then + docker_required=true fi - docker_release_scheduled=false - if { [[ "$RELEASE_RERUN_GROUP_INPUT" == "live-e2e" ]] || - { [[ "$RELEASE_RERUN_GROUP_INPUT" == "all" ]] && [[ "$run_release_soak" == "true" ]]; }; } && - [[ -z "${repo_live_suite_filter// }" ]]; then - docker_release_scheduled=true + package_required=false + if [[ "$cross_os_scheduled" == "true" || + "$package_acceptance_scheduled" == "true" || + "$docker_required" == "true" ]]; then + package_required=true fi { @@ -631,14 +668,18 @@ jobs: printf 'release_profile=%s\n' "$release_profile" printf 'run_release_soak=%s\n' "$run_release_soak" printf 'qa_live_scheduled=%s\n' "$qa_live_scheduled" + printf 'release_check_groups_json=%s\n' "$release_check_groups_json" + printf 'package_required=%s\n' "$package_required" + printf 'docker_required=%s\n' "$docker_required" + printf 'install_smoke_scheduled=%s\n' "$install_smoke_scheduled" printf 'fail_fast=%s\n' "$fail_fast" printf 'run_maturity_scorecard=%s\n' "$run_maturity_scorecard" printf 'allow_unreleased_changelog=%s\n' "$allow_unreleased_changelog" printf 'skip_package_telegram_e2e=%s\n' "$skip_package_telegram_e2e" printf 'rerun_group=%s\n' "$RELEASE_RERUN_GROUP_INPUT" - printf 'live_suite_filter=%s\n' "$RELEASE_LIVE_SUITE_FILTER_INPUT" + printf 'live_suite_filter=%s\n' "$live_suite_filter" printf 'repo_live_suite_filter=%s\n' "$repo_live_suite_filter" - printf 'cross_os_suite_filter=%s\n' "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT" + printf 'cross_os_suite_filter=%s\n' "$cross_os_suite_filter" printf 'qa_live_matrix_enabled=%s\n' "$qa_live_matrix_enabled" printf 'qa_live_buzz_enabled=%s\n' "$qa_live_buzz_enabled" printf 'qa_live_telegram_enabled=%s\n' "$qa_live_telegram_enabled" @@ -649,7 +690,9 @@ jobs: printf 'package_acceptance_package_spec=%s\n' "$RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT" printf 'codex_plugin_spec=%s\n' "$codex_plugin_spec" printf 'cross_os_scheduled=%s\n' "$cross_os_scheduled" - printf 'docker_release_scheduled=%s\n' "$docker_release_scheduled" + printf 'live_e2e_scheduled=%s\n' "$live_e2e_scheduled" + printf 'package_acceptance_scheduled=%s\n' "$package_acceptance_scheduled" + printf 'qa_parity_scheduled=%s\n' "$qa_parity_scheduled" } >> "$GITHUB_OUTPUT" - name: Summarize validated ref @@ -662,6 +705,9 @@ jobs: RELEASE_PROFILE: ${{ steps.inputs.outputs.release_profile }} RUN_RELEASE_SOAK: ${{ steps.inputs.outputs.run_release_soak }} QA_LIVE_SCHEDULED: ${{ steps.inputs.outputs.qa_live_scheduled }} + RELEASE_CHECK_GROUPS_JSON: ${{ steps.inputs.outputs.release_check_groups_json }} + PACKAGE_REQUIRED: ${{ steps.inputs.outputs.package_required }} + DOCKER_REQUIRED: ${{ steps.inputs.outputs.docker_required }} FAIL_FAST: ${{ steps.inputs.outputs.fail_fast }} RUN_MATURITY_SCORECARD: ${{ steps.inputs.outputs.run_maturity_scorecard }} SKIP_PACKAGE_TELEGRAM_E2E: ${{ steps.inputs.outputs.skip_package_telegram_e2e }} @@ -683,6 +729,9 @@ jobs: echo "- Cross-OS mode: \`${RELEASE_MODE}\`" echo "- Release profile: \`${RELEASE_PROFILE}\`" echo "- Release soak lanes: \`${RUN_RELEASE_SOAK}\`" + echo "- Concrete release-check groups: \`${RELEASE_CHECK_GROUPS_JSON}\`" + echo "- Shared package artifact required: \`${PACKAGE_REQUIRED}\`" + echo "- Docker release-path prep required: \`${DOCKER_REQUIRED}\`" echo "- QA-live scheduled: \`${QA_LIVE_SCHEDULED}\`" echo "- Matrix QA fail fast: \`${FAIL_FAST}\`" echo "- Maturity scorecard docs: \`${RUN_MATURITY_SCORECARD}\`" @@ -721,7 +770,7 @@ jobs: prepare_release_package: name: Prepare release package artifact needs: [resolve_target] - if: needs.resolve_target.outputs.cross_os_scheduled == 'true' || needs.resolve_target.outputs.docker_release_scheduled == 'true' || needs.resolve_target.outputs.rerun_group == 'package' + if: needs.resolve_target.outputs.package_required == 'true' runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: @@ -773,7 +822,7 @@ jobs: shell: bash env: CROSS_OS_SCHEDULED: ${{ needs.resolve_target.outputs.cross_os_scheduled }} - DOCKER_RELEASE_SCHEDULED: ${{ needs.resolve_target.outputs.docker_release_scheduled }} + DOCKER_REQUIRED: ${{ needs.resolve_target.outputs.docker_required }} PACKAGE_REF: ${{ needs.resolve_target.outputs.revision }} PROVIDER: ${{ needs.resolve_target.outputs.provider }} RELEASE_PACKAGE_SPEC: ${{ needs.resolve_target.outputs.release_package_spec }} @@ -796,7 +845,7 @@ jobs: )" fi docker_packages='[]' - if [[ "$DOCKER_RELEASE_SCHEDULED" == "true" && -z "${RELEASE_PACKAGE_SPEC// }" ]]; then + if [[ "$DOCKER_REQUIRED" == "true" && -z "${RELEASE_PACKAGE_SPEC// }" ]]; then export OPENCLAW_DOCKER_ALL_PROFILE=release-path export OPENCLAW_DOCKER_ALL_PLAN_RELEASE_ALL=1 export OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI="${{ needs.resolve_target.outputs.release_profile != 'beta' }}" @@ -1033,7 +1082,7 @@ jobs: install_smoke_release_checks: needs: [resolve_target] - if: contains(fromJSON('["all","install-smoke"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.install_smoke_scheduled == 'true' permissions: actions: read contents: read @@ -1080,7 +1129,7 @@ jobs: live_repo_e2e_release_checks: name: Run repo/live E2E validation needs: [resolve_target] - if: needs.resolve_target.outputs.rerun_group == 'live-e2e' || (needs.resolve_target.outputs.rerun_group == 'all' && needs.resolve_target.outputs.run_release_soak == 'true') + if: needs.resolve_target.outputs.live_e2e_scheduled == 'true' permissions: actions: read contents: read @@ -1155,7 +1204,7 @@ jobs: docker_e2e_release_checks: name: Run Docker release-path validation needs: [resolve_target, prepare_release_package] - if: needs.resolve_target.outputs.docker_release_scheduled == 'true' + if: needs.resolve_target.outputs.docker_required == 'true' permissions: actions: read contents: read @@ -1202,7 +1251,7 @@ jobs: package_acceptance_release_checks: name: Run package acceptance needs: [resolve_target, prepare_release_package] - if: contains(fromJSON('["all","package"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.package_acceptance_scheduled == 'true' permissions: actions: read contents: read @@ -1302,7 +1351,7 @@ jobs: qa_lab_parity_lane_release_checks: name: Run QA Lab parity lane (${{ matrix.lane }}) needs: [resolve_target] - if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.qa_parity_scheduled == 'true' continue-on-error: true runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -1470,7 +1519,7 @@ jobs: qa_lab_parity_report_release_checks: name: Run QA Lab parity report needs: [resolve_target, qa_lab_parity_lane_release_checks] - if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.qa_parity_scheduled == 'true' continue-on-error: true runs-on: ubuntu-24.04 timeout-minutes: 20 @@ -1596,7 +1645,7 @@ jobs: qa_lab_runtime_pair_lane_release_checks: name: Run QA Lab runtime-pair lane (${{ matrix.lane }}) needs: [resolve_target] - if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.qa_parity_scheduled == 'true' continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 45 @@ -1808,7 +1857,7 @@ jobs: qa_lab_runtime_parity_release_checks: name: Verify QA Lab runtime-pair lanes needs: [resolve_target, qa_lab_runtime_pair_lane_release_checks] - if: always() && contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + if: always() && needs.resolve_target.outputs.qa_parity_scheduled == 'true' continue-on-error: true runs-on: ubuntu-24.04 timeout-minutes: 15 @@ -1925,7 +1974,7 @@ jobs: runtime_tool_coverage_release_checks: name: Enforce QA Lab runtime tool coverage needs: [resolve_target, qa_lab_runtime_parity_release_checks] - if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + if: needs.resolve_target.outputs.qa_parity_scheduled == 'true' runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: diff --git a/docs/help/testing.md b/docs/help/testing.md index a4290ed91df2..2e7b9d3230a3 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -168,10 +168,12 @@ These commands sit beside the main test suites when you need QA-lab realism. CI runs QA Lab in dedicated workflows. Agentic parity is nested under `QA-Lab - All Lanes` and release validation, not a standalone PR workflow. Broad validation should use `Full Release Validation` with -`rerun_group=qa-parity` or the release-checks QA group. Stable/full, -soak-enabled, and explicit `qa`/`qa-live` release checks include the QA-live -Matrix and Telegram lanes. Bounded beta-publish `all` without soak runs parity -but defers those live lanes to postpublish-confidence. `QA-Lab - All Lanes` runs +`rerun_group=qa-parity` for parity or `rerun_group=qa-live` for live QA. +The direct `OpenClaw Release Checks` child alone may use `rerun_group=qa` as a +manual aggregate of both groups. Stable/full, soak-enabled, and explicit +`qa-live` release checks include the QA-live Matrix and Telegram lanes. Bounded +beta-publish `all` without soak runs parity but defers those live lanes to +postpublish-confidence. `QA-Lab - All Lanes` runs nightly on `main` and from manual dispatch with the mock parity lane, live Matrix lane, Convex-managed live Telegram lane, and Convex-managed live Discord lane as parallel jobs. Scheduled QA and selected release checks run the diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index acc5358d9400..fb23cbd9e11f 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -467,7 +467,7 @@ and rerun group and does not cancel prior runs. Parent cancellation leaves adopted children running until the operator cancels the exact child. Pass `reuse_evidence=false` only when a fresh full run is intentionally required. -For bounded recovery, pass `rerun_group` to the umbrella. `all` is the real release-candidate run, `ci` runs only the normal CI child, `plugin-prerelease` runs only the release-only plugin child, `release-checks` runs every release box, and the narrower release groups are `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and `npm-telegram`. Focused `npm-telegram` reruns require `release_package_spec` or `npm_telegram_package_spec`; full/all runs use the canonical package Telegram E2E inside Package Acceptance. Focused cross-OS reruns can add `cross_os_suite_filter=windows/packaged-upgrade` or another OS/suite filter. QA release-check failures block normal release validation, including OpenClaw dynamic tool drift in the core runtime-pair lane. Tideclaw alpha runs may still treat non-package-safety release-check lanes as advisory. With `release_profile=beta`, the `Run repo/live E2E validation` live-provider suites are advisory (warnings, not blockers); stable and full profiles keep them blocking. When `live_suite_filter` explicitly requests a gated QA live lane such as Discord, WhatsApp, or Slack, the matching `OPENCLAW_RELEASE_QA_*_LIVE_CI_ENABLED` repo variable must be enabled; otherwise input capture fails instead of silently skipping the lane. +For bounded recovery, pass `rerun_group` to the umbrella. Supported controller groups are `ci`, `plugin-prerelease`, `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa-parity`, `qa-live`, `npm-telegram`, and `performance`; use `all` only for deliberate full validation. The removed `release-checks` aggregate handle is invalid because it silently selected every release-check lane and its package/Docker setup. `qa` remains available only as a direct `OpenClaw Release Checks` manual aggregate, not as an umbrella/controller retry API. Focused `npm-telegram` reruns require `release_package_spec` or `npm_telegram_package_spec`; full/all runs use the canonical Package Acceptance Telegram E2E. Focused cross-OS reruns can add `cross_os_suite_filter=windows/packaged-upgrade` or another OS/suite filter. Live, QA-live, and cross-OS filters are valid only with their owning group; mismatches fail before scheduling and never become an unfiltered broad run. QA release-check failures block normal release validation, including OpenClaw dynamic tool drift in the core runtime-pair lane. Tideclaw alpha runs may still treat non-package-safety release-check lanes as advisory. With `release_profile=beta`, the `Run repo/live E2E validation` live-provider suites are advisory (warnings, not blockers); stable and full profiles keep them blocking. When `live_suite_filter` explicitly requests a gated QA live lane such as Discord, WhatsApp, or Slack, the matching `OPENCLAW_RELEASE_QA_*_LIVE_CI_ENABLED` repo variable must be enabled; otherwise input capture fails instead of silently skipping the lane. ### Vitest diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index f58c24a155ad..685eee868bb8 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -144,17 +144,17 @@ the `runtime-assets` Docker target with other stages and is enforced by the umbrella verifier; lanes no longer wait for it before dispatching. A narrower `rerun_group` skips this preflight. -| Stage | Details | -| ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Target resolution | **Job:** `Resolve target ref`
**Child workflow:** none
**Proves:** resolves the release branch, tag, or full commit SHA and records selected inputs.
**Rerun:** rerun the umbrella if this fails. | -| Shared candidate | **Job:** `Prepare shared release candidate`
**Child workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Proves:** packs and validates one exact-SHA package, builds one functional Docker image, and records immutable package and image artifact tuples for both package-facing child workflows.
**Rerun:** rerun the affected package, plugin-prerelease, cross-OS, or live/E2E group. | -| Docker assets preflight | **Job:** `Verify Docker runtime image assets`
**Child workflow:** none
**Proves:** the `runtime-assets` Docker build target still succeeds before any other stage dispatches. Runs only for `rerun_group=all`.
**Rerun:** rerun the umbrella with `rerun_group=all`. | -| Vitest and normal CI | **Job:** `Run normal full CI`
**Child workflow:** `CI`
**Proves:** manual full CI graph against the target ref, including Linux Node lanes, bundled plugin shards, plugin and channel contract shards, Node 22 compatibility, `check-*`, `check-additional-*`, built-artifact smoke checks, docs checks, Python skills, Windows, macOS, Control UI i18n, and Android via the umbrella.
**Rerun:** `rerun_group=ci`. | -| Plugin prerelease | **Job:** `Run plugin prerelease validation`
**Child workflow:** `Plugin Prerelease`
**Proves:** release-only plugin static checks, agentic plugin coverage, full plugin batch shards, plugin prerelease Docker lanes, and a non-blocking `plugin-inspector-advisory` artifact for compatibility triage.
**Rerun:** `rerun_group=plugin-prerelease`. | -| Release checks | **Job:** `Run release/live/Docker/QA validation`
**Child workflow:** `OpenClaw Release Checks`
**Proves:** install smoke, cross-OS package checks, Package Acceptance, and QA Lab parity. QA-live Matrix, Buzz, and Telegram plus gated advisory Discord, WhatsApp, and Slack run for stable/full, beta with `run_release_soak=true`, or explicit `qa`/`qa-live` groups. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks.
**Rerun:** `rerun_group=release-checks` or a narrower release-checks handle. | -| Package Telegram | **Job:** `Run package Telegram E2E`
**Child workflow:** `NPM Telegram Beta E2E`
**Proves:** a focused published-package Telegram E2E when `release_package_spec` or `npm_telegram_package_spec` is set. Full candidate validation uses the canonical Package Acceptance Telegram E2E instead.
**Rerun:** `rerun_group=npm-telegram` with `release_package_spec` or `npm_telegram_package_spec`. | -| Product performance | **Job:** `Run product performance evidence`
**Child workflow:** `OpenClaw Performance`
**Proves:** release-profile performance run (`profile=release`, `repeat=3`, `fail_on_regression=true`, `publish_reports=false`) against the target SHA. Kova output stays in workflow artifacts and the child must prove its report publisher was skipped. Required (blocking) only for `rerun_group=all` or `rerun_group=performance`; not required for narrower rerun groups.
**Rerun:** `rerun_group=performance`. | -| Umbrella verifier | **Job:** `Verify full validation`
**Child workflow:** none
**Proves:** re-checks recorded child run conclusions and appends slowest-job tables from child workflows.
**Rerun:** rerun only this job after rerunning a failed child to green. | +| Stage | Details | +| ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Target resolution | **Job:** `Resolve target ref`
**Child workflow:** none
**Proves:** resolves the release branch, tag, or full commit SHA and records selected inputs.
**Rerun:** rerun the umbrella if this fails. | +| Shared candidate | **Job:** `Prepare shared release candidate`
**Child workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Proves:** packs and validates one exact-SHA package, builds one functional Docker image, and records immutable package and image artifact tuples for both package-facing child workflows.
**Rerun:** rerun the affected package, plugin-prerelease, cross-OS, or live/E2E group. | +| Docker assets preflight | **Job:** `Verify Docker runtime image assets`
**Child workflow:** none
**Proves:** the `runtime-assets` Docker build target still succeeds before any other stage dispatches. Runs only for `rerun_group=all`.
**Rerun:** rerun the umbrella with `rerun_group=all`. | +| Vitest and normal CI | **Job:** `Run normal full CI`
**Child workflow:** `CI`
**Proves:** manual full CI graph against the target ref, including Linux Node lanes, bundled plugin shards, plugin and channel contract shards, Node 22 compatibility, `check-*`, `check-additional-*`, built-artifact smoke checks, docs checks, Python skills, Windows, macOS, Control UI i18n, and Android via the umbrella.
**Rerun:** `rerun_group=ci`. | +| Plugin prerelease | **Job:** `Run plugin prerelease validation`
**Child workflow:** `Plugin Prerelease`
**Proves:** release-only plugin static checks, agentic plugin coverage, full plugin batch shards, plugin prerelease Docker lanes, and a non-blocking `plugin-inspector-advisory` artifact for compatibility triage.
**Rerun:** `rerun_group=plugin-prerelease`. | +| Release checks | **Job:** `Run release/live/Docker/QA validation`
**Child workflow:** `OpenClaw Release Checks`
**Proves:** install smoke, cross-OS package checks, Package Acceptance, and QA Lab parity. QA-live Matrix, Buzz, and Telegram plus gated advisory Discord, WhatsApp, and Slack run for stable/full, beta with `run_release_soak=true`, an explicit `qa-live` controller retry, or the direct child's manual `qa` aggregate. Stable and full profiles also run exhaustive live/E2E suites and Docker release-path chunks.
**Rerun:** classify the failed surface and select one concrete release-check group. | +| Package Telegram | **Job:** `Run package Telegram E2E`
**Child workflow:** `NPM Telegram Beta E2E`
**Proves:** a focused published-package Telegram E2E when `release_package_spec` or `npm_telegram_package_spec` is set. Full candidate validation uses the canonical Package Acceptance Telegram E2E instead.
**Rerun:** `rerun_group=npm-telegram` with `release_package_spec` or `npm_telegram_package_spec`. | +| Product performance | **Job:** `Run product performance evidence`
**Child workflow:** `OpenClaw Performance`
**Proves:** release-profile performance run (`profile=release`, `repeat=3`, `fail_on_regression=true`, `publish_reports=false`) against the target SHA. Kova output stays in workflow artifacts and the child must prove its report publisher was skipped. Required (blocking) only for `rerun_group=all` or `rerun_group=performance`; not required for narrower rerun groups.
**Rerun:** `rerun_group=performance`. | +| Umbrella verifier | **Job:** `Verify full validation`
**Child workflow:** none
**Proves:** re-checks recorded child run conclusions and appends slowest-job tables from child workflows.
**Rerun:** rerun only this job after rerunning a failed child to green. | The umbrella always dispatches product performance in artifact-only mode. `OpenClaw Performance` permits report publication only for scheduled runs or a @@ -189,20 +189,20 @@ artifact when package or Docker-facing stages need it. | Stage | Details | | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Release target | **Job:** `Resolve target ref`
**Backing workflow:** none
**Tests:** selected ref, optional expected Validation SHA, profile, rerun group, and focused live suite filter.
**Rerun:** `rerun_group=release-checks`. | +| Release target | **Job:** `Resolve target ref`
**Backing workflow:** none
**Tests:** selected ref, optional expected Validation SHA, profile, concrete release-check groups, and focused live suite filter.
**Rerun:** select the concrete group for the failed surface. | | Package artifact | **Job:** `Prepare release package artifact`
**Backing workflow:** none
**Tests:** validates the umbrella's immutable package tuple, or packs one candidate tarball for a direct/focused Release Checks dispatch, then exposes it to downstream package-facing checks.
**Rerun:** the affected package, cross-OS, or live/E2E group. | | Install smoke | **Job:** `Run install smoke`
**Backing workflow:** `Install Smoke`
**Tests:** full install path with root Dockerfile smoke image reuse, QR package install, root and gateway Docker smokes, installer Docker tests, and Bun global install image-provider smoke.
**Rerun:** `rerun_group=install-smoke`. | | Cross-OS | **Job:** `cross_os_release_checks`
**Backing workflow:** `OpenClaw Cross-OS Release Checks (Reusable)`
**Tests:** fresh and upgrade lanes on Linux, Windows, and macOS for the selected provider and mode, using the candidate tarball plus a baseline package.
**Rerun:** `rerun_group=cross-os`. | | Repo and live E2E | **Job:** `Run repo/live E2E validation`
**Backing workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Tests:** repository E2E, live cache, OpenAI websocket streaming, native live provider and plugin shards, and Docker-backed live model/backend/gateway harnesses selected by `release_profile`.
**Runs:** `run_release_soak=true`, `release_profile=full`, or focused `rerun_group=live-e2e`.
**Rerun:** `rerun_group=live-e2e`, optionally with `live_suite_filter`. | | Docker release path | **Job:** `Run Docker release-path validation`
**Backing workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Tests:** release-path Docker chunks against the shared package artifact.
**Runs:** `run_release_soak=true`, `release_profile=full`, or focused `rerun_group=live-e2e`.
**Rerun:** `rerun_group=live-e2e`. | | Package Acceptance | **Job:** `Run package acceptance`
**Backing workflow:** `Package Acceptance`
**Tests:** offline plugin package fixtures, plugin update, the canonical mock-OpenAI Telegram package E2E, and published-upgrade survivor checks against the same tarball. Blocking release checks use the default latest published baseline; soak checks (`run_release_soak=true`) expand to the last 4 stable npm releases plus 3 pinned historical versions (`2026.4.23`, `2026.5.2`, `2026.4.15`), run against reported-issue upgrade fixtures.
**Rerun:** `rerun_group=package`. | -| Maturity scorecard | **Job:** `Render maturity scorecard release docs`
**Backing workflow:** `maturity-scorecard.yml`
**Tests:** renders the advisory maturity scorecard docs against the target ref. Only runs when `run_maturity_scorecard=true` is passed.
**Rerun:** `rerun_group=qa` with `run_maturity_scorecard=true`. | -| QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`
**Backing workflow:** direct jobs
**Tests:** candidate and baseline agentic parity packs, then the parity report.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | -| QA runtime parity | **Job:** `Verify QA Lab runtime-pair lanes`
**Backing workflow:** direct job
**Tests:** the canonical core `openclaw`/`codex` lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex --runtime-pair-lane core`) and, with `run_release_soak=true`, the soak lane. Advisory: individual lane jobs do not block the release-check verifier.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | -| QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`
**Backing workflow:** direct job
**Tests:** dynamic tool drift between `openclaw` and `codex` in the canonical core runtime-pair lane (`pnpm openclaw qa coverage --tools`), using that lane's output. Blocking: this job is not advisory-overridable.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | -| QA live Matrix | **Job:** `Run QA Live Matrix catalog`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** catalog-derived YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment, distributed across deterministic shards.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-matrix` for a focused Matrix rerun. | -| QA live Buzz | **Job:** `Run QA Lab live Buzz lane`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** signed canary and mention-gating round trips through the real Buzz plugin using dedicated Convex-leased identities and a hosted relay room.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`; use `live_suite_filter=qa-live-buzz` for a focused Buzz rerun. | -| QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** trusted `OpenClaw Release Telegram QA` dispatch
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | +| Maturity scorecard | **Job:** `Render maturity scorecard release docs`
**Backing workflow:** `maturity-scorecard.yml`
**Tests:** renders the advisory maturity scorecard docs against the target ref. Only runs when `run_maturity_scorecard=true` is passed.
**Rerun:** direct manual `rerun_group=qa` with `run_maturity_scorecard=true`. | +| QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`
**Backing workflow:** direct jobs
**Tests:** candidate and baseline agentic parity packs, then the parity report.
**Rerun:** `rerun_group=qa-parity`; direct manual child dispatch may aggregate with `qa`. | +| QA runtime parity | **Job:** `Verify QA Lab runtime-pair lanes`
**Backing workflow:** direct job
**Tests:** the canonical core `openclaw`/`codex` lane (`pnpm openclaw qa suite --runtime-pair openclaw,codex --runtime-pair-lane core`) and, with `run_release_soak=true`, the soak lane. Advisory: individual lane jobs do not block the release-check verifier.
**Rerun:** `rerun_group=qa-parity`; direct manual child dispatch may aggregate with `qa`. | +| QA runtime tool coverage | **Job:** `Enforce QA Lab runtime tool coverage`
**Backing workflow:** direct job
**Tests:** dynamic tool drift between `openclaw` and `codex` in the canonical core runtime-pair lane (`pnpm openclaw qa coverage --tools`), using that lane's output. Blocking: this job is not advisory-overridable.
**Rerun:** `rerun_group=qa-parity`; direct manual child dispatch may aggregate with `qa`. | +| QA live Matrix | **Job:** `Run QA Live Matrix catalog`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** catalog-derived YAML scenarios through the shared Matrix live adapter in the `qa-live-shared` environment, distributed across deterministic shards.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-matrix`; direct manual child dispatch may aggregate with `qa`. | +| QA live Buzz | **Job:** `Run QA Lab live Buzz lane`
**Backing workflow:** `QA-Lab - All Lanes` reusable workflow
**Tests:** signed canary and mention-gating round trips through the real Buzz plugin using dedicated Convex-leased identities and a hosted relay room.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-buzz`; direct manual child dispatch may aggregate with `qa`. | +| QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** trusted `OpenClaw Release Telegram QA` dispatch
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live`; direct manual child dispatch may aggregate with `qa`. | | QA live Discord | **Job:** `Run QA Lab live Discord lane`
**Backing workflow:** direct advisory job
**Tests:** live Discord QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_DISCORD_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-discord`. | | QA live WhatsApp | **Job:** `Run QA Lab live WhatsApp lane`
**Backing workflow:** direct advisory job
**Tests:** live WhatsApp QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_WHATSAPP_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-whatsapp`. | | QA live Slack | **Job:** `Run QA Lab live Slack lane`
**Backing workflow:** direct advisory job
**Tests:** live Slack QA with Convex CI credential leases when `OPENCLAW_RELEASE_QA_SLACK_LIVE_CI_ENABLED` is enabled.
**Rerun:** `rerun_group=qa-live` with `live_suite_filter=qa-live-slack`. | @@ -234,9 +234,10 @@ commands with package artifact and image reuse inputs when available. It does not remove normal full CI, Plugin Prerelease, install smoke, package acceptance, or QA parity. Stable and full profiles always run exhaustive repo/live E2E, Docker release-path, and QA-live soak coverage. The beta profile -adds those lanes only with `run_release_soak=true` or an explicit `qa` or -`qa-live` rerun. Package Acceptance supplies the canonical package Telegram -E2E for every candidate, so the umbrella does not duplicate that live poller. +adds those lanes only with `run_release_soak=true`, an explicit `qa-live` +controller retry, or the direct child's manual `qa` aggregate. Package +Acceptance supplies the canonical package Telegram E2E for every candidate, so +the umbrella does not duplicate that live poller. | Profile | Intended use | Included live/provider coverage | | -------- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -268,21 +269,26 @@ Use `rerun_group` to avoid repeating unrelated release boxes: | Handle | Scope | | ------------------- | ----------------------------------------------------------------------------------------------- | -| `all` | Phase-default stages; beta without soak excludes broad live/E2E and QA-live. | +| `all` | Deliberate full validation; beta without soak excludes broad live/E2E and QA-live. | | `ci` | Manual full CI child only. | | `plugin-prerelease` | Plugin Prerelease child only. | -| `release-checks` | All OpenClaw Release Checks stages. | | `install-smoke` | Install Smoke through release checks. | | `cross-os` | Cross-OS release checks. | | `live-e2e` | Repo/live E2E and Docker release-path validation. | | `package` | Package Acceptance. | -| `qa` | QA parity plus QA live lanes. | | `qa-parity` | QA parity lanes and report only. | | `qa-live` | QA live Matrix, Buzz, and Telegram plus gated Discord, WhatsApp, and Slack lanes when enabled. | | `npm-telegram` | Published-package Telegram E2E; requires `release_package_spec` or `npm_telegram_package_spec`. | | `performance` | Product performance evidence only. | Use `live_suite_filter` with `rerun_group=live-e2e` when one live suite failed. +The former `release-checks` aggregate retry handle is invalid. It silently +expanded to every release-check lane, including package and Docker setup. Pick +one concrete group after classifying the failed surface. +The umbrella/controller also rejects `qa`; direct `OpenClaw Release Checks` +dispatches may use it only as a deliberate manual aggregate of `qa-parity` and +`qa-live`. Live, QA-live, and cross-OS filters must match their owning group. +Mismatches fail before scheduling and never widen to an unfiltered run. Valid filter ids are defined in the reusable live/E2E workflow, including `docker-live-models`, `live-gateway-docker`, `live-gateway-anthropic-docker`, `live-gateway-google-docker`, @@ -316,8 +322,8 @@ them blocking. When `live_suite_filter` explicitly requests a gated QA live lane such as Discord, WhatsApp, or Slack, the matching `OPENCLAW_RELEASE_QA_*_LIVE_CI_ENABLED` repo variable must be enabled; otherwise input capture fails instead of silently skipping the lane. -Rerun `rerun_group=qa`, `qa-parity`, or `qa-live` when you -need fresh QA evidence. +Use controller groups `qa-parity` or `qa-live` for fresh QA evidence. A direct +manual `OpenClaw Release Checks` dispatch may use `qa` to aggregate both. ## Evidence to keep diff --git a/scripts/full-release-validation-at-sha.mts b/scripts/full-release-validation-at-sha.mts index 058ffc3e5ed3..a9636209c6de 100644 --- a/scripts/full-release-validation-at-sha.mts +++ b/scripts/full-release-validation-at-sha.mts @@ -38,6 +38,19 @@ const RELEASE_CONTEXT_BRANCH_PATTERN = const RELEASE_TAG_PATTERN = /^v([0-9]{4}\.(?:[1-9]|1[0-2])\.[1-9][0-9]*(?:-(?:alpha|beta)\.[1-9][0-9]*)?)$/u; const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const RERUN_GROUPS = new Set([ + "all", + "ci", + "plugin-prerelease", + "install-smoke", + "cross-os", + "live-e2e", + "package", + "qa-parity", + "qa-live", + "npm-telegram", + "performance", +]); const DEFAULT_INPUTS = { provider: "openai", mode: "both", @@ -86,7 +99,9 @@ run. Child workflows collect independent failures by default; pass branch accepts only its final package version or a matching beta prerelease. Exact alpha tags remain supported for Tideclaw. The release profile defaults to beta for beta candidates and exact alpha tags, and stable otherwise; pass --f release_profile=full for the broad advisory sweep.`); +-f release_profile=full for the broad advisory sweep. Focused retries must use +one controller rerun_group; the removed release-checks aggregate and the direct +child's manual qa aggregate are not accepted.`); } function run(command: string, args: string[], options: CommandOptions = {}) { @@ -219,6 +234,9 @@ export function parseArgs(argv: string[]) { ) { throw new Error("release_profile must be beta, stable, or full"); } + if (!RERUN_GROUPS.has(args.inputs.rerun_group)) { + throw new Error(`rerun_group must be one of: ${[...RERUN_GROUPS].join(", ")}`); + } if (Object.hasOwn(args.inputs, "ref")) { throw new Error("SHA-pinned release validation reserves the ref input for --sha"); } diff --git a/scripts/github/validate-release-suite-filters.sh b/scripts/github/validate-release-suite-filters.sh new file mode 100755 index 000000000000..34e6a69f29a3 --- /dev/null +++ b/scripts/github/validate-release-suite-filters.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +normalize_release_suite_filter() { + local raw="$1" + local normalized="" + local token + while IFS= read -r token; do + [[ -z "$token" ]] && continue + normalized+="${normalized:+,}${token}" + done < <(printf '%s\n' "$raw" | tr '[:upper:]' '[:lower:]' | tr ',[:space:]' '\n') + printf '%s' "$normalized" +} + +validate_release_suite_filters() { + local rerun_group="$1" + local raw_live_suite_filter="$2" + local raw_cross_os_suite_filter="$3" + local dispatch_scope="$4" + local live_suite_filter + local cross_os_suite_filter + local qa_filter_seen=false + local repo_filter_seen=false + local -a repo_filter_tokens=() + local token + + case "$dispatch_scope" in + controller) + case "$rerun_group" in + all|ci|plugin-prerelease|install-smoke|cross-os|live-e2e|package|qa-parity|qa-live|npm-telegram|performance) ;; + *) + echo "controller rerun_group is invalid: ${rerun_group}." >&2 + return 1 + ;; + esac + ;; + release-checks) + case "$rerun_group" in + all|install-smoke|cross-os|live-e2e|package|qa|qa-parity|qa-live) ;; + *) + echo "release-checks rerun_group is invalid: ${rerun_group}." >&2 + return 1 + ;; + esac + ;; + *) + echo "release suite filter dispatch scope is invalid: ${dispatch_scope}." >&2 + return 1 + ;; + esac + + live_suite_filter="$(normalize_release_suite_filter "$raw_live_suite_filter")" + cross_os_suite_filter="$(normalize_release_suite_filter "$raw_cross_os_suite_filter")" + if [[ -n "$raw_live_suite_filter" && -z "$live_suite_filter" ]]; then + echo "live_suite_filter must contain at least one suite selector." >&2 + return 1 + fi + if [[ -n "$raw_cross_os_suite_filter" && -z "$cross_os_suite_filter" ]]; then + echo "cross_os_suite_filter must contain at least one suite selector." >&2 + return 1 + fi + if [[ -n "$cross_os_suite_filter" && "$rerun_group" != "cross-os" ]]; then + echo "cross_os_suite_filter requires rerun_group=cross-os; received ${rerun_group}." >&2 + return 1 + fi + + if [[ -n "$live_suite_filter" ]]; then + local -a filter_tokens=() + IFS=',' read -r -a filter_tokens <<< "$live_suite_filter" + for token in "${filter_tokens[@]}"; do + case "$token" in + qa-live|qa-live-all|qa-all|\ + qa-live-non-slack|qa-non-slack|non-slack|no-slack|without-slack|\ + qa-live-matrix|qa-matrix|matrix|\ + qa-live-buzz|qa-buzz|buzz|\ + qa-live-telegram|qa-telegram|telegram|\ + qa-live-discord|qa-discord|discord|\ + qa-live-whatsapp|qa-whatsapp|whatsapp|\ + qa-live-slack|qa-slack|slack) + qa_filter_seen=true + ;; + *) + repo_filter_seen=true + repo_filter_tokens+=("$token") + ;; + esac + done + fi + + if [[ "$qa_filter_seen" == "true" && "$rerun_group" != "qa" && "$rerun_group" != "qa-live" ]]; then + echo "QA live_suite_filter selectors require rerun_group=qa or qa-live; received ${rerun_group}." >&2 + return 1 + fi + if [[ "$repo_filter_seen" == "true" && "$rerun_group" != "live-e2e" ]]; then + echo "Repo live_suite_filter selectors require rerun_group=live-e2e; received ${rerun_group}." >&2 + return 1 + fi + + # Outputs are consumed by the workflow step that sources this helper. + # shellcheck disable=SC2034 + RELEASE_FILTER_LIVE_SUITE_FILTER="$live_suite_filter" + # shellcheck disable=SC2034 + RELEASE_FILTER_CROSS_OS_SUITE_FILTER="$cross_os_suite_filter" + # shellcheck disable=SC2034 + RELEASE_FILTER_REPO_LIVE_SUITE_FILTER="$(IFS=,; printf '%s' "${repo_filter_tokens[*]-}")" + # shellcheck disable=SC2034 + RELEASE_FILTER_QA_FILTER_SEEN="$qa_filter_seen" +} diff --git a/scripts/release-ci-summary.mjs b/scripts/release-ci-summary.mjs index a67aada25627..328cce1c560c 100755 --- a/scripts/release-ci-summary.mjs +++ b/scripts/release-ci-summary.mjs @@ -82,18 +82,21 @@ const RERUN_GROUP_CHILD_KEYS = new Map([ ["all", ["normalCi", "releaseChecks", "pluginPrerelease", "productPerformance"]], ["ci", ["normalCi"]], ["plugin-prerelease", ["pluginPrerelease"]], - ["release-checks", ["releaseChecks"]], ["install-smoke", ["releaseChecks"]], ["cross-os", ["releaseChecks"]], ["live-e2e", ["releaseChecks"]], ["package", ["releaseChecks"]], - ["qa", ["releaseChecks"]], ["qa-parity", ["releaseChecks"]], ["qa-live", ["releaseChecks"]], ["npm-telegram", ["npmTelegram"]], ["performance", ["productPerformance"]], ]); +const HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS = new Map([ + ["release-checks", ["releaseChecks"]], + ["qa", ["releaseChecks"]], +]); + export function runReleaseCiGh(args, params = {}) { const execFileSyncImpl = params.execFileSyncImpl ?? execFileSync; const timeoutMs = params.timeoutMs ?? GH_COMMAND_TIMEOUT_MS; @@ -205,6 +208,16 @@ export function requiredChildKeysForRerunGroup(rerunGroup, validationInputs = {} return selectedKeys; } +function requiredChildKeysForManifest(manifest) { + if ( + [2, 3].includes(manifest.version) && + HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS.has(manifest.rerunGroup) + ) { + return new Set(HISTORICAL_MANIFEST_RERUN_GROUP_CHILD_KEYS.get(manifest.rerunGroup)); + } + return requiredChildKeysForRerunGroup(manifest.rerunGroup, manifest.validationInputs); +} + export function expectedSelectedChildDispatches( parentRunId, parentRunAttempt, @@ -453,7 +466,7 @@ export function validateParentManifest(value, expected) { workflowSha = normalizeSha(expected.workflowSha, "release validation workflow SHA"); } const rerunGroup = String(value.rerunGroup ?? ""); - requiredChildKeysForRerunGroup(rerunGroup); + requiredChildKeysForManifest({ rerunGroup, version: value.version }); const releaseProfile = String(value.releaseProfile ?? ""); if (!["beta", "stable", "full"].includes(releaseProfile)) { throw new Error("release validation manifest release profile is invalid"); @@ -1419,10 +1432,7 @@ export function validateReleaseRunEvidence( ); } } - const selectedKeys = requiredChildKeysForRerunGroup( - rootEvidence.manifest.rerunGroup, - rootEvidence.manifest.validationInputs, - ); + const selectedKeys = requiredChildKeysForManifest(rootEvidence.manifest); const expectedChildren = expectedSelectedChildDispatches( rootEvidence.manifest.runId, rootEvidence.manifest.runAttempt, @@ -1807,10 +1817,7 @@ async function main() { ); } - const selectedKeys = requiredChildKeysForRerunGroup( - sourceManifest.rerunGroup, - sourceManifest.validationInputs, - ); + const selectedKeys = requiredChildKeysForManifest(sourceManifest); const expectedChildren = expectedSelectedChildDispatches( sourceManifest.runId, sourceManifest.runAttempt, diff --git a/test/scripts/full-release-validation-at-sha.test.ts b/test/scripts/full-release-validation-at-sha.test.ts index d160463e7339..0a05232e76b5 100644 --- a/test/scripts/full-release-validation-at-sha.test.ts +++ b/test/scripts/full-release-validation-at-sha.test.ts @@ -221,6 +221,14 @@ describe("full-release-validation-at-sha", () => { expect(() => parseArgs(["--", "-f"])).toThrow("-f requires a value"); }); + it("rejects retry groups that are not controller APIs", () => { + expect(() => parseArgs(["-f", "rerun_group=release-checks"])).toThrow( + "rerun_group must be one of", + ); + expect(() => parseArgs(["-f", "rerun_group=qa"])).toThrow("rerun_group must be one of"); + expect(parseArgs(["-f", "rerun_group=qa-parity"]).inputs.rerun_group).toBe("qa-parity"); + }); + it("infers the release profile from the target package version", () => { const readVersion = (version: string) => () => JSON.stringify({ version }); diff --git a/test/scripts/openclaw-cross-os-release-workflow.test.ts b/test/scripts/openclaw-cross-os-release-workflow.test.ts index 9d52689af2e2..59237119c15d 100644 --- a/test/scripts/openclaw-cross-os-release-workflow.test.ts +++ b/test/scripts/openclaw-cross-os-release-workflow.test.ts @@ -228,23 +228,24 @@ describe("cross-OS release checks workflow", () => { const resolveTarget = job(workflow, "resolve_target"); expect(resolveTarget.outputs).toMatchObject({ cross_os_scheduled: "${{ steps.inputs.outputs.cross_os_scheduled }}", - docker_release_scheduled: "${{ steps.inputs.outputs.docker_release_scheduled }}", + docker_required: "${{ steps.inputs.outputs.docker_required }}", + package_required: "${{ steps.inputs.outputs.package_required }}", }); const capture = step(resolveTarget, "Capture selected inputs"); expect(capture.run).toContain("cross_os_scheduled=false"); - expect(capture.run).toContain("docker_release_scheduled=false"); - expect(capture.run).toContain('"$RELEASE_RERUN_GROUP_INPUT" == "cross-os"'); - expect(capture.run).toContain('[[ -z "${repo_live_suite_filter// }" ]]'); + expect(capture.run).toContain("docker_required=false"); + expect(capture.run).toContain("package_required=false"); + expect(capture.run).toContain("group_selected cross-os && cross_os_scheduled=true"); + expect(capture.run).toContain( + '"$live_e2e_scheduled" == "true" && -z "$repo_live_suite_filter"', + ); const producer = job(workflow, "prepare_release_package"); - expect(producer.if).toContain("needs.resolve_target.outputs.cross_os_scheduled == 'true'"); - expect(producer.if).toContain( - "needs.resolve_target.outputs.docker_release_scheduled == 'true'", - ); + expect(producer.if).toBe("needs.resolve_target.outputs.package_required == 'true'"); const resolvePackage = step(producer, "Resolve release package artifact"); expect(resolvePackage.run).toContain('if [[ "$CROSS_OS_SCHEDULED" == "true" ]]'); expect(resolvePackage.run).toContain( - 'if [[ "$DOCKER_RELEASE_SCHEDULED" == "true" && -z "${RELEASE_PACKAGE_SPEC// }" ]]', + 'if [[ "$DOCKER_REQUIRED" == "true" && -z "${RELEASE_PACKAGE_SPEC// }" ]]', ); expect(resolvePackage.run).toContain("registry_args=()"); expect(resolvePackage.run).toContain("if [[ \"$required_packages\" != '[]' ]]"); @@ -252,7 +253,7 @@ describe("cross-OS release checks workflow", () => { "needs.resolve_target.outputs.cross_os_scheduled == 'true'", ); expect(job(workflow, "docker_e2e_release_checks").if).toBe( - "needs.resolve_target.outputs.docker_release_scheduled == 'true'", + "needs.resolve_target.outputs.docker_required == 'true'", ); }); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index d14cf57b0578..223553a2a866 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -114,6 +114,7 @@ const CRABBOX_CONFIG = ".crabbox.yaml"; const SCHEDULED_LIVE_CHECKS_WORKFLOW = ".github/workflows/openclaw-scheduled-live-checks.yml"; const CI_HYDRATE_LIVE_AUTH_SCRIPT = "scripts/ci-hydrate-live-auth.sh"; const RELEASE_CHECK_ARTIFACT_RESOLVER = "scripts/github/resolve-release-check-artifacts.sh"; +const RELEASE_FILTER_VALIDATOR = "scripts/github/validate-release-suite-filters.sh"; const VERIFY_PROVIDER_SECRETS_SCRIPT = ".agents/skills/release-openclaw-ci/scripts/verify-provider-secrets.mjs"; const UPGRADE_SURVIVOR_RUN_SCRIPT = "scripts/e2e/lib/upgrade-survivor/run.sh"; @@ -397,6 +398,7 @@ function runReleaseChecksInputValidation( GITHUB_OUTPUT: outputPath, PATH: process.env.PATH, RELEASE_FAIL_FAST_INPUT: "false", + RELEASE_FILTER_VALIDATOR: resolve(RELEASE_FILTER_VALIDATOR), RELEASE_LIVE_SUITE_FILTER_INPUT: liveSuiteFilter, RELEASE_MODE_INPUT: "both", RELEASE_PROFILE_INPUT: releaseProfile, @@ -4113,11 +4115,11 @@ describe("package artifact reuse", () => { }, ); - it("schedules only the selected QA-live lane for an all-group QA filter", () => { + it("schedules only the selected QA-live lane for a QA-group filter", () => { const { outputPath, result } = runReleaseChecksInputValidation( "beta", "false", - "all", + "qa", "false", "qa-live-telegram", ); @@ -4131,11 +4133,11 @@ describe("package artifact reuse", () => { } }); - it("does not schedule QA-live for an all-group repo live filter without soak", () => { + it("keeps a focused repo-live filter within the live-E2E group", () => { const { outputPath, result } = runReleaseChecksInputValidation( "beta", "false", - "all", + "live-e2e", "false", "repo-e2e", ); @@ -4144,10 +4146,12 @@ describe("package artifact reuse", () => { const output = readFileSync(outputPath, "utf8"); expect(output).toContain("qa_live_scheduled=false\n"); expect(output).toContain("repo_live_suite_filter=repo-e2e\n"); + expect(output).toContain("package_required=false\n"); + expect(output).toContain("docker_required=false\n"); }); - it("does not let a QA-live filter override an unrelated rerun group", () => { - const { outputPath, result } = runReleaseChecksInputValidation( + it("rejects a QA-live filter for an unrelated rerun group", () => { + const { result } = runReleaseChecksInputValidation( "beta", "false", "install-smoke", @@ -4155,14 +4159,14 @@ describe("package artifact reuse", () => { "qa-live-telegram", ); - expect(result.status, result.stderr).toBe(0); - const output = readFileSync(outputPath, "utf8"); - expect(output).toContain("qa_live_scheduled=false\n"); - expect(output).toContain("qa_live_telegram_enabled=true\n"); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "QA live_suite_filter selectors require rerun_group=qa or qa-live", + ); }); it("summarizes Telegram deferral only when Package Acceptance is scheduled", () => { - const scheduled = runFullReleaseTargetSummary("release-checks", "true"); + const scheduled = runFullReleaseTargetSummary("package", "true"); const unrelated = runFullReleaseTargetSummary("ci", "true"); expect(scheduled.result.status, scheduled.result.stderr).toBe(0); @@ -4178,6 +4182,7 @@ describe("package artifact reuse", () => { it("includes package acceptance in release checks", () => { const workflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8"); + const filterValidator = readFileSync(RELEASE_FILTER_VALIDATOR, "utf8"); const packageAcceptanceWorkflow = parse(readFileSync(PACKAGE_ACCEPTANCE_WORKFLOW, "utf8")) as { on?: { workflow_call?: { inputs?: Record }; @@ -4298,9 +4303,13 @@ describe("package artifact reuse", () => { expect(workflow).toContain("rerun_group:"); expect(workflow).toContain("live_suite_filter:"); expect(workflow).toContain("repo_live_suite_filter:"); - expect(workflow).toContain('repo_filter_tokens+=("$token")'); expect(workflow).toContain( - 'repo_live_suite_filter="$(IFS=,; printf \'%s\' "${repo_filter_tokens[*]-}")"', + "RELEASE_FILTER_VALIDATOR: workflow/scripts/github/validate-release-suite-filters.sh", + ); + expect(workflow).toContain('source "$RELEASE_FILTER_VALIDATOR"'); + expect(filterValidator).toContain('repo_filter_tokens+=("$token")'); + expect(filterValidator).toContain( + 'RELEASE_FILTER_REPO_LIVE_SUITE_FILTER="$(IFS=,; printf \'%s\' "${repo_filter_tokens[*]-}")"', ); expect(workflow).toContain("cross_os_suite_filter:"); expect(workflow).toContain("advisory: false"); @@ -4310,12 +4319,8 @@ describe("package artifact reuse", () => { expect(workflow).toContain( "live_suite_filter: ${{ needs.resolve_target.outputs.repo_live_suite_filter }}", ); - expect(workflow).toContain( - "if: needs.resolve_target.outputs.cross_os_scheduled == 'true' || needs.resolve_target.outputs.docker_release_scheduled == 'true' || needs.resolve_target.outputs.rerun_group == 'package'", - ); - expect(workflow).toContain( - "if: needs.resolve_target.outputs.docker_release_scheduled == 'true'", - ); + expect(workflow).toContain("if: needs.resolve_target.outputs.package_required == 'true'"); + expect(workflow).toContain("if: needs.resolve_target.outputs.docker_required == 'true'"); expect(workflow).toContain( 'if [[ "$release_profile" == "stable" || "$release_profile" == "full" ]]; then\n run_release_soak=true', ); @@ -4323,6 +4328,13 @@ describe("package artifact reuse", () => { expect(workflow).toContain("- live-e2e"); expect(workflow).toContain("- qa-live"); expect(workflow).toContain("disabled_required_lanes=()"); + expect(filterValidator).toContain( + "QA live_suite_filter selectors require rerun_group=qa or qa-live", + ); + expect(filterValidator).toContain( + "Repo live_suite_filter selectors require rerun_group=live-e2e", + ); + expect(filterValidator).toContain("cross_os_suite_filter requires rerun_group=cross-os"); expect(workflow).toContain("live_suite_filter explicitly requested disabled QA live lane(s)"); expect(workflow).toContain("OPENCLAW_RELEASE_QA_*_LIVE_CI_ENABLED"); expect(workflow).not.toContain( @@ -5284,12 +5296,11 @@ describe("package artifact reuse", () => { ]); expect(dispatchStep.run).not.toContain("package_artifact"); expectTextToIncludeAll(workflow, [ - "child_rerun_group=all", - '-f rerun_group="$child_rerun_group"', + '-f rerun_group="$RERUN_GROUP"', 'args+=(-f live_suite_filter="$LIVE_SUITE_FILTER")', 'args+=(-f cross_os_suite_filter="$CROSS_OS_SUITE_FILTER")', 'case "$RERUN_GROUP" in', - "release-checks|install-smoke|cross-os|live-e2e|package|qa|qa-parity|qa-live)", + "install-smoke|cross-os|live-e2e|package|qa-parity|qa-live)", "cancel-in-progress: false", "Verify release checks accepted Tideclaw alpha advisory lanes", "release_checks_advisory_only", diff --git a/test/scripts/release-ci-summary.test.ts b/test/scripts/release-ci-summary.test.ts index 1e4b2735357c..073c6a4e019b 100644 --- a/test/scripts/release-ci-summary.test.ts +++ b/test/scripts/release-ci-summary.test.ts @@ -1595,6 +1595,23 @@ describe("release CI summary child correlation", () => { expect(manifest.rerunGroup).toBe("all"); }); + it.each([ + [2, "release-checks"], + [2, "qa"], + [3, "release-checks"], + [3, "qa"], + ] as const)("keeps historical v%s %s manifests readable", (version, rerunGroup) => { + const workflowSha = version === 3 ? "b".repeat(40) : undefined; + const manifest = validateParentManifest(rawManifest({ rerunGroup, version, workflowSha }), { + runAttempt: 2, + runId: "29090000000", + workflowSha, + }); + + expect(manifest.rerunGroup).toBe(rerunGroup); + expect(manifest.version).toBe(version); + }); + it("binds v3 manifests to their immutable producer workflow SHA", () => { const workflowSha = "b".repeat(40); const manifest = validateParentManifest(rawManifest({ version: 3, workflowSha }), { @@ -1677,6 +1694,12 @@ describe("release CI summary child correlation", () => { }); it("requires the child mapped by rerunGroup and scans only selected in-progress workflows", () => { + expect(() => requiredChildKeysForRerunGroup("release-checks")).toThrow( + "release validation manifest rerun group is invalid: release-checks", + ); + expect(() => requiredChildKeysForRerunGroup("qa")).toThrow( + "release validation manifest rerun group is invalid: qa", + ); const focused = validateParentManifest( { ...rawManifest({ rerunGroup: "npm-telegram" }), diff --git a/test/scripts/release-no-push-workflow.test.ts b/test/scripts/release-no-push-workflow.test.ts index 8ea3d7f70d2b..73b2c6a2acc5 100644 --- a/test/scripts/release-no-push-workflow.test.ts +++ b/test/scripts/release-no-push-workflow.test.ts @@ -27,6 +27,7 @@ const UPDATE_MIGRATION = ".github/workflows/update-migration.yml"; const PERFORMANCE = ".github/workflows/openclaw-performance.yml"; const LIVE_BUILD = "scripts/test-live-build-docker.sh"; const DOCKER_E2E_IMAGE_HELPER = "scripts/lib/docker-e2e-image.sh"; +const RELEASE_FILTER_VALIDATOR = resolve("scripts/github/validate-release-suite-filters.sh"); type WorkflowInput = { default?: boolean | number | string; @@ -191,7 +192,465 @@ function expectReadOnlyPackagePermission(workflowJob: WorkflowJob): void { expect(permissionAt(workflowJob.permissions, "packages", "none")).toBe("read"); } +function executeReleaseGroupCapture( + group: string, + runReleaseSoak = false, + liveSuiteFilter = "", + crossOsSuiteFilter = "", +) { + const root = mkdtempSync(join(tmpdir(), "openclaw-release-groups-")); + const output = join(root, "github-output"); + writeFileSync(output, ""); + try { + const capture = step( + job(readWorkflow(RELEASE_CHECKS), "resolve_target"), + "Capture selected inputs", + ); + const result = spawnSync("bash", ["-euo", "pipefail", "-c", capture.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + CANDIDATE_ARTIFACT_JSON_INPUT: "", + GITHUB_OUTPUT: output, + RELEASE_ALLOW_UNRELEASED_CHANGELOG_INPUT: "false", + RELEASE_CODEX_PLUGIN_SPEC_INPUT: "", + RELEASE_CROSS_OS_SUITE_FILTER_INPUT: crossOsSuiteFilter, + RELEASE_FAIL_FAST_INPUT: "false", + RELEASE_FILTER_VALIDATOR, + RELEASE_LIVE_SUITE_FILTER_INPUT: liveSuiteFilter, + RELEASE_MODE_INPUT: "both", + RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT: "", + RELEASE_PACKAGE_SPEC_INPUT: "", + RELEASE_PROFILE_INPUT: "beta", + RELEASE_PROVIDER_INPUT: "openai", + RELEASE_QA_DISCORD_LIVE_CI_ENABLED: "false", + RELEASE_QA_SLACK_LIVE_CI_ENABLED: "false", + RELEASE_QA_WHATSAPP_LIVE_CI_ENABLED: "false", + RELEASE_REF_INPUT: "main", + RELEASE_RERUN_GROUP_INPUT: group, + RELEASE_RUN_MATURITY_SCORECARD_INPUT: "false", + RELEASE_RUN_RELEASE_SOAK_INPUT: String(runReleaseSoak), + RELEASE_SKIP_PACKAGE_TELEGRAM_E2E_INPUT: "false", + }, + }); + const outputText = readFileSync(output, "utf8").trim(); + const outputs = outputText + ? Object.fromEntries( + outputText.split("\n").map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ) + : {}; + return { outputs, result }; + } finally { + rmSync(root, { force: true, recursive: true }); + } +} + +function runReleaseGroupCapture( + group: string, + runReleaseSoak = false, + liveSuiteFilter = "", + crossOsSuiteFilter = "", +): Record { + const execution = executeReleaseGroupCapture( + group, + runReleaseSoak, + liveSuiteFilter, + crossOsSuiteFilter, + ); + expect(execution.result.status, `${group}: ${execution.result.stderr}`).toBe(0); + return execution.outputs; +} + +function executeParentFilterValidation( + group: string, + liveSuiteFilter = "", + crossOsSuiteFilter = "", +) { + const root = mkdtempSync(join(tmpdir(), "openclaw-parent-filter-normalization-")); + const output = join(root, "github-output"); + writeFileSync(output, ""); + try { + const normalize = step( + job(readWorkflow(FULL_RELEASE), "resolve_target"), + "Validate suite filters", + ); + const result = spawnSync("bash", ["-euo", "pipefail", "-c", normalize.run ?? ""], { + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + RAW_CROSS_OS_SUITE_FILTER: crossOsSuiteFilter, + RAW_LIVE_SUITE_FILTER: liveSuiteFilter, + RELEASE_FILTER_VALIDATOR, + RERUN_GROUP: group, + }, + }); + return { output: readFileSync(output, "utf8"), result }; + } finally { + rmSync(root, { force: true, recursive: true }); + } +} + describe("release validation no-push transport", () => { + it("routes release retries through explicit concrete groups and resource gates", () => { + const full = readWorkflow(FULL_RELEASE); + const release = readWorkflow(RELEASE_CHECKS); + const umbrellaGroups = full.on?.workflow_dispatch?.inputs?.rerun_group?.options ?? []; + const releaseGroups = release.on?.workflow_dispatch?.inputs?.rerun_group?.options ?? []; + const dispatch = step(job(full, "release_checks"), "Dispatch and monitor release checks"); + const capture = step(job(release, "resolve_target"), "Capture selected inputs"); + const parentFilters = step(job(full, "resolve_target"), "Validate suite filters"); + + expect(umbrellaGroups).toEqual([ + "all", + "ci", + "plugin-prerelease", + "install-smoke", + "cross-os", + "live-e2e", + "package", + "qa-parity", + "qa-live", + "npm-telegram", + "performance", + ]); + expect(umbrellaGroups).not.toContain("release-checks"); + expect(umbrellaGroups).not.toContain("qa"); + expect(releaseGroups).not.toContain("release-checks"); + expect(releaseGroups).toContain("qa"); + expect(parentFilters.env?.RELEASE_FILTER_VALIDATOR).toBe( + "workflow/scripts/github/validate-release-suite-filters.sh", + ); + expect(capture.env?.RELEASE_FILTER_VALIDATOR).toBe( + "workflow/scripts/github/validate-release-suite-filters.sh", + ); + expect(dispatch.run).toContain('-f rerun_group="$RERUN_GROUP"'); + expect(dispatch.run).not.toContain("child_rerun_group"); + const candidate = job(full, "prepare_release_candidate"); + expect(candidate.if).not.toContain('"release-checks"'); + expect(candidate.if).toContain( + 'contains(fromJSON(\'["all","plugin-prerelease","cross-os","package"]\'), inputs.rerun_group)', + ); + expect(candidate.if).toContain( + "(inputs.rerun_group == 'live-e2e' && needs.resolve_target.outputs.live_suite_filter == '')", + ); + const verify = step(job(full, "summary"), "Verify child workflow results"); + expect(verify.env?.LIVE_SUITE_FILTER).toBe( + "${{ needs.resolve_target.outputs.live_suite_filter }}", + ); + expect(verify.run).toContain('[[ -n "${LIVE_SUITE_FILTER// }" ]] || candidate_required=1'); + + expect(capture.run).toContain( + "release_check_groups=(install-smoke cross-os package qa-parity)", + ); + expect(capture.run).toContain("release_check_groups=(qa-parity qa-live)"); + expect(capture.run).toContain("release_check_groups_json="); + expect(capture.run).toContain("package_required=false"); + expect(capture.run).toContain("docker_required=false"); + expect(job(release, "prepare_release_package").if).toBe( + "needs.resolve_target.outputs.package_required == 'true'", + ); + expect(job(release, "docker_e2e_release_checks").if).toBe( + "needs.resolve_target.outputs.docker_required == 'true'", + ); + expect(job(release, "install_smoke_release_checks").if).toBe( + "needs.resolve_target.outputs.install_smoke_scheduled == 'true'", + ); + expect(job(release, "qa_lab_parity_lane_release_checks").if).toBe( + "needs.resolve_target.outputs.qa_parity_scheduled == 'true'", + ); + expect(job(release, "qa_live_release_checks").if).toContain( + "needs.resolve_target.outputs.qa_live_scheduled == 'true'", + ); + }); + + it.each([ + { + group: "install-smoke", + groups: ["install-smoke"], + packageRequired: "false", + dockerRequired: "false", + }, + { + group: "qa", + groups: ["qa-parity", "qa-live"], + packageRequired: "false", + dockerRequired: "false", + }, + { + group: "qa-parity", + groups: ["qa-parity"], + packageRequired: "false", + dockerRequired: "false", + }, + { + group: "qa-live", + groups: ["qa-live"], + packageRequired: "false", + dockerRequired: "false", + }, + { + group: "cross-os", + groups: ["cross-os"], + packageRequired: "true", + dockerRequired: "false", + }, + { + group: "package", + groups: ["package"], + packageRequired: "true", + dockerRequired: "false", + }, + { + group: "live-e2e", + groups: ["live-e2e"], + packageRequired: "true", + dockerRequired: "true", + }, + ])( + "maps $group to explicit release resources", + ({ group, groups, packageRequired, dockerRequired }) => { + const outputs = runReleaseGroupCapture(group); + expect(JSON.parse(outputs.release_check_groups_json ?? "null")).toEqual(groups); + expect(outputs.package_required).toBe(packageRequired); + expect(outputs.docker_required).toBe(dockerRequired); + }, + ); + + it("expands all only to the profile-selected concrete groups", () => { + const beta = runReleaseGroupCapture("all"); + const soak = runReleaseGroupCapture("all", true); + + expect(JSON.parse(beta.release_check_groups_json ?? "null")).toEqual([ + "install-smoke", + "cross-os", + "package", + "qa-parity", + ]); + expect(beta.docker_required).toBe("false"); + expect(JSON.parse(soak.release_check_groups_json ?? "null")).toEqual([ + "install-smoke", + "cross-os", + "package", + "qa-parity", + "live-e2e", + "qa-live", + ]); + expect(soak.docker_required).toBe("true"); + }); + + it("skips package and Docker prep for a focused repo live-E2E retry", () => { + const outputs = runReleaseGroupCapture("live-e2e", false, " Repo-E2E,\trepo-smoke "); + + expect(JSON.parse(outputs.release_check_groups_json ?? "null")).toEqual(["live-e2e"]); + expect(outputs.live_e2e_scheduled).toBe("true"); + expect(outputs.live_suite_filter).toBe("repo-e2e,repo-smoke"); + expect(outputs.repo_live_suite_filter).toBe("repo-e2e,repo-smoke"); + expect(outputs.package_required).toBe("false"); + expect(outputs.docker_required).toBe("false"); + }); + + it.each(["\t", " ", ",,,", " \t, , "])( + "rejects raw nonempty live filter %j before install-smoke scheduling", + (filter) => { + const parent = executeParentFilterValidation("install-smoke", filter); + const child = executeReleaseGroupCapture("install-smoke", false, filter); + + expect(parent.result.status).not.toBe(0); + expect(parent.result.stderr).toContain( + "live_suite_filter must contain at least one suite selector", + ); + expect(child.result.status).not.toBe(0); + expect(child.result.stderr).toContain( + "live_suite_filter must contain at least one suite selector", + ); + expect(child.outputs.install_smoke_scheduled).toBeUndefined(); + }, + ); + + it.each([ + "all", + "ci", + "plugin-prerelease", + "install-smoke", + "cross-os", + "live-e2e", + "package", + "qa-parity", + "npm-telegram", + "performance", + ])("parent rejects a QA selector with rerun_group=%s before scheduling", (group) => { + const { output, result } = executeParentFilterValidation(group, "qa-live-matrix"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "QA live_suite_filter selectors require rerun_group=qa or qa-live", + ); + expect(output).toBe(""); + }); + + it.each([ + "all", + "ci", + "plugin-prerelease", + "install-smoke", + "cross-os", + "package", + "qa-parity", + "qa-live", + "npm-telegram", + "performance", + ])("parent rejects a repo-live selector with rerun_group=%s before scheduling", (group) => { + const { output, result } = executeParentFilterValidation(group, "repo-e2e"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "Repo live_suite_filter selectors require rerun_group=live-e2e", + ); + expect(output).toBe(""); + }); + + it.each([ + "all", + "ci", + "plugin-prerelease", + "install-smoke", + "live-e2e", + "package", + "qa-parity", + "qa-live", + "npm-telegram", + "performance", + ])("parent rejects a cross-OS selector with rerun_group=%s before scheduling", (group) => { + const { output, result } = executeParentFilterValidation(group, "", "windows/packaged-upgrade"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("cross_os_suite_filter requires rerun_group=cross-os"); + expect(output).toBe(""); + }); + + it.each([ + ["qa-live", "qa-live-matrix", ""], + ["live-e2e", " Repo-E2E,\trepo-smoke ", ""], + ["cross-os", "", " Windows/Packaged-Upgrade "], + ])( + "parent accepts rerun_group=%s with its owned selector", + (group, liveSuiteFilter, crossOsSuiteFilter) => { + const { output, result } = executeParentFilterValidation( + group, + liveSuiteFilter, + crossOsSuiteFilter, + ); + + expect(result.status, result.stderr).toBe(0); + expect(output).not.toBe(""); + }, + ); + + it.each(["qa", "release-checks", "bogus", ""])( + "parent rejects unsupported controller rerun_group=%j before scheduling", + (group) => { + const { output, result } = executeParentFilterValidation(group); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(`controller rerun_group is invalid: ${group}.`); + expect(output).toBe(""); + }, + ); + + it.each(["\t", " ", ",,,", " \t, , "])( + "rejects raw nonempty live filter %j before live-E2E can widen or require prep", + (filter) => { + const { outputs, result } = executeReleaseGroupCapture("live-e2e", false, filter); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("live_suite_filter must contain at least one suite selector"); + expect(outputs.live_e2e_scheduled).toBeUndefined(); + expect(outputs.package_required).toBeUndefined(); + expect(outputs.docker_required).toBeUndefined(); + }, + ); + + it.each(["\t", " ", ",,,", " \t, , "])( + "rejects raw nonempty cross-OS filter %j before cross-OS scheduling", + (filter) => { + const parent = executeParentFilterValidation("cross-os", "", filter); + const child = executeReleaseGroupCapture("cross-os", false, "", filter); + + expect(parent.result.status).not.toBe(0); + expect(parent.result.stderr).toContain( + "cross_os_suite_filter must contain at least one suite selector", + ); + expect(child.result.status).not.toBe(0); + expect(child.result.stderr).toContain( + "cross_os_suite_filter must contain at least one suite selector", + ); + expect(child.outputs.cross_os_scheduled).toBeUndefined(); + }, + ); + + it("fails before a QA selector can collapse into an unfiltered live-E2E run", () => { + const { outputs, result } = executeReleaseGroupCapture("live-e2e", false, "qa-live-matrix"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "QA live_suite_filter selectors require rerun_group=qa or qa-live", + ); + expect(outputs.repo_live_suite_filter).toBeUndefined(); + expect(outputs.live_e2e_scheduled).toBeUndefined(); + }); + + it.each(["all", "install-smoke", "cross-os", "live-e2e", "package", "qa-parity"])( + "rejects a QA selector with rerun_group=%s", + (group) => { + const { result } = executeReleaseGroupCapture(group, false, "qa-live-matrix"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "QA live_suite_filter selectors require rerun_group=qa or qa-live", + ); + }, + ); + + it.each(["all", "install-smoke", "cross-os", "package", "qa", "qa-parity", "qa-live"])( + "rejects a repo-live selector with rerun_group=%s", + (group) => { + const { result } = executeReleaseGroupCapture(group, false, "repo-e2e"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain( + "Repo live_suite_filter selectors require rerun_group=live-e2e", + ); + }, + ); + + it.each(["all", "install-smoke", "live-e2e", "package", "qa", "qa-parity", "qa-live"])( + "rejects a cross-OS selector with rerun_group=%s", + (group) => { + const { result } = executeReleaseGroupCapture(group, false, "", "windows/packaged-upgrade"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("cross_os_suite_filter requires rerun_group=cross-os"); + }, + ); + + it.each([ + ["qa", "qa-live-matrix"], + ["qa-live", "qa-live-matrix"], + ["live-e2e", "repo-e2e"], + ])("accepts rerun_group=%s with selector %s", (group, filter) => { + const outputs = runReleaseGroupCapture(group, false, filter); + expect(outputs.rerun_group).toBe(group); + }); + + it("accepts a cross-OS selector only for the cross-OS group", () => { + const outputs = runReleaseGroupCapture("cross-os", false, "", "windows/packaged-upgrade"); + expect(outputs.cross_os_suite_filter).toBe("windows/packaged-upgrade"); + }); + it("builds planned live images locally without entering pull fallback", () => { const workflow = readWorkflow(LIVE_E2E); for (const jobName of [