diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index c14a0bad4e3b..23581fc2e4c9 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -1095,7 +1095,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-parity-${{ matrix.lane }}-${{ needs.resolve_target.outputs.revision }} + name: release-qa-parity-${{ matrix.lane }}-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -1175,19 +1175,66 @@ jobs: ref: ${{ needs.resolve_target.outputs.revision }} fetch-depth: 1 + - name: Checkout trusted release artifact resolver + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.sha }} + path: trusted-release-check-artifacts + sparse-checkout: scripts/github/resolve-release-check-artifacts.sh + sparse-checkout-cone-mode: false + fetch-depth: 1 + - name: Setup Node environment uses: ./.github/actions/setup-node-env with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + - name: Resolve parity lane artifacts + id: resolve_parity_lane_artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} + run: | + set -euo pipefail + selection_file=".artifacts/release-check-selection/parity-lanes.json" + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh resolve \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --consumer-attempt "$GITHUB_RUN_ATTEMPT" \ + --target-sha "$RELEASE_CHECK_TARGET_SHA" \ + --pair "qa_lab_parity_lane_release_checks|candidate|release-check-status-qa-parity-candidate-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-parity-candidate-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" \ + --pair "qa_lab_parity_lane_release_checks|baseline|release-check-status-qa-parity-baseline-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-parity-baseline-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" \ + --selection-file "$selection_file" \ + --github-output "$GITHUB_OUTPUT" + + - name: Download parity lane statuses + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.resolve_parity_lane_artifacts.outputs.status_ids }} + path: .artifacts/release-check-status/ + merge-multiple: true + - name: Download parity lane artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: release-qa-parity-*-${{ needs.resolve_target.outputs.revision }} + artifact-ids: ${{ steps.resolve_parity_lane_artifacts.outputs.payload_ids }} path: .artifacts/qa-e2e/ merge-multiple: true + - name: Verify parity lane statuses + shell: bash + run: | + set -euo pipefail + validated_file=".artifacts/release-check-selection/parity-lanes-validated.json" + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh validate \ + --selection-file "${{ steps.resolve_parity_lane_artifacts.outputs.selection_file }}" \ + --status-dir .artifacts/release-check-status \ + --validated-file "$validated_file" + jq -e 'length == 2 and all(.[]; .status == "success")' "$validated_file" >/dev/null + - name: Build private QA runtime env: NODE_OPTIONS: --max-old-space-size=8192 @@ -1209,7 +1256,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-parity-${{ needs.resolve_target.outputs.revision }} + name: release-qa-parity-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -1420,7 +1467,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-runtime-pair-lane-${{ matrix.lane }}-${{ needs.resolve_target.outputs.revision }} + name: release-qa-runtime-pair-lane-${{ matrix.lane }}-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -1454,79 +1501,91 @@ jobs: timeout-minutes: 15 permissions: actions: read + contents: read steps: - - name: Download runtime-pair lane artifacts - id: download_runtime_parity_artifacts - if: always() - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + - name: Checkout trusted release artifact resolver + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: - pattern: release-qa-runtime-pair-lane-*-${{ needs.resolve_target.outputs.revision }} - path: .artifacts/qa-e2e/ - merge-multiple: true + persist-credentials: false + ref: ${{ github.sha }} + path: trusted-release-check-artifacts + sparse-checkout: scripts/github/resolve-release-check-artifacts.sh + sparse-checkout-cone-mode: false + fetch-depth: 1 + + - name: Resolve runtime-pair lane artifacts + id: resolve_runtime_pair_artifacts + if: always() + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} + RUN_RELEASE_SOAK: ${{ needs.resolve_target.outputs.run_release_soak }} + run: | + set -euo pipefail + selection_file=".artifacts/release-check-selection/runtime-pair-lanes.json" + args=( + resolve + --repository "$GITHUB_REPOSITORY" + --run-id "$GITHUB_RUN_ID" + --consumer-attempt "$GITHUB_RUN_ATTEMPT" + --target-sha "$RELEASE_CHECK_TARGET_SHA" + --pair "qa_lab_runtime_pair_lane_release_checks|core|release-internal-status-qa-runtime-pair-lane-core-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-runtime-pair-lane-core-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + --selection-file "$selection_file" + --github-output "$GITHUB_OUTPUT" + ) + if [[ "$RUN_RELEASE_SOAK" == "true" ]]; then + args+=( + --pair "qa_lab_runtime_pair_lane_release_checks|soak|release-internal-status-qa-runtime-pair-lane-soak-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-runtime-pair-lane-soak-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh "${args[@]}" - name: Download runtime-pair lane statuses - id: download_runtime_parity_statuses - if: always() + id: download_runtime_pair_statuses + if: always() && steps.resolve_runtime_pair_artifacts.outcome == 'success' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: release-internal-status-qa-runtime-pair-lane-*-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} + artifact-ids: ${{ steps.resolve_runtime_pair_artifacts.outputs.status_ids }} path: .artifacts/release-check-status/ merge-multiple: true + - name: Download runtime-pair lane artifacts + id: download_runtime_pair_artifacts + if: always() && steps.resolve_runtime_pair_artifacts.outcome == 'success' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.resolve_runtime_pair_artifacts.outputs.payload_ids }} + path: .artifacts/qa-e2e/ + merge-multiple: true + - name: Verify runtime-pair lane statuses id: verify_runtime_parity_statuses if: always() shell: bash env: - RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} RUN_RELEASE_SOAK: ${{ needs.resolve_target.outputs.run_release_soak }} run: | set -euo pipefail - lanes=(core) + validated_file=".artifacts/release-check-selection/runtime-pair-lanes-validated.json" + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh validate \ + --selection-file "${{ steps.resolve_runtime_pair_artifacts.outputs.selection_file }}" \ + --status-dir .artifacts/release-check-status \ + --validated-file "$validated_file" + expected_count=1 if [[ "$RUN_RELEASE_SOAK" == "true" ]]; then - lanes+=(soak) + expected_count=2 fi - status_field() { - local file="$1" - local key="$2" - awk -v key="$key" ' - index($0, key "=") == 1 { - count += 1 - value = substr($0, length(key) + 2) - } - END { - if (count != 1) { - exit 1 - } - print value - } - ' "$file" - } - for lane in "${lanes[@]}"; do - status_path=".artifacts/release-check-status/qa_lab_runtime_pair_lane_release_checks-${lane}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.env" - run_id="$(status_field "$status_path" run_id)" - run_attempt="$(status_field "$status_path" run_attempt)" - target_sha="$(status_field "$status_path" target_sha)" - job_name="$(status_field "$status_path" job)" - variant="$(status_field "$status_path" variant)" - status="$(status_field "$status_path" status)" - if [[ "$run_id" != "$GITHUB_RUN_ID" || - "$run_attempt" != "$GITHUB_RUN_ATTEMPT" || - "$target_sha" != "$RELEASE_CHECK_TARGET_SHA" || - "$job_name" != "qa_lab_runtime_pair_lane_release_checks" || - "$variant" != "$lane" || - "$status" != "success" ]]; then - echo "Runtime-pair lane ${lane} did not produce matching successful evidence." >&2 - exit 1 - fi - done + jq -e --argjson expected_count "$expected_count" \ + 'length == $expected_count and all(.[]; .status == "success")' \ + "$validated_file" >/dev/null - name: Upload runtime parity artifacts id: upload_runtime_parity_artifacts if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }} + name: release-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -1538,7 +1597,7 @@ jobs: RELEASE_CHECK_JOB: qa_lab_runtime_parity_release_checks RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} JOB_STATUS: ${{ job.status }} - RELEASE_CHECK_STEP_OUTCOMES: ${{ steps.download_runtime_parity_artifacts.outcome }} ${{ steps.download_runtime_parity_statuses.outcome }} ${{ steps.verify_runtime_parity_statuses.outcome }} ${{ steps.upload_runtime_parity_artifacts.outcome }} + RELEASE_CHECK_STEP_OUTCOMES: ${{ steps.resolve_runtime_pair_artifacts.outcome }} ${{ steps.download_runtime_pair_statuses.outcome }} ${{ steps.download_runtime_pair_artifacts.outcome }} ${{ steps.verify_runtime_parity_statuses.outcome }} ${{ steps.upload_runtime_parity_artifacts.outcome }} run: *record_release_check_advisory_status - name: Upload advisory status @@ -1570,55 +1629,61 @@ jobs: ref: ${{ needs.resolve_target.outputs.revision }} fetch-depth: 1 + - name: Checkout trusted release artifact resolver + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.sha }} + path: trusted-release-check-artifacts + sparse-checkout: scripts/github/resolve-release-check-artifacts.sh + sparse-checkout-cone-mode: false + fetch-depth: 1 + - name: Setup Node environment uses: ./.github/actions/setup-node-env with: node-version: ${{ env.NODE_VERSION }} install-bun: "true" + - name: Resolve runtime parity artifacts + id: resolve_runtime_parity_artifacts + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} + run: | + set -euo pipefail + selection_file=".artifacts/release-check-selection/runtime-parity.json" + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh resolve \ + --repository "$GITHUB_REPOSITORY" \ + --run-id "$GITHUB_RUN_ID" \ + --consumer-attempt "$GITHUB_RUN_ATTEMPT" \ + --target-sha "$RELEASE_CHECK_TARGET_SHA" \ + --pair "qa_lab_runtime_parity_release_checks||release-check-status-qa-runtime-parity-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-runtime-parity-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" \ + --selection-file "$selection_file" \ + --github-output "$GITHUB_OUTPUT" + - name: Download runtime parity status uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: release-check-status-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} + artifact-ids: ${{ steps.resolve_runtime_parity_artifacts.outputs.status_ids }} path: .artifacts/release-check-status/ - name: Verify runtime parity producer status id: verify_runtime_parity_status shell: bash - env: - RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} run: | set -euo pipefail - status_path=".artifacts/release-check-status/qa_lab_runtime_parity_release_checks-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.env" - status_field() { - local key="$1" - awk -v key="$key" ' - index($0, key "=") == 1 { - count += 1 - value = substr($0, length(key) + 2) - } - END { - if (count != 1) { - exit 1 - } - print value - } - ' "$status_path" - } - run_id="$(status_field run_id)" - run_attempt="$(status_field run_attempt)" - target_sha="$(status_field target_sha)" - job_name="$(status_field job)" - variant="$(status_field variant)" - if [[ "$run_id" != "$GITHUB_RUN_ID" || - "$run_attempt" != "$GITHUB_RUN_ATTEMPT" || - "$target_sha" != "$RELEASE_CHECK_TARGET_SHA" || - "$job_name" != "qa_lab_runtime_parity_release_checks" || - -n "$variant" ]]; then - echo "Runtime parity producer status does not match the current run, attempt, target, job, and variant." >&2 + validated_file=".artifacts/release-check-selection/runtime-parity-validated.json" + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh validate \ + --selection-file "${{ steps.resolve_runtime_parity_artifacts.outputs.selection_file }}" \ + --status-dir .artifacts/release-check-status \ + --validated-file "$validated_file" + status="$(jq -r 'if length == 1 then .[0].status else empty end' "$validated_file")" + [[ -n "$status" ]] || { + echo "Runtime parity producer status selection is incomplete." >&2 exit 1 - fi - status="$(sed -n 's/^status=//p' "$status_path" | tail -n 1)" + } if [[ "$status" != "success" ]]; then echo "Runtime parity producer status is ${status:-missing}; skipping coverage artifact consumer." echo "ready=false" >> "$GITHUB_OUTPUT" @@ -1630,7 +1695,7 @@ jobs: if: steps.verify_runtime_parity_status.outputs.ready == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: release-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }} + artifact-ids: ${{ steps.resolve_runtime_parity_artifacts.outputs.payload_ids }} path: .artifacts/qa-e2e/ - name: Enforce core runtime tool coverage @@ -1906,7 +1971,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-live-discord-${{ needs.resolve_target.outputs.revision }} + name: release-qa-live-discord-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -2005,7 +2070,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-live-whatsapp-${{ needs.resolve_target.outputs.revision }} + name: release-qa-live-whatsapp-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -2101,7 +2166,7 @@ jobs: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: release-qa-live-slack-${{ needs.resolve_target.outputs.revision }} + name: release-qa-live-slack-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} path: .artifacts/qa-e2e/ retention-days: 14 if-no-files-found: error @@ -2150,20 +2215,108 @@ jobs: runs-on: ubuntu-24.04 permissions: actions: read + contents: read timeout-minutes: 5 steps: - - name: Download advisory status artifacts + - name: Checkout trusted release artifact resolver + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.sha }} + path: trusted-release-check-artifacts + sparse-checkout: scripts/github/resolve-release-check-artifacts.sh + sparse-checkout-cone-mode: false + fetch-depth: 1 + + - name: Resolve advisory evidence artifacts + id: resolve_advisory_evidence if: always() continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_parity_lane_release_checks.result }} + QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_parity_report_release_checks.result }} + QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: ${{ needs.qa_lab_runtime_parity_release_checks.result }} + QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }} + QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_whatsapp_release_checks.result }} + QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_slack_release_checks.result }} + RELEASE_CHECK_TARGET_SHA: ${{ needs.resolve_target.outputs.revision }} + run: | + set -euo pipefail + selection_file=".artifacts/release-check-selection/advisory-evidence.json" + args=( + resolve + --repository "$GITHUB_REPOSITORY" + --run-id "$GITHUB_RUN_ID" + --consumer-attempt "$GITHUB_RUN_ATTEMPT" + --target-sha "$RELEASE_CHECK_TARGET_SHA" + ) + if [[ "$QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_lab_parity_lane_release_checks|candidate|release-check-status-qa-parity-candidate-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-parity-candidate-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + --pair "qa_lab_parity_lane_release_checks|baseline|release-check-status-qa-parity-baseline-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-parity-baseline-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + if [[ "$QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_lab_parity_report_release_checks||release-check-status-qa-parity-report-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-parity-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + if [[ "$QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_lab_runtime_parity_release_checks||release-check-status-qa-runtime-parity-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-runtime-parity-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + if [[ "$QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_live_discord_release_checks||release-check-status-qa-live-discord-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-live-discord-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + if [[ "$QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_live_whatsapp_release_checks||release-check-status-qa-live-whatsapp-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-live-whatsapp-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + if [[ "$QA_LIVE_SLACK_RELEASE_CHECKS_RESULT" != "skipped" ]]; then + args+=( + --pair "qa_live_slack_release_checks||release-check-status-qa-live-slack-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}|release-qa-live-slack-${RELEASE_CHECK_TARGET_SHA}-${GITHUB_RUN_ID}" + ) + fi + args+=( + --selection-file "$selection_file" + --github-output "$GITHUB_OUTPUT" + ) + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh "${args[@]}" + + - name: Download advisory status artifacts + id: download_advisory_statuses + if: steps.resolve_advisory_evidence.outcome == 'success' && steps.resolve_advisory_evidence.outputs.status_ids != '' + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: release-check-status-*-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }} + artifact-ids: ${{ steps.resolve_advisory_evidence.outputs.status_ids }} path: .artifacts/release-check-status merge-multiple: true + - name: Validate advisory status artifacts + id: validate_advisory_statuses + if: steps.resolve_advisory_evidence.outcome == 'success' + continue-on-error: true + shell: bash + run: | + set -euo pipefail + mkdir -p .artifacts/release-check-status + bash trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh validate \ + --selection-file "${{ steps.resolve_advisory_evidence.outputs.selection_file }}" \ + --status-dir .artifacts/release-check-status \ + --validated-file .artifacts/release-check-selection/advisory-evidence-validated.json + - name: Verify release check results shell: bash env: + RESOLVE_ADVISORY_EVIDENCE_OUTCOME: ${{ steps.resolve_advisory_evidence.outcome }} + VALIDATE_ADVISORY_STATUSES_OUTCOME: ${{ steps.validate_advisory_statuses.outcome }} RESOLVE_TARGET_RESULT: ${{ needs.resolve_target.result }} PREPARE_RELEASE_PACKAGE_RESULT: ${{ needs.prepare_release_package.result }} INSTALL_SMOKE_RELEASE_CHECKS_RESULT: ${{ needs.install_smoke_release_checks.result }} @@ -2210,106 +2363,37 @@ jobs: "qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}" "qa_live_slack_release_checks=${QA_LIVE_SLACK_RELEASE_CHECKS_RESULT}" ) - advisory_status_items=( - "qa_lab_parity_lane_release_checks=${QA_LAB_PARITY_LANE_RELEASE_CHECKS_RESULT}" - "qa_lab_parity_report_release_checks=${QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT}" - "qa_lab_runtime_parity_release_checks=${QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT}" - "qa_live_discord_release_checks=${QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT}" - "qa_live_whatsapp_release_checks=${QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT}" - "qa_live_slack_release_checks=${QA_LIVE_SLACK_RELEASE_CHECKS_RESULT}" - ) tideclaw_alpha=false if [[ "${WORKFLOW_REF}" =~ ^refs/heads/tideclaw/alpha/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$ ]]; then tideclaw_alpha=true fi - status_field() { - local file="$1" - local key="$2" - awk -v key="$key" ' - index($0, key "=") == 1 { - count += 1 - value = substr($0, length(key) + 2) - } - END { - if (count != 1) { - exit 1 - } - print value - } - ' "$file" - } - validate_status_file() { - local file="$1" - local expected_job="$2" - local expected_variant="$3" - local actual_run_id actual_run_attempt actual_target_sha actual_job actual_variant - if [[ ! -f "$file" ]]; then - echo "::error::Missing advisory status file: ${file}" >&2 - return 1 - fi - actual_run_id="$(status_field "$file" run_id)" || { - echo "::error::Advisory status file ${file} has invalid run_id metadata." >&2 - return 1 - } - actual_run_attempt="$(status_field "$file" run_attempt)" || { - echo "::error::Advisory status file ${file} has invalid run_attempt metadata." >&2 - return 1 - } - actual_target_sha="$(status_field "$file" target_sha)" || { - echo "::error::Advisory status file ${file} has invalid target_sha metadata." >&2 - return 1 - } - actual_job="$(status_field "$file" job)" || { - echo "::error::Advisory status file ${file} has invalid job metadata." >&2 - return 1 - } - actual_variant="$(status_field "$file" variant)" || { - echo "::error::Advisory status file ${file} has invalid variant metadata." >&2 - return 1 - } - if [[ "$actual_run_id" != "$RELEASE_CHECK_RUN_ID" || - "$actual_run_attempt" != "$RELEASE_CHECK_RUN_ATTEMPT" || - "$actual_target_sha" != "$RELEASE_CHECK_TARGET_SHA" || - "$actual_job" != "$expected_job" || - "$actual_variant" != "$expected_variant" ]]; then - echo "::error::Advisory status file ${file} does not match the current run, attempt, target, job, and variant." >&2 - return 1 - fi - } - expected_status_artifact_count() { - local name="$1" - local fallback="$2" - if [[ "$fallback" == "skipped" ]]; then - printf '0\n' - return - fi - if [[ "$name" == "qa_lab_parity_lane_release_checks" ]]; then - printf '2\n' - else - printf '1\n' - fi - } - expected_status_count=0 - for item in "${advisory_status_items[@]}"; do - name="${item%%=*}" - raw_result="${item#*=}" - expected_count="$(expected_status_artifact_count "$name" "$raw_result")" - expected_status_count=$((expected_status_count + expected_count)) - done - status_dir=".artifacts/release-check-status" - actual_status_count=0 - if [[ -d "$status_dir" ]]; then - actual_status_count="$(find "$status_dir" -maxdepth 1 -type f -name '*.env' | wc -l | tr -d '[:space:]')" - fi - if [[ "$actual_status_count" != "$expected_status_count" ]]; then - status_count_message="Expected ${expected_status_count} advisory status artifacts for run ${RELEASE_CHECK_RUN_ID} attempt ${RELEASE_CHECK_RUN_ATTEMPT}, found ${actual_status_count}." + validated_status_file=".artifacts/release-check-selection/advisory-evidence-validated.json" + if [[ "$RESOLVE_ADVISORY_EVIDENCE_OUTCOME" != "success" || + "$VALIDATE_ADVISORY_STATUSES_OUTCOME" != "success" ]]; then + status_resolution_message="Advisory evidence resolution or validation failed for run ${RELEASE_CHECK_RUN_ID} through attempt ${RELEASE_CHECK_RUN_ATTEMPT}." if [[ "$tideclaw_alpha" == "true" ]]; then - echo "::warning::${status_count_message} Tideclaw alpha treats non-package-safety release-check lanes as advisory." + echo "::warning::${status_resolution_message} Tideclaw alpha treats non-package-safety release-check lanes as advisory." else - echo "::error::${status_count_message}" + echo "::error::${status_resolution_message}" failed=1 fi fi + validated_status() { + local expected_job="$1" + local expected_variant="$2" + [[ -f "$validated_status_file" ]] || return 1 + jq -er \ + --arg job "$expected_job" \ + --arg variant "$expected_variant" \ + ' + [.[] | select(.job == $job and .variant == $variant)] as $matches | + if ($matches | length) == 1 and + ($matches[0].status | type) == "string" + then $matches[0].status + else error("missing or duplicate validated status") + end + ' "$validated_status_file" + } release_check_result() { local name="$1" local fallback="$2" @@ -2328,17 +2412,11 @@ jobs: variants=(candidate baseline) fi for variant in "${variants[@]}"; do - suffix="" - if [[ -n "$variant" ]]; then - suffix="-${variant}" - fi - file="${status_dir}/${name}${suffix}-${RELEASE_CHECK_RUN_ID}-${RELEASE_CHECK_RUN_ATTEMPT}.env" - if ! validate_status_file "$file" "$name" "$variant"; then + if ! status="$(validated_status "$name" "$variant")"; then saw_failure=1 continue fi saw=1 - status="$(status_field "$file" status)" || status="" case "$status" in success) ;; skipped) saw_skipped=1 ;; diff --git a/scripts/github/resolve-release-check-artifacts.sh b/scripts/github/resolve-release-check-artifacts.sh new file mode 100755 index 000000000000..b4f60835f1ea --- /dev/null +++ b/scripts/github/resolve-release-check-artifacts.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +tool_name="resolve-release-check-artifacts" +tmp_file="" +cleanup() { + local exit_code=$? + [[ -z "$tmp_file" ]] || rm -f "$tmp_file" + if [[ "$exit_code" -ne 0 ]]; then + echo "[${tool_name}] FAILED (exit ${exit_code})" >&2 + fi +} +trap cleanup EXIT + +fail() { + echo "${tool_name}: $*" >&2 + exit 1 +} + +command_name="${1:-}" +[[ -n "$command_name" ]] || fail "usage: ${tool_name} [options]" +shift + +repository="" +run_id="" +consumer_attempt="" +target_sha="" +selection_file="" +status_dir="" +validated_file="" +github_output="" +pairs=() +while [[ $# -gt 0 ]]; do + option="$1" + [[ "$option" == --* && -n "${2:-}" ]] || fail "${option} requires a value" + value="$2" + shift 2 + case "$option" in + --repository) repository="$value" ;; + --run-id) run_id="$value" ;; + --consumer-attempt) consumer_attempt="$value" ;; + --target-sha) target_sha="$value" ;; + --selection-file) selection_file="$value" ;; + --status-dir) status_dir="$value" ;; + --validated-file) validated_file="$value" ;; + --github-output) github_output="$value" ;; + --pair) pairs+=("$value") ;; + *) fail "unknown ${command_name} argument: ${option}" ;; + esac +done + +case "$command_name" in + resolve) + [[ "$repository" =~ ^[^/]+/[^/]+$ ]] || + fail "--repository must use owner/repository form" + [[ "$run_id" =~ ^[1-9][0-9]*$ ]] || fail "--run-id must be a positive integer" + [[ "$consumer_attempt" =~ ^[1-9][0-9]*$ ]] || + fail "--consumer-attempt must be a positive integer" + [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]] || + fail "--target-sha must be a lowercase full commit SHA" + [[ -n "$selection_file" ]] || fail "--selection-file is required" + mkdir -p "$(dirname "$selection_file")" + + pairs_json="[]" + if [[ "${#pairs[@]}" -gt 0 ]]; then + if ! pairs_json="$( + printf '%s\n' "${pairs[@]}" | jq -Rn ' + [inputs | split("|")] | + if all(.[]; + length == 4 and .[0] != "" and .[2] != "" and .[3] != "" + ) then + map({job: .[0], variant: .[1], status_base: .[2], payload_base: .[3]}) + else + error("invalid --pair value") + end + ' + )"; then + fail "invalid artifact pair specification" + fi + fi + + if [[ "$pairs_json" == "[]" ]]; then + printf '[]\n' > "$selection_file" + elif ! gh api --paginate \ + "repos/${repository}/actions/runs/${run_id}/artifacts?per_page=100" | + jq -e -s \ + --argjson pairs "$pairs_json" \ + --arg run_id "$run_id" \ + --arg target_sha "$target_sha" \ + --argjson consumer_attempt "$consumer_attempt" ' + def selected_artifact($matches; $name; $label): + if ($matches | length) != 1 then + error("\($label) requires exactly one \($name) artifact; found \($matches | length)") + else + $matches[0] + | if ((.id | type) == "number" and .id > 0 and (.id | floor) == .id) then . + else error("\($name) has a missing or invalid artifact id") end + | if ((.workflow_run.id | tostring) == $run_id) then . + else error("\($name) does not belong to workflow run \($run_id)") end + | if .expired == false then . + else error("\($name) is expired or has invalid expiry metadata") end + end; + def select_pair($artifacts; $pair): + ($pair.job + (if $pair.variant == "" then "" else "/" + $pair.variant end)) as $label + | ($pair.status_base + "-") as $status_prefix + | ($pair.payload_base + "-") as $payload_prefix + | [ + $artifacts[] + | select((.name | type) == "string") + | select((.name | startswith($status_prefix)) or + (.name | startswith($payload_prefix))) + | ( + if (.name | startswith($status_prefix)) + then .name | ltrimstr($status_prefix) + else .name | ltrimstr($payload_prefix) + end) as $attempt + | if ($attempt | test("^[1-9][0-9]*$")) then ($attempt | tonumber) + else error("\($label) has malformed producer attempt in artifact \(.name)") end + ] as $attempts + | [$attempts[] | select(. <= $consumer_attempt)] as $eligible + | if ($eligible | length) == 0 then + error("no \($label) evidence exists through consumer attempt \($consumer_attempt)") + else + ($eligible | max) as $attempt + | ($pair.status_base + "-" + ($attempt | tostring)) as $status_name + | ($pair.payload_base + "-" + ($attempt | tostring)) as $payload_name + | selected_artifact( + [$artifacts[] | select(.name == $status_name)]; + $status_name; $label) as $status + | selected_artifact( + [$artifacts[] | select(.name == $payload_name)]; + $payload_name; $label) as $payload + | {run_id: $run_id, target_sha: $target_sha, job: $pair.job, + variant: $pair.variant, producer_attempt: $attempt, + status_id: $status.id, status_name: $status_name, + payload_id: $payload.id, payload_name: $payload_name} + end; + if all(.[]; (.artifacts | type) == "array") then + [.[].artifacts[]] as $artifacts + | [$pairs[] | select_pair($artifacts; .)] as $selected + | [$selected[] | .status_id, .payload_id] as $ids + | if ($ids | length) == ($ids | unique | length) then $selected + else error("logical producers reuse a selected artifact id") end + else + error("invalid workflow artifact response") + end + ' > "$selection_file"; then + fail "artifact resolution failed" + fi + + if [[ -n "$github_output" ]]; then + { + printf 'selection_file=%s\n' "$selection_file" + printf 'status_ids=%s\n' \ + "$(jq -r 'map(.status_id | tostring) | join(",")' "$selection_file")" + printf 'payload_ids=%s\n' \ + "$(jq -r 'map(.payload_id | tostring) | join(",")' "$selection_file")" + } >> "$github_output" + fi + ;; + + validate) + [[ -f "$selection_file" ]] || fail "selection file is missing: ${selection_file}" + [[ -d "$status_dir" ]] || fail "status directory is missing: ${status_dir}" + [[ -n "$validated_file" ]] || fail "--validated-file is required" + if ! jq -e ' + type == "array" and + all(.[]; + (.run_id | type) == "string" and (.target_sha | test("^[0-9a-f]{40}$")) and + (.job | type) == "string" and (.job | length) > 0 and (.variant | type) == "string" and + (.producer_attempt | type) == "number" and .producer_attempt > 0 and + (.producer_attempt | floor) == .producer_attempt and + (.status_id | type) == "number" and .status_id > 0 and + (.payload_id | type) == "number" and .payload_id > 0 + ) and + ([.[] | [.job, .variant] | @tsv] | length) == + ([.[] | [.job, .variant] | @tsv] | unique | length) + ' "$selection_file" >/dev/null; then + fail "selection file has invalid or duplicate entries" + fi + + mkdir -p "$(dirname "$validated_file")" + tmp_file="$(mktemp)" + while IFS= read -r selection; do + expected_name="$( + jq -r ' + .job + + (if .variant == "" then "" else "-" + .variant end) + + "-" + .run_id + "-" + (.producer_attempt | tostring) + ".env" + ' <<< "$selection" + )" + status_file="${status_dir}/${expected_name}" + [[ -f "$status_file" ]] || + fail "${expected_name} requires exactly one downloaded status file; found 0" + if ! jq -Rsc --argjson selection "$selection" ' + def field($key): + [split("\n")[] | select(startswith($key + "=")) | .[($key | length) + 1:]] + | if length == 1 then .[0] + else error("\($key) is duplicate or missing") end; + { + run_id: field("run_id"), + run_attempt: field("run_attempt"), + target_sha: field("target_sha"), + job: field("job"), + variant: field("variant"), + status: field("status") + } as $actual + | if ( + $actual.run_id == $selection.run_id and + $actual.run_attempt == ($selection.producer_attempt | tostring) and + $actual.target_sha == $selection.target_sha and + $actual.job == $selection.job and + $actual.variant == $selection.variant and + ($actual.status | IN("success", "failure", "cancelled", "skipped")) + ) + then $selection + {status: $actual.status} + else error("status metadata does not match selected evidence") + end + ' "$status_file" >> "$tmp_file"; then + fail "${expected_name} has invalid status metadata" + fi + done < <(jq -c '.[]' "$selection_file") + jq -s '.' "$tmp_file" > "$validated_file" + if [[ -n "$github_output" ]]; then + printf 'validated_file=%s\n' "$validated_file" >> "$github_output" + fi + ;; + + *) + fail "usage: ${tool_name} [options]" + ;; +esac diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index e17c30932f61..ac893679e44e 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -99,6 +99,7 @@ const CRABBOX_HYDRATE_WORKFLOW = ".github/workflows/crabbox-hydrate.yml"; const CRABBOX_CONFIG = ".crabbox.yaml"; const SCHEDULED_LIVE_CHECKS_WORKFLOW = ".github/workflows/openclaw-scheduled-live-checks.yml"; const CI_HYDRATE_LIVE_AUTH_SCRIPT = "scripts/ci-hydrate-live-auth.sh"; +const RELEASE_CHECK_ARTIFACT_RESOLVER = "scripts/github/resolve-release-check-artifacts.sh"; const VERIFY_PROVIDER_SECRETS_SCRIPT = ".agents/skills/release-openclaw-ci/scripts/verify-provider-secrets.mjs"; const UPGRADE_SURVIVOR_RUN_SCRIPT = "scripts/e2e/lib/upgrade-survivor/run.sh"; @@ -590,11 +591,168 @@ function runOpenClawNpmTrustedRefGuard(overrides: Record) { }); } +type ReleaseCheckArtifact = { + expired: boolean; + id: number; + name: string; + workflow_run: { id: number }; +}; + +type ReleaseCheckArtifactPair = { + job: string; + payloadBase: string; + statusBase: string; + variant?: string; +}; + +type ResolvedReleaseCheckArtifact = { + job: string; + payload_id: number; + payload_name: string; + producer_attempt: number; + run_id: string; + status_id: number; + status_name: string; + target_sha: string; + variant: string; +}; + +function releaseCheckArtifact(params: { + expired?: boolean; + id: number; + name: string; + runId?: string; +}): ReleaseCheckArtifact { + return { + expired: params.expired ?? false, + id: params.id, + name: params.name, + workflow_run: { id: Number(params.runId ?? "123456") }, + }; +} + +function runReleaseCheckArtifactResolve(params: { + artifacts: ReleaseCheckArtifact[]; + consumerAttempt: string; + pairs: ReleaseCheckArtifactPair[]; + runId?: string; + targetSha?: string; +}) { + const workdir = tempDirs.make("release-check-artifact-resolver-"); + const binDir = resolve(workdir, "bin"); + mkdirSync(binDir, { recursive: true }); + const ghPath = resolve(binDir, "gh"); + writeFileSync(ghPath, "#!/bin/sh\nprintf '%s\\n' \"$MOCK_ARTIFACT_RESPONSE\"\n"); + chmodSync(ghPath, 0o755); + const runId = params.runId ?? "123456"; + const targetSha = params.targetSha ?? "a".repeat(40); + const selectionFile = resolve(workdir, "selection.json"); + const githubOutput = resolve(workdir, "github-output"); + const args = [ + resolve(REPO_ROOT, RELEASE_CHECK_ARTIFACT_RESOLVER), + "resolve", + "--repository", + "openclaw/openclaw", + "--run-id", + runId, + "--consumer-attempt", + params.consumerAttempt, + "--target-sha", + targetSha, + ]; + for (const pair of params.pairs) { + args.push( + "--pair", + [pair.job, pair.variant ?? "", pair.statusBase, pair.payloadBase].join("|"), + ); + } + args.push("--selection-file", selectionFile, "--github-output", githubOutput); + const result = spawnSync("bash", args, { + cwd: workdir, + encoding: "utf8", + env: { + MOCK_ARTIFACT_RESPONSE: JSON.stringify({ artifacts: params.artifacts }), + PATH: `${binDir}:${process.env.PATH}`, + }, + }); + const selection = + result.status === 0 + ? (JSON.parse(readFileSync(selectionFile, "utf8")) as ResolvedReleaseCheckArtifact[]) + : []; + return { result, selection, selectionFile, targetSha, workdir }; +} + +function releaseCheckStatusText( + selection: ResolvedReleaseCheckArtifact, + status: "cancelled" | "failure" | "skipped" | "success" = "success", +): string { + return [ + `run_id=${selection.run_id}`, + `run_attempt=${selection.producer_attempt}`, + `target_sha=${selection.target_sha}`, + `job=${selection.job}`, + `variant=${selection.variant}`, + `status=${status}`, + "job_status=success", + "step_outcomes=success", + "", + ].join("\n"); +} + +function runReleaseCheckArtifactValidation(params: { + selection: ResolvedReleaseCheckArtifact[]; + statusText?: (selection: ResolvedReleaseCheckArtifact) => string; +}) { + const workdir = tempDirs.make("release-check-artifact-validation-"); + const selectionFile = resolve(workdir, "selection.json"); + const statusDir = resolve(workdir, "statuses"); + const validatedFile = resolve(workdir, "validated.json"); + mkdirSync(statusDir, { recursive: true }); + writeFileSync(selectionFile, JSON.stringify(params.selection)); + for (const selection of params.selection) { + const variant = selection.variant ? `-${selection.variant}` : ""; + writeFileSync( + resolve( + statusDir, + `${selection.job}${variant}-${selection.run_id}-${selection.producer_attempt}.env`, + ), + params.statusText?.(selection) ?? releaseCheckStatusText(selection), + ); + } + const result = spawnSync( + "bash", + [ + resolve(REPO_ROOT, RELEASE_CHECK_ARTIFACT_RESOLVER), + "validate", + "--selection-file", + selectionFile, + "--status-dir", + statusDir, + "--validated-file", + validatedFile, + ], + { + cwd: workdir, + encoding: "utf8", + env: { PATH: process.env.PATH }, + }, + ); + const validated = + result.status === 0 + ? (JSON.parse(readFileSync(validatedFile, "utf8")) as Array< + ResolvedReleaseCheckArtifact & { status: string } + >) + : []; + return { result, validated }; +} + function runReleaseChecksSummary(params: { currentAttempt: string; currentResult: "cancelled" | "failure" | "skipped" | "success"; + discordResult?: "failure" | "skipped" | "success"; resolveResult?: "failure" | "success"; telegramSelected?: boolean; + validatedStatuses?: Array<{ job: string; status: string; variant: string }>; workflowRef?: string; }) { const summary = workflowJob(RELEASE_CHECKS_WORKFLOW, "summary"); @@ -605,6 +763,12 @@ function runReleaseChecksSummary(params: { const runId = "123456"; const targetSha = "a".repeat(40); const workdir = tempDirs.make("openclaw-release-check-status-"); + const selectionDir = resolve(workdir, ".artifacts/release-check-selection"); + mkdirSync(selectionDir, { recursive: true }); + writeFileSync( + resolve(selectionDir, "advisory-evidence-validated.json"), + JSON.stringify(params.validatedStatuses ?? []), + ); return spawnSync("bash", ["-c", script], { cwd: workdir, encoding: "utf8", @@ -623,7 +787,7 @@ function runReleaseChecksSummary(params: { QA_LAB_PARITY_REPORT_RELEASE_CHECKS_RESULT: "skipped", QA_LAB_RUNTIME_PARITY_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: "skipped", - QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: "skipped", + QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: params.discordResult ?? "skipped", QA_LIVE_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: "skipped", QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: params.currentResult, @@ -632,8 +796,10 @@ function runReleaseChecksSummary(params: { RELEASE_CHECK_RUN_ATTEMPT: params.currentAttempt, RELEASE_CHECK_RUN_ID: runId, RELEASE_CHECK_TARGET_SHA: targetSha, + RESOLVE_ADVISORY_EVIDENCE_OUTCOME: "success", RESOLVE_TARGET_RESULT: params.resolveResult ?? "success", RUNTIME_TOOL_COVERAGE_RELEASE_CHECKS_RESULT: "skipped", + VALIDATE_ADVISORY_STATUSES_OUTCOME: "success", WORKFLOW_REF: params.workflowRef ?? "refs/heads/release/2026.7.1", }, }); @@ -3410,15 +3576,27 @@ describe("package artifact reuse", () => { "qa_lab_runtime_pair_lane_release_checks", ]); expect(collectorJob.name).toBe("Verify QA Lab runtime-pair lanes"); + expect(workflowStep(collectorJob, "Resolve runtime-pair lane artifacts").run).toContain( + "qa_lab_runtime_pair_lane_release_checks|core", + ); + expect(workflowStep(collectorJob, "Resolve runtime-pair lane artifacts").run).toContain( + "qa_lab_runtime_pair_lane_release_checks|soak", + ); expect(workflowStep(collectorJob, "Download runtime-pair lane artifacts").with).toMatchObject({ - pattern: "release-qa-runtime-pair-lane-*-${{ needs.resolve_target.outputs.revision }}", + "artifact-ids": "${{ steps.resolve_runtime_pair_artifacts.outputs.payload_ids }}", "merge-multiple": true, }); + expect(workflowStep(collectorJob, "Download runtime-pair lane artifacts").if).toBe( + "always() && steps.resolve_runtime_pair_artifacts.outcome == 'success'", + ); + expect(workflowStep(collectorJob, "Download runtime-pair lane statuses").if).toBe( + "always() && steps.resolve_runtime_pair_artifacts.outcome == 'success'", + ); expect(workflowStep(collectorJob, "Verify runtime-pair lane statuses").run).toContain( - "lanes=(core)", + "resolve-release-check-artifacts.sh validate", ); expect(workflowStep(collectorJob, "Upload runtime parity artifacts").with?.name).toBe( - "release-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }}", + "release-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }}", ); }); @@ -3991,6 +4169,287 @@ describe("package artifact reuse", () => { ]); }); + describe("release check artifact resolver", () => { + const runId = "123456"; + const targetSha = "a".repeat(40); + const pair = (job: string, variant: string, slug: string): ReleaseCheckArtifactPair => ({ + job, + payloadBase: `release-payload-${slug}-${targetSha}-${runId}`, + statusBase: `release-status-${slug}-${targetSha}-${runId}`, + variant, + }); + const artifactsFor = ( + artifactPair: ReleaseCheckArtifactPair, + attempt: number, + firstId: number, + options: { expiredPayload?: boolean; expiredStatus?: boolean } = {}, + ): ReleaseCheckArtifact[] => [ + releaseCheckArtifact({ + expired: options.expiredStatus, + id: firstId, + name: `${artifactPair.statusBase}-${attempt}`, + runId, + }), + releaseCheckArtifact({ + expired: options.expiredPayload, + id: firstId + 1, + name: `${artifactPair.payloadBase}-${attempt}`, + runId, + }), + ]; + + it.each([ + { + artifacts: [1, 2].flatMap((attempt, index) => + artifactsFor(pair("qa_job", "candidate", "candidate"), attempt, index * 10 + 1), + ), + consumerAttempt: "2", + expectedAttempt: 2, + name: "selects the current producer attempt", + }, + { + artifacts: artifactsFor(pair("qa_job", "candidate", "candidate"), 1, 1), + consumerAttempt: "2", + expectedAttempt: 1, + name: "carries attempt 1 into consumer attempt 2", + }, + { + artifacts: [2, 10].flatMap((attempt, index) => + artifactsFor(pair("qa_job", "candidate", "candidate"), attempt, index * 10 + 1), + ), + consumerAttempt: "10", + expectedAttempt: 10, + name: "orders producer attempts numerically", + }, + { + artifacts: [2, 3].flatMap((attempt, index) => + artifactsFor(pair("qa_job", "candidate", "candidate"), attempt, index * 10 + 1), + ), + consumerAttempt: "2", + expectedAttempt: 2, + name: "excludes future producer attempts", + }, + ])("$name", ({ artifacts, consumerAttempt, expectedAttempt }) => { + const result = runReleaseCheckArtifactResolve({ + artifacts, + consumerAttempt, + pairs: [pair("qa_job", "candidate", "candidate")], + runId, + targetSha, + }); + + expect(result.result.status, result.result.stderr).toBe(0); + expect(result.selection).toHaveLength(1); + expect(result.selection[0]?.producer_attempt).toBe(expectedAttempt); + }); + + it("selects candidate and baseline attempts independently", () => { + const candidate = pair("qa_lab_parity_lane_release_checks", "candidate", "candidate"); + const baseline = pair("qa_lab_parity_lane_release_checks", "baseline", "baseline"); + const result = runReleaseCheckArtifactResolve({ + artifacts: [...artifactsFor(candidate, 1, 1), ...artifactsFor(baseline, 2, 11)], + consumerAttempt: "2", + pairs: [candidate, baseline], + runId, + targetSha, + }); + + expect(result.result.status, result.result.stderr).toBe(0); + expect( + Object.fromEntries( + result.selection.map((selection) => [selection.variant, selection.producer_attempt]), + ), + ).toEqual({ baseline: 2, candidate: 1 }); + }); + + it("selects core and soak attempts independently", () => { + const core = pair("qa_lab_runtime_pair_lane_release_checks", "core", "core"); + const soak = pair("qa_lab_runtime_pair_lane_release_checks", "soak", "soak"); + const result = runReleaseCheckArtifactResolve({ + artifacts: [...artifactsFor(core, 2, 1), ...artifactsFor(soak, 1, 11)], + consumerAttempt: "2", + pairs: [core, soak], + runId, + targetSha, + }); + + expect(result.result.status, result.result.stderr).toBe(0); + expect( + Object.fromEntries( + result.selection.map((selection) => [selection.variant, selection.producer_attempt]), + ), + ).toEqual({ core: 2, soak: 1 }); + }); + + it("fails when the latest producer attempt has no complete pair", () => { + const candidate = pair("qa_job", "candidate", "candidate"); + const result = runReleaseCheckArtifactResolve({ + artifacts: [ + ...artifactsFor(candidate, 1, 1), + releaseCheckArtifact({ + id: 11, + name: `${candidate.statusBase}-2`, + runId, + }), + ], + consumerAttempt: "2", + pairs: [candidate], + runId, + targetSha, + }); + + expect(result.result.status).toBe(1); + expect(result.result.stderr).toContain( + `requires exactly one ${candidate.payloadBase}-2 artifact; found 0`, + ); + expect(result.result.stderr.trimEnd()).toMatch( + /\[resolve-release-check-artifacts\] FAILED \(exit 1\)$/u, + ); + }); + + it("fails on duplicate artifacts at the latest producer attempt", () => { + const candidate = pair("qa_job", "candidate", "candidate"); + const artifacts = artifactsFor(candidate, 2, 1); + artifacts.push( + releaseCheckArtifact({ + id: 11, + name: `${candidate.statusBase}-2`, + runId, + }), + ); + const result = runReleaseCheckArtifactResolve({ + artifacts, + consumerAttempt: "2", + pairs: [candidate], + runId, + targetSha, + }); + + expect(result.result.status).toBe(1); + expect(result.result.stderr).toContain( + `requires exactly one ${candidate.statusBase}-2 artifact; found 2`, + ); + }); + + it.each([ + { + artifacts: (candidate: ReleaseCheckArtifactPair) => [ + ...artifactsFor(candidate, 1, 1), + releaseCheckArtifact({ + id: 11, + name: `${candidate.statusBase}-broken`, + runId, + }), + ], + expected: "has malformed producer attempt", + name: "malformed newer evidence", + }, + { + artifacts: (candidate: ReleaseCheckArtifactPair) => [ + ...artifactsFor(candidate, 1, 1), + ...artifactsFor(candidate, 2, 11, { expiredPayload: true }), + ], + expected: "is expired or has invalid expiry metadata", + name: "expired newer evidence", + }, + ])("does not fall back past $name", ({ artifacts, expected }) => { + const candidate = pair("qa_job", "candidate", "candidate"); + const result = runReleaseCheckArtifactResolve({ + artifacts: artifacts(candidate), + consumerAttempt: "2", + pairs: [candidate], + runId, + targetSha, + }); + + expect(result.result.status).toBe(1); + expect(result.result.stderr).toContain(expected); + }); + + it.each([ + { + mutate: (text: string) => text.replace("status=success", "status=success\nstatus=failure"), + name: "duplicate status fields", + }, + { + mutate: (text: string) => text.replace("run_attempt=2", "run_attempt=bogus"), + name: "malformed status metadata", + }, + ])("rejects $name", ({ mutate }) => { + const candidate = pair("qa_job", "candidate", "candidate"); + const resolved = runReleaseCheckArtifactResolve({ + artifacts: artifactsFor(candidate, 2, 1), + consumerAttempt: "2", + pairs: [candidate], + runId, + targetSha, + }); + expect(resolved.result.status, resolved.result.stderr).toBe(0); + + const validated = runReleaseCheckArtifactValidation({ + selection: resolved.selection, + statusText: (selection) => mutate(releaseCheckStatusText(selection)), + }); + expect(validated.result.status).toBe(1); + }); + + it("sets runtime parity ready=false for validated non-success evidence", () => { + const runtimePair = pair("qa_lab_runtime_parity_release_checks", "", "runtime-parity"); + const resolved = runReleaseCheckArtifactResolve({ + artifacts: artifactsFor(runtimePair, 2, 1), + consumerAttempt: "2", + pairs: [runtimePair], + runId, + targetSha, + }); + expect(resolved.result.status, resolved.result.stderr).toBe(0); + + const workdir = tempDirs.make("runtime-parity-ready-"); + const trustedScript = resolve( + workdir, + "trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh", + ); + mkdirSync(resolve(trustedScript, ".."), { recursive: true }); + symlinkSync(resolve(REPO_ROOT, RELEASE_CHECK_ARTIFACT_RESOLVER), trustedScript); + const selectionFile = resolve(workdir, "selection.json"); + writeFileSync(selectionFile, JSON.stringify(resolved.selection)); + const statusDir = resolve(workdir, ".artifacts/release-check-status"); + mkdirSync(statusDir, { recursive: true }); + const selection = resolved.selection[0]!; + writeFileSync( + resolve( + statusDir, + `${selection.job}-${selection.run_id}-${selection.producer_attempt}.env`, + ), + releaseCheckStatusText(selection, "failure"), + ); + const outputFile = resolve(workdir, "github-output"); + const runtimeCoverage = workflowJob( + RELEASE_CHECKS_WORKFLOW, + "runtime_tool_coverage_release_checks", + ); + const script = workflowStep( + runtimeCoverage, + "Verify runtime parity producer status", + ).run?.replace( + "${{ steps.resolve_runtime_parity_artifacts.outputs.selection_file }}", + selectionFile, + ); + expect(script).toBeTruthy(); + const result = spawnSync("bash", ["-c", script!], { + cwd: workdir, + encoding: "utf8", + env: { + GITHUB_OUTPUT: outputFile, + PATH: process.env.PATH, + }, + }); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(outputFile, "utf8")).toContain("ready=false"); + }); + }); + it("keeps release QA status artifacts blocking in the verifier", () => { const advisoryJobNames = [ "qa_lab_parity_lane_release_checks", @@ -4031,6 +4490,39 @@ describe("package artifact reuse", () => { expect(uploadStep.with?.["if-no-files-found"], jobName).toBe("error"); } + for (const [jobName, stepName] of [ + ["qa_lab_parity_lane_release_checks", "Upload parity lane artifacts"], + ["qa_lab_parity_report_release_checks", "Upload parity artifacts"], + ["qa_lab_runtime_pair_lane_release_checks", "Upload runtime-pair lane artifacts"], + ["qa_lab_runtime_parity_release_checks", "Upload runtime parity artifacts"], + ["qa_live_discord_release_checks", "Upload Discord QA artifacts"], + ["qa_live_whatsapp_release_checks", "Upload WhatsApp QA artifacts"], + ["qa_live_slack_release_checks", "Upload Slack QA artifacts"], + ] as const) { + const upload = workflowStep(workflowJob(RELEASE_CHECKS_WORKFLOW, jobName), stepName); + expect(upload.with?.name, `${jobName}/${stepName}`).toContain( + "${{ github.run_id }}-${{ github.run_attempt }}", + ); + } + + for (const jobName of [ + "qa_lab_parity_report_release_checks", + "qa_lab_runtime_parity_release_checks", + "runtime_tool_coverage_release_checks", + "summary", + ]) { + const checkout = workflowStep( + workflowJob(RELEASE_CHECKS_WORKFLOW, jobName), + "Checkout trusted release artifact resolver", + ); + expect(checkout.with).toMatchObject({ + path: "trusted-release-check-artifacts", + ref: "${{ github.sha }}", + "sparse-checkout": RELEASE_CHECK_ARTIFACT_RESOLVER, + "sparse-checkout-cone-mode": false, + }); + } + const telegramCaller = workflowJob(RELEASE_CHECKS_WORKFLOW, "qa_live_telegram_release_checks"); const telegramDispatch = workflowStep(telegramCaller, "Dispatch and await trusted Telegram QA"); expect(telegramDispatch.run).toContain('workflow="openclaw-release-telegram-qa.yml"'); @@ -4071,13 +4563,23 @@ describe("package artifact reuse", () => { const summary = workflowJob(RELEASE_CHECKS_WORKFLOW, "summary"); expect(summary.needs).toContain("resolve_target"); expect(summary.permissions?.actions).toBe("read"); + expect(summary.permissions?.contents).toBe("read"); + const resolveStep = workflowStep(summary, "Resolve advisory evidence artifacts"); + expect(resolveStep["continue-on-error"]).toBe(true); + expect(resolveStep.run).toContain( + "trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh", + ); + expect(resolveStep.run).toContain('--consumer-attempt "$GITHUB_RUN_ATTEMPT"'); + expect(resolveStep.run).toContain("qa_lab_parity_lane_release_checks|candidate"); + expect(resolveStep.run).toContain("qa_lab_parity_lane_release_checks|baseline"); const downloadStep = workflowStep(summary, "Download advisory status artifacts"); expect(downloadStep["continue-on-error"]).toBe(true); expect(downloadStep.uses).toBe(DOWNLOAD_ARTIFACT_V8); - expect(downloadStep.with?.pattern).toBe( - "release-check-status-*-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }}", + expect(downloadStep.with?.["artifact-ids"]).toBe( + "${{ steps.resolve_advisory_evidence.outputs.status_ids }}", ); expect(downloadStep.with?.["merge-multiple"]).toBe(true); + expect(downloadStep.with?.pattern).toBeUndefined(); const verifyStep = workflowStep(summary, "Verify release check results"); expect(verifyStep.env).toMatchObject({ @@ -4086,18 +4588,15 @@ describe("package artifact reuse", () => { RELEASE_CHECK_RUN_ATTEMPT: "${{ github.run_attempt }}", RELEASE_CHECK_RUN_ID: "${{ github.run_id }}", RELEASE_CHECK_TARGET_SHA: "${{ needs.resolve_target.outputs.revision }}", + RESOLVE_ADVISORY_EVIDENCE_OUTCOME: "${{ steps.resolve_advisory_evidence.outcome }}", + VALIDATE_ADVISORY_STATUSES_OUTCOME: "${{ steps.validate_advisory_statuses.outcome }}", }); expectTextToIncludeAll(verifyStep.run, [ "release_check_result()", - "validate_status_file()", - "expected_status_artifact_count()", - 'actual_run_id="$(status_field "$file" run_id)"', - 'actual_run_attempt="$(status_field "$file" run_attempt)"', - 'actual_target_sha="$(status_field "$file" target_sha)"', - 'actual_job="$(status_field "$file" job)"', - 'actual_variant="$(status_field "$file" variant)"', - "Expected ${expected_status_count} advisory status artifacts", - "::warning::${status_count_message} Tideclaw alpha treats non-package-safety release-check lanes as advisory.", + "validated_status()", + "advisory-evidence-validated.json", + "missing or duplicate validated status", + "Advisory evidence resolution or validation failed", 'elif [[ "$fallback" != "success" && "$fallback" != "skipped" ]]; then', 'elif [[ "$fallback" == "success" ]]; then', "advisory_status_override_allowed()", @@ -4111,18 +4610,26 @@ describe("package artifact reuse", () => { expect(verifyStep.run).not.toContain( "QA release-check lanes are advisory and do not block release validation.", ); + expect(verifyStep.run).not.toContain("expected_status_artifact_count"); + expect(verifyStep.run).not.toContain("actual_status_count"); const runtimeCoverage = workflowJob( RELEASE_CHECKS_WORKFLOW, "runtime_tool_coverage_release_checks", ); - expect(workflowStep(runtimeCoverage, "Download runtime parity status").with?.name).toBe( - "release-check-status-qa-runtime-parity-${{ needs.resolve_target.outputs.revision }}-${{ github.run_id }}-${{ github.run_attempt }}", + expect(workflowStep(runtimeCoverage, "Resolve runtime parity artifacts").run).toContain( + "trusted-release-check-artifacts/scripts/github/resolve-release-check-artifacts.sh", ); + expect( + workflowStep(runtimeCoverage, "Download runtime parity status").with?.["artifact-ids"], + ).toBe("${{ steps.resolve_runtime_parity_artifacts.outputs.status_ids }}"); expectTextToIncludeAll( workflowStep(runtimeCoverage, "Verify runtime parity producer status").run, - ["run_id", "run_attempt", "target_sha", "job_name", "variant"], + ["resolve-release-check-artifacts.sh validate", "ready=false", "ready=true"], ); + expect( + workflowStep(runtimeCoverage, "Download runtime parity artifacts").with?.["artifact-ids"], + ).toBe("${{ steps.resolve_runtime_parity_artifacts.outputs.payload_ids }}"); }); it.each([ @@ -4183,6 +4690,30 @@ describe("package artifact reuse", () => { } }); + it.each(["cancelled", "failure"] as const)( + "does not mask a later %s advisory status with an older successful job result", + (status) => { + const result = runReleaseChecksSummary({ + currentAttempt: "2", + currentResult: "skipped", + discordResult: "success", + telegramSelected: false, + validatedStatuses: [ + { + job: "qa_live_discord_release_checks", + status, + variant: "", + }, + ], + }); + + expect(result.status).toBe(1); + expect(`${result.stdout}\n${result.stderr}`).toContain( + `::error::qa_live_discord_release_checks ended with ${status}`, + ); + }, + ); + it("summarizes start delay separately from execution time in full validation", () => { const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8"); const parsedWorkflow = readWorkflow(FULL_RELEASE_VALIDATION_WORKFLOW);