ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(release): bound validation retries and soak

Browse Source
This commit is contained in:
Vincent Koc
2026-08-10 20:46:25 +08:00
parent fbebfc713f
commit 1f591bba56
27 changed files with 1399 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/full-release-validation.yml
+34 -24
View File
@@ -8,6 +8,11 @@ on:
required: true
default: main
type: string
expected_sha:
description: Optional full Validation SHA that ref must resolve to
required: false
default: ""
type: string
target_context_ref:
description: Optional canonical release branch or tag context for an exact-SHA target
required: false
@@ -143,8 +148,8 @@ permissions:
contents: read
concurrency:
group: full-release-validation-${{ inputs.ref }}-${{ inputs.rerun_group }}
cancel-in-progress: ${{ (inputs.ref == 'main' && inputs.rerun_group == 'all') || startsWith(inputs.ref, 'tideclaw/alpha/') || startsWith(inputs.ref, 'release/') }}
group: full-release-validation-${{ inputs.expected_sha || inputs.ref }}-${{ github.sha }}-${{ inputs.rerun_group }}
cancel-in-progress: false
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
@@ -175,9 +180,11 @@ jobs:
env:
TARGET_REF: ${{ inputs.ref }}
TARGET_CONTEXT_REF: ${{ inputs.target_context_ref }}
EXPECTED_SHA: ${{ inputs.expected_sha }}
run: |
bash workflow/scripts/github/resolve-openclaw-ref.sh \
--ref "$TARGET_REF" \
--expected-sha "$EXPECTED_SHA" \
--github-output "$GITHUB_OUTPUT"
- name: Validate release inputs
@@ -195,7 +202,7 @@ jobs:
env:
TARGET_REF: ${{ inputs.ref }}
TARGET_SHA: ${{ steps.resolve.outputs.sha }}
CHILD_WORKFLOW_REF: ${{ github.ref_name }}
TOOLING_SHA: ${{ github.sha }}
NPM_TELEGRAM_PACKAGE_SPEC: ${{ inputs.npm_telegram_package_spec }}
RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }}
EVIDENCE_PACKAGE_SPEC: ${{ inputs.evidence_package_spec }}
@@ -214,8 +221,8 @@ jobs:
echo "## Full release validation"
echo
echo "- Target ref: \`${TARGET_REF}\`"
echo "- Target SHA: \`${TARGET_SHA}\`"
echo "- Child workflow ref: \`${CHILD_WORKFLOW_REF}\`"
echo "- Validation SHA: \`${TARGET_SHA}\`"
echo "- Tooling SHA: \`${TOOLING_SHA}\`"
echo "- Release soak lanes: \`${RUN_RELEASE_SOAK}\`"
echo "- Fail fast: \`${FAIL_FAST}\`"
echo "- Package Acceptance Telegram E2E deferred: \`${SKIP_PACKAGE_TELEGRAM_E2E}\`"
@@ -456,7 +463,8 @@ jobs:
needs: [resolve_target, evidence_reuse]
if: ${{ always() && needs.resolve_target.result == 'success' && contains(fromJSON('["all","ci"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ inputs.release_profile != 'beta' && 240 || 60 }}
# The child owns lane timeouts; this monitor also covers queue delay and iOS.
timeout-minutes: 240
outputs:
run_id: ${{ steps.dispatch.outputs.run_id }}
url: ${{ steps.dispatch.outputs.url }}
@@ -678,23 +686,23 @@ jobs:
fi
cancel_child() {
if [[ -n "${active_child_run_id:-}" ]]; then
echo "Cancelling child workflow ${active_child_workflow}: ${active_child_run_id}" >&2
gh run cancel "$active_child_run_id" >/dev/null 2>&1 || true
if [[ -n "${run_id:-}" ]]; then
echo "Cancelling child workflow ${workflow}: ${run_id}" >&2
gh run cancel "$run_id" >/dev/null 2>&1 || true
fi
}
run_json="$(validate_child_run "$run_id")"
# Identity fields prove cancellation ownership before workflow-SHA validation.
# EXIT traps run after function locals unwind; preserve only adopted child identity.
active_child_workflow="$workflow"
active_child_run_id="$run_id"
trap cancel_child EXIT INT TERM
# Generic monitor failures and parent cancellation leave an
# identity-checked child running for explicit operator recovery.
{
echo "- Adopted child: \`${workflow}\` run \`${run_id}\`"
echo "- Parent cancellation leaves this child running; cancel it explicitly if no longer needed."
} >> "$GITHUB_STEP_SUMMARY"
child_head_sha="$(jq -r '.head_sha // ""' <<< "$run_json")"
if [[ "$child_head_sha" != "$PARENT_WORKFLOW_SHA" ]]; then
echo "::error::${workflow} child run used workflow SHA ${child_head_sha}, expected parent workflow SHA ${PARENT_WORKFLOW_SHA}." >&2
cancel_child
trap - EXIT INT TERM
exit 1
fi
@@ -760,7 +768,6 @@ jobs:
fi
jq '.[] | {name, conclusion, url: (.url // .html_url)}' <<< "$failed_jobs_json"
cancel_child
trap - EXIT INT TERM
exit 1
fi
}
@@ -781,7 +788,6 @@ jobs:
fi
sleep 60
done
trap - EXIT INT TERM
if [[ "$workflow" == "openclaw-release-checks.yml" ]]; then
jobs_json="$(fetch_child_jobs | jq -s '{jobs: [.[] | {name, conclusion, url: .html_url}]}')"
@@ -983,7 +989,8 @@ jobs:
needs: [resolve_target, evidence_reuse, prepare_release_candidate]
if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","plugin-prerelease"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ inputs.release_profile == 'full' && 300 || inputs.release_profile == 'stable' && 240 || 60 }}
# The child owns lane timeouts; this monitor also covers queue delay.
timeout-minutes: ${{ inputs.release_profile == 'full' && 300 || 240 }}
outputs:
run_id: ${{ steps.dispatch.outputs.run_id }}
url: ${{ steps.dispatch.outputs.url }}
@@ -1006,10 +1013,10 @@ jobs:
name: Run release/live/Docker/QA validation
needs: [resolve_target, evidence_reuse, prepare_release_candidate]
if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","release-checks","install-smoke","cross-os","live-e2e","package","qa","qa-parity","qa-live"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
runs-on: ubuntu-24.04
# The child owns lane timeouts; this monitor must also tolerate queue delay
# so it does not cancel healthy release checks before their final verifier.
timeout-minutes: 240
runs-on: blacksmith-4vcpu-ubuntu-2404
# The bounded package critical path tops out at 310 minutes; 420 leaves
# queue/API margin. Parent timeout preserves the adopted child for exact cancellation.
timeout-minutes: 420
outputs:
run_id: ${{ steps.dispatch.outputs.run_id }}
url: ${{ steps.dispatch.outputs.url }}
@@ -1047,7 +1054,8 @@ jobs:
if: ${{ always() && needs.resolve_target.result == 'success' && contains(fromJSON('["all","npm-telegram"]'), inputs.rerun_group) && (inputs.npm_telegram_package_spec != '' || inputs.release_package_spec != '') && needs.evidence_reuse.outputs.reuse != 'true' }}
continue-on-error: ${{ startsWith(github.ref, 'refs/heads/tideclaw/alpha/') }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ inputs.release_profile == 'full' && 360 || 60 }}
# The child owns lane timeouts; this monitor also covers queue delay.
timeout-minutes: ${{ inputs.release_profile == 'full' && 360 || 120 }}
outputs:
run_id: ${{ steps.dispatch.outputs.run_id }}
url: ${{ steps.dispatch.outputs.url }}
@@ -1074,7 +1082,9 @@ jobs:
# Keep this monitor off the four-slot GitHub-hosted pool so performance starts
# with the other child workflows instead of extending the critical path.
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: ${{ inputs.release_profile == 'full' && 360 || 120 }}
# Artifact-only and publish paths top out at 255 and 280 minutes; 360 leaves
# queue/API margin. Parent timeout preserves the adopted child for exact cancellation.
timeout-minutes: 360
outputs:
run_id: ${{ steps.dispatch.outputs.run_id }}
url: ${{ steps.dispatch.outputs.url }}
.github/workflows/install-smoke-reusable.yml
+8
View File
@@ -38,6 +38,7 @@ env:
jobs:
preflight:
runs-on: ubuntu-24.04
timeout-minutes: 15
outputs:
docs_only: ${{ steps.manifest.outputs.docs_only }}
run_install_smoke: ${{ steps.manifest.outputs.run_install_smoke }}
@@ -118,6 +119,7 @@ jobs:
needs: [preflight]
if: needs.preflight.outputs.run_fast_install_smoke == 'true' && needs.preflight.outputs.run_full_install_smoke != 'true'
runs-on: ubuntu-24.04
timeout-minutes: 120
env:
DOCKER_BUILD_SUMMARY: "false"
DOCKER_BUILD_RECORD_UPLOAD: "false"
@@ -228,6 +230,7 @@ jobs:
needs: [preflight]
if: needs.preflight.outputs.run_full_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 60
permissions:
contents: read
packages: read
@@ -332,6 +335,7 @@ jobs:
needs: [preflight, root_dockerfile_image]
if: always() && needs.preflight.result == 'success' && needs.preflight.outputs.run_full_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 5
permissions:
contents: read
steps:
@@ -349,6 +353,7 @@ jobs:
needs: [preflight]
if: needs.preflight.outputs.run_full_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- name: Checkout CLI
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
@@ -365,6 +370,7 @@ jobs:
needs: [preflight, root_dockerfile_image, root_dockerfile_image_ready]
if: needs.preflight.outputs.run_full_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 90
env:
OPENCLAW_DOCKER_E2E_REQUIRE_LOCAL_IMAGE: "1"
steps:
@@ -534,6 +540,7 @@ jobs:
needs: [preflight, root_dockerfile_image, root_dockerfile_image_ready]
if: needs.preflight.outputs.run_full_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 150
env:
DOCKER_BUILD_SUMMARY: "false"
DOCKER_BUILD_RECORD_UPLOAD: "false"
@@ -700,6 +707,7 @@ jobs:
needs: [preflight, root_dockerfile_image, root_dockerfile_image_ready]
if: needs.preflight.outputs.run_full_install_smoke == 'true' && needs.preflight.outputs.run_bun_global_install_smoke == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 60
env:
OPENCLAW_DOCKER_E2E_REQUIRE_LOCAL_IMAGE: "1"
steps:
.github/workflows/openclaw-cross-os-release-checks-reusable.yml
+1
View File
@@ -249,6 +249,7 @@ env:
jobs:
prepare:
runs-on: ubuntu-24.04
timeout-minutes: 90
continue-on-error: ${{ inputs.advisory }}
outputs:
baseline_artifact_digest: ${{ steps.upload_baseline.outputs.artifact-digest }}
.github/workflows/openclaw-live-and-e2e-checks-reusable.yml
+3
View File
@@ -542,6 +542,7 @@ env:
jobs:
validate_selected_ref:
runs-on: ubuntu-24.04
timeout-minutes: 30
outputs:
selected_sha: ${{ steps.validate.outputs.selected_sha }}
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
@@ -776,6 +777,7 @@ jobs:
validate_live_suite_filter:
runs-on: ubuntu-24.04
if: inputs.live_suite_filter != ''
timeout-minutes: 10
steps:
- name: Validate focused live suite filter
shell: bash
@@ -892,6 +894,7 @@ jobs:
plan_release_workflow_matrices:
needs: validate_selected_ref
runs-on: ubuntu-24.04
timeout-minutes: 10
outputs:
docker_e2e_count: ${{ steps.plan.outputs.docker_e2e_count }}
docker_e2e_matrix: ${{ steps.plan.outputs.docker_e2e_matrix }}
.github/workflows/openclaw-release-checks.yml
+19 -9
View File
@@ -143,6 +143,7 @@ jobs:
mode: ${{ steps.inputs.outputs.mode }}
release_profile: ${{ steps.inputs.outputs.release_profile }}
run_release_soak: ${{ steps.inputs.outputs.run_release_soak }}
qa_live_scheduled: ${{ steps.inputs.outputs.qa_live_scheduled }}
fail_fast: ${{ steps.inputs.outputs.fail_fast }}
run_maturity_scorecard: ${{ steps.inputs.outputs.run_maturity_scorecard }}
allow_unreleased_changelog: ${{ steps.inputs.outputs.allow_unreleased_changelog }}
@@ -430,10 +431,10 @@ jobs:
codex_plugin_spec="npm:@openclaw/codex@${BASH_REMATCH[1]}"
fi
qa_filter_seen=false
filter="$(printf '%s' "$RELEASE_LIVE_SUITE_FILTER_INPUT" | tr '[:upper:]' '[:lower:]')"
repo_live_suite_filter="$filter"
if [[ -n "${filter// }" ]]; then
qa_filter_seen=false
repo_filter_tokens=()
matrix_selected=false
buzz_selected=false
@@ -525,6 +526,12 @@ jobs:
fi
fi
qa_live_scheduled=false
if [[ "$RELEASE_RERUN_GROUP_INPUT" == "qa" || "$RELEASE_RERUN_GROUP_INPUT" == "qa-live" ]] ||
[[ "$RELEASE_RERUN_GROUP_INPUT" == "all" && ( "$run_release_soak" == "true" || "$qa_filter_seen" == "true" ) ]]; then
qa_live_scheduled=true
fi
cross_os_scheduled=false
if [[ "$RELEASE_RERUN_GROUP_INPUT" == "all" || "$RELEASE_RERUN_GROUP_INPUT" == "cross-os" ]]; then
cross_os_scheduled=true
@@ -542,6 +549,7 @@ jobs:
printf 'mode=%s\n' "$RELEASE_MODE_INPUT"
printf 'release_profile=%s\n' "$release_profile"
printf 'run_release_soak=%s\n' "$run_release_soak"
printf 'qa_live_scheduled=%s\n' "$qa_live_scheduled"
printf 'fail_fast=%s\n' "$fail_fast"
printf 'run_maturity_scorecard=%s\n' "$run_maturity_scorecard"
printf 'allow_unreleased_changelog=%s\n' "$allow_unreleased_changelog"
@@ -572,6 +580,7 @@ jobs:
RELEASE_MODE: ${{ inputs.mode }}
RELEASE_PROFILE: ${{ steps.inputs.outputs.release_profile }}
RUN_RELEASE_SOAK: ${{ steps.inputs.outputs.run_release_soak }}
QA_LIVE_SCHEDULED: ${{ steps.inputs.outputs.qa_live_scheduled }}
FAIL_FAST: ${{ steps.inputs.outputs.fail_fast }}
RUN_MATURITY_SCORECARD: ${{ steps.inputs.outputs.run_maturity_scorecard }}
SKIP_PACKAGE_TELEGRAM_E2E: ${{ steps.inputs.outputs.skip_package_telegram_e2e }}
@@ -593,6 +602,7 @@ jobs:
echo "- Cross-OS mode: \`${RELEASE_MODE}\`"
echo "- Release profile: \`${RELEASE_PROFILE}\`"
echo "- Release soak lanes: \`${RUN_RELEASE_SOAK}\`"
echo "- QA-live scheduled: \`${QA_LIVE_SCHEDULED}\`"
echo "- Matrix QA fail fast: \`${FAIL_FAST}\`"
echo "- Maturity scorecard docs: \`${RUN_MATURITY_SCORECARD}\`"
echo "- Package Acceptance Telegram E2E deferred: \`${SKIP_PACKAGE_TELEGRAM_E2E}\`"
@@ -604,7 +614,7 @@ jobs:
if [[ -n "${RELEASE_CROSS_OS_SUITE_FILTER// }" ]]; then
echo "- Cross-OS suite filter: \`${RELEASE_CROSS_OS_SUITE_FILTER}\`"
fi
echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Buzz \`${{ steps.inputs.outputs.qa_live_buzz_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`"
echo "- QA-live lane eligibility: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Buzz \`${{ steps.inputs.outputs.qa_live_buzz_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Discord \`${{ steps.inputs.outputs.qa_live_discord_enabled }}\`, WhatsApp \`${{ steps.inputs.outputs.qa_live_whatsapp_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`"
if [[ -n "${RELEASE_PACKAGE_SPEC// }" ]]; then
echo "- Release package spec: \`${RELEASE_PACKAGE_SPEC}\`"
fi
@@ -1916,7 +1926,7 @@ jobs:
qa_live_release_checks:
name: Run QA Live Matrix catalog
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true'
permissions:
contents: read
pull-requests: read
@@ -1934,7 +1944,7 @@ jobs:
qa_live_buzz_release_checks:
name: Run QA Lab live Buzz lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_buzz_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_buzz_enabled == 'true'
permissions:
contents: read
pull-requests: read
@@ -1954,7 +1964,7 @@ jobs:
qa_live_telegram_release_checks:
name: Run QA Lab live Telegram lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 210
permissions:
@@ -2079,7 +2089,7 @@ jobs:
qa_live_discord_release_checks:
name: Run QA Lab live Discord lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_discord_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_discord_enabled == 'true'
continue-on-error: true
runs-on: ubuntu-24.04
timeout-minutes: 60
@@ -2194,7 +2204,7 @@ jobs:
qa_live_whatsapp_release_checks:
name: Run QA Lab live WhatsApp lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_whatsapp_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_whatsapp_enabled == 'true'
continue-on-error: true
runs-on: ubuntu-24.04
timeout-minutes: 60
@@ -2293,7 +2303,7 @@ jobs:
qa_live_slack_release_checks:
name: Run QA Lab live Slack lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_slack_enabled == 'true'
if: needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_slack_enabled == 'true'
continue-on-error: true
runs-on: ubuntu-24.04
timeout-minutes: 60
@@ -2528,7 +2538,7 @@ jobs:
QA_LIVE_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_release_checks.result }}
QA_LIVE_BUZZ_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_buzz_release_checks.result }}
QA_LIVE_TELEGRAM_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_telegram_release_checks.result }}
QA_LIVE_TELEGRAM_SELECTED: ${{ contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }}
QA_LIVE_TELEGRAM_SELECTED: ${{ needs.resolve_target.outputs.qa_live_scheduled == 'true' && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' }}
QA_LIVE_DISCORD_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_discord_release_checks.result }}
QA_LIVE_WHATSAPP_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_whatsapp_release_checks.result }}
QA_LIVE_SLACK_RELEASE_CHECKS_RESULT: ${{ needs.qa_live_slack_release_checks.result }}
.github/workflows/qa-live-transports-convex.yml
+2
View File
@@ -120,6 +120,7 @@ jobs:
authorize_actor:
name: Authorize workflow actor
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 10
outputs:
authorized: ${{ steps.permission.outputs.authorized }}
steps:
@@ -158,6 +159,7 @@ jobs:
needs: authorize_actor
if: needs.authorize_actor.outputs.authorized == 'true'
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 30
outputs:
selected_revision: ${{ steps.validate.outputs.selected_revision }}
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}