ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

improve: shorten release candidate validation (#110316)

Browse Source 
* ci(release): shorten candidate validation critical path

* ci(test): parallelize memory extension lane

* ci(release): preserve read-only candidate validation
This commit is contained in:
Peter Steinberger
2026-07-18 03:39:03 +01:00
committed by GitHub
parent bbaaac4b44
commit 6a6d3465d8
16 changed files with 530 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/full-release-validation.yml
+59 -13
View File
@@ -382,6 +382,27 @@ jobs:
--build-arg OPENCLAW_EXTENSIONS="diagnostics-otel,codex" \
.
prepare_release_candidate:
name: Prepare shared release candidate
needs: [resolve_target, evidence_reuse]
if: ${{ always() && needs.resolve_target.result == 'success' && needs.evidence_reuse.outputs.reuse != 'true' && inputs.release_package_spec == '' && inputs.package_acceptance_package_spec == '' && contains(fromJSON('["all","plugin-prerelease","release-checks","cross-os","live-e2e","package"]'), inputs.rerun_group) }}
permissions:
actions: read
contents: read
packages: write
uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml
with:
ref: ${{ needs.resolve_target.outputs.sha }}
prepare_only: true
include_repo_e2e: false
include_release_path_suites: false
include_openwebui: false
include_live_suites: false
allow_unreleased_changelog: ${{ inputs.allow_unreleased_changelog || (inputs.target_context_ref == '' && (inputs.ref == 'main' || inputs.ref == 'refs/heads/main')) }}
release_test_profile: ${{ inputs.release_profile }}
shared_image_artifact_namespace: full-release
shared_image_policy: no-push-artifact
normal_ci:
name: Run normal full CI
needs: [resolve_target, evidence_reuse]
@@ -578,8 +599,8 @@ jobs:
plugin_prerelease:
name: Run plugin prerelease validation
needs: [resolve_target, evidence_reuse]
if: ${{ always() && needs.resolve_target.result == 'success' && contains(fromJSON('["all","plugin-prerelease"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
needs: [resolve_target, evidence_reuse, prepare_release_candidate]
if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","plugin-prerelease"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ inputs.release_profile == 'full' && 300 || inputs.release_profile == 'stable' && 240 || 60 }}
outputs:
@@ -595,6 +616,7 @@ jobs:
TARGET_SHA: ${{ needs.resolve_target.outputs.sha }}
CHILD_WORKFLOW_REF: ${{ github.ref_name }}
PARENT_WORKFLOW_SHA: ${{ github.sha }}
CANDIDATE_ARTIFACT_JSON: ${{ needs.prepare_release_candidate.outputs.candidate_artifact_json }}
run: |
set -euo pipefail
@@ -759,12 +781,16 @@ jobs:
dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-plugin-prerelease"
dispatch_run_name="Plugin Prerelease ${dispatch_id}"
dispatch_and_wait plugin-prerelease.yml "$dispatch_run_name" -f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" -f full_release_validation=true -f dispatch_id="$dispatch_id"
args=(-f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" -f full_release_validation=true -f dispatch_id="$dispatch_id")
if [[ -n "${CANDIDATE_ARTIFACT_JSON// }" ]]; then
args+=(-f candidate_artifact_json="$CANDIDATE_ARTIFACT_JSON")
fi
dispatch_and_wait plugin-prerelease.yml "$dispatch_run_name" "${args[@]}"
release_checks:
name: Run release/live/Docker/QA validation
needs: [resolve_target, evidence_reuse]
if: ${{ always() && needs.resolve_target.result == 'success' && contains(fromJSON('["all","release-checks","install-smoke","cross-os","live-e2e","package","qa","qa-parity","qa-live"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
needs: [resolve_target, evidence_reuse, prepare_release_candidate]
if: ${{ always() && needs.resolve_target.result == 'success' && (needs.prepare_release_candidate.result == 'success' || needs.prepare_release_candidate.result == 'skipped') && contains(fromJSON('["all","release-checks","install-smoke","cross-os","live-e2e","package","qa","qa-parity","qa-live"]'), inputs.rerun_group) && needs.evidence_reuse.outputs.reuse != 'true' }}
runs-on: ubuntu-24.04
timeout-minutes: ${{ inputs.release_profile != 'beta' && 240 || 60 }}
outputs:
@@ -792,6 +818,7 @@ jobs:
RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }}
PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }}
CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }}
CANDIDATE_ARTIFACT_JSON: ${{ needs.prepare_release_candidate.outputs.candidate_artifact_json }}
run: |
set -euo pipefail
@@ -1099,6 +1126,9 @@ jobs:
if [[ -n "${CODEX_PLUGIN_SPEC// }" ]]; then
args+=(-f codex_plugin_spec="$CODEX_PLUGIN_SPEC")
fi
if [[ -n "${CANDIDATE_ARTIFACT_JSON// }" ]]; then
args+=(-f candidate_artifact_json="$CANDIDATE_ARTIFACT_JSON")
fi
dispatch_id="full-release-validation-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-release-checks"
dispatch_run_name="OpenClaw Release Checks ${dispatch_id}"
@@ -1463,6 +1493,7 @@ jobs:
resolve_target,
evidence_reuse,
docker_runtime_assets_preflight,
prepare_release_candidate,
normal_ci,
plugin_prerelease,
release_checks,
@@ -1488,6 +1519,9 @@ jobs:
PERFORMANCE_RESULT: ${{ needs.performance.result }}
RELEASE_PROFILE: ${{ inputs.release_profile }}
DOCKER_RUNTIME_ASSETS_PREFLIGHT_RESULT: ${{ needs.docker_runtime_assets_preflight.result }}
PREPARE_RELEASE_CANDIDATE_RESULT: ${{ needs.prepare_release_candidate.result }}
RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }}
PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }}
EVIDENCE_REUSE: ${{ needs.evidence_reuse.outputs.reuse }}
EVIDENCE_ROOT_RUN_ID: ${{ needs.evidence_reuse.outputs.evidence_root_run_id }}
EVIDENCE_RUN_URL: ${{ needs.evidence_reuse.outputs.evidence_run_url }}
@@ -1682,26 +1716,26 @@ jobs:
| .[])
' || echo "_Unable to summarize jobs for run ${run_id}._"
echo
echo "### Longest queues: ${label}"
echo "### Longest start delays: ${label}"
echo
gh_with_retry api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" --jq ".jobs[] | @json" | jq -sr '
def ts: fromdateiso8601;
"| Job | Result | Queue minutes | Run minutes |",
"| Job | Result | Start delay minutes | Run minutes |",
"| --- | --- | ---: | ---: |",
([.[]
| select(.created_at != null and .started_at != null)
| . + {
queueMin: ((((.started_at | ts) - (.created_at | ts)) / 60) * 10 | round / 10),
startDelayMin: ((((.started_at | ts) - (.created_at | ts)) / 60) * 10 | round / 10),
durationMin: (if .completed_at == null then null else ((((.completed_at | ts) - (.started_at | ts)) / 60) * 10 | round / 10) end)
}
| select(.queueMin > 0)
| {name, conclusion, queueMin, durationMin}]
| sort_by(.queueMin)
| select(.startDelayMin > 0)
| {name, conclusion, startDelayMin, durationMin}]
| sort_by(.startDelayMin)
| reverse
| .[0:10]
| map("| `" + (.name | gsub("\\|"; "\\|")) + "` | `" + ((.conclusion // "") | tostring) + "` | " + (.queueMin | tostring) + " | " + ((.durationMin // "") | tostring) + " |")
| map("| `" + (.name | gsub("\\|"; "\\|")) + "` | `" + ((.conclusion // "") | tostring) + "` | " + (.startDelayMin | tostring) + " | " + ((.durationMin // "") | tostring) + " |")
| .[])
' || echo "_Unable to summarize queue times for run ${run_id}._"
' || echo "_Unable to summarize start delays for run ${run_id}._"
} >> "$GITHUB_STEP_SUMMARY"
}
@@ -1758,6 +1792,18 @@ jobs:
plugin_prerelease_required=0
release_checks_required=0
performance_required=0
candidate_required=0
if [[ -z "${RELEASE_PACKAGE_SPEC// }" && -z "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then
case "$RERUN_GROUP" in
all|plugin-prerelease|release-checks|cross-os|live-e2e|package)
candidate_required=1
;;
esac
fi
if [[ "$candidate_required" == "1" && "$EVIDENCE_REUSE" != "true" && "$PREPARE_RELEASE_CANDIDATE_RESULT" != "success" ]]; then
echo "::error::Shared release candidate preparation ended with ${PREPARE_RELEASE_CANDIDATE_RESULT}."
failed=1
fi
if [[ "$RERUN_GROUP" == "all" && "$EVIDENCE_REUSE" == "true" ]]; then
# Lanes were skipped because a prior green validation covers this
# target; re-verify the chain-root run and its recorded child runs
.github/workflows/openclaw-live-and-e2e-checks-reusable.yml
+216 -32
View File
@@ -130,6 +130,55 @@ on:
default: false
type: boolean
workflow_call:
outputs:
candidate_artifact_json:
description: Immutable package and Docker image artifact tuple
value: ${{ jobs.prepare_docker_e2e_image.outputs.candidate_artifact_json }}
package_artifact_name:
description: Immutable package artifact name
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_artifact_name }}
package_artifact_id:
description: Immutable package artifact id
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_artifact_id }}
package_artifact_digest:
description: Package artifact service digest
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_artifact_digest }}
package_artifact_run_id:
description: Package artifact producer run id
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_artifact_run_id }}
package_artifact_run_attempt:
description: Package artifact producer run attempt
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_artifact_run_attempt }}
package_file_name:
description: Package tarball filename
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_file_name }}
package_source_sha:
description: Package source commit
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_source_sha }}
package_sha256:
description: Package tarball SHA-256
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_sha256 }}
package_version:
description: Package version
value: ${{ jobs.prepare_docker_e2e_image.outputs.package_version }}
shared_image_artifact_name:
description: Immutable Docker image artifact name
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_artifact_name }}
shared_image_artifact_id:
description: Immutable Docker image artifact id
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_artifact_id }}
shared_image_artifact_digest:
description: Docker image artifact service digest
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_artifact_digest }}
shared_image_artifact_run_id:
description: Docker image artifact producer run id
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_artifact_run_id }}
shared_image_artifact_run_attempt:
description: Docker image artifact producer run attempt
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_artifact_run_attempt }}
shared_image_archive_sha256:
description: Docker image archive SHA-256
value: ${{ jobs.prepare_docker_e2e_image.outputs.image_archive_sha256 }}
inputs:
advisory:
description: Treat failures as advisory for the caller
@@ -146,6 +195,11 @@ on:
required: false
default: false
type: boolean
prepare_only:
description: Prepare immutable package and Docker image artifacts without running validation lanes
required: false
default: false
type: boolean
ref:
description: Ref, tag, or SHA to validate
required: true
@@ -250,6 +304,36 @@ on:
required: false
default: direct
type: string
shared_image_artifact_name:
description: Existing immutable Docker image artifact name
required: false
default: ""
type: string
shared_image_artifact_id:
description: Existing immutable Docker image artifact id
required: false
default: ""
type: string
shared_image_artifact_digest:
description: Existing Docker image artifact service SHA-256 digest
required: false
default: ""
type: string
shared_image_artifact_run_id:
description: Producer run id for the Docker image artifact
required: false
default: ""
type: string
shared_image_artifact_run_attempt:
description: Producer run attempt for the Docker image artifact
required: false
default: ""
type: string
shared_image_archive_sha256:
description: SHA-256 of the Docker image archive inside the artifact
required: false
default: ""
type: string
docker_e2e_bare_image:
description: Existing bare Docker E2E image to reuse; blank derives from package SHA/ref
required: false
@@ -464,6 +548,12 @@ jobs:
PROVIDED_BARE_IMAGE: ${{ inputs.docker_e2e_bare_image }}
PROVIDED_FUNCTIONAL_IMAGE: ${{ inputs.docker_e2e_functional_image }}
SHARED_IMAGE_ARTIFACT_NAMESPACE: ${{ inputs.shared_image_artifact_namespace }}
SHARED_IMAGE_ARTIFACT_DIGEST: ${{ inputs.shared_image_artifact_digest }}
SHARED_IMAGE_ARTIFACT_ID: ${{ inputs.shared_image_artifact_id }}
SHARED_IMAGE_ARTIFACT_NAME: ${{ inputs.shared_image_artifact_name }}
SHARED_IMAGE_ARTIFACT_RUN_ATTEMPT: ${{ inputs.shared_image_artifact_run_attempt }}
SHARED_IMAGE_ARTIFACT_RUN_ID: ${{ inputs.shared_image_artifact_run_id }}
SHARED_IMAGE_ARCHIVE_SHA256: ${{ inputs.shared_image_archive_sha256 }}
SHARED_IMAGE_POLICY: ${{ inputs.shared_image_policy }}
shell: bash
run: |
@@ -526,6 +616,34 @@ jobs:
}
fi
image_tuple_present=0
for value in \
"$SHARED_IMAGE_ARTIFACT_DIGEST" \
"$SHARED_IMAGE_ARTIFACT_ID" \
"$SHARED_IMAGE_ARTIFACT_NAME" \
"$SHARED_IMAGE_ARTIFACT_RUN_ATTEMPT" \
"$SHARED_IMAGE_ARTIFACT_RUN_ID" \
"$SHARED_IMAGE_ARCHIVE_SHA256"; do
if [[ -n "${value// }" ]]; then
image_tuple_present=1
fi
done
if [[ "$image_tuple_present" == "1" ]]; then
[[ "$SHARED_IMAGE_ARTIFACT_DIGEST" =~ ^[0-9a-f]{64}$ &&
"$SHARED_IMAGE_ARTIFACT_ID" =~ ^[1-9][0-9]*$ &&
-n "${SHARED_IMAGE_ARTIFACT_NAME// }" &&
"$SHARED_IMAGE_ARTIFACT_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ &&
"$SHARED_IMAGE_ARTIFACT_RUN_ID" =~ ^[1-9][0-9]*$ &&
"$SHARED_IMAGE_ARCHIVE_SHA256" =~ ^[0-9a-f]{64}$ ]] || {
echo "Docker image artifact selection requires the complete immutable artifact tuple." >&2
exit 1
}
[[ "$SHARED_IMAGE_ARTIFACT_NAME" == *"-${SHARED_IMAGE_ARTIFACT_RUN_ID}-${SHARED_IMAGE_ARTIFACT_RUN_ATTEMPT}" ]] || {
echo "Docker image artifact name does not bind the declared producer run attempt." >&2
exit 1
}
fi
case "$SHARED_IMAGE_POLICY" in
existing-only)
if [[ -z "${PROVIDED_BARE_IMAGE// }" && -z "${PROVIDED_FUNCTIONAL_IMAGE// }" ]]; then
@@ -546,6 +664,10 @@ jobs:
echo "shared_image_policy=no-push-artifact builds local image artifacts and rejects provided images." >&2
exit 1
}
if [[ "$image_tuple_present" == "1" && "$package_tuple_present" != "1" ]]; then
echo "Reusing a Docker image artifact requires its immutable package identity tuple." >&2
exit 1
fi
;;
*)
echo "shared_image_policy must be existing-only or no-push-artifact." >&2
@@ -1773,7 +1895,7 @@ jobs:
prepare_docker_e2e_image:
needs: validate_selected_ref
if: inputs.include_release_path_suites || inputs.include_openwebui || inputs.docker_lanes != ''
if: inputs.prepare_only || inputs.include_release_path_suites || inputs.include_openwebui || inputs.docker_lanes != ''
continue-on-error: ${{ inputs.advisory }}
runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }}
timeout-minutes: ${{ inputs.release_test_profile == 'full' && 90 || 60 }}
@@ -1782,12 +1904,13 @@ jobs:
contents: read
packages: read
outputs:
candidate_artifact_json: ${{ steps.candidate_manifest.outputs.json }}
image: ${{ steps.image.outputs.image }}
bare_image: ${{ steps.image.outputs.bare_image }}
functional_image: ${{ steps.image.outputs.functional_image }}
needs_bare_image: ${{ steps.plan.outputs.needs_bare_image }}
needs_bare_image: ${{ inputs.shared_image_artifact_id != '' && '1' || steps.plan.outputs.needs_bare_image }}
needs_e2e_image: ${{ steps.plan.outputs.needs_e2e_image }}
needs_functional_image: ${{ steps.plan.outputs.needs_functional_image }}
needs_functional_image: ${{ inputs.shared_image_artifact_id != '' && '1' || steps.plan.outputs.needs_functional_image }}
needs_live_image: ${{ steps.plan.outputs.needs_live_image }}
needs_package: ${{ steps.plan.outputs.needs_package }}
bare_exists: ${{ steps.image_exists.outputs.bare_exists }}
@@ -1802,12 +1925,12 @@ jobs:
package_artifact_run_attempt: ${{ steps.upload_package.outputs.artifact-id && github.run_attempt || steps.input_package_artifact.outputs.run_attempt }}
package_file_name: ${{ steps.package.outputs.file_name }}
package_source_sha: ${{ steps.package.outputs.source_sha }}
image_artifact_name: ${{ steps.image_artifact.outputs.artifact_name }}
image_archive_sha256: ${{ steps.image_artifact.outputs.archive_sha256 }}
image_artifact_id: ${{ steps.upload_image_artifact.outputs.artifact-id }}
image_artifact_digest: ${{ steps.upload_image_artifact.outputs.artifact-digest }}
image_artifact_run_id: ${{ github.run_id }}
image_artifact_run_attempt: ${{ github.run_attempt }}
image_artifact_name: ${{ steps.image_artifact.outputs.artifact_name || inputs.shared_image_artifact_name }}
image_archive_sha256: ${{ steps.image_artifact.outputs.archive_sha256 || inputs.shared_image_archive_sha256 }}
image_artifact_id: ${{ steps.upload_image_artifact.outputs.artifact-id || inputs.shared_image_artifact_id }}
image_artifact_digest: ${{ steps.upload_image_artifact.outputs.artifact-digest || inputs.shared_image_artifact_digest }}
image_artifact_run_id: ${{ steps.upload_image_artifact.outputs.artifact-id && github.run_id || inputs.shared_image_artifact_run_id }}
image_artifact_run_attempt: ${{ steps.upload_image_artifact.outputs.artifact-id && github.run_attempt || inputs.shared_image_artifact_run_attempt }}
env:
DOCKER_BUILD_SUMMARY: "false"
DOCKER_BUILD_RECORD_UPLOAD: "false"
@@ -1834,6 +1957,7 @@ jobs:
shell: bash
env:
LANES: ${{ inputs.docker_lanes }}
PREPARE_ONLY: ${{ inputs.prepare_only }}
INCLUDE_RELEASE_PATH_SUITES: ${{ inputs.include_release_path_suites }}
INCLUDE_OPENWEBUI: ${{ inputs.include_openwebui }}
RELEASE_TEST_PROFILE: ${{ inputs.release_test_profile }}
@@ -1844,7 +1968,7 @@ jobs:
set -euo pipefail
mkdir -p .artifacts/docker-tests
if [[ "$INCLUDE_RELEASE_PATH_SUITES" == "true" ]]; then
if [[ "$PREPARE_ONLY" == "true" || "$INCLUDE_RELEASE_PATH_SUITES" == "true" ]]; then
export OPENCLAW_DOCKER_ALL_PROFILE=release-path
export OPENCLAW_DOCKER_ALL_PLAN_RELEASE_ALL=1
elif [[ -n "$LANES" ]]; then
@@ -1858,6 +1982,13 @@ jobs:
plan_path=".artifacts/docker-tests/plan.json"
node .release-harness/scripts/test-docker-all.mjs --plan-json > "$plan_path"
node .release-harness/scripts/docker-e2e.mjs github-outputs "$plan_path" >> "$GITHUB_OUTPUT"
if [[ "$PREPARE_ONLY" == "true" ]]; then
grep -Fxq "needs_bare_image=1" "$GITHUB_OUTPUT" &&
grep -Fxq "needs_functional_image=1" "$GITHUB_OUTPUT" || {
echo "Shared release candidate preparation requires both Docker image variants." >&2
exit 1
}
fi
echo "plan_json=$plan_path" >> "$GITHUB_OUTPUT"
- name: Setup Node environment
@@ -2066,7 +2197,7 @@ jobs:
- name: Upload OpenClaw Docker E2E package
id: upload_package
if: steps.plan.outputs.needs_package == '1' && (inputs.package_artifact_id == '' || inputs.package_artifact_run_id != github.run_id)
if: steps.plan.outputs.needs_package == '1' && inputs.package_artifact_id == ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: docker-e2e-package-${{ github.run_id }}-${{ github.run_attempt }}
@@ -2086,17 +2217,24 @@ jobs:
set -euo pipefail
repository="${GITHUB_REPOSITORY,,}"
image_tag="${PACKAGE_TAG:-$SELECTED_SHA}"
bare_context_sha="$(
sed '/^FROM bare AS build$/,$d' scripts/e2e/Dockerfile | sha256sum | awk '{print $1}'
)"
if [[ "$SHARED_IMAGE_POLICY" == "no-push-artifact" ]]; then
bare_image="openclaw-docker-e2e-bare:${image_tag}"
bare_image="openclaw-docker-e2e-bare:base-${bare_context_sha:0:32}"
bare_cache_image="ghcr.io/${repository}-docker-e2e-bare:base-${bare_context_sha:0:32}"
functional_image="openclaw-docker-e2e-functional:${image_tag}"
else
bare_image="${PROVIDED_BARE_IMAGE:-ghcr.io/${repository}-docker-e2e-bare:${image_tag}}"
bare_cache_image=""
functional_image="${PROVIDED_FUNCTIONAL_IMAGE:-ghcr.io/${repository}-docker-e2e-functional:${image_tag}}"
fi
image="$functional_image"
echo "image=$image" >> "$GITHUB_OUTPUT"
echo "bare_image=$bare_image" >> "$GITHUB_OUTPUT"
echo "bare_cache_image=$bare_cache_image" >> "$GITHUB_OUTPUT"
echo "functional_image=$functional_image" >> "$GITHUB_OUTPUT"
echo "bare_context_sha=$bare_context_sha" >> "$GITHUB_OUTPUT"
echo "Shared Docker E2E bare image: \`$bare_image\`" >> "$GITHUB_STEP_SUMMARY"
echo "Shared Docker E2E functional image: \`$functional_image\`" >> "$GITHUB_STEP_SUMMARY"
@@ -2154,52 +2292,56 @@ jobs:
echo "functional_exists=$functional_exists" >> "$GITHUB_OUTPUT"
echo "needs_build=$needs_build" >> "$GITHUB_OUTPUT"
- name: Setup Docker builder
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1'
uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
with:
max-cache-size-mb: 800000
- name: Build bare Docker E2E image artifact
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_bare_image == '1'
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_bare_image == '1' && inputs.shared_image_artifact_id == ''
shell: bash
env:
CACHE_IMAGE_REF: ${{ steps.image.outputs.bare_cache_image }}
IMAGE_REF: ${{ steps.image.outputs.bare_image }}
run: |
set -euo pipefail
# Docker's local image exporter cannot load attestation manifest lists.
timeout --kill-after=30s 45m docker buildx build \
--load \
cache=()
if docker manifest inspect "$CACHE_IMAGE_REF" >/dev/null 2>&1; then
if bash .release-harness/scripts/ci-docker-pull-retry.sh "$CACHE_IMAGE_REF"; then
cache=(--cache-from "$CACHE_IMAGE_REF")
echo "Using package-independent bare image cache: $CACHE_IMAGE_REF"
else
echo "::warning::Bare image cache pull failed; continuing with a cold build."
fi
fi
timeout --kill-after=30s 45m env DOCKER_BUILDKIT=1 docker build \
--file ./scripts/e2e/Dockerfile \
--target bare \
--platform linux/amd64 \
--tag "$IMAGE_REF" \
--sbom=false \
--provenance=false \
--build-arg BUILDKIT_INLINE_CACHE=1 \
"${cache[@]}" \
.
- name: Build functional Docker E2E image artifact
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_functional_image == '1'
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_functional_image == '1' && inputs.shared_image_artifact_id == ''
shell: bash
env:
IMAGE_REF: ${{ steps.image.outputs.functional_image }}
BARE_IMAGE_REF: ${{ steps.image.outputs.bare_image }}
run: |
set -euo pipefail
# Docker's local image exporter cannot load attestation manifest lists.
timeout --kill-after=30s 45m docker buildx build \
--load \
cache=()
if docker image inspect "$BARE_IMAGE_REF" >/dev/null 2>&1; then
cache=(--cache-from "$BARE_IMAGE_REF")
fi
timeout --kill-after=30s 45m env DOCKER_BUILDKIT=1 docker build \
--file ./scripts/e2e/Dockerfile \
--target functional \
--build-context openclaw_package=.artifacts/docker-e2e-package \
--platform linux/amd64 \
--tag "$IMAGE_REF" \
--sbom=false \
--provenance=false \
"${cache[@]}" \
.
- name: Pack Docker E2E image artifact
id: image_artifact
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1'
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1' && inputs.shared_image_artifact_id == ''
shell: bash
env:
BARE_IMAGE: ${{ steps.image.outputs.bare_image }}
@@ -2234,7 +2376,7 @@ jobs:
- name: Upload Docker E2E image artifact
id: upload_image_artifact
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1'
if: inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1' && inputs.shared_image_artifact_id == ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ steps.image_artifact.outputs.artifact_name }}
@@ -2243,6 +2385,48 @@ jobs:
compression-level: 0
retention-days: 7
- name: Emit immutable release candidate tuple
id: candidate_manifest
if: steps.plan.outputs.needs_e2e_image == '1'
env:
IMAGE_ARCHIVE_SHA256: ${{ steps.image_artifact.outputs.archive_sha256 || inputs.shared_image_archive_sha256 }}
IMAGE_ARTIFACT_DIGEST: ${{ steps.upload_image_artifact.outputs.artifact-digest || inputs.shared_image_artifact_digest }}
IMAGE_ARTIFACT_ID: ${{ steps.upload_image_artifact.outputs.artifact-id || inputs.shared_image_artifact_id }}
IMAGE_ARTIFACT_NAME: ${{ steps.image_artifact.outputs.artifact_name || inputs.shared_image_artifact_name }}
IMAGE_ARTIFACT_RUN_ATTEMPT: ${{ steps.upload_image_artifact.outputs.artifact-id && github.run_attempt || inputs.shared_image_artifact_run_attempt }}
IMAGE_ARTIFACT_RUN_ID: ${{ steps.upload_image_artifact.outputs.artifact-id && github.run_id || inputs.shared_image_artifact_run_id }}
PACKAGE_ARTIFACT_DIGEST: ${{ steps.upload_package.outputs.artifact-digest || inputs.package_artifact_digest }}
PACKAGE_ARTIFACT_ID: ${{ steps.upload_package.outputs.artifact-id || inputs.package_artifact_id }}
PACKAGE_ARTIFACT_NAME: ${{ steps.upload_package.outputs.artifact-id && format('docker-e2e-package-{0}-{1}', github.run_id, github.run_attempt) || inputs.package_artifact_name }}
PACKAGE_ARTIFACT_RUN_ATTEMPT: ${{ steps.upload_package.outputs.artifact-id && github.run_attempt || inputs.package_artifact_run_attempt }}
PACKAGE_ARTIFACT_RUN_ID: ${{ steps.upload_package.outputs.artifact-id && github.run_id || inputs.package_artifact_run_id }}
PACKAGE_FILE_NAME: ${{ steps.package.outputs.file_name }}
PACKAGE_SHA256: ${{ steps.package.outputs.sha256 }}
PACKAGE_SOURCE_SHA: ${{ steps.package.outputs.source_sha }}
PACKAGE_VERSION: ${{ steps.package.outputs.version }}
shell: bash
run: |
set -euo pipefail
json="$(jq -cn \
--arg packageArtifactName "$PACKAGE_ARTIFACT_NAME" \
--arg packageArtifactId "$PACKAGE_ARTIFACT_ID" \
--arg packageArtifactDigest "$PACKAGE_ARTIFACT_DIGEST" \
--arg packageArtifactRunId "$PACKAGE_ARTIFACT_RUN_ID" \
--arg packageArtifactRunAttempt "$PACKAGE_ARTIFACT_RUN_ATTEMPT" \
--arg packageFileName "$PACKAGE_FILE_NAME" \
--arg packageSourceSha "$PACKAGE_SOURCE_SHA" \
--arg packageSha256 "$PACKAGE_SHA256" \
--arg packageVersion "$PACKAGE_VERSION" \
--arg imageArtifactName "$IMAGE_ARTIFACT_NAME" \
--arg imageArtifactId "$IMAGE_ARTIFACT_ID" \
--arg imageArtifactDigest "$IMAGE_ARTIFACT_DIGEST" \
--arg imageArtifactRunId "$IMAGE_ARTIFACT_RUN_ID" \
--arg imageArtifactRunAttempt "$IMAGE_ARTIFACT_RUN_ATTEMPT" \
--arg imageArchiveSha256 "$IMAGE_ARCHIVE_SHA256" \
'$ARGS.named')"
jq -e 'all(.[]; type == "string" and length > 0)' <<< "$json" >/dev/null
echo "json=$json" >> "$GITHUB_OUTPUT"
docker_e2e_image_ready:
needs: prepare_docker_e2e_image
if: always() && needs.prepare_docker_e2e_image.result != 'skipped'
.github/workflows/openclaw-release-checks.yml
+56 -9
View File
@@ -105,6 +105,11 @@ on:
required: false
default: ""
type: string
candidate_artifact_json:
description: Immutable package and Docker image artifact tuple from Full Release Validation
required: false
default: ""
type: string
concurrency:
group: openclaw-release-checks-${{ inputs.expected_sha || inputs.ref }}-${{ inputs.rerun_group }}
@@ -552,15 +557,15 @@ jobs:
contents: read
packages: read
outputs:
artifact_digest: ${{ steps.release_package_upload.outputs.artifact-digest }}
artifact_id: ${{ steps.release_package_upload.outputs.artifact-id }}
artifact_name: ${{ steps.artifact.outputs.name }}
artifact_run_attempt: ${{ steps.artifact.outputs.run_attempt }}
artifact_run_id: ${{ steps.artifact.outputs.run_id }}
package_file_name: ${{ steps.artifact.outputs.file_name }}
package_sha256: ${{ steps.package.outputs.sha256 }}
package_version: ${{ steps.package.outputs.package_version }}
source_sha: ${{ steps.package.outputs.source_sha }}
artifact_digest: ${{ steps.release_package_upload.outputs.artifact-digest || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactDigest }}
artifact_id: ${{ steps.release_package_upload.outputs.artifact-id || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactId }}
artifact_name: ${{ steps.artifact.outputs.name || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactName }}
artifact_run_attempt: ${{ steps.artifact.outputs.run_attempt || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunAttempt }}
artifact_run_id: ${{ steps.artifact.outputs.run_id || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunId }}
package_file_name: ${{ steps.artifact.outputs.file_name || fromJSON(inputs.candidate_artifact_json || '{}').packageFileName }}
package_sha256: ${{ steps.package.outputs.sha256 || fromJSON(inputs.candidate_artifact_json || '{}').packageSha256 }}
package_version: ${{ steps.package.outputs.package_version || fromJSON(inputs.candidate_artifact_json || '{}').packageVersion }}
source_sha: ${{ steps.package.outputs.source_sha || fromJSON(inputs.candidate_artifact_json || '{}').packageSourceSha }}
steps:
- name: Checkout trusted workflow ref
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -572,6 +577,7 @@ jobs:
- name: Set artifact metadata
id: artifact
if: inputs.candidate_artifact_json == ''
run: |
{
echo "file_name=openclaw-current.tgz"
@@ -581,6 +587,7 @@ jobs:
} >> "$GITHUB_OUTPUT"
- name: Setup Node environment
if: inputs.candidate_artifact_json == ''
uses: ./.github/actions/setup-node-env
with:
node-version: ${{ env.NODE_VERSION }}
@@ -589,6 +596,7 @@ jobs:
- name: Resolve release package artifact
id: package
if: inputs.candidate_artifact_json == ''
shell: bash
env:
PACKAGE_REF: ${{ needs.resolve_target.outputs.revision }}
@@ -627,6 +635,7 @@ jobs:
- name: Upload release package artifact
id: release_package_upload
if: inputs.candidate_artifact_json == ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ steps.artifact.outputs.name }}
@@ -637,6 +646,7 @@ jobs:
if-no-files-found: error
- name: Validate release package artifact binding
if: inputs.candidate_artifact_json == ''
env:
ARTIFACT_DIGEST: ${{ steps.release_package_upload.outputs.artifact-digest }}
ARTIFACT_ID: ${{ steps.release_package_upload.outputs.artifact-id }}
@@ -674,6 +684,36 @@ jobs:
exit 1
}
- name: Validate shared release candidate identity
if: inputs.candidate_artifact_json != ''
env:
CANDIDATE_ARTIFACT_JSON: ${{ inputs.candidate_artifact_json }}
SELECTED_SHA: ${{ needs.resolve_target.outputs.revision }}
run: |
set -euo pipefail
jq -e \
--arg sha "$SELECTED_SHA" \
'def digits: test("^[1-9][0-9]*$");
def hex40: test("^[a-f0-9]{40}$");
def hex64: test("^[a-f0-9]{64}$");
(.packageArtifactName | type == "string" and length > 0) and
(.packageArtifactId | tostring | digits) and
(.packageArtifactDigest | hex64) and
(.packageArtifactRunId | tostring | digits) and
(.packageArtifactRunAttempt | tostring | digits) and
(.packageFileName | test("^[A-Za-z0-9][A-Za-z0-9._-]*\\.tgz$")) and
(.packageSourceSha | hex40) and
.packageSourceSha == $sha and
(.packageSha256 | hex64) and
(.packageVersion | type == "string" and length > 0) and
(.imageArtifactName | type == "string" and length > 0) and
(.imageArtifactId | tostring | digits) and
(.imageArtifactDigest | hex64) and
(.imageArtifactRunId | tostring | digits) and
(.imageArtifactRunAttempt | tostring | digits) and
(.imageArchiveSha256 | hex64)'
<<< "$CANDIDATE_ARTIFACT_JSON" >/dev/null
install_smoke_release_checks:
needs: [resolve_target]
if: contains(fromJSON('["all","install-smoke"]'), needs.resolve_target.outputs.rerun_group)
@@ -825,6 +865,12 @@ jobs:
package_version: ${{ needs.prepare_release_package.outputs.package_version }}
codex_plugin_spec: ${{ needs.resolve_target.outputs.codex_plugin_spec }}
shared_image_artifact_namespace: release-docker
shared_image_artifact_name: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactName || '' }}
shared_image_artifact_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactId || '' }}
shared_image_artifact_digest: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactDigest || '' }}
shared_image_artifact_run_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunId || '' }}
shared_image_artifact_run_attempt: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunAttempt || '' }}
shared_image_archive_sha256: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArchiveSha256 || '' }}
shared_image_policy: no-push-artifact
secrets: *live_e2e_release_secrets
@@ -860,6 +906,7 @@ jobs:
telegram_mode: mock-openai
telegram_advisory: ${{ needs.resolve_target.outputs.release_profile == 'beta' }}
shared_image_artifact_namespace: release-package
candidate_artifact_json: ${{ inputs.candidate_artifact_json }}
shared_image_policy: no-push-artifact
secrets:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
.github/workflows/package-acceptance.yml
+11
View File
@@ -243,6 +243,11 @@ on:
required: false
default: package-acceptance
type: string
candidate_artifact_json:
description: Immutable Docker image artifact tuple from Full Release Validation
required: false
default: ""
type: string
suite_profile:
description: "Acceptance profile: smoke, package, product, full, or custom"
required: false
@@ -825,6 +830,12 @@ jobs:
include_live_suites: ${{ needs.resolve_package.outputs.include_live_suites == 'true' }}
live_models_only: false
shared_image_artifact_namespace: ${{ inputs.shared_image_artifact_namespace }}
shared_image_artifact_name: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactName || '' }}
shared_image_artifact_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactId || '' }}
shared_image_artifact_digest: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactDigest || '' }}
shared_image_artifact_run_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunId || '' }}
shared_image_artifact_run_attempt: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunAttempt || '' }}
shared_image_archive_sha256: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArchiveSha256 || '' }}
shared_image_policy: ${{ inputs.shared_image_policy }}
secrets: &docker_acceptance_secrets
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
.github/workflows/plugin-prerelease.yml
+31 -4
View File
@@ -25,6 +25,11 @@ on:
required: false
default: ""
type: string
candidate_artifact_json:
description: Immutable package and Docker image artifact tuple from Full Release Validation
required: false
default: ""
type: string
permissions:
contents: read
@@ -121,9 +126,16 @@ jobs:
}).map((shard) => ({
check_name: shard.checkName,
extensions_csv: shard.extensionIds.join(","),
runner: [0, 1, 2, 3].includes(shard.index)
? "blacksmith-8vcpu-ubuntu-2404"
: "blacksmith-4vcpu-ubuntu-2404",
vitest_max_workers: shard.extensionIds.some((extensionId) =>
extensionId.startsWith("memory-"),
)
? 4
: 1,
runner: shard.extensionIds.some((extensionId) => extensionId.startsWith("memory-"))
? "blacksmith-16vcpu-ubuntu-2404"
: [0, 1, 2, 3].includes(shard.index)
? "blacksmith-8vcpu-ubuntu-2404"
: "blacksmith-4vcpu-ubuntu-2404",
shard_index: shard.index + 1,
task: "extensions-batch",
}));
@@ -349,7 +361,7 @@ jobs:
env:
NODE_OPTIONS: --max-old-space-size=8192
OPENCLAW_EXTENSION_BATCH_PARALLEL: 2
OPENCLAW_VITEST_MAX_WORKERS: 1
OPENCLAW_VITEST_MAX_WORKERS: ${{ matrix.vitest_max_workers }}
OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }}
run: pnpm test:extensions:batch "$OPENCLAW_EXTENSION_BATCH" -- --retry=1 --exclude extensions/codex/src/app-server/run-attempt.test.ts
@@ -553,6 +565,21 @@ jobs:
include_live_suites: false
live_models_only: false
shared_image_artifact_namespace: plugin-prerelease
package_artifact_name: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactName || '' }}
package_artifact_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactId || '' }}
package_artifact_digest: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactDigest || '' }}
package_artifact_run_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunId || '' }}
package_artifact_run_attempt: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunAttempt || '' }}
package_file_name: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageFileName || '' }}
package_source_sha: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageSourceSha || '' }}
package_sha256: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageSha256 || '' }}
package_version: ${{ fromJSON(inputs.candidate_artifact_json || '{}').packageVersion || '' }}
shared_image_artifact_name: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactName || '' }}
shared_image_artifact_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactId || '' }}
shared_image_artifact_digest: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactDigest || '' }}
shared_image_artifact_run_id: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunId || '' }}
shared_image_artifact_run_attempt: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArtifactRunAttempt || '' }}
shared_image_archive_sha256: ${{ fromJSON(inputs.candidate_artifact_json || '{}').imageArchiveSha256 || '' }}
shared_image_policy: no-push-artifact
plugin-prerelease-suite:
docs/reference/full-release-validation.md
+13 -3
View File
@@ -77,6 +77,14 @@ run. Evidence reuse runs only from `main` or a canonical SHA-pinned
`release-ci/*` ref whose workflow commit remains on trusted `main` lineage;
other workflow refs run the selected lanes fresh.
Fresh package-facing validation prepares one immutable tarball plus one Docker
image artifact before dispatching Plugin Prerelease and OpenClaw Release Checks.
Both children verify the same package SHA, artifact IDs, service digests,
producer run attempt, and Docker archive digest before use. The package-independent
bare Docker layer uses a content-addressed GHCR cache; candidate-specific images
remain immutable GitHub artifacts. Focused runs with an explicit published
package spec keep the existing package path instead.
Also for `rerun_group=all`, a `Verify Docker runtime image assets` job builds
the `runtime-assets` Docker target with
`OPENCLAW_EXTENSIONS=diagnostics-otel,codex`. It runs in parallel with the
@@ -86,6 +94,7 @@ it before dispatching. A narrower `rerun_group` skips this preflight.
| Stage | Details |
| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Target resolution | **Job:** `Resolve target ref`<br />**Child workflow:** none<br />**Proves:** resolves the release branch, tag, or full commit SHA and records selected inputs.<br />**Rerun:** rerun the umbrella if this fails. |
| Shared candidate | **Job:** `Prepare shared release candidate`<br />**Child workflow:** `OpenClaw Live And E2E Checks (Reusable)`<br />**Proves:** packs and validates one exact-SHA package, builds one functional Docker image, and records immutable package and image artifact tuples for both package-facing child workflows.<br />**Rerun:** rerun the affected package, plugin-prerelease, cross-OS, or live/E2E group. |
| Docker assets preflight | **Job:** `Verify Docker runtime image assets`<br />**Child workflow:** none<br />**Proves:** the `runtime-assets` Docker build target still succeeds before any other stage dispatches. Runs only for `rerun_group=all`.<br />**Rerun:** rerun the umbrella with `rerun_group=all`. |
| Vitest and normal CI | **Job:** `Run normal full CI`<br />**Child workflow:** `CI`<br />**Proves:** manual full CI graph against the target ref, including Linux Node lanes, bundled plugin shards, plugin and channel contract shards, Node 22 compatibility, `check-*`, `check-additional-*`, built-artifact smoke checks, docs checks, Python skills, Windows, macOS, Control UI i18n, and Android via the umbrella.<br />**Rerun:** `rerun_group=ci`. |
| Plugin prerelease | **Job:** `Run plugin prerelease validation`<br />**Child workflow:** `Plugin Prerelease`<br />**Proves:** release-only plugin static checks, agentic plugin coverage, full plugin batch shards, plugin prerelease Docker lanes, and a non-blocking `plugin-inspector-advisory` artifact for compatibility triage.<br />**Rerun:** `rerun_group=plugin-prerelease`. |
@@ -122,13 +131,14 @@ cancel each other.
## Release checks stages
`OpenClaw Release Checks` is the largest child workflow. It resolves the target
once and prepares a shared `release-package-under-test` artifact when package
or Docker-facing stages need it.
once and validates the umbrella's shared package artifact when available. A
direct or focused dispatch prepares its own `release-package-under-test`
artifact when package or Docker-facing stages need it.
| Stage | Details |
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Release target | **Job:** `Resolve target ref`<br />**Backing workflow:** none<br />**Tests:** selected ref, optional expected SHA, profile, rerun group, and focused live suite filter.<br />**Rerun:** `rerun_group=release-checks`. |
| Package artifact | **Job:** `Prepare release package artifact`<br />**Backing workflow:** none<br />**Tests:** packs or resolves one candidate tarball and uploads `release-package-under-test` for downstream package-facing checks.<br />**Rerun:** the affected package, cross-OS, or live/E2E group. |
| Package artifact | **Job:** `Prepare release package artifact`<br />**Backing workflow:** none<br />**Tests:** validates the umbrella's immutable package tuple, or packs one candidate tarball for a direct/focused Release Checks dispatch, then exposes it to downstream package-facing checks.<br />**Rerun:** the affected package, cross-OS, or live/E2E group. |
| Install smoke | **Job:** `Run install smoke`<br />**Backing workflow:** `Install Smoke`<br />**Tests:** full install path with root Dockerfile smoke image reuse, QR package install, root and gateway Docker smokes, installer Docker tests, and Bun global install image-provider smoke.<br />**Rerun:** `rerun_group=install-smoke`. |
| Cross-OS | **Job:** `cross_os_release_checks`<br />**Backing workflow:** `OpenClaw Cross-OS Release Checks (Reusable)`<br />**Tests:** fresh and upgrade lanes on Linux, Windows, and macOS for the selected provider and mode, using the candidate tarball plus a baseline package.<br />**Rerun:** `rerun_group=cross-os`. |
| Repo and live E2E | **Job:** `Run repo/live E2E validation`<br />**Backing workflow:** `OpenClaw Live And E2E Checks (Reusable)`<br />**Tests:** repository E2E, live cache, OpenAI websocket streaming, native live provider and plugin shards, and Docker-backed live model/backend/gateway harnesses selected by `release_profile`.<br />**Runs:** `run_release_soak=true`, `release_profile=full`, or focused `rerun_group=live-e2e`.<br />**Rerun:** `rerun_group=live-e2e`, optionally with `live_suite_filter`. |
scripts/ci-run-timings.d.mts
+1 -1
View File
@@ -14,7 +14,7 @@ export function summarizeRunTimings(
limit?: number,
): {
byDuration: Array<{ name: string; durationSeconds: number }>;
byQueue: Array<{ name: string; queueSeconds: number }>;
byStartDelay: Array<{ name: string; startDelaySeconds: number }>;
conclusion: unknown;
status: unknown;
wallSeconds: number | null;
scripts/ci-run-timings.mjs
+17 -9
View File
@@ -124,7 +124,9 @@ function collectRunTimingContext(run) {
conclusion: job.conclusion ?? "",
durationSeconds: secondsBetween(started, completed),
name: job.name,
queueSeconds: secondsBetween(created, started),
// Actions exposes job start time, but not the split between `needs`
// dependency wait and runner queue. Keep the combined delay honest.
startDelaySeconds: secondsBetween(created, started),
started,
completed,
status: job.status,
@@ -146,9 +148,9 @@ export function summarizeRunTimings(run, limit = 15) {
.filter((job) => job.durationSeconds !== null)
.toSorted((left, right) => right.durationSeconds - left.durationSeconds)
.slice(0, limit);
const byQueue = [...jobs]
.filter((job) => job.queueSeconds !== null && (job.durationSeconds ?? 0) > 5)
.toSorted((left, right) => right.queueSeconds - left.queueSeconds)
const byStartDelay = [...jobs]
.filter((job) => job.startDelaySeconds !== null && (job.durationSeconds ?? 0) > 5)
.toSorted((left, right) => right.startDelaySeconds - left.startDelaySeconds)
.slice(0, limit);
const badJobs = jobs.filter(
(job) => job.conclusion && !["success", "skipped", "cancelled"].includes(job.conclusion),
@@ -156,7 +158,7 @@ export function summarizeRunTimings(run, limit = 15) {
return {
byDuration,
byQueue,
byStartDelay,
conclusion: run.conclusion ?? "",
status: run.status ?? "",
wallSeconds: secondsBetween(created, updated),
@@ -347,7 +349,9 @@ function summarizeJobs(run) {
Number.isFinite(firstStart) && Number.isFinite(lastComplete)
? secondsBetween(firstStart, lastComplete)
: null,
firstQueueSeconds: Number.isFinite(firstStart) ? secondsBetween(created, firstStart) : null,
firstStartDelaySeconds: Number.isFinite(firstStart)
? secondsBetween(created, firstStart)
: null,
jobCount: successfulDurations.length,
maxDurationSeconds: successfulDurations.length === 0 ? null : Math.max(...successfulDurations),
p90DurationSeconds: percentile(successfulDurations, 0.9),
@@ -360,7 +364,7 @@ function printSection(title, jobs, metric) {
console.log(title);
for (const job of jobs) {
console.log(
`${String(job.name).padEnd(48)} ${formatSeconds(job[metric]).padStart(6)} queue=${formatSeconds(job.queueSeconds).padStart(6)} ${job.status}/${job.conclusion}`,
`${String(job.name).padEnd(48)} ${formatSeconds(job[metric]).padStart(6)} start-delay=${formatSeconds(job.startDelaySeconds).padStart(6)} ${job.status}/${job.conclusion}`,
);
}
}
@@ -447,7 +451,7 @@ async function main() {
run.headSha.slice(0, 10),
`wall=${formatSeconds(summary.wallSeconds)}`,
`exec=${formatSeconds(summary.executionWindowSeconds)}`,
`firstQueue=${formatSeconds(summary.firstQueueSeconds)}`,
`firstStartDelay=${formatSeconds(summary.firstStartDelaySeconds)}`,
`jobs=${summary.jobCount}`,
`avg=${formatSeconds(summary.avgDurationSeconds)}`,
`p90=${formatSeconds(summary.p90DurationSeconds)}`,
@@ -486,7 +490,11 @@ async function main() {
);
}
printSection("\nSlowest jobs", summary.byDuration, "durationSeconds");
printSection("\nLongest queues", summary.byQueue, "queueSeconds");
printSection(
"\nLongest start delays (dependencies + runner queue)",
summary.byStartDelay,
"startDelaySeconds",
);
if (summary.badJobs.length > 0) {
console.log("\nFailed jobs");
for (const job of summary.badJobs) {
scripts/lib/extension-test-plan.mjs
+1 -1
View File
@@ -53,7 +53,7 @@ const EXTENSION_TEST_COST_MULTIPLIERS = {
"test/vitest/vitest.extension-matrix.config.ts": 0.28,
"test/vitest/vitest.extension-mattermost.config.ts": 0.75,
"test/vitest/vitest.extension-media.config.ts": 0.7,
"test/vitest/vitest.extension-memory.config.ts": 0.25,
"test/vitest/vitest.extension-memory.config.ts": 1,
"test/vitest/vitest.extension-messaging.config.ts": 0.4,
"test/vitest/vitest.extension-misc.config.ts": 0.7,
"test/vitest/vitest.extension-msteams.config.ts": 0.5,
test/scripts/ci-run-timings.test.ts
+2 -2
View File
@@ -10,7 +10,7 @@ import {
} from "../../scripts/ci-run-timings.mjs";
describe("scripts/ci-run-timings.mjs", () => {
it("separates queue time from job duration", () => {
it("separates start delay from job duration without mislabeling dependency wait", () => {
const summary = summarizeRunTimings(
{
conclusion: "success",
@@ -49,7 +49,7 @@ describe("scripts/ci-run-timings.mjs", () => {
["slow", 60],
["queued", 10],
]);
expect(summary.byQueue.map((job) => [job.name, job.queueSeconds])).toEqual([
expect(summary.byStartDelay.map((job) => [job.name, job.startDelaySeconds])).toEqual([
["queued", 50],
["slow", 20],
]);
test/scripts/openclaw-cross-os-release-workflow.test.ts
+18 -9
View File
@@ -83,15 +83,24 @@ describe("cross-OS release checks workflow", () => {
const release = readWorkflow(RELEASE_CHECKS_PATH);
const producer = job(release, "prepare_release_package");
expect(producer.outputs).toMatchObject({
artifact_digest: "${{ steps.release_package_upload.outputs.artifact-digest }}",
artifact_id: "${{ steps.release_package_upload.outputs.artifact-id }}",
artifact_name: "${{ steps.artifact.outputs.name }}",
artifact_run_attempt: "${{ steps.artifact.outputs.run_attempt }}",
artifact_run_id: "${{ steps.artifact.outputs.run_id }}",
package_file_name: "${{ steps.artifact.outputs.file_name }}",
package_sha256: "${{ steps.package.outputs.sha256 }}",
package_version: "${{ steps.package.outputs.package_version }}",
source_sha: "${{ steps.package.outputs.source_sha }}",
artifact_digest:
"${{ steps.release_package_upload.outputs.artifact-digest || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactDigest }}",
artifact_id:
"${{ steps.release_package_upload.outputs.artifact-id || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactId }}",
artifact_name:
"${{ steps.artifact.outputs.name || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactName }}",
artifact_run_attempt:
"${{ steps.artifact.outputs.run_attempt || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunAttempt }}",
artifact_run_id:
"${{ steps.artifact.outputs.run_id || fromJSON(inputs.candidate_artifact_json || '{}').packageArtifactRunId }}",
package_file_name:
"${{ steps.artifact.outputs.file_name || fromJSON(inputs.candidate_artifact_json || '{}').packageFileName }}",
package_sha256:
"${{ steps.package.outputs.sha256 || fromJSON(inputs.candidate_artifact_json || '{}').packageSha256 }}",
package_version:
"${{ steps.package.outputs.package_version || fromJSON(inputs.candidate_artifact_json || '{}').packageVersion }}",
source_sha:
"${{ steps.package.outputs.source_sha || fromJSON(inputs.candidate_artifact_json || '{}').packageSourceSha }}",
});
expect(step(producer, "Checkout trusted workflow ref").with).toMatchObject({
ref: "${{ github.sha }}",
test/scripts/package-acceptance-workflow.test.ts
+63 -4
View File
@@ -35,6 +35,7 @@ const DOCKER_E2E_PLAN_ACTION = ".github/actions/docker-e2e-plan/action.yml";
const RELEASE_CHECKS_WORKFLOW = ".github/workflows/openclaw-release-checks.yml";
const RELEASE_TELEGRAM_QA_WORKFLOW = ".github/workflows/openclaw-release-telegram-qa.yml";
const RELEASE_PUBLISH_WORKFLOW = ".github/workflows/openclaw-release-publish.yml";
const PLUGIN_PRERELEASE_WORKFLOW = ".github/workflows/plugin-prerelease.yml";
const OPENCLAW_NPM_RELEASE_WORKFLOW = ".github/workflows/openclaw-npm-release.yml";
const PLUGIN_CLAWHUB_RELEASE_WORKFLOW = ".github/workflows/plugin-clawhub-release.yml";
const PLUGIN_NPM_RELEASE_WORKFLOW = ".github/workflows/plugin-npm-release.yml";
@@ -1561,6 +1562,7 @@ describe("package artifact reuse", () => {
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS:
"${{ inputs.published_upgrade_survivor_baselines }}",
OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS: "${{ inputs.published_upgrade_survivor_scenarios }}",
PREPARE_ONLY: "${{ inputs.prepare_only }}",
RELEASE_TEST_PROFILE: "${{ inputs.release_test_profile }}",
});
expect(workflow).toContain("plan_docker_lane_groups:");
@@ -1645,15 +1647,72 @@ describe("package artifact reuse", () => {
expect(dockerE2ePlanAction).not.toContain('docker pull "${OPENCLAW_DOCKER_E2E_');
});
it("uses Blacksmith Docker build caching for prepared E2E images", () => {
it("reuses a content-addressed bare image for prepared E2E images", () => {
const workflow = readFileSync(LIVE_E2E_WORKFLOW, "utf8");
expect(workflow).toContain("bare_context_sha=");
expect(workflow).toContain("-docker-e2e-bare:base-${bare_context_sha:0:32}");
expect(workflow).toContain('docker manifest inspect "$CACHE_IMAGE_REF"');
expect(workflow).toContain('cache=(--cache-from "$CACHE_IMAGE_REF")');
expect(workflow).not.toContain('docker tag "$CACHE_IMAGE_REF" "$IMAGE_REF"');
expect(workflow).toContain(
"Shared release candidate preparation requires both Docker image variants.",
);
expect(workflow).toContain(
"inputs.shared_image_artifact_id != '' && '1' || steps.plan.outputs.needs_bare_image",
);
expect(workflow).toContain("env DOCKER_BUILDKIT=1 docker build");
expect(workflow).toContain(
'if bash .release-harness/scripts/ci-docker-pull-retry.sh "$CACHE_IMAGE_REF"; then',
);
expect(workflow).toContain("Bare image cache pull failed; continuing with a cold build.");
expect(workflow).toContain("--build-context openclaw_package=.artifacts/docker-e2e-package");
expect(workflow).toContain('cache=(--cache-from "$BARE_IMAGE_REF")');
expect(workflow).not.toContain('docker push "$CACHE_IMAGE_REF"');
expect(workflow).toContain("uses: useblacksmith/setup-docker-builder@");
expect(workflow).toContain("uses: useblacksmith/build-push-action@");
expect(workflow).not.toContain("cache-from: type=gha,scope=docker-e2e");
expect(workflow).not.toContain("cache-to: type=gha,mode=max,scope=docker-e2e");
});
it("prepares one immutable candidate for release validation children", () => {
const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8");
const prepare = workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, "prepare_release_candidate");
const pluginDispatch = workflowStep(
workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, "plugin_prerelease"),
"Dispatch and monitor plugin prerelease",
);
const releaseDispatch = workflowStep(
workflowJob(FULL_RELEASE_VALIDATION_WORKFLOW, "release_checks"),
"Dispatch and monitor release checks",
);
expect(prepare.uses).toBe("./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml");
expect(prepare.with).toMatchObject({
prepare_only: true,
shared_image_policy: "no-push-artifact",
});
expect(pluginDispatch.run).toContain(
'args+=(-f candidate_artifact_json="$CANDIDATE_ARTIFACT_JSON")',
);
expect(releaseDispatch.run).toContain(
'args+=(-f candidate_artifact_json="$CANDIDATE_ARTIFACT_JSON")',
);
expect(workflow).toContain("Shared release candidate preparation ended with");
});
it("gives memory extension shards enough CPU without lowering their planner cost", () => {
const workflow = readFileSync(PLUGIN_PRERELEASE_WORKFLOW, "utf8");
expect(workflow).toContain('extensionId.startsWith("memory-")');
expect(workflow).toContain('"blacksmith-16vcpu-ubuntu-2404"');
expect(workflow).toContain("vitest_max_workers:");
expect(workflow).toContain("OPENCLAW_VITEST_MAX_WORKERS: ${{ matrix.vitest_max_workers }}");
expect(readFileSync("scripts/lib/extension-test-plan.mjs", "utf8")).toContain(
'"test/vitest/vitest.extension-memory.config.ts": 1',
);
});
it("shards broad native live tests instead of one serial live-all job", () => {
const workflow = readFileSync(LIVE_E2E_WORKFLOW, "utf8");
const retryHelper = readFileSync("scripts/ci-live-command-retry.sh", "utf8");
@@ -3462,21 +3521,21 @@ describe("package artifact reuse", () => {
expect(output).toContain("Tideclaw alpha");
});
it("summarizes queue time separately from execution time in full validation", () => {
it("summarizes start delay separately from execution time in full validation", () => {
const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8");
const parsedWorkflow = readWorkflow(FULL_RELEASE_VALIDATION_WORKFLOW);
const summaryJob = parsedWorkflow.jobs?.summary;
const manifestStep = workflowStep(summaryJob ?? {}, "Write release validation manifest");
expect(workflow).toContain("### Slowest jobs: ${label}");
expect(workflow).toContain("### Longest queues: ${label}");
expect(workflow).toContain("### Longest start delays: ${label}");
expect(workflow).toContain("Write release validation manifest");
expect(workflow).toContain("PERFORMANCE_RUN_ID: ${{ needs.performance.outputs.run_id }}");
expect(workflow).toContain("Upload release validation manifest");
expect(workflow).toContain("Failed child detail: ${label}");
expect(workflow).toContain("actions/runs/${run_id}/artifacts?per_page=100");
expect(workflow).toContain("full-release-validation-${{ github.run_id }}");
expect(workflow).toContain("| Job | Result | Queue minutes | Run minutes |");
expect(workflow).toContain("| Job | Result | Start delay minutes | Run minutes |");
expect(workflow).toContain(
'gh_with_retry api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100"',
);
test/scripts/plugin-prerelease-test-plan.test.ts
+8 -2
View File
@@ -435,7 +435,13 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => {
expect(normalCiScript).toContain('dispatch_and_wait ci.yml "$dispatch_run_name" "${args[@]}"');
expect(normalCiScript).not.toContain("full_release_validation=true");
expect(pluginPrereleaseScript).toContain(
'dispatch_and_wait plugin-prerelease.yml "$dispatch_run_name" -f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" -f full_release_validation=true -f dispatch_id="$dispatch_id"',
'args=(-f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" -f full_release_validation=true -f dispatch_id="$dispatch_id")',
);
expect(pluginPrereleaseScript).toContain(
'args+=(-f candidate_artifact_json="$CANDIDATE_ARTIFACT_JSON")',
);
expect(pluginPrereleaseScript).toContain(
'dispatch_and_wait plugin-prerelease.yml "$dispatch_run_name" "${args[@]}"',
);
expect(pluginManifestScript).toContain("await import(");
expect(pluginManifestScript).toContain('"./scripts/lib/plugin-prerelease-test-plan.mjs"');
@@ -546,7 +552,7 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => {
(step: WorkflowStep) => step.name === "Run plugin prerelease static shard",
).run,
).toContain('bash -c "$PLUGIN_PRERELEASE_COMMAND"');
expect(dockerSuite).toEqual({
expect(dockerSuite).toMatchObject({
if: "${{ inputs.full_release_validation && needs.preflight.outputs.run_plugin_prerelease_docker == 'true' }}",
name: "plugin-prerelease-docker-suite",
needs: ["preflight"],
test/scripts/release-no-push-workflow.test.ts
+22 -6
View File
@@ -583,9 +583,17 @@ describe("release validation no-push transport", () => {
expect(producer.outputs?.image_archive_sha256).toContain("archive_sha256");
expect(producer.outputs?.image_artifact_id).toContain("artifact-id");
expect(producer.outputs?.image_artifact_digest).toContain("artifact-digest");
expect(producer.outputs?.image_artifact_run_id).toBe("${{ github.run_id }}");
expect(producer.outputs?.image_artifact_run_attempt).toBe("${{ github.run_attempt }}");
}
expect(dockerProducer.outputs?.image_artifact_run_id).toContain("github.run_id");
expect(dockerProducer.outputs?.image_artifact_run_id).toContain(
"inputs.shared_image_artifact_run_id",
);
expect(dockerProducer.outputs?.image_artifact_run_attempt).toContain("github.run_attempt");
expect(dockerProducer.outputs?.image_artifact_run_attempt).toContain(
"inputs.shared_image_artifact_run_attempt",
);
expect(liveProducer.outputs?.image_artifact_run_id).toBe("${{ github.run_id }}");
expect(liveProducer.outputs?.image_artifact_run_attempt).toBe("${{ github.run_attempt }}");
expect(dockerProducer.outputs?.package_artifact_id).toContain("artifact-id");
expect(dockerProducer.outputs?.package_artifact_digest).toContain("artifact-digest");
expect(dockerProducer.outputs?.package_artifact_run_attempt).toContain("run_attempt");
@@ -631,13 +639,21 @@ describe("release validation no-push transport", () => {
]) {
const build = step(dockerProducer, name);
expect(build.if).toContain("shared_image_policy == 'no-push-artifact'");
expect(build.run).toContain("--load");
expect(build.run).toContain("--sbom=false");
expect(build.run).toContain("--provenance=false");
expect(build.run).not.toContain("--push");
expect(build.run).not.toContain("--sbom=true");
expect(build.run).not.toContain("--provenance=mode=max");
}
const bareBuild = step(dockerProducer, "Build bare Docker E2E image artifact");
expect(bareBuild.run).toContain("docker build");
expect(bareBuild.run).toContain("--target bare");
expect(bareBuild.run).toContain('--tag "$IMAGE_REF"');
const functionalBuild = step(dockerProducer, "Build functional Docker E2E image artifact");
expect(functionalBuild.run).toContain("docker build");
expect(functionalBuild.run).toContain("--target functional");
expect(functionalBuild.run).toContain(
"--build-context openclaw_package=.artifacts/docker-e2e-package",
);
expect(functionalBuild.run).toContain('--tag "$IMAGE_REF"');
const packDockerArtifact = step(dockerProducer, "Pack Docker E2E image artifact");
expect(packDockerArtifact.env?.PACKAGE_SHA256).toBe("${{ steps.package.outputs.sha256 }}");
expect(packDockerArtifact.run).toContain("shared-image-artifact.sh");
@@ -693,7 +709,7 @@ describe("release validation no-push transport", () => {
}
expect(step(dockerProducer, "Upload Docker E2E image artifact")).toMatchObject({
id: "upload_image_artifact",
if: "inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1'",
if: "inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1' && inputs.shared_image_artifact_id == ''",
with: { "if-no-files-found": "error" },
});
expect(step(liveProducer, "Pack live-test image artifact").run).toContain(
test/scripts/release-workflow-matrix-plan.test.ts
+7
View File
@@ -58,6 +58,7 @@ function requiredJob(definition: WorkflowDocument, name: string): WorkflowJob {
// Direct dispatches build from the selected ref. Only trusted workflow callers
// may provide the complete immutable package artifact tuple.
const WORKFLOW_CALL_ONLY_INPUTS = new Set([
"prepare_only",
"package_artifact_name",
"package_artifact_id",
"package_artifact_digest",
@@ -67,6 +68,12 @@ const WORKFLOW_CALL_ONLY_INPUTS = new Set([
"package_source_sha",
"package_sha256",
"package_version",
"shared_image_artifact_name",
"shared_image_artifact_id",
"shared_image_artifact_digest",
"shared_image_artifact_run_id",
"shared_image_artifact_run_attempt",
"shared_image_archive_sha256",
]);
const PROFILE_EXPECTATIONS = [
test/vitest-scoped-config.test.ts
+5
View File
@@ -866,6 +866,11 @@ describe("scoped vitest configs", () => {
it("normalizes memory extension include patterns relative to the scoped dir", () => {
const testConfig = requireTestConfig(defaultExtensionMemoryConfig);
expect(testConfig.dir).toBe(path.join(process.cwd(), "extensions"));
expect(normalizeConfigPaths(testConfig.setupFiles)).toEqual([
"test/setup.ts",
"test/setup.extensions.ts",
"test/setup-openclaw-runtime.ts",
]);
expect(testConfig.include).toEqual([
"memory-core/**/*.test.ts",
"memory-lancedb/**/*.test.ts",