diff --git a/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs b/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs index 02df54eaffb9..e79bf2d2b7a7 100644 --- a/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs +++ b/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs @@ -14,6 +14,7 @@ import { const repo = "openclaw/openclaw"; const githubSnapshotSchemaVersion = 1; +const githubSnapshotCheckpointInterval = 25; const commitAssociationQueryBatchSize = 20; const excludedHandles = new Set(["openclaw", "clawsweeper", "claude", "codex", "steipete"]); const nonEditorialTypes = new Set([ @@ -187,10 +188,10 @@ function parseArgs(argv) { return options; } -function run(command, args) { +function run(command, args, options = {}) { return execFileSync(command, args, { encoding: "utf8", - env: { ...process.env, NO_COLOR: "1" }, + env: { ...process.env, NO_COLOR: "1", ...options.env }, maxBuffer: 16 * 1024 * 1024, stdio: ["ignore", "pipe", "pipe"], }); @@ -240,7 +241,12 @@ function gitCommit(ref, required = false) { function fetchGithubApi(args) { try { - return JSON.parse(run("ghx", ["api", ...args]).replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, "")); + return JSON.parse( + run("ghx", ["api", ...args], { env: { GHX_NO_CACHE: "1" } }).replace( + /\u001B\[[0-?]*[ -/]*[@-~]/g, + "", + ), + ); } catch (error) { if (typeof error.stdout === "string" && error.stdout.trim() !== "") { return JSON.parse(error.stdout.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, "")); @@ -251,11 +257,15 @@ function fetchGithubApi(args) { export function createGithubSnapshotState({ base, + checkpointEvery = githubSnapshotCheckpointInterval, filePath, refresh = false, repository = repo, target, }) { + if (!Number.isSafeInteger(checkpointEvery) || checkpointEvery < 1) { + fail("GitHub snapshot checkpoint interval must be a positive integer"); + } let responses = {}; if (!refresh && existsSync(filePath)) { let parsed; @@ -285,6 +295,7 @@ export function createGithubSnapshotState({ } return { base, + checkpointEvery, dirty: refresh && existsSync(filePath), filePath, hits: 0, @@ -292,6 +303,7 @@ export function createGithubSnapshotState({ repository, responses, target, + writesSincePersist: 0, }; } @@ -318,6 +330,10 @@ export function githubApiWithSnapshot(args, fetchApi, snapshotState) { } snapshotState.responses[key] = structuredClone(response); snapshotState.dirty = true; + snapshotState.writesSincePersist += 1; + if (snapshotState.writesSincePersist >= snapshotState.checkpointEvery) { + persistGithubSnapshot(snapshotState); + } return response; } @@ -342,6 +358,7 @@ export function persistGithubSnapshot(snapshotState) { writeFileSync(tempPath, output); renameSync(tempPath, snapshotState.filePath); snapshotState.dirty = false; + snapshotState.writesSincePersist = 0; } finally { rmSync(tempPath, { force: true }); } @@ -351,16 +368,20 @@ function githubApi(args) { return githubApiWithSnapshot(args, fetchGithubApi, githubSnapshotState); } +export function defaultGithubSnapshotPath(base, target, gitCommonDir) { + const defaultName = `verify-release-notes-${base}-${target}.json`; + return path.resolve(gitCommonDir, "openclaw-release-cache", defaultName); +} + function initializeGithubSnapshot(options) { if (options.noGithubSnapshot) { return undefined; } const base = git(["rev-parse", `${options.base}^{commit}`]); const target = git(["rev-parse", `${options.target}^{commit}`]); - const defaultName = `verify-release-notes-${base}-${target}.json`; const filePath = path.resolve( options.githubSnapshotPath ?? - git(["rev-parse", "--git-path", `openclaw-release-cache/${defaultName}`]), + defaultGithubSnapshotPath(base, target, git(["rev-parse", "--git-common-dir"])), ); const state = createGithubSnapshotState({ base, diff --git a/.agents/skills/release-openclaw-ci/SKILL.md b/.agents/skills/release-openclaw-ci/SKILL.md index e2fe645422e3..5f93e8d5988d 100644 --- a/.agents/skills/release-openclaw-ci/SKILL.md +++ b/.agents/skills/release-openclaw-ci/SKILL.md @@ -39,6 +39,10 @@ Use this with `$release-openclaw-maintainer` and `$openclaw-testing` when a rele task-owned box and warm a fresh one before testing. Testbox source sync is relative to the warmed source tree; continuing can mix an old base file with a new candidate diff and produce false lockfile or Docker failures. +- Reused Testboxes are provenance-gated after their first successful run. + Source-only edits may reuse the lease; base, dependency, wrapper, or Testbox + workflow drift requires a fresh lease. Do not set + `OPENCLAW_TESTBOX_ALLOW_STALE=1` for release evidence. - For a committed release candidate, warm the box with `blacksmith testbox warmup ... --ref `. Do not rely on source sync to overlay committed branch changes onto the workflow's @@ -109,6 +113,12 @@ gh workflow run full-release-validation.yml \ -f rerun_group=all ``` +For immutable workflow proof on a moving `main`, use +`pnpm ci:full-release --sha `. Its canonical `release-ci/*` ref +keeps exact-target evidence reuse enabled after proving the workflow commit is +still on trusted `main` lineage. Pass `-f reuse_evidence=false` only when the +operator intentionally needs a fresh full run. + Use `release_profile=stable` unless the operator explicitly asks for the broad advisory provider/media matrix. Stable and full profiles force the release soak; the beta profile may opt in with `run_release_soak=true`. Use narrow `rerun_group` after focused fixes. Publish with `openclaw-release-publish.yml` using `release_profile=from-validation` unless a maintainer intentionally wants to cross-check a specific profile; the @@ -116,16 +126,16 @@ publish workflow reads the effective profile from the full-validation manifest. ## Watch -Use the summary helper instead of repeated raw polling: +Use the transition-only summary watcher instead of repeated raw polling: ```bash -node .agents/skills/release-openclaw-ci/scripts/release-ci-summary.mjs +node scripts/release-ci-summary.mjs --watch ``` -Then watch only when useful: +For a one-shot snapshot: ```bash -gh run watch --repo openclaw/openclaw --exit-status +node scripts/release-ci-summary.mjs ``` Stop watchers before ending the turn or switching strategy. diff --git a/.agents/skills/release-openclaw-ci/scripts/release-ci-summary.mjs b/.agents/skills/release-openclaw-ci/scripts/release-ci-summary.mjs deleted file mode 100755 index 79c2caebe1f0..000000000000 --- a/.agents/skills/release-openclaw-ci/scripts/release-ci-summary.mjs +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env node -/** - * Release CI summary helper that prints parent and child workflow status for a - * full release run. - */ -import { execFileSync } from "node:child_process"; -import process from "node:process"; -import { plainGhEnv, resolvePlainGhBin } from "../../../../scripts/lib/plain-gh.mjs"; - -const runId = process.argv[2]; -const repo = process.env.OPENCLAW_RELEASE_REPO || "openclaw/openclaw"; - -if (!runId) { - console.error("usage: release-ci-summary.mjs "); - process.exit(2); -} - -function gh(args) { - return execFileSync(resolvePlainGhBin(), args, { - encoding: "utf8", - env: plainGhEnv(), - stdio: ["ignore", "pipe", "pipe"], - }); -} - -function jsonGh(args) { - return JSON.parse(gh(args)); -} - -function githubRestJson(pathSuffix) { - const result = execFileSync( - "bash", - [ - "-lc", - [ - "set -euo pipefail", - 'token="$("$OPENCLAW_PLAIN_GH_BIN" auth token)"', - 'curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "${OPENCLAW_GITHUB_REST_URL}"', - ].join("\n"), - ], - { - encoding: "utf8", - env: { - ...plainGhEnv(), - OPENCLAW_PLAIN_GH_BIN: resolvePlainGhBin(), - OPENCLAW_GITHUB_REST_URL: `https://api.github.com/repos/${repo}/${pathSuffix}`, - }, - maxBuffer: 16 * 1024 * 1024, - stdio: ["ignore", "pipe", "pipe"], - }, - ); - return JSON.parse(result); -} - -function rate() { - try { - return jsonGh(["api", "rate_limit"]).resources.core; - } catch { - return undefined; - } -} - -const core = rate(); -if (core) { - const reset = new Date(core.reset * 1000).toISOString(); - console.log(`rate: remaining=${core.remaining}/${core.limit} reset=${reset}`); - if (core.remaining < 20) { - console.error("rate too low for CI summary; wait for reset before polling"); - process.exit(3); - } -} - -const parent = jsonGh([ - "run", - "view", - runId, - "--repo", - repo, - "--json", - "status,conclusion,createdAt,headSha,url,jobs", -]); - -console.log(`parent: ${runId} ${parent.status}/${parent.conclusion || "none"}`); -console.log(`sha: ${parent.headSha}`); -console.log(`url: ${parent.url}`); - -for (const job of parent.jobs ?? []) { - const marker = job.conclusion || job.status; - console.log(`parent-job: ${marker} ${job.name}`); -} - -const since = parent.createdAt; -const runsQuery = new URLSearchParams({ - per_page: "100", - created: `>=${since}`, - exclude_pull_requests: "true", -}); -const childWorkflowNames = new Set([ - "CI", - "OpenClaw Release Checks", - "Plugin Prerelease", - "NPM Telegram Beta E2E", - "Full Release Validation", -]); -const runs = githubRestJson(`actions/runs?${runsQuery.toString()}`).workflow_runs ?? []; -const runList = runs - .filter( - (run) => - run.created_at >= since && - run.head_sha === parent.headSha && - childWorkflowNames.has(run.name), - ) - .map((run) => - [run.id, run.name, run.status, run.conclusion ?? "", run.head_sha, run.html_url].join("\t"), - ) - .join("\n"); - -if (!runList) { - console.log("children: none found yet"); - process.exit(0); -} - -console.log("children:"); -for (const line of runList.split("\n")) { - const [id, name, status, conclusion, sha, url] = line.split("\t"); - console.log(`child: ${id} ${name} ${status}/${conclusion || "none"} sha=${sha}`); - console.log(`child-url: ${url}`); -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c43f6e24ac9..ead0c1a53a5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -968,6 +968,7 @@ jobs: export OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS=180000 export NODE_OPTIONS=--max-old-space-size=16384 node scripts/package-openclaw-for-docker.mjs \ + --allow-unreleased-changelog \ --output-dir "$package_dir" \ --output-name openclaw-current.tgz export OPENCLAW_CURRENT_PACKAGE_TGZ="$PWD/$package_dir/openclaw-current.tgz" diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index ad0628c0b53f..b23f6e7f3594 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -236,7 +236,7 @@ jobs: evidence_reuse: name: Check for reusable validation evidence needs: [resolve_target] - if: inputs.rerun_group == 'all' && inputs.reuse_evidence + if: inputs.rerun_group == 'all' && inputs.reuse_evidence && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release-ci/')) runs-on: ubuntu-24.04 timeout-minutes: 10 outputs: @@ -271,6 +271,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} TARGET_SHA: ${{ needs.resolve_target.outputs.sha }} + WORKFLOW_REF: ${{ github.ref_name }} RELEASE_PROFILE: ${{ inputs.release_profile }} RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'stable' || inputs.release_profile == 'full' }} PROVIDER: ${{ inputs.provider }} @@ -304,6 +305,7 @@ jobs: bash workflow/scripts/github/find-reusable-release-validation.sh \ --target-sha "$TARGET_SHA" \ --workflow-sha "$GITHUB_SHA" \ + --workflow-ref "$WORKFLOW_REF" \ --release-profile "$RELEASE_PROFILE" \ --run-release-soak "$RUN_RELEASE_SOAK" \ --inputs-json "$inputs_json" \ diff --git a/.github/workflows/openclaw-release-publish.yml b/.github/workflows/openclaw-release-publish.yml index 1d68ea8245d9..cabb99245e16 100644 --- a/.github/workflows/openclaw-release-publish.yml +++ b/.github/workflows/openclaw-release-publish.yml @@ -347,9 +347,13 @@ jobs: run: | set -euo pipefail tooling_dir="${RUNNER_TEMP}/release-validation-tooling" - mkdir -p "$tooling_dir" + mkdir -p "${tooling_dir}/lib" gh api "repos/${GITHUB_REPOSITORY}/contents/scripts/validate-full-release-validation-evidence.mjs?ref=${WORKFLOW_SHA}" \ --jq .content | base64 --decode > "${tooling_dir}/validate-full-release-validation-evidence.mjs" + gh api "repos/${GITHUB_REPOSITORY}/contents/scripts/release-ci-summary.mjs?ref=${WORKFLOW_SHA}" \ + --jq .content | base64 --decode > "${tooling_dir}/release-ci-summary.mjs" + gh api "repos/${GITHUB_REPOSITORY}/contents/scripts/lib/plain-gh.mjs?ref=${WORKFLOW_SHA}" \ + --jq .content | base64 --decode > "${tooling_dir}/lib/plain-gh.mjs" - name: Checkout release tag uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -419,6 +423,7 @@ jobs: RUN_JSON_FILE: ${{ runner.temp }}/full-release-validation-run.json TRUSTED_MAIN_REF: refs/remotes/origin/main VALIDATOR_FILE: ${{ runner.temp }}/release-validation-tooling/validate-full-release-validation-evidence.mjs + STRICT_VALIDATOR_FILE: ${{ runner.temp }}/release-validation-tooling/release-ci-summary.mjs run: | set -euo pipefail manifest="${RUNNER_TEMP}/full-release-validation-manifest/full-release-validation-manifest.json" diff --git a/docs/ci.md b/docs/ci.md index cfa46943fea5..5246467be315 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -278,9 +278,11 @@ pnpm ci:full-release --sha GitHub workflow dispatch refs must be branches or tags, not raw commit SHAs. The helper pushes a temporary `release-ci/-...` branch at a trusted `main` workflow SHA, passes the requested target SHA through the workflow `ref` input, -verifies every child workflow `headSha` matches the trusted workflow SHA, and -deletes the temporary branch when the run completes. The umbrella verifier also -fails if any child workflow ran at a different workflow SHA. +reuses strict exact-target evidence when available, verifies every child +workflow `headSha` matches the trusted workflow SHA, and deletes the temporary +branch when the run completes. Pass `-f reuse_evidence=false` to force fresh +validation. The umbrella verifier also fails if any child workflow ran at a +different workflow SHA. `release_profile` controls live/provider breadth passed into release checks. The manual release workflows default to `stable`; use `full` only when you diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index ff6ff949269c..a6243abf201a 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -168,7 +168,7 @@ This checklist is the public shape of the release flow. Private credentials, sig 6. Run `OpenClaw NPM Release` with `preflight_only=true`. Before a tag exists, a full 40-character release-branch SHA is allowed for validation-only preflight. The preflight generates dependency release evidence for the exact checked-out dependency graph and stores it in the npm preflight artifact. Save the successful `preflight_run_id`. 7. Kick off all pre-release tests with `Full Release Validation` for the release branch, tag, or full commit SHA. This is the one manual entrypoint for the four big release test boxes: Vitest, Docker, QA Lab, and Package. Save the `full_release_validation_run_id` and exact `full_release_validation_run_attempt`; both are required inputs for `OpenClaw NPM Release` and `OpenClaw Release Publish`. 8. If validation fails, fix on the release branch and rerun the smallest failed file, lane, workflow job, package profile, provider, or model allowlist that proves the fix. Rerun the full umbrella only when the changed surface makes prior evidence stale. -9. For a tagged beta candidate, run `pnpm release:candidate -- --tag vYYYY.M.PATCH-beta.N` from the matching `release/YYYY.M.PATCH` branch. For stable, also pass the required Windows source release: `pnpm release:candidate -- --tag vYYYY.M.PATCH --windows-node-tag vX.Y.Z`. Before it dispatches the full validation matrix, the helper deterministically renders the exact tag's GitHub release body and rejects a missing version heading, an over-limit body that cannot use the canonical compact form, or contribution-record base/target provenance that is not reachable from the tag. It also validates any explicit shipped-baseline exclusion metadata against the referenced cumulative tag records. It then runs the local generated-release checks, dispatches or verifies full release validation and npm preflight evidence, runs Parallels fresh/update proof against the exact prepared tarball plus Telegram package proof, records plugin npm and ClawHub plans, and prints the exact `OpenClaw Release Publish` command only after the evidence bundle is green. +9. For a tagged beta candidate, run `pnpm release:candidate -- --tag vYYYY.M.PATCH-beta.N` from the matching `release/YYYY.M.PATCH` branch. For stable, also pass the required Windows source release: `pnpm release:candidate -- --tag vYYYY.M.PATCH --windows-node-tag vX.Y.Z`. The helper checkpoints immutable candidate/tooling identity and dispatched run IDs in `.artifacts/release-candidate//release-candidate-state.json`; rerunning the same command resumes those exact runs, while any candidate, tooling, profile, or option drift fails closed. Before it dispatches the full validation matrix, the helper deterministically renders the exact tag's GitHub release body and rejects a missing version heading, an over-limit body that cannot use the canonical compact form, or contribution-record base/target provenance that is not reachable from the tag. It also validates any explicit shipped-baseline exclusion metadata against the referenced cumulative tag records. It then runs the local generated-release checks, dispatches or verifies full release validation and npm preflight evidence, runs Parallels fresh/update proof against the exact prepared tarball plus Telegram package proof, records plugin npm and ClawHub plans, and prints the exact `OpenClaw Release Publish` command only after the evidence bundle is green. `OpenClaw Release Publish` dispatches the selected or all-publishable plugin packages to npm and the same set to ClawHub in parallel, then promotes the prepared OpenClaw npm preflight artifact with the matching dist-tag once plugin npm publish succeeds. Before any publish child starts, it renders and caches the exact GitHub release body. When the complete matching `CHANGELOG.md` section fits GitHub's 125,000-character limit and the renderer's matching 125,000-byte safety ceiling, the page contains that exact `## YYYY.M.PATCH` section including its heading. When the source section does not fit, the page keeps the exact grouped editorial notes and replaces the oversized contribution record with a stable link to the full record in the tag-pinned `CHANGELOG.md`; partial records and truncated bullets are never published. The workflow chooses that full or compact body before adding `### Release verification`; if the proof tail would exceed the limit, it keeps the canonical body and relies on the immutable attached evidence instead. Stable releases published to npm `latest` become the GitHub latest release, while stable maintenance releases kept on npm `beta` are created with GitHub `latest=false`. The workflow also uploads the preflight dependency evidence, the full-validation manifest, and postpublish registry verification evidence to the GitHub release for post-release incident response. It prints child run IDs immediately, auto-approves release environment gates the workflow token is allowed to approve, summarizes failed child jobs with log tails, closes out the GitHub release and dependency evidence as soon as OpenClaw npm publish succeeds, waits for ClawHub whenever OpenClaw npm is being published, then runs `pnpm release:verify-beta` and uploads postpublish evidence for the GitHub release, npm package, selected plugin npm packages, selected ClawHub packages, child workflow run IDs, and optional NPM Telegram run ID. The ClawHub path retries transient CLI dependency install failures, publishes preview-passing plugins even when one preview cell flakes, and ends with registry verification for every expected plugin version so partial publishes stay visible and retryable. @@ -280,7 +280,7 @@ A legacy fallback correction tag may reuse base-package evidence only when the c pnpm ci:full-release --sha ``` -The helper fetches current `origin/main`, pushes `release-ci/-...` at that trusted workflow commit, dispatches `Full Release Validation` from the temporary branch with `ref=` and `reuse_evidence=false`, verifies every child workflow `headSha` matches the pinned parent workflow SHA, then deletes the temporary branch. Pass `--workflow-sha ` to pin an older commit that is still reachable from current `origin/main`. The workflow itself never writes repository refs. This keeps main-only release tooling available without adding tooling commits to the candidate and avoids proving a newer `main` child run by accident. +The helper fetches current `origin/main`, pushes `release-ci/-...` at that trusted workflow commit, dispatches `Full Release Validation` from the temporary branch with `ref=`, reuses strict exact-target evidence when available, verifies every child workflow `headSha` matches the pinned parent workflow SHA, then deletes the temporary branch. Pass `-f reuse_evidence=false` to force a fresh run or `--workflow-sha ` to pin an older commit that is still reachable from current `origin/main`. The workflow itself never writes repository refs. This keeps main-only release tooling available without adding tooling commits to the candidate and avoids proving a newer `main` child run by accident. For release branch or tag validation, run it from the trusted `main` workflow ref and pass the release branch or tag as `ref`: diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index b9ae790a0424..efebe44f56f0 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -24,9 +24,23 @@ gh workflow run full-release-validation.yml \ ``` `provider` also accepts `anthropic` or `minimax` for cross-OS onboarding and the -end-to-end agent turn. Child workflows use the trusted workflow ref for the -harness and the input `ref` for the candidate under test, so new validation -logic stays available when validating an older release branch or tag. +end-to-end agent turn. Reusable child jobs resolve the called workflow harness +from `job.workflow_repository` and `job.workflow_sha`, while the input `ref` +selects the candidate under test. This keeps current trusted validation logic +available when validating an older release branch or tag. + +Every dispatched child must report the same workflow SHA as the parent +`Full Release Validation` run. If `main` moves between the parent and child +dispatches, the umbrella fails closed even when the child itself succeeds. For +an immutable exact-commit proof, use +`pnpm ci:full-release --sha `. The helper creates a temporary +`release-ci/*` ref pinned to current trusted `origin/main`, passes the target +SHA only as the candidate `ref`, reuses strict exact-target evidence when +available, and deletes the ref after validation. Pass +`-f reuse_evidence=false` to force a fresh run or +`--workflow-sha ` to select an older workflow commit still +reachable from current `origin/main`. The workflow never creates or updates +repository refs itself. `release_profile=stable` and `release_profile=full` always run the exhaustive live/Docker soak. Pass `run_release_soak=true` to include the same soak lanes @@ -49,12 +63,16 @@ that plugin, then runs Codex CLI preflight and same-session OpenAI agent turns. ## Top-level stages For `rerun_group=all`, a `Check for reusable validation evidence` job runs -first: it looks for the newest prior green full validation whose target differs -from the current target only by release metadata paths (changelog, version -stamps; see `RELEASE_METADATA_PATHS` in `scripts/changed-lanes.mjs`). When such -evidence exists, every lane is skipped and the umbrella verifier re-checks the -evidence run instead, so changelog-only commits do not re-drive hours of -validation. Pass `reuse_evidence=false` to force a fresh full run. +first: it looks for the newest prior green full validation for the exact same +target SHA, release profile, effective soak setting, and validation inputs. +When such evidence exists, every lane is skipped and the umbrella verifier +re-checks the immutable parent artifact, child runs, and dispatch logs. This is +same-candidate rerun recovery only; it does not authorize cross-SHA reuse. For +a changed candidate, rerun every package, artifact, install, Docker, or provider +gate affected by that delta. Pass `reuse_evidence=false` to force a fresh full +run. Evidence reuse runs only from `main` or a canonical SHA-pinned +`release-ci/*` ref whose workflow commit remains on trusted `main` lineage; +other workflow refs run the selected lanes fresh. Also for `rerun_group=all`, a `Verify Docker runtime image assets` job builds the `runtime-assets` Docker target with diff --git a/docs/reference/test.md b/docs/reference/test.md index 6e671433871a..b44abcda821f 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -24,6 +24,14 @@ stop it before handoff: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --keep --timing-json ``` +After the first successful reuse, the wrapper records the lease's base, +dependency, and Testbox workflow fingerprint under `.crabbox/testbox-leases/`. +Source-only edits keep reusing the warmed box. A changed merge base, lockfile, +package-manager input, wrapper, or Testbox workflow fails closed and requires a +fresh lease. Every run still syncs the current checkout. +`OPENCLAW_TESTBOX_ALLOW_STALE=1` is only for intentional diagnostics, not +release proof. + Local test commands below are for human workflows or an explicit agent fallback requested by the user. Remote-provider unavailability must be reported; it is not permission to silently run a broad local gate. diff --git a/scripts/crabbox-wrapper.mjs b/scripts/crabbox-wrapper.mjs index d40ff1ca30cf..ae0d81482f7b 100755 --- a/scripts/crabbox-wrapper.mjs +++ b/scripts/crabbox-wrapper.mjs @@ -24,6 +24,10 @@ import { isProviderAdvertised, parseProvidersFromHelp, } from "./crabbox-wrapper-providers.mjs"; +import { + prepareTestboxLeaseFreshness, + recordTestboxLeaseFreshness, +} from "./testbox-lease-freshness.mjs"; import { resolvePathEnvKey, resolveWindowsCmdExePath } from "./windows-cmd-helpers.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -365,7 +369,11 @@ function buildBatchCommandLine(command, commandArgs) { return `"${[escapedCommand, ...escapedArgs].join(" ")}"`; } -function checkedOutput(command, commandArgs, timeoutMs = resolveMetadataProbeTimeoutMs(process.env)) { +function checkedOutput( + command, + commandArgs, + timeoutMs = resolveMetadataProbeTimeoutMs(process.env), +) { const invocation = spawnInvocation(command, commandArgs, process.env, process.platform); const result = spawnSync(invocation.command, invocation.args, { cwd: repoRoot, @@ -451,10 +459,12 @@ function satisfiesMinimumCrabboxVersion(version, minimum) { function gitOutput(commandArgs) { const gitBinary = resolvePathBinary("git", process.env, process.platform) ?? "git"; - const invocation = spawnInvocation(gitBinary, commandArgs, process.env, process.platform); + const gitEnv = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null" }; + const invocation = spawnInvocation(gitBinary, commandArgs, gitEnv, process.platform); const result = spawnSync(invocation.command, invocation.args, { cwd: repoRoot, encoding: "utf8", + env: gitEnv, stdio: ["ignore", "pipe", "pipe"], windowsVerbatimArguments: invocation.windowsVerbatimArguments, }); @@ -2929,7 +2939,8 @@ function isSparseCheckout() { } function isWorktreeClean() { - return gitOutput(["status", "--porcelain=v1"]).stdout === ""; + const status = gitOutput(["status", "--porcelain=v1"]); + return status.status === 0 && status.stdout === ""; } function shouldUseFullCheckoutForCleanRemoteSync(commandArgs, _providerName) { @@ -3173,6 +3184,23 @@ function injectFullCheckoutLeaseReclaim(commandArgs) { return normalizedArgs; } +function injectRemoteTestboxCi(commandArgs, providerName) { + if (commandArgs[0] !== "run" || canonicalProviderName(providerName) !== "blacksmith-testbox") { + return commandArgs; + } + const normalizedArgs = [...commandArgs]; + const { start } = runCommandBounds(normalizedArgs); + if (start < 0) { + return normalizedArgs; + } + if (hasOption(normalizedArgs, "--shell")) { + normalizedArgs[start] = `export CI=true; ${normalizedArgs[start]}`; + } else { + normalizedArgs.splice(start, 0, "env", "CI=true"); + } + return normalizedArgs; +} + const version = probeCrabboxMetadata(binary, ["--version"]); const help = probeCrabboxMetadata(binary, ["run", "--help"]); const providers = parseProvidersFromHelp(help.text); @@ -3253,6 +3281,19 @@ if (canonicalProvider === "blacksmith-testbox") { enforceCrabboxOwnedBlacksmithLease(normalizedArgs); } +let testboxLeaseFreshness; +try { + testboxLeaseFreshness = prepareTestboxLeaseFreshness({ + args: normalizedArgs, + env: { ...process.env, CI: process.env.CI || "true" }, + provider: canonicalProvider, + repoRoot, + }); +} catch (error) { + console.error(`[crabbox] ${error instanceof Error ? error.message : String(error)}`); + process.exit(2); +} + let childCwd = repoRoot; let cleanupChildCwd = () => {}; let fullCheckout = null; @@ -3329,6 +3370,9 @@ if (normalizedArgs[0] === "run" && isBrokeredWsl2RemoteTarget(normalizedArgs, pr } const childEnv = { ...process.env }; +if (canonicalProvider === "blacksmith-testbox" && !childEnv.CI) { + childEnv.CI = "true"; +} if ( isLocalContainerProvider(provider) && !childEnv.CRABBOX_LOCAL_CONTAINER_DOCKER_SOCKET && @@ -3364,7 +3408,7 @@ try { cleanupOnce(); throw error; } -const childArgs = +const childArgs = injectRemoteTestboxCi( childCwd === repoRoot ? injectRemoteWindowsHydratedNodeModulesBootstrap( injectRemoteAwsMacosSwiftBootstrap( @@ -3384,7 +3428,9 @@ const childArgs = provider, ), remoteChangedGateBase, - ); + ), + provider, +); let fullCheckoutKeepaliveIntervalMsValue = 0; if (fullCheckout) { try { @@ -3437,16 +3483,27 @@ child.on("exit", (code, signal) => { if (childTreeShutdownStarted) { return; } + let exitCode = code; let fullCheckoutAvailable = true; if (fullCheckout) { fullCheckoutAvailable = assertFullCheckoutAvailableBeforeExit(fullCheckout.dir); } + if (!signal && code === 0) { + try { + recordTestboxLeaseFreshness(testboxLeaseFreshness); + } catch (error) { + console.error( + `[crabbox] failed to record Testbox lease freshness: ${error instanceof Error ? error.message : String(error)}`, + ); + exitCode = 2; + } + } cleanupOnce(); if (signal) { process.exit(signalExitCodes.get(signal) ?? 1); return; } - process.exit(fullCheckoutAvailable ? (code ?? 1) : 1); + process.exit(fullCheckoutAvailable ? (exitCode ?? 1) : 1); }); child.on("error", (error) => { diff --git a/scripts/full-release-validation-at-sha.mjs b/scripts/full-release-validation-at-sha.mjs index ec100f3099f0..be199a5c30ab 100755 --- a/scripts/full-release-validation-at-sha.mjs +++ b/scripts/full-release-validation-at-sha.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node // Dispatches full release validation against a temporary SHA-pinned branch. import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { pathToFileURL } from "node:url"; const WORKFLOW = "full-release-validation.yml"; @@ -9,7 +12,7 @@ const DEFAULT_INPUTS = { mode: "both", release_profile: "full", rerun_group: "all", - reuse_evidence: "false", + reuse_evidence: "true", }; function usage() { @@ -18,8 +21,9 @@ function usage() { Creates a temporary remote branch pinned to trusted main release tooling, dispatches Full Release Validation with the target commit as its ref input, watches the parent run, verifies all child workflow head SHAs match the trusted -workflow SHA, then deletes the temporary branch by default. Exact-target -evidence reuse is disabled because it is trusted only from main.`); +workflow lineage through the release evidence manifest, then deletes the +temporary branch by default. Exact-target evidence reuse stays enabled; pass +-f reuse_evidence=false to force a fresh run.`); } function run(command, args, options = {}) { @@ -119,8 +123,8 @@ export function parseArgs(argv) { throw new Error(`Unknown argument: ${arg}`); } - if (args.inputs.reuse_evidence !== "false") { - throw new Error("SHA-pinned release validation always disables evidence reuse"); + if (!["true", "false"].includes(args.inputs.reuse_evidence)) { + throw new Error("reuse_evidence must be true or false"); } if (Object.hasOwn(args.inputs, "ref")) { throw new Error("SHA-pinned release validation reserves the ref input for --sha"); @@ -177,47 +181,53 @@ function findLatestRunId(branch, sha) { return match?.databaseId ? String(match.databaseId) : ""; } -function childRunIds(parentRunId) { - const jobsJson = run("gh", ["run", "view", parentRunId, "--json", "jobs"]); - const jobs = JSON.parse(jobsJson).jobs ?? []; - const summaryJob = jobs.find((job) => job.name === "Verify full validation"); - if (!summaryJob?.databaseId) { - return []; +export function releaseEvidenceVerificationArgs(parentRunId) { + if (!/^[1-9][0-9]*$/u.test(String(parentRunId))) { + throw new Error("parent run ID must be a positive decimal"); } - const log = run("gh", [ - "run", - "view", - parentRunId, - "--job", - String(summaryJob.databaseId), - "--log", - ]); - return [...new Set([...log.matchAll(/actions\/runs\/(\d+)/g)].map((match) => match[1]))]; + return ["--validate-run", String(parentRunId), "--trusted-workflow-ref", "main", "--json"]; } -function verifyChildHeads(parentRunId, workflowSha) { - const ids = childRunIds(parentRunId); - if (ids.length === 0) { - throw new Error( - `Could not find child workflow run ids in parent verifier logs for ${parentRunId}.`, - ); +export function releaseEvidenceVerifierPath(worktreeRoot) { + const candidates = [ + join(worktreeRoot, "scripts", "release-ci-summary.mjs"), + join( + worktreeRoot, + ".agents", + "skills", + "release-openclaw-ci", + "scripts", + "release-ci-summary.mjs", + ), + ]; + const verifier = candidates.find((candidate) => existsSync(candidate)); + if (!verifier) { + throw new Error("trusted workflow checkout does not contain a release evidence verifier"); } + return verifier; +} - let failed = false; - for (const id of ids) { - const json = run("gh", ["run", "view", id, "--json", "name,status,conclusion,headSha,url"]); - const child = JSON.parse(json); - const ok = - child.headSha === workflowSha && - child.status === "completed" && - child.conclusion === "success"; - console.log( - `${ok ? "ok" : "bad"} ${child.name} ${child.status}/${child.conclusion} ${child.headSha} ${child.url}`, +function verifyReleaseEvidence(parentRunId, workflowSha) { + const verifierWorktree = mkdtempSync(join(tmpdir(), "openclaw-release-verifier-")); + try { + run("git", ["worktree", "add", "--detach", verifierWorktree, workflowSha], { + stdio: ["ignore", "ignore", "inherit"], + }); + const verifier = releaseEvidenceVerifierPath(verifierWorktree); + const evidence = JSON.parse( + run(process.execPath, [verifier, ...releaseEvidenceVerificationArgs(parentRunId)]), ); - failed ||= !ok; - } - if (failed) { - throw new Error(`One or more child workflows failed or did not run at ${workflowSha}.`); + if (evidence.valid !== true) { + throw new Error(`Full Release Validation evidence is invalid for run ${parentRunId}.`); + } + console.log( + `ok release evidence current=${evidence.current.runId} root=${evidence.root.runId} reused=${Boolean(evidence.evidenceReuse)}`, + ); + } finally { + runStatus("git", ["worktree", "remove", "--force", verifierWorktree], { + stdio: ["ignore", "ignore", "ignore"], + }); + rmSync(verifierWorktree, { force: true, recursive: true }); } } @@ -280,7 +290,7 @@ function main() { `Full Release Validation failed: https://github.com/openclaw/openclaw/actions/runs/${parentRunId}`, ); } - verifyChildHeads(parentRunId, workflowSha); + verifyReleaseEvidence(parentRunId, workflowSha); } finally { if (!args.keepBranch) { run("git", ["push", "origin", `:${remoteBranchRef}`], { diff --git a/scripts/github/find-reusable-release-validation.sh b/scripts/github/find-reusable-release-validation.sh index 3c10ccee2521..f7ffeb747e82 100755 --- a/scripts/github/find-reusable-release-validation.sh +++ b/scripts/github/find-reusable-release-validation.sh @@ -1,15 +1,16 @@ #!/usr/bin/env bash set -euo pipefail -# Finds a prior green Full Release Validation run whose evidence still covers -# the target SHA: same rerun scope, equal-or-broader release profile/soak, and -# a target delta that is release-metadata-only per check-release-metadata-only. +# Finds a prior green Full Release Validation run for the exact target SHA. +# Cross-SHA evidence reuse is intentionally left to the granular delta manifest, +# which can require fresh package/install/provider closure per changed artifact. # Always exits 0 with reuse=true/false; callers fail open to a full validation. REPO="${GH_REPO:-}" WORKFLOW_FILE="full-release-validation.yml" TARGET_SHA="" -WORKFLOW_SHA="" +VERIFIER_WORKFLOW_SHA="" +WORKFLOW_REF="" RELEASE_PROFILE="" RUN_RELEASE_SOAK="false" INPUTS_JSON="" @@ -17,22 +18,24 @@ REPO_DIR="." MAX_CANDIDATES=12 GITHUB_OUTPUT_FILE="${GITHUB_OUTPUT:-}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CLASSIFIER="${SCRIPT_DIR}/../check-release-metadata-only.mjs" PREFLIGHT="${SCRIPT_DIR}/../release-preflight.mjs" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +VALIDATOR="${OPENCLAW_RELEASE_CI_SUMMARY_VALIDATOR:-${REPO_ROOT}/scripts/release-ci-summary.mjs}" usage() { cat >&2 <<'EOF' Usage: find-reusable-release-validation.sh --target-sha --workflow-sha \ + --workflow-ref \ --release-profile --inputs-json \ [--run-release-soak ] [--repo ] [--repo-dir ] \ [--workflow ] [--max-candidates ] [--github-output ] -Scans recent successful Full Release Validation runs for a validation manifest -whose targetSha differs from --target-sha only by release metadata paths, whose -recorded lane-selection inputs match --inputs-json exactly, whose harness -(.github/workflows tree at the run's head SHA) matches --workflow-sha, and -whose recorded child runs are still green. Writes reuse=true plus evidence_* -outputs when found; reuse=false otherwise. +Scans recent successful Full Release Validation runs for an exact-target +validation manifest whose recorded lane-selection inputs match --inputs-json +and whose normalized strict-v3 evidence is accepted by the current trusted-main +verifier identified by --workflow-sha. The historical producer workflow SHA +remains independent. Writes reuse=true plus evidence_* outputs when found; +reuse=false otherwise. EOF } @@ -43,7 +46,11 @@ while [[ $# -gt 0 ]]; do shift 2 ;; --workflow-sha) - WORKFLOW_SHA="${2:-}" + VERIFIER_WORKFLOW_SHA="${2:-}" + shift 2 + ;; + --workflow-ref) + WORKFLOW_REF="${2:-}" shift 2 ;; --release-profile) @@ -107,74 +114,64 @@ no_reuse() { exit 0 } -profile_rank() { - case "$1" in - beta) echo 1 ;; - stable) echo 2 ;; - full) echo 3 ;; - *) echo 0 ;; - esac -} - if [[ ! "$TARGET_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "Expected --target-sha to be a full lowercase commit SHA; got: ${TARGET_SHA}" >&2 exit 2 fi -if [[ ! "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then - echo "Expected --workflow-sha to be a full lowercase commit SHA; got: ${WORKFLOW_SHA}" >&2 +if [[ ! "$VERIFIER_WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "Expected --workflow-sha to be a full lowercase commit SHA; got: ${VERIFIER_WORKFLOW_SHA}" >&2 exit 2 fi +if [[ "$WORKFLOW_REF" != "main" ]]; then + expected_release_ref="release-ci/${VERIFIER_WORKFLOW_SHA:0:12}-" + if [[ ! "$WORKFLOW_REF" =~ ^release-ci/[0-9a-f]{12}-[1-9][0-9]*$ ]] || + [[ "$WORKFLOW_REF" != "$expected_release_ref"* ]]; then + no_reuse "workflow ref is not a canonical SHA-pinned release ref" + fi +fi if [[ -z "$REPO" ]]; then echo "Expected --repo or GH_REPO." >&2 exit 2 fi -current_rank="$(profile_rank "$RELEASE_PROFILE")" -if [[ "$current_rank" == "0" ]]; then - no_reuse "unknown release profile ${RELEASE_PROFILE}" +if [[ "$RUN_RELEASE_SOAK" != "true" && "$RUN_RELEASE_SOAK" != "false" ]]; then + echo "Expected --run-release-soak to be true or false; got: ${RUN_RELEASE_SOAK}" >&2 + exit 2 fi +case "$RELEASE_PROFILE" in + beta|stable|full) ;; + *) no_reuse "unknown release profile ${RELEASE_PROFILE}" ;; +esac expected_inputs="" -if ! expected_inputs="$(jq -Sc . <<< "$INPUTS_JSON" 2>/dev/null)" || [[ -z "$expected_inputs" ]]; then +if ! expected_inputs="$(jq -Sc 'if type == "object" then . else error("expected object") end' <<< "$INPUTS_JSON" 2>/dev/null)" || [[ -z "$expected_inputs" ]]; then echo "Expected --inputs-json to be a JSON object of lane-selection inputs." >&2 exit 2 fi -# A metadata-only diff can still leave the target's version stamps mutually -# inconsistent (for example package.json bumped without the macOS plist); -# validate the target state before trusting any prior evidence. +workflow_lineage="" +if ! workflow_lineage="$( + gh api "repos/${REPO}/compare/${VERIFIER_WORKFLOW_SHA}...main" +)"; then + no_reuse "could not verify workflow SHA against trusted main" +fi +if ! jq -e \ + --arg workflow_sha "$VERIFIER_WORKFLOW_SHA" ' + (.status == "ahead" or .status == "identical") + and .merge_base_commit.sha == $workflow_sha + ' <<< "$workflow_lineage" >/dev/null; then + no_reuse "workflow SHA is not on trusted main lineage" +fi + +# Exact-target reuse still requires internally consistent version stamps +# (for example package.json must agree with the macOS plist). if ! (cd "$REPO_DIR" && node "$PREFLIGHT" --macos-versions-only >&2); then no_reuse "target version metadata is inconsistent" fi -# Evidence must come from an equivalent harness: workflows and their helper -# scripts run from the workflow ref, so the tree diff between the candidate -# run's head SHA and the current workflow SHA must itself be metadata-only. -harness_matches() { - local candidate_sha="$1" - if [[ "$candidate_sha" == "$WORKFLOW_SHA" ]]; then - return 0 - fi - if ! git -C "$REPO_DIR" fetch --quiet --depth=1 origin "$candidate_sha" "$WORKFLOW_SHA"; then - return 1 - fi - local harness_paths - if ! harness_paths="$(git -C "$REPO_DIR" diff --name-only "$candidate_sha" "$WORKFLOW_SHA")"; then - return 1 - fi - if [[ -z "$harness_paths" ]]; then - return 0 - fi - local -a harness_path_list=() - while IFS= read -r harness_path; do - [[ -n "$harness_path" ]] && harness_path_list+=("$harness_path") - done <<< "$harness_paths" - (cd "$REPO_DIR" && node "$CLASSIFIER" --base "$candidate_sha" --head "$WORKFLOW_SHA" -- "${harness_path_list[@]}") -} - runs_json="" if ! runs_json="$( gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs" \ -F status=success -F event=workflow_dispatch -F per_page="$MAX_CANDIDATES" \ - --jq '[.workflow_runs[] | {id, html_url, head_sha}]' + --jq '[.workflow_runs[] | {id}]' )"; then no_reuse "could not list prior successful validation runs" fi @@ -184,157 +181,127 @@ if [[ "$run_count" == "0" ]]; then no_reuse "no prior successful validation runs" fi -work_dir="$(mktemp -d)" -trap 'rm -rf "$work_dir"' EXIT - for ((index = 0; index < run_count; index += 1)); do run_id="$(jq -r ".[${index}].id" <<< "$runs_json")" - run_url="$(jq -r ".[${index}].html_url" <<< "$runs_json")" - run_head_sha="$(jq -r ".[${index}].head_sha // \"\"" <<< "$runs_json")" - - if [[ ! "$run_head_sha" =~ ^[0-9a-f]{40}$ ]] || ! harness_matches "$run_head_sha"; then - echo "[evidence-reuse] run ${run_id}: harness differs from the current workflow ref beyond release metadata; skipping" >&2 - continue - fi - - artifact_id="" - if ! artifact_id="$( - gh api "repos/${REPO}/actions/runs/${run_id}/artifacts?per_page=100" \ - --jq "first(.artifacts[] | select(.name == \"full-release-validation-${run_id}\" and .expired == false) | .id)" + validation_record="" + if ! validation_record="$( + node "$VALIDATOR" \ + --validate-run "$run_id" \ + --repo "$REPO" \ + --trusted-workflow-ref main \ + --json )"; then - echo "[evidence-reuse] run ${run_id}: artifact listing failed; skipping" >&2 + echo "[evidence-reuse] run ${run_id}: shared evidence validator rejected the run; skipping" >&2 continue fi - if [[ -z "${artifact_id// }" ]]; then - echo "[evidence-reuse] run ${run_id}: no validation manifest artifact; skipping" >&2 + if ! jq -e \ + --arg repo "$REPO" \ + --arg run_id "$run_id" \ + --arg verifier_sha "$VERIFIER_WORKFLOW_SHA" ' + . as $record + | .schema == "openclaw.release-validation-evidence/v3" + and .valid == true + and .repository == $repo + and .producerOnTrustedMainLineage == true + and .trustedWorkflowRef == "main" + and .trustedWorkflowFullRef == "refs/heads/main" + and .directRoot == true + and .evidenceReuse == null + and .rerunGroup == "all" + and .controls.performanceReportPublication == "artifact-only" + and .conclusions.current == "success" + and .conclusions.root == "success" + and .conclusions.allRequiredSucceeded == true + and (.current == .root) + and (.root.runId | tostring) == $run_id + and (.root.workflowSha | type == "string" and test("^[0-9a-f]{40}$")) + and (.root.targetSha | type == "string" and test("^[0-9a-f]{40}$")) + and (.root.artifact.digest | type == "string" and test("^sha256:[0-9a-f]{64}$")) + and all($record.current, $record.root; + . as $parent + | .producerOnTrustedMainLineage == true + and .workflowRefType == "branch" + and .workflowPath == ".github/workflows/full-release-validation.yml" + and .workflowFullRef == ("refs/heads/" + .workflowRef) + and .workflowQualifiedPath == + (".github/workflows/full-release-validation.yml@" + .workflowFullRef) + and ( + .workflowRunPath == ".github/workflows/full-release-validation.yml" + or .workflowRunPath == .workflowQualifiedPath + ) + and ( + ( + .workflowRef == "main" + and ( + (.manifestVersion == 3 and .workflowRefProof == "manifest-v3-branch") + or ( + .manifestVersion == 2 + and .workflowRefProof == "legacy-v2-main-ancestry" + ) + ) + ) + or ( + .manifestVersion == 3 + and .workflowRefProof == "manifest-v3-sha-pinned-main-ancestry" + and (.workflowRef | test("^release-ci/[0-9a-f]{12}-[1-9][0-9]*$")) + and (.workflowRef | startswith("release-ci/\($parent.workflowSha[0:12])-")) + ) + ) + ) + and (.verifier.schemaVersion == 3) + and (.verifier.sourceSha == $verifier_sha) + and ([.children[].role] | sort) == + ["normalCi", "pluginPrerelease", "productPerformance", "releaseChecks"] + and ([.children[].runId] | length == (unique | length)) + and ([.children[] + | select(.role == "productPerformance") + | .reportPublication] == ["artifact-only"]) + and all(.children[]; + .status == "completed" + and .conclusion == "success" + and .workflowSha == $record.root.workflowSha + and (.sourceParentRunId | tostring) == $run_id + ) + ' <<< "$validation_record" >/dev/null 2>&1; then + echo "[evidence-reuse] run ${run_id}: normalized evidence is not a strict direct-root full validation; skipping" >&2 continue fi - manifest_zip="${work_dir}/manifest-${run_id}.zip" - manifest_path="${work_dir}/manifest-${run_id}.json" - if ! gh api "repos/${REPO}/actions/artifacts/${artifact_id}/zip" > "$manifest_zip"; then - echo "[evidence-reuse] run ${run_id}: manifest download failed; skipping" >&2 - continue - fi - if ! unzip -p "$manifest_zip" full-release-validation-manifest.json > "$manifest_path" 2>/dev/null; then - echo "[evidence-reuse] run ${run_id}: manifest missing from artifact; skipping" >&2 - continue - fi - - if ! jq -e ' - (.version >= 2) - and (.rerunGroup == "all") - and ((.targetSha // "") | test("^[0-9a-f]{40}$")) - ' "$manifest_path" >/dev/null 2>&1; then - echo "[evidence-reuse] run ${run_id}: manifest is not a full-scope v2 manifest; skipping" >&2 - continue - fi - - prior_profile="$(jq -r '.releaseProfile // ""' "$manifest_path")" - prior_rank="$(profile_rank "$prior_profile")" - if (( prior_rank < current_rank )); then - echo "[evidence-reuse] run ${run_id}: profile ${prior_profile} does not cover ${RELEASE_PROFILE}; skipping" >&2 + prior_profile="$(jq -r '.releaseProfile // ""' <<< "$validation_record")" + if [[ "$prior_profile" != "$RELEASE_PROFILE" ]]; then + echo "[evidence-reuse] run ${run_id}: profile ${prior_profile} differs from ${RELEASE_PROFILE}; skipping" >&2 continue fi # Lane selection (provider, mode, filters, package specs) changes what the # prior run proved; only exact-match manifests are reusable. Manifests # written before validationInputs existed never match. - manifest_inputs="$(jq -Sc '.validationInputs // empty' "$manifest_path")" + manifest_inputs="$(jq -Sc '.validationInputs // empty' <<< "$validation_record")" if [[ -z "$manifest_inputs" || "$manifest_inputs" != "$expected_inputs" ]]; then echo "[evidence-reuse] run ${run_id}: validation inputs differ from the current request; skipping" >&2 continue fi - prior_soak="$(jq -r '.runReleaseSoak // "false"' "$manifest_path")" - if [[ "$RUN_RELEASE_SOAK" == "true" && "$prior_soak" != "true" ]]; then - echo "[evidence-reuse] run ${run_id}: no soak evidence; skipping" >&2 + prior_soak="$(jq -r '.runReleaseSoak // false' <<< "$validation_record")" + if [[ "$prior_soak" != "$RUN_RELEASE_SOAK" ]]; then + echo "[evidence-reuse] run ${run_id}: soak ${prior_soak} differs from ${RUN_RELEASE_SOAK}; skipping" >&2 continue fi - prior_sha="$(jq -r '.targetSha' "$manifest_path")" - # Track count/joined separately: empty-array expansion under `set -u` breaks - # on the bash 3.2 that macOS ships. - changed_paths=() - changed_path_count=0 - changed_paths_joined="" + prior_sha="$(jq -r '.root.targetSha' <<< "$validation_record")" if [[ "$prior_sha" != "$TARGET_SHA" ]]; then - compare_json="" - if ! compare_json="$( - gh api "repos/${REPO}/compare/${prior_sha}...${TARGET_SHA}" \ - --jq '{status, file_count: ((.files // []) | length), files: [(.files // [])[].filename]}' - )"; then - echo "[evidence-reuse] run ${run_id}: compare ${prior_sha}...${TARGET_SHA} failed; skipping" >&2 - continue - fi - compare_status="$(jq -r '.status' <<< "$compare_json")" - if [[ "$compare_status" != "ahead" ]]; then - echo "[evidence-reuse] run ${run_id}: target is ${compare_status} of prior evidence, not ahead; skipping" >&2 - continue - fi - file_count="$(jq -r '.file_count' <<< "$compare_json")" - # The compare API truncates at 300 files; a truncated list cannot prove a - # metadata-only delta, so fall back to full validation. - if (( file_count >= 300 )); then - echo "[evidence-reuse] run ${run_id}: delta too large to classify (${file_count} files); skipping" >&2 - continue - fi - while IFS= read -r changed_path; do - if [[ -n "$changed_path" ]]; then - changed_paths+=("$changed_path") - changed_path_count=$((changed_path_count + 1)) - changed_paths_joined="${changed_paths_joined:+${changed_paths_joined} }${changed_path}" - fi - done < <(jq -r '.files[]' <<< "$compare_json") - if (( changed_path_count == 0 )); then - echo "[evidence-reuse] run ${run_id}: delta has no file changes" >&2 - else - if ! git -C "$REPO_DIR" fetch --quiet --depth=1 origin "$prior_sha"; then - echo "[evidence-reuse] run ${run_id}: could not fetch prior SHA ${prior_sha}; skipping" >&2 - continue - fi - if ! (cd "$REPO_DIR" && node "$CLASSIFIER" --base "$prior_sha" --head "$TARGET_SHA" -- "${changed_paths[@]}"); then - echo "[evidence-reuse] run ${run_id}: delta is not release-metadata-only; skipping" >&2 - continue - fi - fi - fi - - # Recorded child runs can be re-run to failure after the parent stays green; - # reuse only evidence whose children are still completed/success, matching - # the recheck the normal summary performs on its own children. - children_healthy=1 - while IFS= read -r child_run_id; do - [[ -n "$child_run_id" ]] || continue - if ! child_state="$(gh api "repos/${REPO}/actions/runs/${child_run_id}" --jq '(.status // "") + "/" + (.conclusion // "")')"; then - echo "[evidence-reuse] run ${run_id}: could not verify child run ${child_run_id}; skipping" >&2 - children_healthy=0 - break - fi - if [[ "$child_state" != "completed/success" ]]; then - echo "[evidence-reuse] run ${run_id}: child run ${child_run_id} is ${child_state}; skipping" >&2 - children_healthy=0 - break - fi - done < <(jq -r '[.childRuns.normalCi // "", .childRuns.pluginPrerelease // "", .childRuns.releaseChecks // "", .childRuns.npmTelegram // "", (.childRuns.productPerformance.runId // "")] | map(select(. != "")) | .[]' "$manifest_path") - if [[ "$children_healthy" != "1" ]]; then + echo "[evidence-reuse] run ${run_id}: target ${prior_sha} differs from ${TARGET_SHA}; cross-SHA reuse requires granular artifact evidence" >&2 continue fi - # A reused run may itself be a reuse manifest; evidenceReuse.runId points at - # the chain root that actually executed the lanes. - evidence_root_run_id="$(jq -r '.evidenceReuse.runId // empty' "$manifest_path")" - if [[ -z "${evidence_root_run_id// }" ]]; then - evidence_root_run_id="$run_id" - fi - - echo "[evidence-reuse] reusing run ${run_id} (${run_url}) for ${TARGET_SHA}: prior sha ${prior_sha}, ${changed_path_count} metadata-only changed files" >&2 + run_url="$(jq -r '.root.url' <<< "$validation_record")" + echo "[evidence-reuse] reusing exact-target run ${run_id} (${run_url}) for ${TARGET_SHA}" >&2 write_output reuse true write_output evidence_run_id "$run_id" - write_output evidence_root_run_id "$evidence_root_run_id" + write_output evidence_root_run_id "$run_id" write_output evidence_run_url "$run_url" write_output evidence_sha "$prior_sha" - write_output changed_path_count "$changed_path_count" - write_output changed_paths "$changed_paths_joined" - write_output evidence_manifest "$(jq -c . "$manifest_path")" + write_output changed_path_count "0" + write_output changed_paths "[]" + write_output evidence_manifest "$(jq -c '.manifest' <<< "$validation_record")" exit 0 done diff --git a/scripts/package-changelog.mjs b/scripts/package-changelog.mjs index 5865e2b4789f..74f48f126f74 100644 --- a/scripts/package-changelog.mjs +++ b/scripts/package-changelog.mjs @@ -22,7 +22,7 @@ const PRERELEASE_VERSION_PATTERN = /** * Resolves acceptable changelog headings for a package version. */ -export function resolvePackageChangelogVersions(packageVersion) { +export function resolvePackageChangelogVersions(packageVersion, options = {}) { const match = RELEASE_VERSION_PATTERN.exec(packageVersion); if (!match) { throw new Error( @@ -32,7 +32,7 @@ export function resolvePackageChangelogVersions(packageVersion) { if (PRERELEASE_VERSION_PATTERN.test(packageVersion)) { return [packageVersion, match[1], UNRELEASED_HEADING]; } - return [packageVersion]; + return options.allowUnreleased ? [packageVersion, UNRELEASED_HEADING] : [packageVersion]; } function splitLines(content) { @@ -61,8 +61,8 @@ function extractPreamble(lines, firstHeadingIndex) { /** * Extracts the current release changelog section for package publishing. */ -export function extractCurrentPackageChangelog(content, packageVersion) { - const targetVersions = resolvePackageChangelogVersions(packageVersion); +export function extractCurrentPackageChangelog(content, packageVersion, options = {}) { + const targetVersions = resolvePackageChangelogVersions(packageVersion, options); const lines = splitLines(content); const headings = findLevelTwoHeadings(lines); const heading = targetVersions @@ -125,11 +125,17 @@ export async function restorePackageChangelog(cwd = process.cwd()) { try { expectedPackaged = extractCurrentPackageChangelog(backup, packageVersion); } catch (error) { - const message = error instanceof Error ? error.message : String(error); - throw new Error( - `Refusing to restore stale packaged changelog backup from ${BACKUP_PATH}: ${message}`, - { cause: error }, - ); + try { + expectedPackaged = extractCurrentPackageChangelog(backup, packageVersion, { + allowUnreleased: true, + }); + } catch { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Refusing to restore stale packaged changelog backup from ${BACKUP_PATH}: ${message}`, + { cause: error }, + ); + } } if (current !== expectedPackaged) { throw new Error( @@ -145,13 +151,13 @@ export async function restorePackageChangelog(cwd = process.cwd()) { /** * Writes packaged changelog content while preserving a restorable backup. */ -export async function preparePackageChangelog(cwd = process.cwd()) { +export async function preparePackageChangelog(cwd = process.cwd(), options = {}) { await restorePackageChangelog(cwd); const changelogPath = path.join(cwd, CHANGELOG_PATH); const backupPath = path.join(cwd, BACKUP_PATH); const original = await readFile(changelogPath, "utf8"); const packageVersion = await readPackageVersion(cwd); - const packaged = extractCurrentPackageChangelog(original, packageVersion); + const packaged = extractCurrentPackageChangelog(original, packageVersion, options); if (packaged === original) { return false; } diff --git a/scripts/package-openclaw-for-docker.mjs b/scripts/package-openclaw-for-docker.mjs index 471a83763f21..b7c1d5181d67 100644 --- a/scripts/package-openclaw-for-docker.mjs +++ b/scripts/package-openclaw-for-docker.mjs @@ -131,6 +131,7 @@ function resolvePackedOpenClawFileName(value) { export function parseArgs(argv) { const options = { + allowUnreleasedChangelog: false, outputDir: "", outputName: "", packJson: "", @@ -147,7 +148,9 @@ export function parseArgs(argv) { }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; - if (arg === "--output-dir") { + if (arg === "--allow-unreleased-changelog") { + setOnce(arg, "allowUnreleasedChangelog", true); + } else if (arg === "--output-dir") { setOnce("--output-dir", "outputDir", readOptionValue(argv, index, arg)); index += 1; } else if (arg?.startsWith("--output-dir=")) { @@ -635,7 +638,12 @@ export async function prepareBundledAiRuntimePackage( export async function packOpenClawPackageForDocker(sourceDir, outputDir, options = {}) { const runCaptureImpl = options.runCaptureImpl ?? runCapture; - const prepareChangelog = options.prepareChangelog ?? preparePackageChangelog; + const prepareChangelog = + options.prepareChangelog ?? + ((cwd) => + preparePackageChangelog(cwd, { + allowUnreleased: options.allowUnreleasedChangelog, + })); const restoreChangelog = options.restoreChangelog ?? restorePackageChangelog; const prepareBundledAiRuntime = options.prepareBundledAiRuntime ?? prepareBundledAiRuntimePackage; console.error("==> Packing OpenClaw package"); @@ -718,6 +726,7 @@ async function main() { ); const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, { + allowUnreleasedChangelog: options.allowUnreleasedChangelog, outputName: options.outputName, packJsonPath: options.packJson, }); diff --git a/scripts/release-candidate-checklist.mjs b/scripts/release-candidate-checklist.mjs index 41b235d42412..94743d3a8825 100644 --- a/scripts/release-candidate-checklist.mjs +++ b/scripts/release-candidate-checklist.mjs @@ -2,12 +2,23 @@ // Coordinates release-candidate validation runs and emits the publish command // only after required local, CI, npm, plugin, and E2E evidence is green. import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { isDeepStrictEqual } from "node:util"; import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mjs"; import { readBoundedResponseText } from "./lib/bounded-response.mjs"; import { + dedicatedSectionVersionForTag, extractChangelogReleaseSections, extractChangelogSection, formatShippedBaselineExclusions, @@ -18,6 +29,7 @@ import { } from "./render-github-release-notes.mjs"; import { isShaPinnedReleaseValidationBranch, + runStrictReleaseEvidenceValidation, validateFullReleaseValidationEvidence, } from "./validate-full-release-validation-evidence.mjs"; @@ -30,7 +42,9 @@ const DEFAULT_TELEGRAM_PROVIDER_MODE = "mock-openai"; const DEFAULT_GITHUB_API_TIMEOUT_MS = 30_000; const DEFAULT_GITHUB_API_RESPONSE_BODY_MAX_BYTES = 16 * 1024 * 1024; const COMMAND_CAPTURE_MAX_BUFFER_BYTES = 16 * 1024 * 1024; -const FULL_RELEASE_WORKFLOW_PATH = ".github/workflows/full-release-validation.yml"; +const TOOLING_ROOT = fileURLToPath(new URL("../", import.meta.url)); +const TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN = + /^tideclaw\/alpha\/[0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}Z$/u; const WINDOWS_NODE_TAG_PATTERN = /^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z]+([.-][0-9A-Za-z]+)*)?$/u; const WINDOWS_NODE_REPO = "openclaw/openclaw-windows-node"; const WINDOWS_NODE_REQUIRED_ASSETS = [ @@ -38,26 +52,25 @@ const WINDOWS_NODE_REQUIRED_ASSETS = [ "OpenClawCompanion-Setup-arm64.exe", ]; const SHA256_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u; - -export function supportsImmutableFullValidationEvidence(workflowSource) { - const v3ManifestPaths = workflowSource.match(/\bversion: 3,/gu)?.length ?? 0; - return ( - v3ManifestPaths >= 2 && - workflowSource.includes( - "full-release-validation-${{ github.run_id }}-${{ github.run_attempt }}", - ) - ); -} - -function requireImmutableFullValidationProducer() { - const workflowSource = readFileSync(FULL_RELEASE_WORKFLOW_PATH, "utf8"); - if (supportsImmutableFullValidationEvidence(workflowSource)) { - return; - } - throw new Error( - "Automatic Full Release Validation dispatch requires a v3 attempt-qualified producer. Pass --full-release-run from the trusted current-main scripts/full-release-validation-at-sha.mjs helper.", - ); -} +const RELEASE_CANDIDATE_STATE_VERSION = 1; +const RELEASE_CANDIDATE_STATE_FILE = "release-candidate-state.json"; +const RELEASE_CANDIDATE_STATE_KEYS = [ + "repo", + "tag", + "targetSha", + "toolingSha", + "workflowRef", + "provider", + "mode", + "releaseProfile", + "npmDistTag", + "pluginPublishScope", + "plugins", + "windowsNodeTag", + "skipParallels", + "skipTelegram", + "telegramProviderMode", +]; function usage() { return `Usage: pnpm release:candidate -- --tag vYYYY.M.PATCH-beta.N [options] @@ -68,7 +81,7 @@ OpenClaw Release Publish command only after everything is green. Options: --tag Release tag to validate. - --workflow-ref Workflow branch/ref. Default: current branch. + --workflow-ref Trusted workflow ref. Default: main; matching Tideclaw branch required for alpha. --repo GitHub repo. Default: ${DEFAULT_REPO} --full-release-run Reuse successful Full Release Validation run. --npm-preflight-run Reuse successful OpenClaw NPM Release preflight run. @@ -200,6 +213,18 @@ export function parseArgs(argv) { if (!options.tag) { throw new Error("--tag is required"); } + if (options.tag.includes("-alpha.")) { + if (!TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN.test(options.workflowRef)) { + throw new Error( + "--workflow-ref must be the matching tideclaw/alpha/YYYY-MM-DD-HHMMZ branch for alpha release candidates", + ); + } + } else { + options.workflowRef ||= "main"; + } + if (!options.tag.includes("-alpha.") && options.workflowRef !== "main") { + throw new Error("--workflow-ref must be main for regular beta and stable release candidates"); + } options.releaseProfile ||= options.tag.includes("-alpha.") || options.tag.includes("-beta.") ? "beta" : "stable"; if (!["beta", "stable", "full"].includes(options.releaseProfile)) { @@ -261,6 +286,74 @@ function readJson(path, label) { } } +export function buildReleaseCandidateState(options, { targetSha, toolingSha }) { + return { + version: RELEASE_CANDIDATE_STATE_VERSION, + phase: "validated", + repo: options.repo, + tag: options.tag, + targetSha, + toolingSha, + workflowRef: options.workflowRef, + provider: options.provider, + mode: options.mode, + releaseProfile: options.releaseProfile, + npmDistTag: options.npmDistTag, + pluginPublishScope: options.pluginPublishScope, + plugins: options.plugins, + windowsNodeTag: options.windowsNodeTag, + skipParallels: options.skipParallels, + skipTelegram: options.skipTelegram, + telegramProviderMode: options.telegramProviderMode, + fullReleaseRunId: options.fullReleaseRunId, + npmPreflightRunId: options.npmPreflightRunId, + }; +} + +export function reconcileReleaseCandidateState(saved, expected) { + if (!saved) { + return expected; + } + if ( + typeof saved !== "object" || + Array.isArray(saved) || + saved.version !== RELEASE_CANDIDATE_STATE_VERSION + ) { + throw new Error("release candidate state has an unsupported schema"); + } + for (const key of RELEASE_CANDIDATE_STATE_KEYS) { + if (!isDeepStrictEqual(saved[key], expected[key])) { + throw new Error( + `release candidate state mismatch for ${key}: saved=${JSON.stringify(saved[key])} current=${JSON.stringify(expected[key])}`, + ); + } + } + for (const key of ["fullReleaseRunId", "npmPreflightRunId"]) { + if (saved[key] && expected[key] && saved[key] !== expected[key]) { + throw new Error(`release candidate state mismatch for ${key}`); + } + } + return { + ...expected, + phase: typeof saved.phase === "string" ? saved.phase : expected.phase, + fullReleaseRunId: expected.fullReleaseRunId || saved.fullReleaseRunId || "", + npmPreflightRunId: expected.npmPreflightRunId || saved.npmPreflightRunId || "", + }; +} + +function writeReleaseCandidateState(path, state) { + mkdirSync(join(path, ".."), { recursive: true }); + const temporaryPath = `${path}.tmp-${process.pid}`; + writeFileSync(temporaryPath, `${JSON.stringify(state, null, 2)}\n`); + renameSync(temporaryPath, path); +} + +function updateReleaseCandidateState(path, state, phase, runIds = {}) { + const next = { ...state, ...runIds, phase }; + writeReleaseCandidateState(path, next); + return next; +} + function githubApiTimeoutMs() { const raw = process.env.OPENCLAW_RELEASE_CANDIDATE_GITHUB_API_TIMEOUT_MS; if (!raw) { @@ -368,24 +461,100 @@ export async function validateWindowsSourceRelease(tag, options = {}) { }; } -function currentBranch() { - return run("git", ["branch", "--show-current"], { capture: true }).trim(); +function gitRevParse(ref, cwd) { + return run("git", ["rev-parse", ref], { capture: true, cwd }).trim(); } -function gitRevParse(ref) { - return run("git", ["rev-parse", ref], { capture: true }).trim(); +function gitTopLevel(cwd) { + return run("git", ["rev-parse", "--show-toplevel"], { capture: true, cwd }).trim(); } -export function validateCandidateCheckout({ targetSha, headSha, trackedStatus }) { - if (headSha !== targetSha) { - throw new Error(`release candidate tag resolves to ${targetSha}, but HEAD is ${headSha}`); +function gitTrackedStatus(cwd) { + return run("git", ["status", "--porcelain=v1", "--untracked-files=no"], { + capture: true, + cwd, + }); +} + +function fetchTrustedWorkflowSha(workflowRef, toolingRoot) { + const remoteRef = `refs/remotes/origin/${workflowRef}`; + run("git", ["fetch", "--no-tags", "origin", `+refs/heads/${workflowRef}:${remoteRef}`], { + cwd: toolingRoot, + }); + return gitRevParse(`${remoteRef}^{commit}`, toolingRoot); +} + +function runFromTrustedTooling(argv, { targetRoot, workflowRef }) { + const trustedToolingSha = fetchTrustedWorkflowSha(workflowRef, targetRoot); + const tempRoot = mkdtempSync(join(tmpdir(), "openclaw-release-tooling-")); + const toolingRoot = join(tempRoot, "checkout"); + let worktreeAdded = false; + try { + run("git", ["worktree", "add", "--detach", toolingRoot, trustedToolingSha], { + cwd: targetRoot, + }); + worktreeAdded = true; + const result = spawnSync( + process.execPath, + [join(toolingRoot, "scripts/release-candidate-checklist.mjs"), ...argv], + { + cwd: targetRoot, + env: process.env, + stdio: "inherit", + }, + ); + if (result.status !== 0) { + throw new Error( + `trusted release candidate tooling failed with ${result.status ?? result.signal}`, + ); + } + } finally { + if (worktreeAdded) { + const cleanup = spawnSync("git", ["worktree", "remove", "--force", toolingRoot], { + cwd: targetRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + if (cleanup.status !== 0) { + console.warn( + `could not remove temporary trusted tooling worktree: ${cleanup.stderr?.trim() || cleanup.signal || cleanup.status}`, + ); + } + } + rmSync(tempRoot, { force: true, recursive: true }); } - if (trackedStatus.trim()) { +} + +export function validateCandidateCheckout({ + targetSha, + targetHeadSha, + targetTrackedStatus, + toolingSha, + trustedToolingSha, + toolingTrackedStatus, + workflowRef, +}) { + if (targetHeadSha !== targetSha) { throw new Error( - "release candidate validation requires a clean tracked worktree so the checked tooling matches the tag", + `release candidate tag resolves to ${targetSha}, but target worktree HEAD is ${targetHeadSha}`, ); } - return { status: "passed", targetSha }; + if (targetTrackedStatus.trim()) { + throw new Error( + "release candidate validation requires a clean tracked target worktree at the release tag", + ); + } + if (toolingSha !== trustedToolingSha) { + throw new Error( + `release candidate tooling HEAD ${toolingSha} does not match trusted ${workflowRef} ${trustedToolingSha}`, + ); + } + if (toolingTrackedStatus.trim()) { + throw new Error( + "release candidate validation requires a clean tracked tooling checkout at the trusted workflow ref", + ); + } + return { status: "passed", targetSha, toolingSha, workflowRef }; } function gitIsAncestor(ancestor, target) { @@ -513,16 +682,31 @@ export function validateCandidateChangelogProvenance({ isAncestor = gitIsAncestor, loadShippedBaseline = loadCandidateShippedBaseline, }) { + // Validate the same section the renderer publishes: alpha and correction + // tags may carry their own heading, and alpha tags may fall back to + // Unreleased. let section; + let sectionVersion = version; let usesAlphaUnreleasedFallback = false; - try { - section = extractChangelogSection(changelog, version); - } catch (error) { - if (!/-alpha\.[1-9][0-9]*$/u.test(tag)) { - throw error; + const dedicatedVersion = dedicatedSectionVersionForTag(tag); + if (dedicatedVersion && dedicatedVersion !== version) { + try { + section = extractChangelogSection(changelog, dedicatedVersion); + sectionVersion = dedicatedVersion; + } catch { + // No dedicated section; validate the base section. + } + } + if (section === undefined) { + try { + section = extractChangelogSection(changelog, version); + } catch (error) { + if (!/-alpha\.[1-9][0-9]*$/u.test(tag)) { + throw error; + } + section = releaseNotesSectionForTag(changelog, version, tag); + usesAlphaUnreleasedFallback = true; } - section = releaseNotesSectionForTag(changelog, version, tag); - usesAlphaUnreleasedFallback = true; } const recordStart = section.search(/\n### Complete contribution record\r?$/m); if (recordStart < 0) { @@ -533,12 +717,14 @@ export function validateCandidateChangelogProvenance({ shippedBaselines: [], }; } - throw new Error(`CHANGELOG.md ## ${version} is missing ### Complete contribution record`); + throw new Error( + `CHANGELOG.md ## ${sectionVersion} is missing ### Complete contribution record`, + ); } const record = section.slice(recordStart); const recordedPullRequests = candidateContributionRecordPullRequests( section, - `CHANGELOG.md ## ${version}`, + `CHANGELOG.md ## ${sectionVersion}`, ); const provenance = record.match( /^This audited record covers the complete (?\S+)\.\.(?[0-9a-f]{40}) history:/mu, @@ -547,7 +733,7 @@ export function validateCandidateChangelogProvenance({ const recordedTarget = provenance?.groups?.target; if (!base || !recordedTarget) { throw new Error( - `CHANGELOG.md ## ${version} is missing exact complete contribution record provenance`, + `CHANGELOG.md ## ${sectionVersion} is missing exact complete contribution record provenance`, ); } const shippedBaselines = parseShippedBaselineExclusions(record); @@ -860,6 +1046,12 @@ function shellQuote(value) { * Builds the final release publish workflow command once validation evidence is ready. */ export function buildPublishCommand(options) { + const workflowRef = options.tag.includes("-alpha.") ? options.workflowRef : "main"; + if (options.tag.includes("-alpha.") && !TIDECLAW_ALPHA_WORKFLOW_REF_PATTERN.test(workflowRef)) { + throw new Error( + "alpha release publish requires a matching tideclaw/alpha/YYYY-MM-DD-HHMMZ workflow ref", + ); + } const fields = [ ["tag", options.tag], ["preflight_run_id", options.npmPreflightRunId], @@ -891,7 +1083,7 @@ export function buildPublishCommand(options) { "--repo", options.repo, "--ref", - options.workflowRef, + workflowRef, ...fields.flatMap(([key, value]) => ["-f", `${key}=${value}`]), ] .map(shellQuote) @@ -1033,16 +1225,37 @@ async function runTelegramIfNeeded(options, artifactName) { async function main() { const options = parseArgs(process.argv.slice(2)); - options.workflowRef ||= currentBranch(); + const targetRoot = gitTopLevel(process.cwd()); + const toolingRoot = gitTopLevel(TOOLING_ROOT); + if (targetRoot === toolingRoot) { + runFromTrustedTooling(process.argv.slice(2), { + targetRoot, + workflowRef: options.workflowRef, + }); + return; + } options.outputDir ||= join(".artifacts", "release-candidate", options.tag); - const targetSha = gitRevParse(`${options.tag}^{}`); + const targetSha = gitRevParse(`${options.tag}^{}`, targetRoot); + const toolingSha = gitRevParse("HEAD", TOOLING_ROOT); + const trustedToolingSha = fetchTrustedWorkflowSha(options.workflowRef, TOOLING_ROOT); validateCandidateCheckout({ targetSha, - headSha: gitRevParse("HEAD"), - trackedStatus: run("git", ["status", "--porcelain=v1", "--untracked-files=no"], { - capture: true, - }), + targetHeadSha: gitRevParse("HEAD", targetRoot), + targetTrackedStatus: gitTrackedStatus(targetRoot), + toolingSha, + trustedToolingSha, + toolingTrackedStatus: gitTrackedStatus(TOOLING_ROOT), + workflowRef: options.workflowRef, }); + const statePath = join(options.outputDir, RELEASE_CANDIDATE_STATE_FILE); + const expectedState = buildReleaseCandidateState(options, { targetSha, toolingSha }); + let candidateState = reconcileReleaseCandidateState( + existsSync(statePath) ? readJson(statePath, "release candidate state") : undefined, + expectedState, + ); + options.fullReleaseRunId = candidateState.fullReleaseRunId; + options.npmPreflightRunId = candidateState.npmPreflightRunId; + writeReleaseCandidateState(statePath, candidateState); const releaseChangelog = run("git", ["show", `${targetSha}:CHANGELOG.md`], { capture: true }); const releaseNotesVersion = releaseNotesVersionForTag(options.tag); const releaseNotesCheck = validateCandidateReleaseNotes({ @@ -1069,9 +1282,6 @@ async function main() { const localGeneratedCheck = runLocalGeneratedCheckIfNeeded(options); if (!options.fullReleaseRunId && !options.skipDispatch) { - // Older release branches cannot produce the immutable evidence consumed below. - // Fail before an expensive run; their candidate must use the trusted-main helper. - requireImmutableFullValidationProducer(); const workflowFile = "full-release-validation.yml"; options.fullReleaseRunId = dispatchWorkflow(options.repo, workflowFile, options.workflowRef, { ref: options.tag, @@ -1081,7 +1291,9 @@ async function main() { run_release_soak: options.releaseProfile === "stable" || options.releaseProfile === "full" ? "true" : "false", rerun_group: "all", - reuse_evidence: "false", + }); + candidateState = updateReleaseCandidateState(statePath, candidateState, "dispatching", { + fullReleaseRunId: options.fullReleaseRunId, }); } @@ -1092,7 +1304,14 @@ async function main() { preflight_only: "true", npm_dist_tag: options.npmDistTag, }); + candidateState = updateReleaseCandidateState(statePath, candidateState, "dispatching", { + npmPreflightRunId: options.npmPreflightRunId, + }); } + candidateState = updateReleaseCandidateState(statePath, candidateState, "waiting", { + fullReleaseRunId: options.fullReleaseRunId, + npmPreflightRunId: options.npmPreflightRunId, + }); const fullRun = await waitForSuccessfulRun(options.repo, options.fullReleaseRunId, { workflowName: "Full Release Validation", @@ -1138,6 +1357,8 @@ async function main() { expectedTargetSha: targetSha, expectedWorkflowBranch: options.workflowRef, isTrustedMainAncestor: (sha) => gitIsAncestor(sha, "refs/remotes/origin/main"), + validateEvidenceReuseStrictly: ({ repository, runId }) => + runStrictReleaseEvidenceValidation({ repository, runId }), }); if (fullValidationEvidence.source === "direct" && fullRun.headSha !== targetSha) { throw new Error(`run SHA mismatch: tag=${targetSha} full=${fullRun.headSha}`); @@ -1273,6 +1494,7 @@ async function main() { "", ].join("\n"), ); + updateReleaseCandidateState(statePath, candidateState, "completed"); console.log(`release candidate evidence: ${evidencePath}`); console.log(`release candidate summary: ${evidenceMarkdownPath}`); diff --git a/scripts/release-ci-summary.mjs b/scripts/release-ci-summary.mjs new file mode 100755 index 000000000000..a5d526dc4823 --- /dev/null +++ b/scripts/release-ci-summary.mjs @@ -0,0 +1,1792 @@ +#!/usr/bin/env node +/** + * Release CI summary helper that prints parent and child workflow status for a + * full release run. + */ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; +import { plainGhEnv, resolvePlainGhBin } from "./lib/plain-gh.mjs"; + +const DEFAULT_REPO = process.env.OPENCLAW_RELEASE_REPO || "openclaw/openclaw"; +const RELEASE_EVIDENCE_SCHEMA = "openclaw.release-validation-evidence/v3"; +const SHA_PINNED_BRANCH_PATTERN = /^release-ci\/[a-f0-9]{12}-[1-9][0-9]*$/u; +const RELEASE_EVIDENCE_SCRIPT = "scripts/release-ci-summary.mjs"; +const RELEASE_EVIDENCE_FILE = fileURLToPath(import.meta.url); +const RELEASE_EVIDENCE_REPO_ROOT = resolve(dirname(RELEASE_EVIDENCE_FILE), ".."); +const MANIFEST_ARTIFACT_ENTRY = "full-release-validation-manifest.json"; +const MAX_MANIFEST_ARTIFACT_ZIP_BYTES = 256 * 1024; +const MAX_MANIFEST_JSON_BYTES = 128 * 1024; +const MAX_MANIFEST_ENTRY_LIST_BYTES = 8 * 1024; + +const CHILD_DISPATCHES = [ + { + manifestKey: "normalCi", + name: "CI", + parentJobName: "Run normal full CI", + suffix: "-ci", + trustedRef: "parent", + workflow: "ci.yml", + }, + { + manifestKey: "releaseChecks", + name: "OpenClaw Release Checks", + parentJobName: "Run release/live/Docker/QA validation", + suffix: "-release-checks", + trustedRef: "parent", + workflow: "openclaw-release-checks.yml", + }, + { + manifestKey: "pluginPrerelease", + name: "Plugin Prerelease", + parentJobName: "Run plugin prerelease validation", + suffix: "-plugin-prerelease", + trustedRef: "parent", + workflow: "plugin-prerelease.yml", + }, + { + manifestKey: "npmTelegram", + name: "NPM Telegram Beta E2E", + parentJobName: "Run package Telegram E2E", + suffix: "-npm-telegram", + trustedRef: "parent", + workflow: "npm-telegram-beta-e2e.yml", + }, + { + manifestKey: "productPerformance", + name: "OpenClaw Performance", + parentJobName: "Run product performance evidence", + suffix: "", + trustedRef: "parent", + workflow: "openclaw-performance.yml", + }, +]; + +const EVIDENCE_REUSE_POLICY = "exact-target-full-validation-v1"; + +const RERUN_GROUP_CHILD_KEYS = new Map([ + ["all", ["normalCi", "releaseChecks", "pluginPrerelease", "productPerformance"]], + ["ci", ["normalCi"]], + ["plugin-prerelease", ["pluginPrerelease"]], + ["release-checks", ["releaseChecks"]], + ["install-smoke", ["releaseChecks"]], + ["cross-os", ["releaseChecks"]], + ["live-e2e", ["releaseChecks"]], + ["package", ["releaseChecks"]], + ["qa", ["releaseChecks"]], + ["qa-parity", ["releaseChecks"]], + ["qa-live", ["releaseChecks"]], + ["npm-telegram", ["npmTelegram"]], + ["performance", ["productPerformance"]], +]); + +function gh(args) { + return execFileSync(resolvePlainGhBin(), args, { + encoding: "utf8", + env: plainGhEnv(), + maxBuffer: 64 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function jsonGh(args) { + return JSON.parse(gh(args)); +} + +function githubRestJson(pathSuffix, repository = DEFAULT_REPO) { + const result = execFileSync( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + 'token="$("$OPENCLAW_PLAIN_GH_BIN" auth token)"', + 'curl -fsS -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" "${OPENCLAW_GITHUB_REST_URL}"', + ].join("\n"), + ], + { + encoding: "utf8", + env: { + ...plainGhEnv(), + OPENCLAW_PLAIN_GH_BIN: resolvePlainGhBin(), + OPENCLAW_GITHUB_REST_URL: `https://api.github.com/repos/${repository}/${pathSuffix}`, + }, + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + return JSON.parse(result); +} + +function downloadArtifactZip(artifactId, destination, repository = DEFAULT_REPO) { + execFileSync( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + 'token="$("$OPENCLAW_PLAIN_GH_BIN" auth token)"', + 'curl -fsSL -H "Authorization: Bearer ${token}" -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" --output "$OPENCLAW_GITHUB_ARTIFACT_DESTINATION" "$OPENCLAW_GITHUB_ARTIFACT_URL"', + ].join("\n"), + ], + { + env: { + ...plainGhEnv(), + OPENCLAW_GITHUB_ARTIFACT_DESTINATION: destination, + OPENCLAW_GITHUB_ARTIFACT_URL: `https://api.github.com/repos/${repository}/actions/artifacts/${artifactId}/zip`, + OPENCLAW_PLAIN_GH_BIN: resolvePlainGhBin(), + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); +} + +function rate() { + try { + return jsonGh(["api", "rate_limit"]).resources.core; + } catch { + return undefined; + } +} + +export function validateParentRunBinding(parentView, parentRest, expectedRunId) { + const boundWorkflowPath = String(parentRest.path ?? "").split("@", 1)[0]; + if ( + String(parentRest.id) !== String(expectedRunId) || + parentRest.event !== "workflow_dispatch" || + boundWorkflowPath !== ".github/workflows/full-release-validation.yml" || + Number(parentRest.run_attempt) !== Number(parentView.attempt) || + parentRest.head_branch !== parentView.headBranch || + parentRest.head_sha !== parentView.headSha + ) { + throw new Error(`full release parent run binding mismatch: ${expectedRunId}`); + } + return parentRest; +} + +export function expectedChildDispatches(parentRunId, parentRunAttempt, parentWorkflowRef) { + if (!/^[1-9][0-9]*$/u.test(String(parentRunId))) { + throw new Error("parent run ID must be a positive decimal"); + } + if (!Number.isSafeInteger(parentRunAttempt) || parentRunAttempt < 1) { + throw new Error("parent run attempt must be a positive integer"); + } + if (typeof parentWorkflowRef !== "string" || parentWorkflowRef.length === 0) { + throw new Error("parent workflow ref is required"); + } + const dispatchPrefix = `full-release-validation-${parentRunId}-${parentRunAttempt}`; + return CHILD_DISPATCHES.map((child) => ({ + ...child, + displayTitle: `${child.name} ${dispatchPrefix}${child.suffix}`, + headBranch: child.trustedRef === "main" ? "main" : parentWorkflowRef, + })); +} + +export function requiredChildKeysForRerunGroup(rerunGroup) { + const childKeys = RERUN_GROUP_CHILD_KEYS.get(rerunGroup); + if (!childKeys) { + throw new Error(`release validation manifest rerun group is invalid: ${rerunGroup}`); + } + return new Set(childKeys); +} + +export function expectedSelectedChildDispatches( + parentRunId, + parentRunAttempt, + parentWorkflowRef, + selectedKeys, +) { + return expectedChildDispatches(parentRunId, parentRunAttempt, parentWorkflowRef).filter((child) => + selectedKeys.has(child.manifestKey), + ); +} + +export function selectExactChildRun(runs, expectedDisplayTitle, expectedHeadBranch) { + const matches = runs.filter( + (run) => + run.event === "workflow_dispatch" && + run.display_title === expectedDisplayTitle && + run.head_branch === expectedHeadBranch, + ); + if (matches.length > 1) { + throw new Error( + `multiple child runs have exact dispatch title and branch: ${expectedDisplayTitle} (${expectedHeadBranch})`, + ); + } + return matches[0]; +} + +export function selectExactChildRunFromPages(runPages, expectedDisplayTitle, expectedHeadBranch) { + let exactMatch; + for (const runs of runPages) { + const match = selectExactChildRun(runs, expectedDisplayTitle, expectedHeadBranch); + if (match) { + if (exactMatch) { + throw new Error( + `multiple child runs have exact dispatch title and branch: ${expectedDisplayTitle} (${expectedHeadBranch})`, + ); + } + exactMatch = match; + } + if (runs.length < 100) { + break; + } + } + return exactMatch; +} + +function findExactChildRun(child, repository = DEFAULT_REPO) { + const runPages = []; + for (let page = 1; page <= 10; page += 1) { + const query = new URLSearchParams({ + event: "workflow_dispatch", + branch: child.headBranch, + page: String(page), + per_page: "100", + }); + const runs = + githubRestJson(`actions/workflows/${child.workflow}/runs?${query.toString()}`, repository) + .workflow_runs ?? []; + runPages.push(runs); + if (runs.length < 100) { + break; + } + } + return selectExactChildRunFromPages(runPages, child.displayTitle, child.headBranch); +} + +function findParentJobsAll(parentRunId, repository = DEFAULT_REPO) { + const jobs = []; + for (let page = 1; page <= 10; page += 1) { + const query = new URLSearchParams({ + filter: "all", + page: String(page), + per_page: "100", + }); + const pageJobs = + githubRestJson(`actions/runs/${parentRunId}/jobs?${query.toString()}`, repository).jobs ?? []; + jobs.push(...pageJobs); + if (pageJobs.length < 100) { + break; + } + } + return jobs; +} + +function parentJobLog(jobId, repository = DEFAULT_REPO) { + return gh(["api", `repos/${repository}/actions/jobs/${jobId}/logs`]); +} + +function normalizeOptionalRunId(value, label) { + if (value === "") { + return ""; + } + if (!/^[1-9][0-9]*$/u.test(String(value))) { + throw new Error(`${label} must be empty or a positive decimal run ID`); + } + return String(value); +} + +function normalizeRequiredRunId(value, label) { + const runId = normalizeOptionalRunId(value, label); + if (!runId) { + throw new Error(`${label} is required`); + } + return runId; +} + +function normalizeRepository(value) { + const repository = String(value ?? ""); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("repository must use the owner/name form"); + } + return repository; +} + +function normalizeWorkflowRef(value, label) { + const workflowRef = String(value ?? ""); + const hasForbiddenCharacter = Array.from(workflowRef).some((character) => { + const codePoint = character.codePointAt(0) ?? 0; + return ( + codePoint <= 0x1f || + codePoint === 0x7f || + character.trim() === "" || + "~^:?*[\\".includes(character) + ); + }); + if (workflowRef.length === 0 || workflowRef.length > 255 || hasForbiddenCharacter) { + throw new Error(`${label} is invalid`); + } + return workflowRef; +} + +function normalizeSha(value, label) { + const sha = String(value ?? ""); + if (!/^[a-f0-9]{40}$/u.test(sha)) { + throw new Error(`${label} is invalid`); + } + return sha; +} + +function normalizePositiveInteger(value, label) { + const number = Number(value); + if (!Number.isSafeInteger(number) || number < 1) { + throw new Error(`${label} must be a positive integer`); + } + return number; +} + +function normalizeJsonObject(value, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} is invalid`); + } + return value; +} + +function canonicalJson(value) { + if (Array.isArray(value)) { + return value.map(canonicalJson); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, canonicalJson(entry)]), + ); + } + return value; +} + +function manifestEvidenceIdentity(manifest) { + return canonicalJson({ + childRunIds: manifest.childRunIds, + controls: manifest.controls, + releaseProfile: manifest.releaseProfile, + rerunGroup: manifest.rerunGroup, + runReleaseSoak: manifest.runReleaseSoak, + validationInputs: manifest.validationInputs, + }); +} + +export function validateParentManifest(value, expected) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("release validation manifest must be an object"); + } + if (![2, 3].includes(value.version) || value.workflowName !== "Full Release Validation") { + throw new Error("release validation manifest schema is unsupported"); + } + if (String(value.runId) !== String(expected.runId)) { + throw new Error("release validation manifest run ID mismatch"); + } + if ( + !/^[1-9][0-9]*$/u.test(String(value.runAttempt)) || + (expected.runAttempt !== undefined && Number(value.runAttempt) !== Number(expected.runAttempt)) + ) { + throw new Error("release validation manifest run attempt mismatch"); + } + const targetSha = normalizeSha(value.targetSha, "release validation manifest target SHA"); + if (typeof value.workflowRef !== "string" || value.workflowRef.length === 0) { + throw new Error("release validation manifest workflow ref is invalid"); + } + if (expected.workflowRef !== undefined && value.workflowRef !== expected.workflowRef) { + throw new Error("release validation manifest workflow ref mismatch"); + } + let workflowSha; + let workflowFullRef; + let workflowRefType; + if (value.version === 3) { + workflowSha = normalizeSha(value.workflowSha, "release validation manifest workflow SHA"); + if (expected.workflowSha !== undefined && workflowSha !== expected.workflowSha) { + throw new Error("release validation manifest workflow SHA mismatch"); + } + workflowFullRef = String(value.workflowFullRef ?? ""); + workflowRefType = String(value.workflowRefType ?? ""); + if ( + !["branch", "tag"].includes(workflowRefType) || + workflowFullRef !== + `refs/${workflowRefType === "branch" ? "heads" : "tags"}/${value.workflowRef}` + ) { + throw new Error("release validation manifest workflow full ref is invalid"); + } + } else if (expected.workflowSha !== undefined) { + workflowSha = normalizeSha(expected.workflowSha, "release validation workflow SHA"); + } + const rerunGroup = String(value.rerunGroup ?? ""); + requiredChildKeysForRerunGroup(rerunGroup); + const releaseProfile = String(value.releaseProfile ?? ""); + if (!["beta", "stable", "full"].includes(releaseProfile)) { + throw new Error("release validation manifest release profile is invalid"); + } + const runReleaseSoak = String(value.runReleaseSoak ?? ""); + if (!["true", "false"].includes(runReleaseSoak)) { + throw new Error("release validation manifest release soak value is invalid"); + } + const controls = normalizeJsonObject(value.controls, "release validation manifest controls"); + if (value.version === 3 && controls.performanceReportPublication !== "artifact-only") { + throw new Error("release validation manifest performance report publication mode is invalid"); + } + const validationInputs = + value.validationInputs === undefined + ? undefined + : normalizeJsonObject( + value.validationInputs, + "release validation manifest validation inputs", + ); + const childRuns = value.childRuns; + if (!childRuns || typeof childRuns !== "object" || Array.isArray(childRuns)) { + throw new Error("release validation manifest childRuns is invalid"); + } + const childRunIds = { + normalCi: normalizeOptionalRunId(childRuns.normalCi, "normal CI run ID"), + npmTelegram: normalizeOptionalRunId(childRuns.npmTelegram, "npm Telegram run ID"), + pluginPrerelease: normalizeOptionalRunId( + childRuns.pluginPrerelease, + "plugin prerelease run ID", + ), + productPerformance: normalizeOptionalRunId( + childRuns.productPerformance?.runId ?? "", + "performance run ID", + ), + releaseChecks: normalizeOptionalRunId(childRuns.releaseChecks, "release checks run ID"), + }; + let evidenceReuse; + if (value.evidenceReuse !== undefined) { + const reuse = normalizeJsonObject( + value.evidenceReuse, + "release validation manifest evidence reuse", + ); + if (reuse.policy !== EVIDENCE_REUSE_POLICY) { + throw new Error("release validation manifest evidence reuse policy is invalid"); + } + if (!/^[a-f0-9]{40}$/u.test(String(reuse.evidenceSha))) { + throw new Error("release validation manifest evidence SHA is invalid"); + } + if ( + !Array.isArray(reuse.changedPaths) || + reuse.changedPaths.some( + (changedPath) => typeof changedPath !== "string" || changedPath.length === 0, + ) || + new Set(reuse.changedPaths).size !== reuse.changedPaths.length + ) { + throw new Error("release validation manifest evidence changed paths are invalid"); + } + evidenceReuse = { + changedPaths: reuse.changedPaths, + evidenceSha: String(reuse.evidenceSha), + policy: reuse.policy, + runId: normalizeRequiredRunId(reuse.runId, "evidence reuse root run ID"), + selectedRunId: normalizeRequiredRunId(reuse.selectedRunId, "evidence reuse selected run ID"), + }; + } + return { + childRunIds, + controls, + evidenceReuse, + releaseProfile, + rerunGroup, + runAttempt: Number(value.runAttempt), + runId: String(value.runId), + runReleaseSoak, + targetRef: String(value.targetRef ?? ""), + targetSha, + validationInputs, + version: value.version, + workflowFullRef, + workflowSha, + workflowRef: value.workflowRef, + workflowRefType, + }; +} + +export function validateEvidenceReuseChain(currentManifest, selectedManifest, rootManifest) { + const reuse = currentManifest.evidenceReuse; + if (!reuse) { + throw new Error("release validation manifest does not authorize evidence reuse"); + } + if (reuse.changedPaths.length !== 0) { + throw new Error("full release evidence reuse requires an exact target with no changed paths"); + } + if (rootManifest.evidenceReuse || selectedManifest.evidenceReuse) { + throw new Error("evidence reuse must select a root execution manifest"); + } + if ( + !currentManifest.validationInputs || + !selectedManifest.validationInputs || + !rootManifest.validationInputs + ) { + throw new Error("evidence reuse manifests must record validation inputs"); + } + if (rootManifest.runId !== reuse.runId) { + throw new Error("evidence reuse root manifest run ID mismatch"); + } + if (selectedManifest.runId !== reuse.selectedRunId) { + throw new Error("evidence reuse selected manifest run ID mismatch"); + } + if (selectedManifest.targetSha !== reuse.evidenceSha) { + throw new Error("evidence reuse selected manifest SHA mismatch"); + } + if ( + currentManifest.targetSha !== reuse.evidenceSha || + rootManifest.targetSha !== reuse.evidenceSha + ) { + throw new Error("full release evidence reuse target SHA mismatch"); + } + if (selectedManifest.runId !== rootManifest.runId) { + throw new Error("evidence reuse selected manifest is not the chain root"); + } + + const rootIdentity = JSON.stringify(manifestEvidenceIdentity(rootManifest)); + for (const [label, manifest] of [ + ["selected", selectedManifest], + ["current", currentManifest], + ]) { + if (JSON.stringify(manifestEvidenceIdentity(manifest)) !== rootIdentity) { + throw new Error(`evidence reuse ${label} manifest policy differs from the chain root`); + } + } + return rootManifest.targetSha; +} + +export function selectedChildKeys(parentJobs) { + return new Set( + CHILD_DISPATCHES.filter((child) => { + const parentJob = parentJobs.find((job) => job.name === child.parentJobName); + return parentJob && parentJob.conclusion !== "skipped"; + }).map((child) => child.manifestKey), + ); +} + +export function manifestChildEntries(manifest, children, selectedKeys) { + return children.flatMap((child) => { + const runId = manifest.childRunIds[child.manifestKey]; + if (!runId) { + if (selectedKeys.has(child.manifestKey)) { + throw new Error(`selected child is missing from manifest: ${child.name}`); + } + return []; + } + return [{ child, runId }]; + }); +} + +function childDispatchAttempt(displayTitle, child, parentRunId, parentRunAttempt) { + const prefix = `${child.name} full-release-validation-${parentRunId}-`; + if (!displayTitle.startsWith(prefix) || !displayTitle.endsWith(child.suffix)) { + return undefined; + } + const attemptEnd = child.suffix ? -child.suffix.length : undefined; + const attemptText = displayTitle.slice(prefix.length, attemptEnd); + if (!/^[1-9][0-9]*$/u.test(attemptText)) { + return undefined; + } + const attempt = Number(attemptText); + if (!Number.isSafeInteger(attempt) || attempt > parentRunAttempt) { + return undefined; + } + return attempt; +} + +function parentJobExecutionFingerprint(job) { + return canonicalJson({ + completedAt: job.completed_at, + conclusion: job.conclusion, + name: job.name, + startedAt: job.started_at, + status: job.status, + steps: (job.steps ?? []).map((step) => ({ + completedAt: step.completed_at, + conclusion: step.conclusion, + name: step.name, + number: step.number, + startedAt: step.started_at, + status: step.status, + })), + }); +} + +function selectedAttemptParentJob(parentJobs, child, parentManifest) { + const slotJobs = parentJobs.filter((job) => job.name === child.parentJobName); + if (slotJobs.length === 0) { + throw new Error(`manifest parent job is missing: ${child.name}`); + } + const latestAttempt = Math.max(...slotJobs.map((job) => Number(job.run_attempt))); + if (latestAttempt !== parentManifest.runAttempt) { + throw new Error(`manifest parent job latest attempt mismatch: ${child.name}`); + } + const currentJobs = slotJobs.filter( + (job) => Number(job.run_attempt) === parentManifest.runAttempt, + ); + if (currentJobs.length !== 1) { + throw new Error(`manifest parent job is not unique at the selected attempt: ${child.name}`); + } + const currentJob = currentJobs[0]; + if (currentJob.status !== "completed" || currentJob.conclusion !== "success") { + throw new Error(`manifest parent job is not completed/success: ${child.name}`); + } + return { currentJob, slotJobs }; +} + +export function resolveManifestChildOriginAttempt(run, child, parentManifest, parentJobs) { + const correlatedAttempt = childDispatchAttempt( + String(run.display_title ?? ""), + child, + parentManifest.runId, + parentManifest.runAttempt, + ); + if (correlatedAttempt !== undefined) { + return correlatedAttempt; + } + if (run.display_title !== child.name) { + return undefined; + } + + const { currentJob, slotJobs } = selectedAttemptParentJob(parentJobs, child, parentManifest); + const currentFingerprint = JSON.stringify(parentJobExecutionFingerprint(currentJob)); + const carriedOriginAttempts = slotJobs + .filter( + (job) => + Number(job.run_attempt) < parentManifest.runAttempt && + job.status === "completed" && + job.conclusion === "success" && + JSON.stringify(parentJobExecutionFingerprint(job)) === currentFingerprint, + ) + .map((job) => Number(job.run_attempt)); + return carriedOriginAttempts.length > 0 + ? Math.min(...carriedOriginAttempts) + : parentManifest.runAttempt; +} + +export function selectManifestParentJob(parentJobs, child, parentManifest, originAttempt) { + const { currentJob, slotJobs } = selectedAttemptParentJob(parentJobs, child, parentManifest); + if (originAttempt === parentManifest.runAttempt) { + return currentJob; + } + + const originJobs = slotJobs.filter((job) => Number(job.run_attempt) === originAttempt); + if (originJobs.length !== 1) { + throw new Error(`manifest parent job origin is not unique: ${child.name}`); + } + const originJob = originJobs[0]; + if (originJob.status !== "completed" || originJob.conclusion !== "success") { + throw new Error(`manifest parent job origin is not completed/success: ${child.name}`); + } + if ( + JSON.stringify(parentJobExecutionFingerprint(currentJob)) !== + JSON.stringify(parentJobExecutionFingerprint(originJob)) + ) { + throw new Error(`manifest parent job carry-forward fingerprint mismatch: ${child.name}`); + } + return currentJob; +} + +function childRunIdsFromParentLog(log, repository = DEFAULT_REPO) { + const escapedRepo = repository.replaceAll(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const pattern = new RegExp( + `https://github\\.com/${escapedRepo}/actions/runs/([1-9][0-9]*)`, + "gu", + ); + return new Set(Array.from(log.matchAll(pattern), (match) => match[1])); +} + +export function validateManifestChildRun( + run, + child, + runId, + parentManifest, + parentJobs, + selectedParentJobLog, + repository = DEFAULT_REPO, +) { + if (String(run.id) !== String(runId)) { + throw new Error(`manifest child run ID mismatch: ${child.name}`); + } + const originAttempt = resolveManifestChildOriginAttempt(run, child, parentManifest, parentJobs); + if ( + run.event !== "workflow_dispatch" || + run.head_branch !== child.headBranch || + (child.trustedRef === "parent" && run.head_sha !== parentManifest.workflowSha) || + !/^[a-f0-9]{40}$/u.test(String(run.head_sha)) || + run.actor?.login !== "github-actions[bot]" || + run.triggering_actor?.login !== "github-actions[bot]" || + !Number.isSafeInteger(Number(run.run_attempt)) || + Number(run.run_attempt) < 1 || + originAttempt === undefined + ) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const childWorkflowPath = String(run.path ?? "").split("@", 1)[0]; + if (childWorkflowPath !== `.github/workflows/${child.workflow}`) { + throw new Error(`manifest child workflow mismatch: ${child.name}`); + } + selectManifestParentJob(parentJobs, child, parentManifest, originAttempt); + const emittedChildRunIds = childRunIdsFromParentLog(selectedParentJobLog, repository); + if (emittedChildRunIds.size !== 1 || !emittedChildRunIds.has(String(runId))) { + throw new Error(`manifest child run is not uniquely emitted by its parent job: ${child.name}`); + } + if ( + child.manifestKey !== "npmTelegram" && + !selectedParentJobLog.includes(`TARGET_SHA: ${parentManifest.targetSha}`) + ) { + throw new Error(`manifest parent job target SHA mismatch: ${child.name}`); + } + if ( + child.manifestKey === "productPerformance" && + !selectedParentJobLog.includes("-f publish_reports=false") + ) { + throw new Error("manifest performance child is not dispatched in artifact-only mode"); + } + return run; +} + +export function validatePerformanceArtifactOnlyJobs(jobs, runAttempt) { + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "performance run attempt"); + const currentJobs = jobs.filter((job) => Number(job.run_attempt) === normalizedRunAttempt); + const guards = currentJobs.filter((job) => job.name === "Verify artifact-only report mode"); + if ( + guards.length !== 1 || + guards[0].status !== "completed" || + guards[0].conclusion !== "success" + ) { + throw new Error("performance artifact-only guard is missing or unsuccessful"); + } + const unsafePublisher = currentJobs.find( + (job) => + String(job.name ?? "").startsWith("Publish ") && + String(job.name ?? "").endsWith(" report") && + job.conclusion !== "skipped", + ); + if (unsafePublisher) { + throw new Error(`performance report publisher was not skipped: ${unsafePublisher.name}`); + } + return guards[0]; +} + +function manifestArtifactName(runId, runAttempt) { + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "full release run attempt"); + return `full-release-validation-${normalizedRunId}-${normalizedRunAttempt}`; +} + +function legacyManifestArtifactName(runId) { + return `full-release-validation-${normalizeRequiredRunId(runId, "full release run ID")}`; +} + +export function validateManifestArtifactIdentity( + artifact, + { artifactDigest, artifactId, runAttempt, runId }, +) { + const normalizedArtifactId = normalizeRequiredRunId(artifactId, "manifest artifact ID"); + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const normalizedRunAttempt = normalizePositiveInteger(runAttempt, "full release run attempt"); + const normalizedDigest = String(artifactDigest ?? ""); + if (!/^sha256:[a-f0-9]{64}$/u.test(normalizedDigest)) { + throw new Error(`release validation manifest artifact digest is invalid: ${normalizedRunId}`); + } + const canonicalName = manifestArtifactName(normalizedRunId, normalizedRunAttempt); + const legacyName = legacyManifestArtifactName(normalizedRunId); + const validName = + artifact.name === canonicalName || (normalizedRunAttempt === 1 && artifact.name === legacyName); + if ( + String(artifact.id) !== normalizedArtifactId || + !validName || + artifact.digest !== normalizedDigest || + artifact.expired !== false || + String(artifact.workflow_run?.id) !== normalizedRunId || + !Number.isSafeInteger(Number(artifact.size_in_bytes)) || + Number(artifact.size_in_bytes) < 1 + ) { + throw new Error(`release validation manifest artifact identity mismatch: ${normalizedRunId}`); + } + return artifact; +} + +export function selectManifestArtifact(artifacts, runId, runAttempt) { + const expectedName = manifestArtifactName(runId, runAttempt); + const canonicalMatches = artifacts.filter( + (artifact) => + artifact.name === expectedName && + artifact.expired === false && + String(artifact.workflow_run?.id) === String(runId), + ); + if (canonicalMatches.length > 1) { + throw new Error(`multiple release validation manifest artifacts found: ${runId}`); + } + const canonicalArtifact = canonicalMatches[0]; + if (canonicalArtifact) { + return validateManifestArtifactIdentity(canonicalArtifact, { + artifactDigest: canonicalArtifact.digest, + artifactId: canonicalArtifact.id, + runAttempt, + runId, + }); + } + + const legacyName = legacyManifestArtifactName(runId); + const legacyMatches = artifacts.filter( + (artifact) => + artifact.name === legacyName && + artifact.expired === false && + String(artifact.workflow_run?.id) === String(runId), + ); + if (legacyMatches.length > 1) { + throw new Error(`multiple legacy release validation manifest artifacts found: ${runId}`); + } + const legacyArtifact = legacyMatches[0]; + if (!legacyArtifact) { + return undefined; + } + if (Number(runAttempt) !== 1) { + throw new Error(`legacy release validation manifest requires run attempt 1: ${runId}`); + } + return validateManifestArtifactIdentity(legacyArtifact, { + artifactDigest: legacyArtifact.digest, + artifactId: legacyArtifact.id, + runAttempt, + runId, + }); +} + +export function validateManifestArtifactCompatibility(artifact, manifest, runId, runAttempt) { + if (artifact.name === manifestArtifactName(runId, runAttempt)) { + return artifact; + } + if ( + Number(runAttempt) === 1 && + artifact.name === legacyManifestArtifactName(runId) && + manifest?.version === 2 + ) { + return artifact; + } + throw new Error(`legacy release validation manifest artifact is not compatible: ${runId}`); +} + +export function readManifestArtifactArchive(archivePath, expectedDigest) { + const archiveSize = statSync(archivePath).size; + if ( + !Number.isSafeInteger(archiveSize) || + archiveSize < 1 || + archiveSize > MAX_MANIFEST_ARTIFACT_ZIP_BYTES + ) { + throw new Error("release validation manifest artifact compressed size is invalid"); + } + const archiveBytes = readFileSync(archivePath); + if (archiveBytes.byteLength !== archiveSize) { + throw new Error("release validation manifest artifact changed while being verified"); + } + const actualDigest = `sha256:${createHash("sha256").update(archiveBytes).digest("hex")}`; + if (actualDigest !== expectedDigest) { + throw new Error("release validation manifest artifact digest mismatch"); + } + + let entryList; + try { + entryList = execFileSync("unzip", ["-Z", "-1", archivePath], { + encoding: "utf8", + maxBuffer: MAX_MANIFEST_ENTRY_LIST_BYTES, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch { + throw new Error("release validation manifest artifact entry list is invalid"); + } + const entries = entryList.split(/\r?\n/u).filter((entry) => entry.length > 0); + if (entries.length !== 1 || entries[0] !== MANIFEST_ARTIFACT_ENTRY) { + throw new Error( + `release validation manifest artifact must contain only ${MANIFEST_ARTIFACT_ENTRY}`, + ); + } + + let manifestBytes; + try { + manifestBytes = execFileSync("unzip", ["-p", archivePath, MANIFEST_ARTIFACT_ENTRY], { + maxBuffer: MAX_MANIFEST_JSON_BYTES + 1, + stdio: ["ignore", "pipe", "pipe"], + }); + } catch { + throw new Error("release validation manifest artifact entry could not be read safely"); + } + if (manifestBytes.byteLength < 1 || manifestBytes.byteLength > MAX_MANIFEST_JSON_BYTES) { + throw new Error("release validation manifest artifact entry size is invalid"); + } + return JSON.parse(manifestBytes.toString("utf8")); +} + +function downloadParentManifestEvidence(runId, runAttempt, repository, manifestPath) { + const targetRepository = repository ?? DEFAULT_REPO; + const artifacts = []; + for (let page = 1; page <= 10; page += 1) { + const pageArtifacts = + githubRestJson(`actions/runs/${runId}/artifacts?per_page=100&page=${page}`, targetRepository) + .artifacts ?? []; + artifacts.push(...pageArtifacts); + if (pageArtifacts.length < 100) { + break; + } + } + const listedArtifact = selectManifestArtifact(artifacts, runId, runAttempt); + if (!listedArtifact) { + return undefined; + } + const artifact = validateManifestArtifactIdentity( + githubRestJson(`actions/artifacts/${listedArtifact.id}`, targetRepository), + { + artifactDigest: listedArtifact.digest, + artifactId: listedArtifact.id, + runAttempt, + runId, + }, + ); + const downloadDir = mkdtempSync(join(tmpdir(), "openclaw-release-ci-summary-")); + try { + const archivePath = join(downloadDir, "manifest.zip"); + downloadArtifactZip(String(artifact.id), archivePath, targetRepository); + const manifest = readManifestArtifactArchive(archivePath, artifact.digest); + validateManifestArtifactCompatibility(artifact, manifest, runId, runAttempt); + if (manifestPath) { + const providedManifest = JSON.parse(readFileSync(resolve(manifestPath), "utf8")); + if ( + JSON.stringify(canonicalJson(providedManifest)) !== JSON.stringify(canonicalJson(manifest)) + ) { + throw new Error("provided release validation manifest differs from the run artifact"); + } + } + return { artifact, manifest }; + } finally { + rmSync(downloadDir, { force: true, recursive: true }); + } +} + +function tryDownloadParentManifest(runId, runAttempt, repository = DEFAULT_REPO) { + return downloadParentManifestEvidence(runId, runAttempt, repository)?.manifest; +} + +function workflowPath(run) { + return String(run.path ?? "").split("@", 1)[0]; +} + +function normalizedManifestArtifact(artifact, runAttempt) { + return { + digest: artifact.digest, + id: String(artifact.id), + name: artifact.name, + runAttempt, + sizeInBytes: Number(artifact.size_in_bytes), + }; +} + +function validateManifestArtifactBinding(artifact, manifest, parentRun, runId) { + validateManifestArtifactCompatibility(artifact, manifest, runId, parentRun.run_attempt); + if ( + String(artifact.workflow_run?.id) !== String(runId) || + artifact.workflow_run?.head_branch !== parentRun.head_branch || + artifact.workflow_run?.head_sha !== parentRun.head_sha + ) { + throw new Error(`release validation manifest artifact binding mismatch: ${runId}`); + } +} + +function validateCompletedParentRun(parentView, parentRest, repository, runId) { + validateParentRunBinding(parentView, parentRest, runId); + if ( + parentView.status !== "completed" || + parentView.conclusion !== "success" || + parentRest.status !== "completed" || + parentRest.conclusion !== "success" || + parentRest.repository?.full_name !== repository + ) { + throw new Error(`full release parent run is not completed/success: ${runId}`); + } +} + +export function createReleaseEvidenceClient(repository = DEFAULT_REPO) { + const normalizedRepository = normalizeRepository(repository); + return { + compareCommits(base, head) { + return githubRestJson(`compare/${base}...${head}`, normalizedRepository); + }, + getJobLog(jobId) { + return parentJobLog(jobId, normalizedRepository); + }, + getParentJobs(runId) { + return findParentJobsAll(runId, normalizedRepository); + }, + getRun(runId) { + return githubRestJson(`actions/runs/${runId}`, normalizedRepository); + }, + getRunView(runId) { + return jsonGh([ + "run", + "view", + String(runId), + "--repo", + normalizedRepository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + }, + loadManifest(runId, runAttempt, manifestPath) { + return downloadParentManifestEvidence(runId, runAttempt, normalizedRepository, manifestPath); + }, + }; +} + +function loadValidatedParentEvidence({ client, manifestPath, repository, runId }) { + const parentView = client.getRunView(runId); + const parentRun = client.getRun(runId); + validateCompletedParentRun(parentView, parentRun, repository, runId); + + const manifestEvidence = client.loadManifest(runId, parentRun.run_attempt, manifestPath); + if (!manifestEvidence) { + throw new Error(`successful parent run is missing its release validation manifest: ${runId}`); + } + const manifest = validateParentManifest(manifestEvidence.manifest, { + runAttempt: parentRun.run_attempt, + runId, + workflowRef: parentRun.head_branch, + workflowSha: parentRun.head_sha, + }); + validateManifestArtifactBinding(manifestEvidence.artifact, manifest, parentRun, runId); + + return { + artifact: manifestEvidence.artifact, + manifest, + manifestJson: canonicalJson(manifestEvidence.manifest), + parentRun, + parentView, + }; +} + +function trustedWorkflowFullRef(workflowRef) { + return `refs/heads/${workflowRef}`; +} + +function normalizeWorkflowPathRef(ref) { + if (!ref || ref.startsWith("refs/")) { + return ref; + } + return `refs/heads/${ref}`; +} + +export function validateTrustedProducerIdentity(evidence, client, verifier, trustedWorkflowRef) { + const { manifest, parentRun } = evidence; + // Keep this predicate local: verifier source identity covers this file only. + const shaPinned = SHA_PINNED_BRANCH_PATTERN.test(manifest.workflowRef ?? ""); + if (manifest.workflowRef !== trustedWorkflowRef && !shaPinned) { + throw new Error( + `release evidence producer must run from trusted workflow ref: ${trustedWorkflowRef}`, + ); + } + if (shaPinned) { + if (manifest.version !== 3) { + throw new Error("SHA-pinned release evidence requires a v3 manifest"); + } + if (!manifest.workflowRef.startsWith(`release-ci/${manifest.workflowSha.slice(0, 12)}-`)) { + throw new Error("SHA-pinned release evidence branch does not match its workflow SHA"); + } + if (manifest.targetRef !== manifest.targetSha) { + throw new Error("SHA-pinned release evidence target ref must equal its target SHA"); + } + } + const expectedFullRef = trustedWorkflowFullRef(manifest.workflowRef); + const runPath = String(parentRun.path ?? ""); + const [runWorkflowPath, runWorkflowFullRef] = runPath.split("@", 2); + if (runWorkflowPath !== ".github/workflows/full-release-validation.yml") { + throw new Error("release evidence producer workflow path is not trusted"); + } + if (runWorkflowFullRef && normalizeWorkflowPathRef(runWorkflowFullRef) !== expectedFullRef) { + throw new Error("release evidence producer workflow full ref is not trusted"); + } + + let workflowRefProof = "legacy-v2-main-ancestry"; + if (manifest.version === 3) { + if (manifest.workflowRefType !== "branch" || manifest.workflowFullRef !== expectedFullRef) { + throw new Error("release evidence producer workflow full ref is not trusted"); + } + workflowRefProof = shaPinned ? "manifest-v3-sha-pinned-main-ancestry" : "manifest-v3-branch"; + } + + const comparison = client.compareCommits(manifest.workflowSha, verifier.sourceSha); + if ( + !["ahead", "identical"].includes(String(comparison.status)) || + comparison.merge_base_commit?.sha !== manifest.workflowSha + ) { + throw new Error("release evidence producer is not on the trusted main verifier lineage"); + } + + return { + producerOnTrustedMainLineage: true, + workflowFullRef: expectedFullRef, + workflowQualifiedPath: `${runWorkflowPath}@${expectedFullRef}`, + workflowRefProof, + workflowRefType: "branch", + workflowRunPath: runPath, + }; +} + +function normalizedParentTuple(evidence, identity) { + const { manifest, parentRun } = evidence; + return { + artifact: normalizedManifestArtifact(evidence.artifact, manifest.runAttempt), + conclusion: parentRun.conclusion, + manifest: evidence.manifestJson, + manifestVersion: manifest.version, + runAttempt: manifest.runAttempt, + runId: manifest.runId, + status: parentRun.status, + targetSha: manifest.targetSha, + url: parentRun.html_url ?? evidence.parentView.url, + ...identity, + workflowPath: workflowPath(parentRun), + workflowRef: manifest.workflowRef, + workflowSha: manifest.workflowSha, + }; +} + +export function resolveVerifierIdentity( + sourceSha, + verifierSourceContent, + repositoryRoot = RELEASE_EVIDENCE_REPO_ROOT, +) { + let normalizedSourceSha = sourceSha ?? process.env.GITHUB_SHA; + if (!/^[a-f0-9]{40}$/u.test(String(normalizedSourceSha ?? ""))) { + try { + normalizedSourceSha = execFileSync("git", ["-C", repositoryRoot, "rev-parse", "HEAD"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + normalizedSourceSha = null; + } + } + if (!/^[a-f0-9]{40}$/u.test(String(normalizedSourceSha ?? ""))) { + throw new Error("release evidence verifier source SHA is unavailable"); + } + const script = readFileSync(RELEASE_EVIDENCE_FILE); + const scriptSha256 = createHash("sha256").update(script).digest("hex"); + let sourceScript; + if (verifierSourceContent !== undefined) { + sourceScript = Buffer.from(verifierSourceContent); + } else { + try { + sourceScript = execFileSync( + "git", + ["-C", repositoryRoot, "show", `${normalizedSourceSha}:${RELEASE_EVIDENCE_SCRIPT}`], + { + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + } catch { + throw new Error("release evidence verifier source blob is unavailable"); + } + } + const sourceScriptSha256 = createHash("sha256").update(sourceScript).digest("hex"); + if (scriptSha256 !== sourceScriptSha256) { + throw new Error("release evidence verifier script differs from its source SHA"); + } + return { + schemaVersion: 3, + script: RELEASE_EVIDENCE_SCRIPT, + scriptSha256, + sourceSha: normalizedSourceSha, + }; +} + +function validateStrictChildRun({ child, client, parentEvidence, parentJobs, repository, runId }) { + const run = client.getRun(runId); + const originAttempt = resolveManifestChildOriginAttempt( + run, + child, + parentEvidence.manifest, + parentJobs, + ); + if (originAttempt === undefined) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const parentJob = selectManifestParentJob( + parentJobs, + child, + parentEvidence.manifest, + originAttempt, + ); + validateManifestChildRun( + run, + child, + runId, + parentEvidence.manifest, + parentJobs, + client.getJobLog(parentJob.id), + repository, + ); + if ( + run.repository?.full_name !== repository || + run.status !== "completed" || + run.conclusion !== "success" || + run.head_sha !== parentEvidence.manifest.workflowSha + ) { + throw new Error(`manifest child run is not exact completed/success evidence: ${child.name}`); + } + if (child.manifestKey === "productPerformance") { + validatePerformanceArtifactOnlyJobs(client.getParentJobs(runId), run.run_attempt); + } + + return { + conclusion: run.conclusion, + dispatchNonce: `full-release-validation-${parentEvidence.manifest.runId}-${originAttempt}${child.suffix}`, + displayTitle: run.display_title, + event: run.event, + headBranch: run.head_branch, + parentJobId: String(parentJob.id), + path: workflowPath(run), + role: child.manifestKey, + runAttempt: normalizePositiveInteger(run.run_attempt, `${child.name} run attempt`), + runId: String(run.id), + sourceParentAttempt: originAttempt, + sourceParentRunId: parentEvidence.manifest.runId, + status: run.status, + url: run.html_url, + workflowSha: run.head_sha, + ...(child.manifestKey === "productPerformance" ? { reportPublication: "artifact-only" } : {}), + }; +} + +export function validateReleaseRunEvidence( + { + manifestPath, + repository = DEFAULT_REPO, + runId, + trustedWorkflowRef = "main", + verifierSourceContent, + verifierSourceSha, + }, + client, +) { + const normalizedRepository = normalizeRepository(repository); + const normalizedRunId = normalizeRequiredRunId(runId, "full release run ID"); + const normalizedTrustedWorkflowRef = normalizeWorkflowRef( + trustedWorkflowRef, + "trusted workflow ref", + ); + const evidenceClient = client ?? createReleaseEvidenceClient(normalizedRepository); + const verifier = resolveVerifierIdentity(verifierSourceSha, verifierSourceContent); + const currentEvidence = loadValidatedParentEvidence({ + client: evidenceClient, + manifestPath, + repository: normalizedRepository, + runId: normalizedRunId, + }); + const producerIdentities = new Map([ + [ + currentEvidence.manifest.runId, + validateTrustedProducerIdentity( + currentEvidence, + evidenceClient, + verifier, + normalizedTrustedWorkflowRef, + ), + ], + ]); + + let rootEvidence = currentEvidence; + let selectedEvidence = currentEvidence; + const reuse = currentEvidence.manifest.evidenceReuse; + if (reuse) { + rootEvidence = loadValidatedParentEvidence({ + client: evidenceClient, + repository: normalizedRepository, + runId: reuse.runId, + }); + selectedEvidence = + reuse.selectedRunId === reuse.runId + ? rootEvidence + : loadValidatedParentEvidence({ + client: evidenceClient, + repository: normalizedRepository, + runId: reuse.selectedRunId, + }); + validateEvidenceReuseChain( + currentEvidence.manifest, + selectedEvidence.manifest, + rootEvidence.manifest, + ); + } + + for (const evidence of [currentEvidence, selectedEvidence, rootEvidence]) { + if (!producerIdentities.has(evidence.manifest.runId)) { + producerIdentities.set( + evidence.manifest.runId, + validateTrustedProducerIdentity( + evidence, + evidenceClient, + verifier, + normalizedTrustedWorkflowRef, + ), + ); + } + } + const selectedKeys = requiredChildKeysForRerunGroup(rootEvidence.manifest.rerunGroup); + const expectedChildren = expectedSelectedChildDispatches( + rootEvidence.manifest.runId, + rootEvidence.manifest.runAttempt, + rootEvidence.manifest.workflowRef, + selectedKeys, + ); + const parentJobs = evidenceClient.getParentJobs(rootEvidence.manifest.runId); + const children = manifestChildEntries(rootEvidence.manifest, expectedChildren, selectedKeys).map( + ({ child, runId: childRunId }) => + validateStrictChildRun({ + child, + client: evidenceClient, + parentEvidence: rootEvidence, + parentJobs, + repository: normalizedRepository, + runId: childRunId, + }), + ); + + const current = normalizedParentTuple( + currentEvidence, + producerIdentities.get(currentEvidence.manifest.runId), + ); + const root = normalizedParentTuple( + rootEvidence, + producerIdentities.get(rootEvidence.manifest.runId), + ); + const childConclusions = Object.fromEntries( + children.map((child) => [child.role, child.conclusion]), + ); + return canonicalJson({ + children, + conclusions: { + allRequiredSucceeded: children.every((child) => child.conclusion === "success"), + children: childConclusions, + current: current.conclusion, + root: root.conclusion, + }, + controls: rootEvidence.manifest.controls, + current, + directRoot: !reuse, + evidenceReuse: reuse + ? { + changedPaths: reuse.changedPaths, + evidenceSha: reuse.evidenceSha, + policy: reuse.policy, + rootRunId: reuse.runId, + selectedRunId: reuse.selectedRunId, + } + : null, + manifest: rootEvidence.manifestJson, + releaseProfile: rootEvidence.manifest.releaseProfile, + repository: normalizedRepository, + rerunGroup: rootEvidence.manifest.rerunGroup, + root, + runReleaseSoak: rootEvidence.manifest.runReleaseSoak === "true", + schema: RELEASE_EVIDENCE_SCHEMA, + producerOnTrustedMainLineage: true, + trustedWorkflowFullRef: trustedWorkflowFullRef(normalizedTrustedWorkflowRef), + trustedWorkflowRef: normalizedTrustedWorkflowRef, + valid: true, + validationInputs: rootEvidence.manifest.validationInputs ?? null, + verifier, + }); +} + +export function parseReleaseCiSummaryArgs(argv) { + const options = { + intervalMs: 30_000, + json: false, + manifestPath: undefined, + repository: DEFAULT_REPO, + runId: undefined, + trustedWorkflowRef: "main", + validate: false, + verifierSourceFile: undefined, + verifierSourceSha: undefined, + watch: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === "--validate-run") { + options.validate = true; + options.runId = argv[++index]; + } else if (argument === "--repo") { + options.repository = argv[++index]; + } else if (argument === "--manifest") { + options.manifestPath = argv[++index]; + } else if (argument === "--trusted-workflow-ref") { + options.trustedWorkflowRef = argv[++index]; + } else if (argument === "--verifier-source-sha") { + options.verifierSourceSha = argv[++index]; + } else if (argument === "--verifier-source-file") { + options.verifierSourceFile = argv[++index]; + } else if (argument === "--json") { + options.json = true; + } else if (argument === "--watch") { + options.watch = true; + } else if (argument === "--interval") { + const seconds = argv[++index]; + if (!/^[1-9][0-9]*$/u.test(seconds ?? "")) { + throw new Error("--interval requires a positive number of seconds"); + } + options.intervalMs = Number(seconds) * 1000; + } else if (!argument.startsWith("-") && !options.runId && !options.validate) { + options.runId = argument; + } else { + throw new Error(`unknown or incomplete argument: ${argument}`); + } + } + if (!options.validate && options.manifestPath) { + throw new Error("--manifest requires --validate-run"); + } + if (options.validate && options.watch) { + throw new Error("--watch cannot be combined with --validate-run"); + } + if (options.verifierSourceFile && !options.verifierSourceSha) { + throw new Error("--verifier-source-file requires --verifier-source-sha"); + } + if (!options.runId) { + throw new Error("full release run ID is required"); + } + return options; +} + +function printUsage() { + console.error( + [ + "usage: release-ci-summary.mjs ", + " release-ci-summary.mjs --watch [--interval seconds]", + " release-ci-summary.mjs --validate-run [--repo owner/name] [--trusted-workflow-ref main] [--manifest path] [--verifier-source-sha sha --verifier-source-file path] --json", + ].join("\n"), + ); +} + +export function releaseCiWatchFingerprint(parent) { + return JSON.stringify({ + attempt: parent.attempt, + conclusion: parent.conclusion ?? "", + jobs: (parent.jobs ?? []) + .map((job) => ({ + conclusion: job.conclusion ?? "", + name: job.name, + status: job.status, + })) + .toSorted((left, right) => left.name.localeCompare(right.name)), + status: parent.status, + }); +} + +function summarizeReleaseCiRun(options) { + execFileSync( + process.execPath, + [ + RELEASE_EVIDENCE_FILE, + options.runId, + "--repo", + options.repository, + "--trusted-workflow-ref", + options.trustedWorkflowRef, + ], + { stdio: "inherit" }, + ); +} + +export async function watchReleaseCiRun(options, overrides = {}) { + const fetchParent = + overrides.fetchParent ?? + (() => + jsonGh([ + "run", + "view", + options.runId, + "--repo", + options.repository, + "--json", + "status,conclusion,attempt,jobs", + ])); + const summarize = overrides.summarize ?? (() => summarizeReleaseCiRun(options)); + const sleep = + overrides.sleep ?? + ((milliseconds) => + new Promise((complete) => { + setTimeout(complete, milliseconds); + })); + let previousFingerprint; + while (true) { + const parent = fetchParent(); + const fingerprint = releaseCiWatchFingerprint(parent); + if (fingerprint !== previousFingerprint) { + summarize(); + previousFingerprint = fingerprint; + } + if (parent.status === "completed") { + if (parent.conclusion !== "success") { + throw new Error( + `full release run ${options.runId} completed with ${parent.conclusion || "no conclusion"}`, + ); + } + return; + } + await sleep(options.intervalMs); + } +} + +async function main() { + let options; + try { + options = parseReleaseCiSummaryArgs(process.argv.slice(2)); + } catch (error) { + printUsage(); + console.error(error instanceof Error ? error.message : String(error)); + process.exit(2); + } + const { repository, runId } = options; + + if (options.validate) { + try { + const evidence = validateReleaseRunEvidence({ + manifestPath: options.manifestPath, + repository, + runId, + trustedWorkflowRef: options.trustedWorkflowRef, + verifierSourceContent: options.verifierSourceFile + ? readFileSync(options.verifierSourceFile) + : undefined, + verifierSourceSha: options.verifierSourceSha, + }); + console.log(JSON.stringify(evidence, null, options.json ? 2 : 0)); + } catch (error) { + const failure = { + error: error instanceof Error ? error.message : String(error), + schema: RELEASE_EVIDENCE_SCHEMA, + valid: false, + }; + if (options.json) { + console.log(JSON.stringify(failure, null, 2)); + } else { + console.error(failure.error); + } + process.exit(1); + } + return; + } + if (options.watch) { + await watchReleaseCiRun(options); + return; + } + + const core = rate(); + if (core) { + const reset = new Date(core.reset * 1000).toISOString(); + console.log(`rate: remaining=${core.remaining}/${core.limit} reset=${reset}`); + if (core.remaining < 20) { + console.error("rate too low for CI summary; wait for reset before polling"); + process.exit(3); + } + } + + const parent = jsonGh([ + "run", + "view", + runId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding(parent, githubRestJson(`actions/runs/${runId}`, repository), runId); + + console.log(`parent: ${runId} ${parent.status}/${parent.conclusion || "none"}`); + console.log(`workflow-ref: ${parent.headBranch}`); + console.log(`workflow-sha: ${parent.headSha}`); + console.log(`url: ${parent.url}`); + + for (const job of parent.jobs ?? []) { + const marker = job.conclusion || job.status; + console.log(`parent-job: ${marker} ${job.name}`); + } + + const currentManifestRaw = tryDownloadParentManifest(runId, parent.attempt, repository); + let children; + if (currentManifestRaw) { + const currentManifest = validateParentManifest(currentManifestRaw, { + runAttempt: parent.attempt, + runId, + workflowRef: parent.headBranch, + workflowSha: parent.headSha, + }); + console.log(`candidate-sha: ${currentManifest.targetSha}`); + console.log(`manifest-run: ${currentManifest.runId}/${currentManifest.runAttempt}`); + + let sourceManifest = currentManifest; + let sourceParent = parent; + if (currentManifest.evidenceReuse) { + const rootRunId = currentManifest.evidenceReuse.runId; + const rootParent = jsonGh([ + "run", + "view", + rootRunId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding( + rootParent, + githubRestJson(`actions/runs/${rootRunId}`, repository), + rootRunId, + ); + if (rootParent.status !== "completed" || rootParent.conclusion !== "success") { + throw new Error(`evidence root run is not completed/success: ${rootRunId}`); + } + const rootManifestRaw = tryDownloadParentManifest(rootRunId, rootParent.attempt, repository); + if (!rootManifestRaw) { + throw new Error(`evidence root manifest is unavailable: ${rootRunId}`); + } + const rootManifest = validateParentManifest(rootManifestRaw, { + runAttempt: rootParent.attempt, + runId: rootRunId, + workflowRef: rootParent.headBranch, + workflowSha: rootParent.headSha, + }); + + const selectedRunId = currentManifest.evidenceReuse.selectedRunId; + let selectedManifest = rootManifest; + if (selectedRunId !== rootRunId) { + const selectedParent = jsonGh([ + "run", + "view", + selectedRunId, + "--repo", + repository, + "--json", + "status,conclusion,attempt,headBranch,headSha,url,jobs", + ]); + validateParentRunBinding( + selectedParent, + githubRestJson(`actions/runs/${selectedRunId}`, repository), + selectedRunId, + ); + if (selectedParent.status !== "completed" || selectedParent.conclusion !== "success") { + throw new Error(`selected evidence run is not completed/success: ${selectedRunId}`); + } + const selectedManifestRaw = tryDownloadParentManifest( + selectedRunId, + selectedParent.attempt, + repository, + ); + if (!selectedManifestRaw) { + throw new Error(`selected evidence manifest is unavailable: ${selectedRunId}`); + } + selectedManifest = validateParentManifest(selectedManifestRaw, { + runAttempt: selectedParent.attempt, + runId: selectedRunId, + workflowRef: selectedParent.headBranch, + workflowSha: selectedParent.headSha, + }); + } + + const evidenceSha = validateEvidenceReuseChain( + currentManifest, + selectedManifest, + rootManifest, + ); + sourceManifest = rootManifest; + sourceParent = rootParent; + console.log(`evidence-selected-run: ${selectedRunId}`); + console.log(`evidence-root-run: ${rootRunId}`); + console.log(`evidence-sha: ${evidenceSha}`); + } + + const expectedChildren = expectedSelectedChildDispatches( + sourceManifest.runId, + sourceManifest.runAttempt, + sourceManifest.workflowRef, + requiredChildKeysForRerunGroup(sourceManifest.rerunGroup), + ); + const sourceParentJobs = findParentJobsAll(sourceManifest.runId, repository); + children = manifestChildEntries( + sourceManifest, + expectedChildren, + requiredChildKeysForRerunGroup(sourceManifest.rerunGroup), + ).map(({ child, runId: childRunId }) => { + const run = githubRestJson(`actions/runs/${childRunId}`, repository); + const originAttempt = resolveManifestChildOriginAttempt( + run, + child, + sourceManifest, + sourceParentJobs, + ); + if (originAttempt === undefined) { + throw new Error(`manifest child dispatch tuple mismatch: ${child.name}`); + } + const parentJob = selectManifestParentJob( + sourceParentJobs, + child, + sourceManifest, + originAttempt, + ); + const validatedRun = validateManifestChildRun( + run, + child, + childRunId, + { ...sourceManifest, workflowSha: sourceParent.headSha }, + sourceParentJobs, + parentJobLog(parentJob.id, repository), + repository, + ); + if (child.manifestKey === "productPerformance") { + validatePerformanceArtifactOnlyJobs( + findParentJobsAll(childRunId, repository), + run.run_attempt, + ); + } + return { child, run: validatedRun }; + }); + } else { + console.log("candidate-sha: unavailable (release validation manifest not uploaded)"); + if (parent.status === "completed" && parent.conclusion === "success") { + throw new Error("successful parent run is missing its release validation manifest"); + } + const selectedKeys = selectedChildKeys(parent.jobs ?? []); + children = expectedSelectedChildDispatches( + runId, + parent.attempt, + parent.headBranch, + selectedKeys, + ) + .map((child) => { + const run = findExactChildRun(child, repository); + if (!run) { + console.log( + `child-missing: ${child.name} title=${child.displayTitle} branch=${child.headBranch}`, + ); + } + return { child, run }; + }) + .filter((entry) => entry.run); + } + if (children.length === 0) { + console.log("children: none found yet"); + return; + } + + console.log("children:"); + for (const { child, run } of children) { + console.log( + `child: ${run.id} ${child.name} ${run.status}/${run.conclusion || "none"} branch=${run.head_branch} workflow_sha=${run.head_sha}`, + ); + console.log(`child-url: ${run.html_url}`); + } +} + +if (process.argv[1]?.endsWith("release-ci-summary.mjs")) { + await main().catch( + /** @param {unknown} error */ (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, + ); +} diff --git a/scripts/testbox-lease-freshness.mjs b/scripts/testbox-lease-freshness.mjs new file mode 100644 index 000000000000..84ce5915f9f9 --- /dev/null +++ b/scripts/testbox-lease-freshness.mjs @@ -0,0 +1,131 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + renameSync, + statSync, + writeFileSync, +} from "node:fs"; +import { resolve } from "node:path"; + +const STATE_VERSION = 1; +const DEPENDENCY_INPUTS = ["package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc"]; +const ENVIRONMENT_INPUTS = [ + ".crabbox.yaml", + ".github/workflows/ci-check-testbox.yml", + ".node-version", + "scripts/crabbox-wrapper.mjs", +]; + +function optionValue(args, name, fallback = "") { + const shortName = name.replace(/^--/u, "-"); + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (argument === name || argument === shortName) { + return args[index + 1] ?? fallback; + } + if (argument.startsWith(`${name}=`) || argument.startsWith(`${shortName}=`)) { + return argument.slice(argument.indexOf("=") + 1); + } + } + return fallback; +} + +function git(repoRoot, args) { + return execFileSync("git", ["-C", repoRoot, ...args], { + encoding: "utf8", + env: { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null" }, + }).trim(); +} + +function listFiles(path) { + if (!existsSync(path)) { + return []; + } + if (statSync(path).isFile()) { + return [path]; + } + return readdirSync(path, { withFileTypes: true }) + .flatMap((entry) => listFiles(resolve(path, entry.name))) + .toSorted((left, right) => left.localeCompare(right)); +} + +function digestInputs(repoRoot, inputs) { + const hash = createHash("sha256"); + for (const input of inputs) { + for (const path of listFiles(resolve(repoRoot, input))) { + hash.update(path.slice(repoRoot.length)); + hash.update("\0"); + hash.update(readFileSync(path)); + hash.update("\0"); + } + } + return hash.digest("hex"); +} + +export function buildTestboxLeaseFingerprint(repoRoot, args) { + let baseSha; + try { + baseSha = git(repoRoot, ["merge-base", "HEAD", "refs/remotes/origin/main"]); + } catch { + baseSha = git(repoRoot, ["rev-parse", "HEAD"]); + } + return { + version: STATE_VERSION, + baseSha, + headSha: git(repoRoot, ["rev-parse", "HEAD"]), + workingTreeClean: git(repoRoot, ["status", "--porcelain=v1"]) === "", + dependencyDigest: digestInputs(repoRoot, [...DEPENDENCY_INPUTS, "patches"]), + environmentDigest: digestInputs(repoRoot, ENVIRONMENT_INPUTS), + workflow: optionValue(args, "--blacksmith-workflow", ".github/workflows/ci-check-testbox.yml"), + job: optionValue(args, "--blacksmith-job", "check"), + ref: optionValue(args, "--blacksmith-ref", "main"), + }; +} + +export function testboxLeaseStaleReasons(saved, current) { + if (!saved || saved.version !== STATE_VERSION) { + return ["state schema"]; + } + return ["baseSha", "dependencyDigest", "environmentDigest", "workflow", "job", "ref"].filter( + (key) => saved[key] !== current[key], + ); +} + +export function prepareTestboxLeaseFreshness({ args, env, provider, repoRoot }) { + const id = optionValue(args, "--id"); + if (provider !== "blacksmith-testbox" || args[0] !== "run" || !id?.startsWith("tbx_")) { + return null; + } + const configuredStateDir = env.OPENCLAW_TESTBOX_LEASE_STATE_DIR?.trim(); + if (env.VITEST && !configuredStateDir) { + return null; + } + const stateDir = resolve(configuredStateDir || resolve(repoRoot, ".crabbox", "testbox-leases")); + const path = resolve(stateDir, `${id}.json`); + const current = buildTestboxLeaseFingerprint(repoRoot, args); + if (existsSync(path)) { + const saved = JSON.parse(readFileSync(path, "utf8")); + const staleReasons = testboxLeaseStaleReasons(saved, current); + if (staleReasons.length > 0 && env.OPENCLAW_TESTBOX_ALLOW_STALE !== "1") { + throw new Error( + `Testbox ${id} is stale (${staleReasons.join(", ")}); stop it and warm a fresh lease, or set OPENCLAW_TESTBOX_ALLOW_STALE=1 for an intentional diagnostic reuse`, + ); + } + return { current, path }; + } + return { current, path }; +} + +export function recordTestboxLeaseFreshness(prepared) { + if (!prepared) { + return; + } + mkdirSync(resolve(prepared.path, ".."), { recursive: true }); + const temporaryPath = `${prepared.path}.tmp-${process.pid}`; + writeFileSync(temporaryPath, `${JSON.stringify(prepared.current, null, 2)}\n`); + renameSync(temporaryPath, prepared.path); +} diff --git a/scripts/validate-full-release-validation-evidence.mjs b/scripts/validate-full-release-validation-evidence.mjs index be0d3d848920..683b6268d439 100755 --- a/scripts/validate-full-release-validation-evidence.mjs +++ b/scripts/validate-full-release-validation-evidence.mjs @@ -49,6 +49,7 @@ export function validateFullReleaseValidationEvidence({ expectedTargetSha, expectedWorkflowBranch, isTrustedMainAncestor, + validateEvidenceReuseStrictly, }) { const run = normalizeFullReleaseValidationRun(rawRun); const checks = [ @@ -136,17 +137,87 @@ export function validateFullReleaseValidationEvidence({ `SHA-pinned validation target ref mismatch: expected ${expectedTargetSha}, got ${manifest.targetRef ?? ""}.`, ); } - if (Object.hasOwn(manifest, "evidenceReuse")) { - throw new Error("SHA-pinned validation evidence must not reuse another validation run."); - } if (!isTrustedMainAncestor?.(run.headSha)) { throw new Error( `SHA-pinned validation workflow ${run.headSha} is not reachable from current main.`, ); } + if (Object.hasOwn(manifest, "evidenceReuse")) { + const reuse = manifest.evidenceReuse; + if ( + !reuse || + typeof reuse !== "object" || + Array.isArray(reuse) || + reuse.policy !== "exact-target-full-validation-v1" || + reuse.evidenceSha !== expectedTargetSha || + !Array.isArray(reuse.changedPaths) || + reuse.changedPaths.length !== 0 || + !/^[1-9][0-9]*$/u.test(String(reuse.runId ?? "")) || + !/^[1-9][0-9]*$/u.test(String(reuse.selectedRunId ?? "")) + ) { + throw new Error("SHA-pinned validation evidence reuse is invalid."); + } + if (typeof validateEvidenceReuseStrictly !== "function") { + throw new Error("SHA-pinned validation evidence reuse requires strict chain validation."); + } + const strictEvidence = validateEvidenceReuseStrictly({ + repository: expectedRepository, + runId: String(expectedRunId), + targetSha: expectedTargetSha, + }); + if ( + strictEvidence?.schema !== "openclaw.release-validation-evidence/v3" || + strictEvidence.valid !== true || + String(strictEvidence.current?.runId ?? "") !== String(expectedRunId) || + strictEvidence.current?.targetSha !== expectedTargetSha || + strictEvidence.root?.targetSha !== expectedTargetSha || + strictEvidence.evidenceReuse?.evidenceSha !== expectedTargetSha || + String(strictEvidence.evidenceReuse?.rootRunId ?? "") !== String(reuse.runId) || + String(strictEvidence.evidenceReuse?.selectedRunId ?? "") !== String(reuse.selectedRunId) || + strictEvidence.conclusions?.allRequiredSucceeded !== true + ) { + throw new Error("SHA-pinned validation evidence reuse failed strict chain validation."); + } + } return { run, source: "sha-pinned-main" }; } +export function runStrictReleaseEvidenceValidation({ + repository, + runId, + validatorFile = fileURLToPath(new URL("./release-ci-summary.mjs", import.meta.url)), + verifierSourceSha, +}) { + const verifierSourceArgs = verifierSourceSha + ? ["--verifier-source-sha", verifierSourceSha, "--verifier-source-file", validatorFile] + : []; + const result = spawnSync( + process.execPath, + [ + validatorFile, + "--validate-run", + String(runId), + "--repo", + repository, + "--trusted-workflow-ref", + "main", + "--json", + ...verifierSourceArgs, + ], + { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }, + ); + if (result.status !== 0) { + throw new Error( + `Strict release evidence validation failed: ${result.stderr?.trim() || result.signal || result.status}.`, + ); + } + try { + return JSON.parse(result.stdout); + } catch { + throw new Error("Strict release evidence validator returned invalid JSON."); + } +} + function gitIsAncestor(ancestor, target) { const result = spawnSync( "git", @@ -180,6 +251,15 @@ function main() { expectedTargetSha: process.env.EXPECTED_SHA, expectedWorkflowBranch: process.env.EXPECTED_WORKFLOW_BRANCH, isTrustedMainAncestor: (sha) => gitIsAncestor(sha, trustedMainRef), + validateEvidenceReuseStrictly: ({ repository, runId }) => + runStrictReleaseEvidenceValidation({ + repository, + runId, + validatorFile: + process.env.STRICT_VALIDATOR_FILE ?? + fileURLToPath(new URL("./release-ci-summary.mjs", import.meta.url)), + verifierSourceSha: process.env.GITHUB_SHA, + }), }); console.log( `Using full release validation run ${result.run.databaseId} (${result.source}): ${result.run.url}`, diff --git a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts index b3d26f3b418a..1e7132284827 100644 --- a/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/package-openclaw-for-docker.e2e.test.ts @@ -90,9 +90,11 @@ describe("package-openclaw-for-docker", () => { ".artifacts/docker/pack.json", "--source-dir", "/repo", + "--allow-unreleased-changelog", "--skip-build", ]), ).toEqual({ + allowUnreleasedChangelog: true, outputDir: ".artifacts/docker", outputName: "openclaw-current.tgz", packJson: ".artifacts/docker/pack.json", @@ -116,6 +118,11 @@ describe("package-openclaw-for-docker", () => { ["--output-dir", ["--output-dir", "one", "--output-dir=two"]], ["--output-name", ["--output-name", "one.tgz", "--output-name=two.tgz"]], ["--pack-json", ["--pack-json", "one.json", "--pack-json=two.json"]], + [ + "--allow-unreleased-changelog", + ["--allow-unreleased-changelog", "--allow-unreleased-changelog"], + ], + ["--pnpm-pack", ["--pnpm-pack", "--pnpm-pack"]], ["--source-dir", ["--source-dir", "/repo-a", "--source-dir=/repo-b"]], ["--skip-build", ["--skip-build", "--skip-build"]], ] satisfies Array<[string, string[]]>; @@ -387,6 +394,75 @@ describe("package-openclaw-for-docker", () => { ]); }); + it("packages Unreleased notes for explicitly non-publish stable artifacts", async () => { + const sourceDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-unreleased-package-")); + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-unreleased-output-")); + const sourceChangelog = [ + "# Changelog", + "", + "## Unreleased", + "### Fixes", + "- Pending release notes with enough detail.", + "", + "## 2026.5.28", + "- Previous release notes with enough detail.", + "", + ].join("\n"); + fs.writeFileSync( + path.join(sourceDir, "package.json"), + '{"name":"openclaw","version":"2026.5.29"}\n', + ); + fs.writeFileSync(path.join(sourceDir, "CHANGELOG.md"), sourceChangelog); + + try { + const tarball = await packOpenClawPackageForDocker(sourceDir, outputDir, { + allowUnreleasedChangelog: true, + prepareBundledAiRuntime: skipBundledAiRuntime, + runCaptureImpl: async () => { + const packagedChangelog = fs.readFileSync(path.join(sourceDir, "CHANGELOG.md"), "utf8"); + expect(packagedChangelog).toContain("## Unreleased"); + expect(packagedChangelog).not.toContain("## 2026.5.28"); + const packedPath = path.join(outputDir, "openclaw-2026.5.29.tgz"); + fs.writeFileSync(packedPath, "package"); + return "openclaw-2026.5.29.tgz\n"; + }, + }); + + expect(tarball).toBe(path.join(outputDir, "openclaw-2026.5.29.tgz")); + expect(fs.readFileSync(path.join(sourceDir, "CHANGELOG.md"), "utf8")).toBe(sourceChangelog); + } finally { + fs.rmSync(sourceDir, { recursive: true, force: true }); + fs.rmSync(outputDir, { recursive: true, force: true }); + } + }); + + it("uses pnpm pack when requested", async () => { + const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-pnpm-pack-")); + const calls: string[] = []; + const packedPath = path.join(outputDir, "openclaw-2026.5.28.tgz"); + + try { + const tarball = await packOpenClawPackageForDocker("/repo", outputDir, { + pnpmPack: true, + prepareBundledAiRuntime: skipBundledAiRuntime, + prepareChangelog: async () => {}, + restoreChangelog: async () => {}, + runCaptureImpl: async (command: string, args: string[], cwd: string) => { + calls.push(`${command}:${args.join(" ")}:${cwd}`); + fs.writeFileSync(packedPath, "package"); + return `${packedPath}\n`; + }, + }); + + expect(tarball).toBe(packedPath); + expect(calls).toEqual([ + `pnpm:pack --silent --config.ignore-scripts=true --pack-destination ${outputDir}:/repo`, + ]); + } finally { + fs.rmSync(outputDir, { force: true, recursive: true }); + } + }); + it("writes npm pack metadata for renamed package artifacts", async () => { const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-docker-pack-json-")); const packJsonPath = path.join(outputDir, "pack.json"); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 47610640f485..562a1f08aa4f 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -1117,6 +1117,7 @@ describe("ci workflow guards", () => { expect(smokeRunStep.run).toContain("createQaSmokeCiMatrix"); expect(smokeRunStep.run).toContain("--qa-profile smoke-ci"); expect(smokeRunStep.run).toContain("--concurrency 8"); + expect(smokeRunStep.run).toContain("--allow-unreleased-changelog"); expect(smokeRunStep.run).toContain('scenario_args+=(--scenario "$scenario_id")'); expect(smokeRunStep.run).not.toContain("--category"); expect(smokeRunStep.run).not.toContain("--allow-failures"); diff --git a/test/scripts/crabbox-wrapper.test.ts b/test/scripts/crabbox-wrapper.test.ts index 9efd78f7f9b5..1906d780af93 100644 --- a/test/scripts/crabbox-wrapper.test.ts +++ b/test/scripts/crabbox-wrapper.test.ts @@ -760,6 +760,8 @@ describe("scripts/crabbox-wrapper", () => { "--id", "tbx_owned", "--", + "env", + "CI=true", "echo ok", ]); }); @@ -782,10 +784,36 @@ describe("scripts/crabbox-wrapper", () => { "--id", "blue-hermit", "--", + "env", + "CI=true", "echo ok", ]); }); + it("exports CI for complete Blacksmith Testbox shell snippets", () => { + const result = runWrapper( + "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n", + [ + "run", + "--provider", + "blacksmith-testbox", + "--shell", + "--", + "cd packages && pnpm install && pnpm build", + ], + ); + + expect(result.status).toBe(0); + expect(parseFakeCrabboxOutput(result).args).toEqual([ + "run", + "--provider", + "blacksmith-testbox", + "--shell", + "--", + "export CI=true; cd packages && pnpm install && pnpm build", + ]); + }); + it("only forces the short local-container Docker work root on Linux", () => { const result = runWrapper( "provider: hetzner, aws, local-container, blacksmith-testbox, or cloudflare\n", @@ -3166,7 +3194,9 @@ describe("scripts/crabbox-wrapper", () => { expect(result.error).toBeUndefined(); expect(result.status).toBe(0); expect(result.stderr).not.toContain("could not parse provider list"); - expect(result.stderr).not.toContain("selected binary failed basic --version/--help sanity checks"); + expect(result.stderr).not.toContain( + "selected binary failed basic --version/--help sanity checks", + ); expect(result.stderr).toContain( "providers=hetzner,aws,local-container,blacksmith-testbox,cloudflare", ); diff --git a/test/scripts/find-reusable-release-validation.test.ts b/test/scripts/find-reusable-release-validation.test.ts index dc1d9c38a493..fc75854cf4df 100644 --- a/test/scripts/find-reusable-release-validation.test.ts +++ b/test/scripts/find-reusable-release-validation.test.ts @@ -1,7 +1,7 @@ -// Covers the release validation evidence-reuse resolver used by -// full-release-validation.yml to skip lanes on release-metadata-only deltas. +// Covers policy selection layered on the strict normalized release evidence +// produced by release-ci-summary.mjs. Topology validation belongs there. import { execFileSync, spawnSync } from "node:child_process"; -import { chmodSync, mkdirSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createTempDirTracker } from "../helpers/temp-dir.js"; @@ -9,6 +9,103 @@ import { createTempDirTracker } from "../helpers/temp-dir.js"; const SCRIPT_PATH = join(process.cwd(), "scripts/github/find-reusable-release-validation.sh"); const tempDirs = createTempDirTracker(); +const REPOSITORY = "openclaw/openclaw"; +const PRODUCER_SHA = "0".repeat(40); +const VERIFIER_SHA = "c".repeat(40); +const DEFAULT_INPUTS = { + provider: "openai", + mode: "both", + liveSuiteFilter: "", + crossOsSuiteFilter: "", + releasePackageSpec: "", + packageAcceptancePackageSpec: "", + codexPluginSpec: "", +}; + +interface ParentTuple { + artifact: { + digest: string; + id: string; + name: string; + runAttempt: number; + sizeInBytes: number; + }; + conclusion: string; + manifest: Record; + manifestVersion: number; + runAttempt: number; + runId: string; + status: string; + targetSha: string; + url: string; + producerOnTrustedMainLineage: boolean; + workflowFullRef: string; + workflowPath: string; + workflowQualifiedPath: string; + workflowRef: string; + workflowRefProof: string; + workflowRefType: string; + workflowRunPath: string; + workflowSha: string; +} + +interface ChildTuple { + conclusion: string; + dispatchNonce: string; + displayTitle: string; + event: string; + headBranch: string; + parentJobId: string; + path: string; + reportPublication?: string; + role: string; + runAttempt: number; + runId: string; + sourceParentAttempt: number; + sourceParentRunId: string; + status: string; + url: string; + workflowSha: string; +} + +interface NormalizedEvidence { + children: ChildTuple[]; + conclusions: { + allRequiredSucceeded: boolean; + children: Record; + current: string; + root: string; + }; + controls: Record; + current: ParentTuple; + directRoot: boolean; + evidenceReuse: Record | null; + manifest: Record; + releaseProfile: string; + repository: string; + rerunGroup: string; + root: ParentTuple; + runReleaseSoak: boolean; + schema: string; + producerOnTrustedMainLineage: boolean; + trustedWorkflowFullRef: string; + trustedWorkflowRef: string; + valid: boolean; + validationInputs: Record | null; + verifier: { + schemaVersion: number; + script: string; + scriptSha256: string; + sourceSha: string | null; + }; +} + +interface RunFixture { + exitCode?: number; + record?: NormalizedEvidence; + runId: string; +} + afterEach(() => { tempDirs.cleanup(); }); @@ -43,12 +140,11 @@ function plistFor(shortVersion: string, buildVersion: string): string { ].join("\n"); } -function createRepoPair(options: { plistBuildVersion?: string } = {}) { +function createRepo(options: { plistBuildVersion?: string } = {}) { const origin = tempDirs.make("evidence-reuse-origin-"); git(origin, ["init", "-q", "-b", "main"]); - git(origin, ["config", "user.email", "test-user"]); + git(origin, ["config", "user.email", "test-user@example.invalid"]); git(origin, ["config", "user.name", "Test User"]); - // Allows depth-1 fetches of the prior evidence SHA, matching GitHub remotes. git(origin, ["config", "uploadpack.allowReachableSHA1InWant", "true"]); writeFileSync( join(origin, "package.json"), @@ -59,12 +155,13 @@ function createRepoPair(options: { plistBuildVersion?: string } = {}) { join(origin, "apps/macos/Sources/OpenClaw/Resources/Info.plist"), plistFor("2026.7.1", options.plistBuildVersion ?? "2026070100"), ); + mkdirSync(join(origin, "docs/install"), { recursive: true }); + writeFileSync(join(origin, "docs/install/updating.md"), "# Updating\n"); writeFileSync(join(origin, "CHANGELOG.md"), "# Changelog\n"); writeFileSync(join(origin, "index.ts"), "export const value = 1;\n"); git(origin, ["add", "-A"]); git(origin, ["-c", "commit.gpgSign=false", "commit", "-qm", "seed"]); - const priorSha = git(origin, ["rev-parse", "HEAD"]); - return { origin, priorSha }; + return { origin, priorSha: git(origin, ["rev-parse", "HEAD"]) }; } function cloneHead(origin: string): string { @@ -73,6 +170,164 @@ function cloneHead(origin: string): string { return clone; } +function normalizedEvidence(options: { + producerSha?: string; + releaseProfile?: string; + runId?: string; + soak?: boolean; + targetSha: string; + validationInputs?: Record | null; + verifierSha?: string | null; + workflowRef?: string; +}): NormalizedEvidence { + const runId = options.runId ?? "111"; + const producerSha = options.producerSha ?? PRODUCER_SHA; + const releaseProfile = options.releaseProfile ?? "full"; + const soak = options.soak ?? true; + const workflowRef = options.workflowRef ?? "main"; + const workflowFullRef = `refs/heads/${workflowRef}`; + const shaPinned = workflowRef.startsWith("release-ci/"); + const validationInputs = + options.validationInputs === undefined ? DEFAULT_INPUTS : options.validationInputs; + const manifest = { + version: shaPinned ? 3 : 2, + workflowName: "Full Release Validation", + workflowRef, + workflowSha: producerSha, + workflowFullRef, + workflowRefType: "branch", + runId, + runAttempt: "2", + targetRef: "release/2026.7.1", + targetSha: options.targetSha, + rerunGroup: "all", + releaseProfile, + runReleaseSoak: String(soak), + validationInputs, + controls: { + performanceBlocking: true, + performanceReportPublication: "artifact-only", + stableSoakRequired: releaseProfile === "stable" || releaseProfile === "full", + }, + childRuns: { + normalCi: "201", + npmTelegram: "", + pluginPrerelease: "202", + releaseChecks: "203", + productPerformance: { + blocking: true, + conclusion: "success", + runId: "204", + }, + }, + }; + const root: ParentTuple = { + artifact: { + digest: `sha256:${"a".repeat(64)}`, + id: "9001", + name: `full-release-validation-${runId}-2`, + runAttempt: 2, + sizeInBytes: 4096, + }, + conclusion: "success", + manifest, + manifestVersion: shaPinned ? 3 : 2, + runAttempt: 2, + runId, + status: "completed", + targetSha: options.targetSha, + url: `https://example.test/runs/${runId}`, + producerOnTrustedMainLineage: true, + workflowFullRef, + workflowPath: ".github/workflows/full-release-validation.yml", + workflowQualifiedPath: `.github/workflows/full-release-validation.yml@${workflowFullRef}`, + workflowRef, + workflowRefProof: shaPinned + ? "manifest-v3-sha-pinned-main-ancestry" + : "legacy-v2-main-ancestry", + workflowRefType: "branch", + workflowRunPath: shaPinned + ? `.github/workflows/full-release-validation.yml@${workflowFullRef}` + : ".github/workflows/full-release-validation.yml", + workflowSha: producerSha, + }; + const roles = [ + ["normalCi", "201", 1, 1, "CI", "ci.yml", "-ci"], + [ + "pluginPrerelease", + "202", + 2, + 1, + "Plugin Prerelease", + "plugin-prerelease.yml", + "-plugin-prerelease", + ], + [ + "releaseChecks", + "203", + 1, + 2, + "OpenClaw Release Checks", + "openclaw-release-checks.yml", + "-release-checks", + ], + ["productPerformance", "204", 3, 2, "OpenClaw Performance", "openclaw-performance.yml", ""], + ] as const; + const children = roles.map( + ([role, childRunId, runAttempt, sourceParentAttempt, name, workflow, suffix]) => ({ + conclusion: "success", + dispatchNonce: `full-release-validation-${runId}-${sourceParentAttempt}${suffix}`, + displayTitle: `${name} full-release-validation-${runId}-${sourceParentAttempt}${suffix}`, + event: "workflow_dispatch", + headBranch: workflowRef, + parentJobId: `job-${role}`, + path: `.github/workflows/${workflow}`, + role, + runAttempt, + runId: childRunId, + sourceParentAttempt, + sourceParentRunId: runId, + status: "completed", + url: `https://example.test/runs/${childRunId}`, + workflowSha: producerSha, + ...(role === "productPerformance" ? { reportPublication: "artifact-only" } : {}), + }), + ); + return { + children, + conclusions: { + allRequiredSucceeded: true, + children: Object.fromEntries(children.map((child) => [child.role, child.conclusion])), + current: "success", + root: "success", + }, + controls: { + performanceReportPublication: "artifact-only", + }, + current: structuredClone(root), + directRoot: true, + evidenceReuse: null, + manifest, + releaseProfile, + repository: REPOSITORY, + rerunGroup: "all", + root, + runReleaseSoak: soak, + schema: "openclaw.release-validation-evidence/v3", + producerOnTrustedMainLineage: true, + trustedWorkflowFullRef: "refs/heads/main", + trustedWorkflowRef: "main", + valid: true, + validationInputs, + verifier: { + schemaVersion: 3, + script: "scripts/release-ci-summary.mjs", + scriptSha256: "b".repeat(64), + sourceSha: options.verifierSha === undefined ? VERIFIER_SHA : options.verifierSha, + }, + }; +} + const FAKE_GH = `#!/usr/bin/env bash set -euo pipefail [[ "\${1:-}" == "api" ]] || { echo "unexpected gh command: $*" >&2; exit 1; } @@ -87,10 +342,6 @@ while [[ $# -gt 0 ]]; do esac done fixture="\${FAKE_GH_FIXTURES}/$(printf '%s' "$endpoint" | tr '/?' '__')" -if [[ "$endpoint" == */zip ]]; then - [[ -f "\${fixture}.bin" ]] || { echo "no fixture for $endpoint" >&2; exit 1; } - exec cat "\${fixture}.bin" -fi [[ -f "\${fixture}.json" ]] || { echo "no fixture for $endpoint" >&2; exit 1; } if [[ -n "$jq_expr" ]]; then exec jq -r "$jq_expr" "\${fixture}.json" @@ -98,16 +349,42 @@ fi exec cat "\${fixture}.json" `; -interface FixtureOptions { - runId?: string; - headSha: string; - manifest?: Record; - compare?: { base: string; head: string; status: string; files: string[] } | undefined; - childRunStates?: Record; +const FAKE_VALIDATOR = `#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const runIndex = process.argv.indexOf("--validate-run"); +const repoIndex = process.argv.indexOf("--repo"); +const trustedRefIndex = process.argv.indexOf("--trusted-workflow-ref"); +if ( + runIndex < 0 || + repoIndex < 0 || + trustedRefIndex < 0 || + process.argv[repoIndex + 1] !== "openclaw/openclaw" || + process.argv[trustedRefIndex + 1] !== "main" +) { + console.error("validator invocation contract mismatch"); + process.exit(2); +} +const fixture = JSON.parse( + readFileSync(join(process.env.FAKE_VALIDATOR_FIXTURES, \`\${process.argv[runIndex + 1]}.json\`), "utf8"), +); +if (fixture.exitCode) { + console.error("fixture validator rejection"); + process.exit(fixture.exitCode); +} +process.stdout.write(\`\${JSON.stringify(fixture.record)}\\n\`); +`; + +function fixtureName(fixtures: string, endpoint: string): string { + return join(fixtures, `${endpoint.replaceAll(/[/?]/gu, "_")}.json`); } -function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: string } { - const runId = options.runId ?? "111"; +function setUpFixtures(runs: RunFixture[]): { + binDir: string; + fixtures: string; + validatorPath: string; +} { const root = tempDirs.make("evidence-reuse-fixtures-"); const fixtures = join(root, "fixtures"); const binDir = join(root, "bin"); @@ -115,94 +392,46 @@ function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: str mkdirSync(binDir, { recursive: true }); writeFileSync(join(binDir, "gh"), FAKE_GH); chmodSync(join(binDir, "gh"), 0o755); + const validatorPath = join(root, "validator.mjs"); + writeFileSync(validatorPath, FAKE_VALIDATOR); writeFileSync( - join( + fixtureName( fixtures, - "repos_openclaw_openclaw_actions_workflows_full-release-validation.yml_runs.json", + "repos/openclaw/openclaw/actions/workflows/full-release-validation.yml/runs", ), - JSON.stringify({ - workflow_runs: [ - { - id: Number(runId), - html_url: `https://example.test/runs/${runId}`, - head_sha: options.headSha, - }, - ], - }), + JSON.stringify({ workflow_runs: runs.map(({ runId }) => ({ id: Number(runId) })) }), ); - if (options.manifest) { + for (const run of runs) { writeFileSync( - join(fixtures, `repos_openclaw_openclaw_actions_runs_${runId}_artifacts_per_page=100.json`), - JSON.stringify({ - artifacts: [{ id: 999, name: `full-release-validation-${runId}`, expired: false }], - }), - ); - const manifestDir = join(root, "manifest"); - mkdirSync(manifestDir, { recursive: true }); - writeFileSync( - join(manifestDir, "full-release-validation-manifest.json"), - JSON.stringify(options.manifest), - ); - execFileSync( - "zip", - [ - "-q", - "-j", - join(root, "manifest.zip"), - join(manifestDir, "full-release-validation-manifest.json"), - ], - { - encoding: "utf8", - }, - ); - execFileSync("cp", [ - join(root, "manifest.zip"), - join(fixtures, "repos_openclaw_openclaw_actions_artifacts_999_zip.bin"), - ]); - } - if (options.compare) { - writeFileSync( - join( - fixtures, - `repos_openclaw_openclaw_compare_${options.compare.base}...${options.compare.head}.json`, - ), - JSON.stringify({ - status: options.compare.status, - files: options.compare.files.map((filename) => ({ filename })), - }), + join(fixtures, `${run.runId}.json`), + JSON.stringify({ exitCode: run.exitCode ?? 0, record: run.record }), ); } - for (const [childRunId, state] of Object.entries(options.childRunStates ?? {})) { - const [status, conclusion] = state.split("/"); - writeFileSync( - join(fixtures, `repos_openclaw_openclaw_actions_runs_${childRunId}.json`), - JSON.stringify({ status, conclusion }), - ); - } - return { fixtures, binDir }; + return { binDir, fixtures, validatorPath }; } -const DEFAULT_INPUTS = { - provider: "openai", - mode: "both", - liveSuiteFilter: "", - crossOsSuiteFilter: "", - releasePackageSpec: "", - packageAcceptancePackageSpec: "", - codexPluginSpec: "", -}; - function runResolver(args: { - repoDir: string; - targetSha: string; - workflowSha: string; - releaseProfile: string; - runReleaseSoak?: string; - inputs?: Record; - fixtures: string; binDir: string; + fixtures: string; + inputs?: unknown; + releaseProfile?: string; + repoDir: string; + runReleaseSoak?: string; + targetSha: string; + validatorPath: string; + verifierOnMain?: boolean; + verifierSha?: string; + workflowRef?: string; }) { + const verifierSha = args.verifierSha ?? VERIFIER_SHA; + writeFileSync( + fixtureName(args.fixtures, `repos/${REPOSITORY}/compare/${verifierSha}...main`), + JSON.stringify({ + merge_base_commit: { sha: args.verifierOnMain === false ? "f".repeat(40) : verifierSha }, + status: args.verifierOnMain === false ? "diverged" : "ahead", + }), + ); return spawnSync( "bash", [ @@ -210,15 +439,17 @@ function runResolver(args: { "--target-sha", args.targetSha, "--workflow-sha", - args.workflowSha, + verifierSha, + "--workflow-ref", + args.workflowRef ?? "main", "--release-profile", - args.releaseProfile, + args.releaseProfile ?? "full", "--run-release-soak", - args.runReleaseSoak ?? "false", + args.runReleaseSoak ?? "true", "--inputs-json", - JSON.stringify(args.inputs ?? DEFAULT_INPUTS), + JSON.stringify(args.inputs === undefined ? DEFAULT_INPUTS : args.inputs), "--repo", - "openclaw/openclaw", + REPOSITORY, "--repo-dir", args.repoDir, ], @@ -227,9 +458,11 @@ function runResolver(args: { encoding: "utf8", env: { ...process.env, - PATH: `${args.binDir}:${process.env.PATH}`, FAKE_GH_FIXTURES: args.fixtures, + FAKE_VALIDATOR_FIXTURES: args.fixtures, GITHUB_OUTPUT: "", + OPENCLAW_RELEASE_CI_SUMMARY_VALIDATOR: args.validatorPath, + PATH: `${args.binDir}:${process.env.PATH}`, }, }, ); @@ -248,273 +481,442 @@ function parseOutput(output: string): Record { ); } -function manifestFor(targetSha: string, overrides: Record = {}) { - return { - version: 2, - workflowName: "Full Release Validation", - runId: "111", - rerunGroup: "all", - releaseProfile: "stable", - runReleaseSoak: "true", - targetSha, - validationInputs: DEFAULT_INPUTS, - childRuns: { normalCi: "201", productPerformance: { runId: "202" } }, - ...overrides, - }; -} - -const HEALTHY_CHILDREN = { "201": "completed/success", "202": "completed/success" }; - describe("scripts/github/find-reusable-release-validation.sh", () => { - it("reuses evidence when the delta is release-metadata-only", () => { - const { origin, priorSha } = createRepoPair(); - const targetSha = commitFile( - origin, - "CHANGELOG.md", - "# Changelog\n\n- entry\n", - "docs(changelog): refresh", - ); + it("reuses strict direct-root evidence produced by a canonical SHA-pinned run", () => { + const { origin, priorSha } = createRepo(); const clone = cloneHead(origin); - // The candidate ran when the branch was at priorSha; the current dispatch - // runs from the branch tip, so the harness delta equals the target delta. - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha), - compare: { base: priorSha, head: targetSha, status: "ahead", files: ["CHANGELOG.md"] }, - childRunStates: HEALTHY_CHILDREN, + const producerSha = "d".repeat(40); + const producerRef = `release-ci/${producerSha.slice(0, 12)}-122`; + const record = normalizedEvidence({ + producerSha, + targetSha: priorSha, + workflowRef: producerRef, }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); const result = runResolver({ - repoDir: clone, - targetSha, - workflowSha: targetSha, - releaseProfile: "stable", - fixtures, binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + workflowRef: `release-ci/${VERIFIER_SHA.slice(0, 12)}-123`, }); + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ + evidence_run_id: "111", + reuse: "true", + }); + }); + + it("rejects noncanonical release refs and workflow SHAs outside trusted main", () => { + const { origin, priorSha } = createRepo(); + const clone = cloneHead(origin); + const record = normalizedEvidence({ targetSha: priorSha }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); + + const forgedRef = runResolver({ + binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + workflowRef: "release-ci/not-trusted", + }); + expect(parseOutput(forgedRef.stdout)).toMatchObject({ reuse: "false" }); + + const untrustedSha = runResolver({ + binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + verifierOnMain: false, + workflowRef: `release-ci/${VERIFIER_SHA.slice(0, 12)}-123`, + }); + expect(parseOutput(untrustedSha.stdout)).toMatchObject({ reuse: "false" }); + }); + + it("reuses pre-tooling trusted-main evidence for the exact target", () => { + const { origin, priorSha } = createRepo(); + const clone = cloneHead(origin); + const record = normalizedEvidence({ targetSha: priorSha }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); + + const result = runResolver({ + binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + }); + + expect(result.status).toBe(0); + expect(record.root.workflowSha).not.toBe(record.root.targetSha); + expect(record.verifier.sourceSha).not.toBe(record.root.workflowSha); + expect(new Set(record.children.map((child) => child.sourceParentAttempt)).size).toBe(2); const output = parseOutput(result.stdout); expect(output).toMatchObject({ - reuse: "true", - evidence_run_id: "111", + changed_path_count: "0", evidence_root_run_id: "111", + evidence_run_id: "111", evidence_sha: priorSha, - changed_path_count: "1", - changed_paths: "CHANGELOG.md", + reuse: "true", }); + expect(JSON.parse(output.changed_paths ?? "null")).toEqual([]); expect(JSON.parse(output.evidence_manifest ?? "{}")).toMatchObject({ targetSha: priorSha }); }); - it("reuses identical targets without comparing and resolves the chain root", () => { - const { origin, priorSha } = createRepoPair(); + it("accepts exact-target trusted-main evidence without a compare request", () => { + const { origin, priorSha } = createRepo(); const clone = cloneHead(origin); - // No compare fixture: an identical target must not hit the compare API. - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha, { evidenceReuse: { runId: "42" } }), - childRunStates: HEALTHY_CHILDREN, + const record = normalizedEvidence({ + producerSha: priorSha, + targetSha: priorSha, + verifierSha: priorSha, }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); const result = runResolver({ + binDir, + fixtures, repoDir: clone, targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, - binDir, + validatorPath, + verifierSha: priorSha, }); + expect(result.status).toBe(0); expect(parseOutput(result.stdout)).toMatchObject({ - reuse: "true", - evidence_run_id: "111", - evidence_root_run_id: "42", changed_path_count: "0", + changed_paths: "[]", + reuse: "true", }); }); - it("rejects deltas that touch non-metadata paths", () => { - const { origin, priorSha } = createRepoPair(); - const targetSha = commitFile( - origin, - "index.ts", - "export const value = 2;\n", - "fix: change code", - ); + it.each([ + ["beta", "beta"], + ["stable", "stable"], + ["full", "full"], + ] as const)("accepts exact profile identity %s -> %s", (priorProfile, requestedProfile) => { + const { origin, priorSha } = createRepo(); const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha), - compare: { base: priorSha, head: targetSha, status: "ahead", files: ["index.ts"] }, + const record = normalizedEvidence({ + releaseProfile: priorProfile, + targetSha: priorSha, + }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); + + const result = runResolver({ + binDir, + fixtures, + releaseProfile: requestedProfile, + repoDir: clone, + targetSha: priorSha, + validatorPath, }); - // The candidate harness matches the pinned workflow SHA; only the target - // delta is non-metadata here. + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ reuse: "true" }); + }); + + it("skips validator rejection and selects the next strict record", () => { + const { origin, priorSha } = createRepo(); + const clone = cloneHead(origin); + const record = normalizedEvidence({ runId: "111", targetSha: priorSha }); + const { binDir, fixtures, validatorPath } = setUpFixtures([ + { exitCode: 1, runId: "222" }, + { record, runId: "111" }, + ]); + const result = runResolver({ - repoDir: clone, - targetSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, }); + + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ evidence_run_id: "111", reuse: "true" }); + expect(result.stderr).toContain("run 222: shared evidence validator rejected the run"); + }); + + it.each([ + { + label: "reused wrapper", + mutate(record: NormalizedEvidence) { + record.directRoot = false; + record.evidenceReuse = { rootRunId: "42" }; + }, + }, + { + label: "untrusted producer ref", + mutate(record: NormalizedEvidence) { + record.trustedWorkflowRef = "release/2026.7.1"; + }, + }, + { + label: "missing trusted-main lineage proof", + mutate(record: NormalizedEvidence) { + record.producerOnTrustedMainLineage = false; + }, + }, + { + label: "tag-qualified producer workflow", + mutate(record: NormalizedEvidence) { + record.root.workflowFullRef = "refs/tags/main"; + record.current.workflowFullRef = "refs/tags/main"; + }, + }, + { + label: "verifier source drift", + mutate(record: NormalizedEvidence) { + record.verifier.sourceSha = "d".repeat(40); + }, + }, + { + label: "non-full rerun group", + mutate(record: NormalizedEvidence) { + record.rerunGroup = "package"; + }, + }, + { + label: "publishing performance reports", + mutate(record: NormalizedEvidence) { + record.controls.performanceReportPublication = "publish"; + }, + }, + { + label: "missing performance report publication proof", + mutate(record: NormalizedEvidence) { + const performance = record.children.find((child) => child.role === "productPerformance"); + if (!performance) { + throw new Error("missing product performance child"); + } + delete performance.reportPublication; + }, + }, + { + label: "missing required role", + mutate(record: NormalizedEvidence) { + record.children.pop(); + }, + }, + { + label: "duplicate child run id", + mutate(record: NormalizedEvidence) { + record.children[1].runId = record.children[0].runId; + }, + }, + { + label: "failed child", + mutate(record: NormalizedEvidence) { + record.children[0].conclusion = "failure"; + }, + }, + { + label: "extra child role", + mutate(record: NormalizedEvidence) { + record.children.push({ + ...record.children[0], + role: "npmTelegram", + runId: "205", + }); + }, + }, + { + label: "invalid root artifact digest", + mutate(record: NormalizedEvidence) { + record.root.artifact.digest = "sha256:not-a-digest"; + record.current.artifact.digest = "sha256:not-a-digest"; + }, + }, + ])("rejects normalized evidence that is not reusable: $label", ({ mutate }) => { + const { origin, priorSha } = createRepo(); + const clone = cloneHead(origin); + const record = normalizedEvidence({ targetSha: priorSha }); + mutate(record); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); + + const result = runResolver({ + binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + }); + expect(result.status).toBe(0); expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + expect(result.stderr).toContain("not a strict direct-root full validation"); }); - it("rejects evidence from a narrower release profile or diverged history", () => { - const { origin, priorSha } = createRepoPair(); + it.each([ + { + expected: "profile beta differs from stable", + label: "beta evidence for stable", + recordOptions: { releaseProfile: "beta" }, + resolverOptions: { releaseProfile: "stable" }, + }, + { + expected: "profile beta differs from full", + label: "beta evidence for full", + recordOptions: { releaseProfile: "beta" }, + resolverOptions: { releaseProfile: "full" }, + }, + { + expected: "profile stable differs from beta", + label: "stable evidence for beta", + recordOptions: { releaseProfile: "stable" }, + resolverOptions: { releaseProfile: "beta" }, + }, + { + expected: "profile full differs from beta", + label: "full evidence for beta", + recordOptions: { releaseProfile: "full" }, + resolverOptions: { releaseProfile: "beta" }, + }, + { + expected: "profile full differs from stable", + label: "full evidence for stable", + recordOptions: { releaseProfile: "full" }, + resolverOptions: { releaseProfile: "stable" }, + }, + { + expected: "validation inputs differ", + label: "different lane inputs", + recordOptions: { validationInputs: { ...DEFAULT_INPUTS, provider: "anthropic" } }, + resolverOptions: {}, + }, + { + expected: "soak false differs from true", + label: "missing required soak", + recordOptions: { soak: false }, + resolverOptions: { runReleaseSoak: "true" }, + }, + { + expected: "soak true differs from false", + label: "extra soak evidence", + recordOptions: { soak: true }, + resolverOptions: { runReleaseSoak: "false" }, + }, + ])( + "rejects evidence with incompatible policy coverage: $label", + ({ expected, recordOptions, resolverOptions }) => { + const { origin, priorSha } = createRepo(); + const clone = cloneHead(origin); + const record = normalizedEvidence({ targetSha: priorSha, ...recordOptions }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); + + const result = runResolver({ + binDir, + fixtures, + repoDir: clone, + targetSha: priorSha, + validatorPath, + ...resolverOptions, + }); + + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + expect(result.stderr).toContain(expected); + }, + ); + + it("rejects cross-SHA reuse even for a changelog-only delta", () => { + const { origin, priorSha } = createRepo(); const targetSha = commitFile( origin, "CHANGELOG.md", - "# Changelog\n\n- entry\n", + "# Changelog\n\n- beta3\n", "docs(changelog): refresh", ); const clone = cloneHead(origin); + const record = normalizedEvidence({ targetSha: priorSha }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); - const beta = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha, { releaseProfile: "beta" }), - }); - const betaResult = runResolver({ + const result = runResolver({ + binDir, + fixtures, repoDir: clone, targetSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures: beta.fixtures, - binDir: beta.binDir, - }); - expect(betaResult.status).toBe(0); - expect(parseOutput(betaResult.stdout)).toMatchObject({ reuse: "false" }); - - const diverged = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha), - compare: { base: priorSha, head: targetSha, status: "diverged", files: ["CHANGELOG.md"] }, - }); - const divergedResult = runResolver({ - repoDir: clone, - targetSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures: diverged.fixtures, - binDir: diverged.binDir, - }); - expect(divergedResult.status).toBe(0); - expect(parseOutput(divergedResult.stdout)).toMatchObject({ reuse: "false" }); - }); - - it("rejects evidence recorded for different lane-selection inputs", () => { - const { origin, priorSha } = createRepoPair(); - const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha, { - validationInputs: { ...DEFAULT_INPUTS, provider: "anthropic" }, - }), - childRunStates: HEALTHY_CHILDREN, + validatorPath, }); - const result = runResolver({ - repoDir: clone, - targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, - binDir, - }); expect(result.status).toBe(0); expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + expect(result.stderr).toContain("cross-SHA reuse requires granular artifact evidence"); }); - it("rejects evidence whose recorded child runs are no longer green", () => { - const { origin, priorSha } = createRepoPair(); + it("rejects target version metadata that is internally inconsistent", () => { + const { origin, priorSha } = createRepo({ plistBuildVersion: "2026061000" }); const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha), - childRunStates: { "201": "completed/failure", "202": "completed/success" }, - }); + const record = normalizedEvidence({ targetSha: priorSha }); + const { binDir, fixtures, validatorPath } = setUpFixtures([{ record, runId: "111" }]); const result = runResolver({ + binDir, + fixtures, repoDir: clone, targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, - binDir, + validatorPath, }); + expect(result.status).toBe(0); - expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + expect(parseOutput(result.stdout)).toMatchObject({ + reuse: "false", + reuse_reason: "target version metadata is inconsistent", + }); }); - it("rejects evidence whose harness differs beyond release metadata", () => { - const { origin, priorSha } = createRepoPair(); - git(origin, ["checkout", "-q", "-b", "harness-drift"]); - const driftSha = commitFile( - origin, - "index.ts", - "export const value = 3;\n", - "ci: change harness logic", - ); - git(origin, ["checkout", "-q", "main"]); + it.each([ + { inputs: [], label: "array inputs", runReleaseSoak: "true" }, + { inputs: null, label: "null inputs", runReleaseSoak: "true" }, + { inputs: DEFAULT_INPUTS, label: "invalid soak flag", runReleaseSoak: "yes" }, + ])("rejects invalid resolver arguments: $label", ({ inputs, runReleaseSoak }) => { + const { origin, priorSha } = createRepo(); const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ - headSha: driftSha, - manifest: manifestFor(priorSha), - childRunStates: HEALTHY_CHILDREN, - }); + const { binDir, fixtures, validatorPath } = setUpFixtures([]); const result = runResolver({ - repoDir: clone, - targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, binDir, + fixtures, + inputs, + repoDir: clone, + runReleaseSoak, + targetSha: priorSha, + validatorPath, }); - expect(result.status).toBe(0); - expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + + expect(result.status).toBe(2); }); - it("rejects targets whose version stamps are internally inconsistent", () => { - const { origin, priorSha } = createRepoPair({ plistBuildVersion: "2026061000" }); + it("reports no reuse when no prior successful runs exist", () => { + const { origin, priorSha } = createRepo(); const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ - headSha: priorSha, - manifest: manifestFor(priorSha), - childRunStates: HEALTHY_CHILDREN, - }); + const { binDir, fixtures, validatorPath } = setUpFixtures([]); const result = runResolver({ + binDir, + fixtures, repoDir: clone, targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "stable", - fixtures, - binDir, + validatorPath, }); + expect(result.status).toBe(0); - const output = parseOutput(result.stdout); - expect(output.reuse).toBe("false"); - expect(output.reuse_reason).toContain("version metadata"); + expect(parseOutput(result.stdout)).toMatchObject({ + reuse: "false", + reuse_reason: "no prior successful validation runs", + }); }); - it("reports no reuse when no prior runs or manifests exist", () => { - const { origin, priorSha } = createRepoPair(); - const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({ headSha: priorSha }); - - const result = runResolver({ - repoDir: clone, - targetSha: priorSha, - workflowSha: priorSha, - releaseProfile: "beta", - fixtures, - binDir, - }); - expect(result.status).toBe(0); - const output = parseOutput(result.stdout); - expect(output.reuse).toBe("false"); - expect(output.reuse_reason).toContain("no prior validation run covers"); + it("rewrites inherited v3 producer identity to the current immutable workflow SHA", () => { + const workflow = readFileSync(".github/workflows/full-release-validation.yml", "utf8"); + expect(workflow).toContain('--arg workflowSha "$GITHUB_SHA"'); + expect(workflow).toContain("workflowSha: $workflowSha"); + expect(workflow).toContain("ref: ${{ github.sha }}"); }); }); diff --git a/test/scripts/full-release-validation-at-sha.test.ts b/test/scripts/full-release-validation-at-sha.test.ts index 450e6c35323e..cf44371204fb 100644 --- a/test/scripts/full-release-validation-at-sha.test.ts +++ b/test/scripts/full-release-validation-at-sha.test.ts @@ -1,5 +1,12 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { parseArgs } from "../../scripts/full-release-validation-at-sha.mjs"; +import { + parseArgs, + releaseEvidenceVerificationArgs, + releaseEvidenceVerifierPath, +} from "../../scripts/full-release-validation-at-sha.mjs"; describe("full-release-validation-at-sha", () => { it("parses release validation dispatch args", () => { @@ -22,7 +29,7 @@ describe("full-release-validation-at-sha", () => { inputs: { mode: "linux", provider: "anthropic", - reuse_evidence: "false", + reuse_evidence: "true", }, sha: "abc123", workflowSha: "origin/main", @@ -40,9 +47,10 @@ describe("full-release-validation-at-sha", () => { expect(() => parseArgs(["-f", "-h"])).toThrow("-f requires a value"); }); - it("cannot enable evidence reuse on a temporary SHA-pinned workflow ref", () => { - expect(() => parseArgs(["-f", "reuse_evidence=true"])).toThrow( - "always disables evidence reuse", + it("allows exact-target reuse to be disabled for a forced fresh run", () => { + expect(parseArgs(["-f", "reuse_evidence=false"]).inputs.reuse_evidence).toBe("false"); + expect(() => parseArgs(["-f", "reuse_evidence=maybe"])).toThrow( + "reuse_evidence must be true or false", ); }); @@ -50,4 +58,39 @@ describe("full-release-validation-at-sha", () => { expect(() => parseArgs(["-f", "ref=other"])).toThrow("reserves the ref input"); expect(() => parseArgs(["--", "ref=other"])).toThrow("reserves the ref input"); }); + + it("validates direct and reused runs through the strict evidence verifier", () => { + expect(releaseEvidenceVerificationArgs("123")).toEqual([ + "--validate-run", + "123", + "--trusted-workflow-ref", + "main", + "--json", + ]); + expect(() => releaseEvidenceVerificationArgs("")).toThrow("positive decimal"); + }); + + it("supports current and legacy verifier locations in trusted workflow checkouts", () => { + const root = mkdtempSync(join(tmpdir(), "openclaw-release-verifier-path-")); + try { + const legacy = join( + root, + ".agents", + "skills", + "release-openclaw-ci", + "scripts", + "release-ci-summary.mjs", + ); + mkdirSync(join(legacy, ".."), { recursive: true }); + writeFileSync(legacy, ""); + expect(releaseEvidenceVerifierPath(root)).toBe(legacy); + + const current = join(root, "scripts", "release-ci-summary.mjs"); + mkdirSync(join(current, ".."), { recursive: true }); + writeFileSync(current, ""); + expect(releaseEvidenceVerifierPath(root)).toBe(current); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); }); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 632ca38b77f9..1f13145fb3a9 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -2047,11 +2047,14 @@ describe("package artifact reuse", () => { ); expect(trustedTooling.env?.WORKFLOW_SHA).toBe("${{ github.sha }}"); expect(trustedTooling.run).toContain("validate-full-release-validation-evidence.mjs"); + expect(trustedTooling.run).toContain("release-ci-summary.mjs"); + expect(trustedTooling.run).toContain("scripts/lib/plain-gh.mjs"); expect(validateManifest.env).toMatchObject({ RUN_JSON_FILE: "${{ runner.temp }}/full-release-validation-run.json", TRUSTED_MAIN_REF: "refs/remotes/origin/main", VALIDATOR_FILE: "${{ runner.temp }}/release-validation-tooling/validate-full-release-validation-evidence.mjs", + STRICT_VALIDATOR_FILE: "${{ runner.temp }}/release-validation-tooling/release-ci-summary.mjs", }); expect(validateManifest.run).toContain( 'MANIFEST_FILE="$manifest" node "$VALIDATOR_FILE" < "$RUN_JSON_FILE"', diff --git a/test/scripts/package-changelog.test.ts b/test/scripts/package-changelog.test.ts index 776a48bc8614..3361f6d51aa2 100644 --- a/test/scripts/package-changelog.test.ts +++ b/test/scripts/package-changelog.test.ts @@ -124,6 +124,25 @@ Docs: https://docs.openclaw.ai ); }); + it("allows Unreleased notes for explicitly non-publish stable artifacts", () => { + const unreleasedChangelog = cumulativeChangelog.replace( + "- Pending note.", + "- Pending release note with enough detail.", + ); + expect( + extractCurrentPackageChangelog(unreleasedChangelog, "2026.5.29", { + allowUnreleased: true, + }), + ).toBe(changelog` +# Changelog +Docs: https://docs.openclaw.ai + +## Unreleased +### Fixes +- Pending release note with enough detail. +`); + }); + it("fails closed when the packaged changelog is unexpectedly large", () => { const source = changelog` # Changelog diff --git a/test/scripts/release-candidate-checklist.test.ts b/test/scripts/release-candidate-checklist.test.ts index e350afe42697..0496e1716631 100644 --- a/test/scripts/release-candidate-checklist.test.ts +++ b/test/scripts/release-candidate-checklist.test.ts @@ -3,6 +3,7 @@ import { readFileSync } from "node:fs"; import { describe, expect, it, vi } from "vitest"; import { parse } from "yaml"; import { + buildReleaseCandidateState, buildPublishCommand, candidateCumulativeShippedPullRequests, candidateParallelsArgs, @@ -10,6 +11,7 @@ import { githubApi, parseArgs, parseRunIdFromDispatchOutput, + reconcileReleaseCandidateState, resolveArtifactName, requireRunIdFromDispatchOutput, run, @@ -41,6 +43,49 @@ async function withGithubApiTimeoutEnv(value: string, fn: () => Promise): } describe("release candidate checklist", () => { + it("resumes exact workflow runs from matching release candidate state", () => { + const options = parseArgs(["--tag", "v2026.7.1-beta.4"]); + const expected = buildReleaseCandidateState(options, { + targetSha: "a".repeat(40), + toolingSha: "b".repeat(40), + }); + const resumed = reconcileReleaseCandidateState( + JSON.parse( + JSON.stringify({ + ...expected, + phase: "waiting", + fullReleaseRunId: "111", + npmPreflightRunId: "222", + }), + ), + expected, + ); + + expect(resumed).toMatchObject({ + phase: "waiting", + fullReleaseRunId: "111", + npmPreflightRunId: "222", + }); + }); + + it("rejects stale or conflicting release candidate state", () => { + const options = parseArgs(["--tag", "v2026.7.1-beta.4"]); + const expected = buildReleaseCandidateState(options, { + targetSha: "a".repeat(40), + toolingSha: "b".repeat(40), + }); + + expect(() => + reconcileReleaseCandidateState({ ...expected, targetSha: "c".repeat(40) }, expected), + ).toThrow("state mismatch for targetSha"); + expect(() => + reconcileReleaseCandidateState( + { ...expected, fullReleaseRunId: "111" }, + { ...expected, fullReleaseRunId: "333" }, + ), + ).toThrow("state mismatch for fullReleaseRunId"); + }); + it("captures changelogs larger than the Node spawnSync default buffer", () => { const output = run( process.execPath, @@ -516,6 +561,7 @@ describe("release candidate checklist", () => { expect(source).toContain( "const fullValidationEvidence = validateFullReleaseValidationEvidence({", ); + expect(source).toContain("runStrictReleaseEvidenceValidation({ repository, runId })"); expect(source).toContain("refs/heads/main:refs/remotes/origin/main"); expect(source).toContain( 'fullValidationEvidence.source === "direct" && fullRun.headSha !== targetSha', diff --git a/test/scripts/release-ci-summary.test.ts b/test/scripts/release-ci-summary.test.ts new file mode 100644 index 000000000000..a7769ac55d1c --- /dev/null +++ b/test/scripts/release-ci-summary.test.ts @@ -0,0 +1,1705 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + expectedChildDispatches, + expectedSelectedChildDispatches, + manifestChildEntries, + parseReleaseCiSummaryArgs, + readManifestArtifactArchive, + releaseCiWatchFingerprint, + requiredChildKeysForRerunGroup, + resolveManifestChildOriginAttempt, + selectExactChildRun, + selectExactChildRunFromPages, + selectManifestArtifact, + selectManifestParentJob, + selectedChildKeys, + validateEvidenceReuseChain, + validateManifestArtifactCompatibility, + validateManifestArtifactIdentity, + validateManifestChildRun, + validateParentManifest, + validateParentRunBinding, + validatePerformanceArtifactOnlyJobs, + validateReleaseRunEvidence, + validateTrustedProducerIdentity, + watchReleaseCiRun, +} from "../../scripts/release-ci-summary.mjs"; + +const SCRIPT = "scripts/release-ci-summary.mjs"; +const MANIFEST_ARTIFACT_ENTRY = "full-release-validation-manifest.json"; + +function crc32(input: Buffer): number { + let crc = 0xffffffff; + for (const byte of input) { + crc ^= byte; + for (let bit = 0; bit < 8; bit += 1) { + crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + } + return (crc ^ 0xffffffff) >>> 0; +} + +function u16(value: number): Buffer { + const buffer = Buffer.alloc(2); + buffer.writeUInt16LE(value); + return buffer; +} + +function u32(value: number): Buffer { + const buffer = Buffer.alloc(4); + buffer.writeUInt32LE(value); + return buffer; +} + +function makeStoredZip(files: Record): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let offset = 0; + + for (const [name, contents] of Object.entries(files)) { + const nameBuffer = Buffer.from(name, "utf8"); + const contentsBuffer = Buffer.from(contents, "utf8"); + const checksum = crc32(contentsBuffer); + const localHeader = Buffer.concat([ + u32(0x04034b50), + u16(20), + u16(0), + u16(0), + u16(0), + u16(0), + u32(checksum), + u32(contentsBuffer.length), + u32(contentsBuffer.length), + u16(nameBuffer.length), + u16(0), + nameBuffer, + ]); + localParts.push(localHeader, contentsBuffer); + centralParts.push( + Buffer.concat([ + u32(0x02014b50), + u16(20), + u16(20), + u16(0), + u16(0), + u16(0), + u16(0), + u32(checksum), + u32(contentsBuffer.length), + u32(contentsBuffer.length), + u16(nameBuffer.length), + u16(0), + u16(0), + u16(0), + u16(0), + u32((0o100644 << 16) >>> 0), + u32(offset), + nameBuffer, + ]), + ); + offset += localHeader.length + contentsBuffer.length; + } + + const localData = Buffer.concat(localParts); + const centralDirectory = Buffer.concat(centralParts); + return Buffer.concat([ + localData, + centralDirectory, + u32(0x06054b50), + u16(0), + u16(0), + u16(Object.keys(files).length), + u16(Object.keys(files).length), + u32(centralDirectory.length), + u32(localData.length), + u16(0), + ]); +} + +function artifactDigest(bytes: Buffer): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +function rawManifest({ + evidenceReuse, + rerunGroup = "all", + runId = "29090000000", + targetSha = "a".repeat(40), + version = 2, + workflowFullRef, + workflowRefType, + workflowSha, +}: { + evidenceReuse?: Record; + rerunGroup?: string; + runId?: string; + targetSha?: string; + version?: 2 | 3; + workflowFullRef?: string; + workflowRefType?: "branch" | "tag"; + workflowSha?: string; +}) { + return { + childRuns: { + normalCi: "101", + npmTelegram: "", + pluginPrerelease: "202", + productPerformance: { blocking: true, conclusion: "success", runId: "303" }, + releaseChecks: "404", + }, + controls: { + performanceBlocking: true, + performanceReportPublication: "artifact-only", + stableSoakRequired: false, + }, + evidenceReuse, + releaseProfile: "beta", + rerunGroup, + runAttempt: "2", + runId, + runReleaseSoak: "false", + targetSha, + validationInputs: { + codexPluginSpec: "", + crossOsSuiteFilter: "", + liveSuiteFilter: "", + mode: "direct", + packageAcceptancePackageSpec: "", + provider: "openai", + releasePackageSpec: "", + }, + version, + workflowName: "Full Release Validation", + workflowRef: "main", + ...(workflowSha ? { workflowSha } : {}), + ...(version === 3 + ? { + workflowFullRef: workflowFullRef ?? "refs/heads/main", + workflowRefType: workflowRefType ?? "branch", + } + : {}), + }; +} + +function trustedMainPackageFixture({ + manifestVersion = 2, + parentPath = ".github/workflows/full-release-validation.yml", + targetSha = "8".repeat(40), + workflowFullRef, + workflowRef = "main", + workflowRefType, + workflowSha = "0".repeat(40), +}: { + manifestVersion?: 2 | 3; + parentPath?: string; + targetSha?: string; + workflowFullRef?: string; + workflowRef?: string; + workflowRefType?: "branch" | "tag"; + workflowSha?: string; +} = {}) { + const runId = "29071366025"; + const childRunId = "29071382629"; + const manifest = rawManifest({ + rerunGroup: "package", + runId, + targetSha, + version: manifestVersion, + workflowFullRef, + workflowRefType, + workflowSha, + }); + manifest.childRuns = { + normalCi: "", + npmTelegram: "", + pluginPrerelease: "", + productPerformance: { blocking: true, conclusion: "", runId: "" }, + releaseChecks: childRunId, + }; + manifest.releaseProfile = "full"; + manifest.runAttempt = "1"; + manifest.runReleaseSoak = "true"; + manifest.workflowRef = workflowRef; + + const parentRun = { + conclusion: "success", + event: "workflow_dispatch", + head_branch: workflowRef, + head_sha: workflowSha, + html_url: `https://github.com/openclaw/openclaw/actions/runs/${runId}`, + id: Number(runId), + path: parentPath, + repository: { full_name: "openclaw/openclaw" }, + run_attempt: 1, + status: "completed", + }; + const parentView = { + attempt: 1, + conclusion: "success", + headBranch: workflowRef, + headSha: workflowSha, + jobs: [], + status: "completed", + url: parentRun.html_url, + }; + const child = expectedChildDispatches(runId, 1, workflowRef).find( + (entry) => entry.manifestKey === "releaseChecks", + ); + if (!child) { + throw new Error("missing release checks child fixture"); + } + const parentJob = { + completed_at: "2026-07-10T01:10:00Z", + conclusion: "success", + id: 86293408710, + name: child.parentJobName, + run_attempt: 1, + started_at: "2026-07-10T01:00:00Z", + status: "completed", + steps: [], + }; + const childRun = { + actor: { login: "github-actions[bot]" }, + conclusion: "success", + display_title: child.displayTitle, + event: "workflow_dispatch", + head_branch: workflowRef, + head_sha: workflowSha, + html_url: `https://github.com/openclaw/openclaw/actions/runs/${childRunId}`, + id: Number(childRunId), + path: ".github/workflows/openclaw-release-checks.yml", + repository: { full_name: "openclaw/openclaw" }, + run_attempt: 1, + status: "completed", + triggering_actor: { login: "github-actions[bot]" }, + }; + const artifact = { + digest: `sha256:${"9".repeat(64)}`, + expired: false, + id: 8220114429, + name: `full-release-validation-${runId}-1`, + size_in_bytes: 507, + workflow_run: { + head_branch: workflowRef, + head_sha: workflowSha, + id: Number(runId), + }, + }; + const client = { + compareCommits(base: string, head: string) { + expect(base).toBe(workflowSha); + return { + merge_base_commit: { sha: workflowSha }, + status: base === head ? "identical" : "ahead", + }; + }, + getJobLog(jobId: number) { + expect(jobId).toBe(parentJob.id); + return [ + `TARGET_SHA: ${targetSha}`, + `Dispatched openclaw-release-checks.yml: ${childRun.html_url}`, + ].join("\n"); + }, + getParentJobs(requestedRunId: string) { + expect(requestedRunId).toBe(runId); + return [parentJob]; + }, + getRun(requestedRunId: string) { + if (String(requestedRunId) === runId) { + return parentRun; + } + if (String(requestedRunId) === childRunId) { + return childRun; + } + throw new Error(`unexpected run: ${requestedRunId}`); + }, + getRunView(requestedRunId: string) { + expect(requestedRunId).toBe(runId); + return parentView; + }, + loadManifest(requestedRunId: string, requestedRunAttempt: number) { + expect(requestedRunId).toBe(runId); + expect(requestedRunAttempt).toBe(1); + return { artifact, manifest }; + }, + }; + + return { artifact, childRun, client, manifest, parentRun, runId, targetSha, workflowSha }; +} + +describe("release CI summary child correlation", () => { + it("parses the reusable strict validation CLI without changing positional summary mode", () => { + expect( + parseReleaseCiSummaryArgs([ + "--validate-run", + "29071366025", + "--repo", + "openclaw/openclaw", + "--manifest", + "/tmp/manifest.json", + "--json", + ]), + ).toEqual({ + json: true, + intervalMs: 30_000, + manifestPath: "/tmp/manifest.json", + repository: "openclaw/openclaw", + runId: "29071366025", + trustedWorkflowRef: "main", + validate: true, + verifierSourceFile: undefined, + verifierSourceSha: undefined, + watch: false, + }); + expect(parseReleaseCiSummaryArgs(["29071366025"])).toMatchObject({ + repository: "openclaw/openclaw", + runId: "29071366025", + trustedWorkflowRef: "main", + validate: false, + }); + expect(parseReleaseCiSummaryArgs(["29071366025", "--watch", "--interval", "15"])).toMatchObject( + { + intervalMs: 15_000, + watch: true, + }, + ); + expect(() => parseReleaseCiSummaryArgs(["29071366025", "--interval", "0"])).toThrow( + "positive number of seconds", + ); + expect(() => parseReleaseCiSummaryArgs(["--validate-run", "29071366025", "--watch"])).toThrow( + "--watch cannot be combined", + ); + expect(() => parseReleaseCiSummaryArgs(["--manifest", "/tmp/manifest.json"])).toThrow( + "--manifest requires --validate-run", + ); + expect(() => + parseReleaseCiSummaryArgs([ + "--validate-run", + "29071366025", + "--verifier-source-file", + "/tmp/verifier.mjs", + ]), + ).toThrow("--verifier-source-file requires --verifier-source-sha"); + expect( + parseReleaseCiSummaryArgs([ + "--validate-run", + "29071366025", + "--verifier-source-sha", + "a".repeat(40), + "--verifier-source-file", + "/tmp/verifier.mjs", + ]), + ).toMatchObject({ + verifierSourceFile: "/tmp/verifier.mjs", + verifierSourceSha: "a".repeat(40), + }); + }); + + it("changes the watch fingerprint only for visible run transitions", () => { + const parent = { + attempt: 1, + conclusion: "", + jobs: [{ name: "Run normal full CI", status: "in_progress", conclusion: "" }], + status: "in_progress", + url: "ignored", + }; + expect(releaseCiWatchFingerprint({ ...parent, url: "changed" })).toBe( + releaseCiWatchFingerprint(parent), + ); + expect( + releaseCiWatchFingerprint({ + ...parent, + jobs: [{ ...parent.jobs[0], conclusion: "success", status: "completed" }], + }), + ).not.toBe(releaseCiWatchFingerprint(parent)); + }); + + it("summarizes only transitions while watching a release run", async () => { + const states = [ + { attempt: 1, conclusion: "", jobs: [], status: "queued" }, + { attempt: 1, conclusion: "", jobs: [], status: "queued" }, + { + attempt: 1, + conclusion: "success", + jobs: [{ name: "Run normal full CI", status: "completed", conclusion: "success" }], + status: "completed", + }, + ]; + let index = 0; + let summaries = 0; + let sleeps = 0; + + await watchReleaseCiRun( + parseReleaseCiSummaryArgs(["29071366025", "--watch", "--interval", "1"]), + { + fetchParent: () => states[index++], + sleep: async () => { + sleeps += 1; + }, + summarize: () => { + summaries += 1; + }, + }, + ); + + expect(summaries).toBe(2); + expect(sleeps).toBe(2); + }); + + it("selects one immutable manifest artifact bound to the exact parent run", () => { + const { artifact, runId } = trustedMainPackageFixture(); + const legacyArtifact = { + ...artifact, + id: artifact.id + 1, + name: `full-release-validation-${runId}`, + }; + expect(selectManifestArtifact([artifact], runId, 1)).toBe(artifact); + expect(selectManifestArtifact([legacyArtifact, artifact], runId, 1)).toBe(artifact); + expect(selectManifestArtifact([legacyArtifact], runId, 1)).toBe(legacyArtifact); + expect(validateManifestArtifactCompatibility(legacyArtifact, { version: 2 }, runId, 1)).toBe( + legacyArtifact, + ); + expect( + selectManifestArtifact( + [{ ...artifact, workflow_run: { ...artifact.workflow_run, id: 1 } }], + runId, + 1, + ), + ).toBeUndefined(); + expect(() => + selectManifestArtifact([artifact, { ...artifact, id: artifact.id + 1 }], runId, 1), + ).toThrow("multiple release validation manifest artifacts"); + expect(() => + selectManifestArtifact( + [legacyArtifact, { ...legacyArtifact, id: legacyArtifact.id + 1 }], + runId, + 1, + ), + ).toThrow("multiple legacy release validation manifest artifacts"); + expect(() => selectManifestArtifact([legacyArtifact], runId, 2)).toThrow( + "legacy release validation manifest requires run attempt 1", + ); + expect(() => + validateManifestArtifactCompatibility(legacyArtifact, { version: 3 }, runId, 1), + ).toThrow("legacy release validation manifest artifact is not compatible"); + expect(selectManifestArtifact([artifact], runId, 2)).toBeUndefined(); + expect(() => selectManifestArtifact([{ ...artifact, digest: undefined }], runId, 1)).toThrow( + "manifest artifact digest is invalid", + ); + expect(() => + validateManifestArtifactIdentity( + { ...artifact, digest: `sha256:${"8".repeat(64)}` }, + { + artifactDigest: artifact.digest, + artifactId: artifact.id, + runAttempt: 1, + runId, + }, + ), + ).toThrow("manifest artifact identity mismatch"); + expect(() => + validateManifestArtifactIdentity( + { ...artifact, id: artifact.id + 1 }, + { + artifactDigest: artifact.digest, + artifactId: artifact.id, + runAttempt: 1, + runId, + }, + ), + ).toThrow("manifest artifact identity mismatch"); + + const source = readFileSync(SCRIPT, "utf8"); + expect(source).toContain("actions/artifacts/${artifactId}/zip"); + expect(source).not.toContain('"--name",'); + expect(source).not.toContain("gh run download"); + expect(source).toContain( + "downloadParentManifestEvidence(runId, runAttempt, normalizedRepository, manifestPath)", + ); + }); + + it("hashes and safely streams one bounded manifest entry from the exact artifact ZIP", () => { + const root = mkdtempSync(join(tmpdir(), "release-manifest-artifact-")); + try { + const archivePath = join(root, "manifest.zip"); + const manifest = { runAttempt: 1, runId: "29071366025" }; + const archive = makeStoredZip({ + [MANIFEST_ARTIFACT_ENTRY]: JSON.stringify(manifest), + }); + writeFileSync(archivePath, archive); + expect(readManifestArtifactArchive(archivePath, artifactDigest(archive))).toEqual(manifest); + expect(() => readManifestArtifactArchive(archivePath, `sha256:${"0".repeat(64)}`)).toThrow( + "artifact digest mismatch", + ); + + const extraEntryArchive = makeStoredZip({ + [MANIFEST_ARTIFACT_ENTRY]: JSON.stringify(manifest), + "unexpected.json": "{}", + }); + writeFileSync(archivePath, extraEntryArchive); + expect(() => + readManifestArtifactArchive(archivePath, artifactDigest(extraEntryArchive)), + ).toThrow(`must contain only ${MANIFEST_ARTIFACT_ENTRY}`); + + const oversizedManifestArchive = makeStoredZip({ + [MANIFEST_ARTIFACT_ENTRY]: "x".repeat(128 * 1024 + 1), + }); + writeFileSync(archivePath, oversizedManifestArchive); + expect(() => + readManifestArtifactArchive(archivePath, artifactDigest(oversizedManifestArchive)), + ).toThrow("artifact entry size is invalid"); + + const oversizedArchive = Buffer.alloc(256 * 1024 + 1); + writeFileSync(archivePath, oversizedArchive); + expect(() => + readManifestArtifactArchive(archivePath, artifactDigest(oversizedArchive)), + ).toThrow("artifact compressed size is invalid"); + + const source = readFileSync(SCRIPT, "utf8"); + expect(source).toContain('execFileSync("unzip", ["-p", archivePath'); + expect(source).not.toContain('execFileSync("unzip", ["-q", archivePath, "-d"'); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("bridges only attempt-one manifest v2 artifacts with the legacy stable name", () => { + const legacyV2 = trustedMainPackageFixture(); + legacyV2.artifact.name = `full-release-validation-${legacyV2.runId}`; + expect( + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: legacyV2.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + legacyV2.client, + ).root.artifact.name, + ).toBe(legacyV2.artifact.name); + + const legacyV3 = trustedMainPackageFixture({ + manifestVersion: 3, + workflowSha: "a".repeat(40), + }); + legacyV3.artifact.name = `full-release-validation-${legacyV3.runId}`; + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: legacyV3.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + legacyV3.client, + ), + ).toThrow("legacy release validation manifest artifact is not compatible"); + }); + + it("normalizes a pre-tooling trusted-main producer separately from the current verifier", () => { + const fixture = trustedMainPackageFixture({ + targetSha: "8".repeat(40), + workflowSha: "0".repeat(40), + }); + const verifierSourceSha = "c".repeat(40); + const evidence = validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha, + }, + fixture.client, + ); + + expect(evidence).toMatchObject({ + directRoot: true, + evidenceReuse: null, + releaseProfile: "full", + repository: "openclaw/openclaw", + rerunGroup: "package", + runReleaseSoak: true, + schema: "openclaw.release-validation-evidence/v3", + producerOnTrustedMainLineage: true, + trustedWorkflowFullRef: "refs/heads/main", + trustedWorkflowRef: "main", + valid: true, + verifier: { + schemaVersion: 3, + sourceSha: verifierSourceSha, + }, + }); + expect(evidence.root).toMatchObject({ + manifestVersion: 2, + runAttempt: 1, + runId: fixture.runId, + targetSha: fixture.targetSha, + producerOnTrustedMainLineage: true, + workflowFullRef: "refs/heads/main", + workflowPath: ".github/workflows/full-release-validation.yml", + workflowQualifiedPath: ".github/workflows/full-release-validation.yml@refs/heads/main", + workflowRef: "main", + workflowRefProof: "legacy-v2-main-ancestry", + workflowRefType: "branch", + workflowSha: fixture.workflowSha, + }); + expect(evidence.root.workflowSha).not.toBe(evidence.root.targetSha); + expect(evidence.verifier.sourceSha).not.toBe(evidence.root.workflowSha); + expect(evidence.children).toEqual([ + expect.objectContaining({ + conclusion: "success", + dispatchNonce: `full-release-validation-${fixture.runId}-1-release-checks`, + headBranch: "main", + role: "releaseChecks", + runAttempt: 1, + runId: String(fixture.childRun.id), + sourceParentAttempt: 1, + workflowSha: fixture.workflowSha, + }), + ]); + expect(evidence.root.artifact).toEqual({ + digest: fixture.artifact.digest, + id: String(fixture.artifact.id), + name: fixture.artifact.name, + runAttempt: 1, + sizeInBytes: fixture.artifact.size_in_bytes, + }); + }); + + it("accepts a trusted-main producer when the candidate is the same main commit", () => { + const sharedSha = "a".repeat(40); + const fixture = trustedMainPackageFixture({ + targetSha: sharedSha, + workflowSha: sharedSha, + }); + expect( + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ).root, + ).toMatchObject({ + targetSha: sharedSha, + workflowRef: "main", + workflowSha: sharedSha, + }); + }); + + it("binds v3 producer evidence to the exact trusted branch ref", () => { + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + workflowSha: "a".repeat(40), + }); + const evidence = validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ); + expect(evidence.root).toMatchObject({ + producerOnTrustedMainLineage: true, + workflowFullRef: "refs/heads/main", + workflowRefProof: "manifest-v3-branch", + workflowRefType: "branch", + workflowRunPath: ".github/workflows/full-release-validation.yml", + }); + }); + + it("accepts a Unicode trusted workflow ref", () => { + const workflowRef = "release/unicode-\u{1f4a5}"; + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + workflowSha: "a".repeat(40), + }); + const evidence = validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + trustedWorkflowRef: workflowRef, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ); + + expect(evidence.root).toMatchObject({ + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + }); + }); + + it("rejects a v3 producer dispatched from a tag named main", () => { + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + workflowFullRef: "refs/tags/main", + workflowRefType: "tag", + workflowSha: "a".repeat(40), + }); + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ), + ).toThrow("producer workflow full ref is not trusted"); + }); + + it("rejects a legacy producer outside the trusted main verifier lineage", () => { + const fixture = trustedMainPackageFixture({ workflowSha: "a".repeat(40) }); + fixture.client.compareCommits = () => ({ + merge_base_commit: { sha: "d".repeat(40) }, + status: "diverged", + }); + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ), + ).toThrow("producer is not on the trusted main verifier lineage"); + }); + + it("rejects a candidate branch producer even when its SHA differs from the target", () => { + const fixture = trustedMainPackageFixture({ + targetSha: "8".repeat(40), + workflowRef: "release/2026.7.1", + workflowSha: "7".repeat(40), + }); + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + trustedWorkflowRef: "main", + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ), + ).toThrow("producer must run from trusted workflow ref: main"); + }); + + it("accepts canonical SHA-pinned v3 evidence on the trusted main lineage", () => { + const workflowSha = "7".repeat(40); + const workflowRef = `release-ci/${workflowSha.slice(0, 12)}-1783705000000`; + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + targetSha: "8".repeat(40), + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + workflowSha, + }); + fixture.manifest.targetRef = fixture.targetSha; + + expect( + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ).root, + ).toMatchObject({ + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + workflowRefProof: "manifest-v3-sha-pinned-main-ancestry", + workflowSha, + }); + }); + + it.each(["main", "refs/heads/main"])( + "accepts a REST workflow path qualified with %s", + (qualifiedRef) => { + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + parentPath: `.github/workflows/full-release-validation.yml@${qualifiedRef}`, + workflowSha: "7".repeat(40), + }); + + expect( + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ).root, + ).toMatchObject({ workflowFullRef: "refs/heads/main" }); + }, + ); + + it("accepts SHA-pinned producer identity with exact-target evidence reuse", () => { + const workflowSha = "7".repeat(40); + const workflowRef = `release-ci/${workflowSha.slice(0, 12)}-1783705000000`; + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + workflowSha, + }); + fixture.manifest.targetRef = fixture.targetSha; + fixture.manifest.evidenceReuse = { + changedPaths: [], + evidenceSha: fixture.targetSha, + policy: "exact-target-full-validation-v1", + runId: "29071366024", + selectedRunId: "29071366024", + }; + + expect( + validateTrustedProducerIdentity( + { + manifest: fixture.manifest, + parentRun: fixture.parentRun, + }, + fixture.client, + { sourceSha: "c".repeat(40) }, + "main", + ), + ).toMatchObject({ + producerOnTrustedMainLineage: true, + workflowRefProof: "manifest-v3-sha-pinned-main-ancestry", + }); + }); + + it("rejects a SHA-pinned evidenceReuse field even when false", () => { + const workflowSha = "7".repeat(40); + const workflowRef = `release-ci/${workflowSha.slice(0, 12)}-1783705000000`; + const fixture = trustedMainPackageFixture({ + manifestVersion: 3, + workflowFullRef: `refs/heads/${workflowRef}`, + workflowRef, + workflowSha, + }); + fixture.manifest.targetRef = fixture.targetSha; + fixture.manifest.evidenceReuse = false; + + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: readFileSync(SCRIPT), + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ), + ).toThrow("evidence reuse is invalid"); + }); + + it("rejects dirty verifier bytes and a forged verifier source SHA", () => { + const fixture = trustedMainPackageFixture(); + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceContent: "different verifier bytes", + verifierSourceSha: "c".repeat(40), + }, + fixture.client, + ), + ).toThrow("verifier script differs from its source SHA"); + expect(() => + validateReleaseRunEvidence( + { + repository: "openclaw/openclaw", + runId: fixture.runId, + verifierSourceSha: "f".repeat(40), + }, + fixture.client, + ), + ).toThrow("verifier source blob is unavailable"); + }); + + it("binds verifier bytes from the repository root even outside the caller cwd", () => { + const repositoryRoot = mkdtempSync(join(tmpdir(), "release-verifier-repo-")); + const outsideCwd = mkdtempSync(join(tmpdir(), "release-verifier-cwd-")); + try { + const scriptPath = join(repositoryRoot, SCRIPT); + mkdirSync(dirname(scriptPath), { recursive: true }); + writeFileSync(scriptPath, readFileSync(SCRIPT)); + execFileSync("git", ["init", "-q"], { cwd: repositoryRoot }); + execFileSync("git", ["add", SCRIPT], { cwd: repositoryRoot }); + execFileSync( + "git", + [ + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@example.invalid", + "-c", + "commit.gpgSign=false", + "commit", + "-qm", + "test verifier", + ], + { cwd: repositoryRoot }, + ); + const sourceSha = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: repositoryRoot, + encoding: "utf8", + }).trim(); + + const moduleUrl = pathToFileURL(resolve(SCRIPT)).href; + const output = execFileSync( + process.execPath, + [ + "--input-type=module", + "--eval", + `import { resolveVerifierIdentity } from ${JSON.stringify(moduleUrl)}; + process.stdout.write(JSON.stringify(resolveVerifierIdentity( + process.env.SOURCE_SHA, + undefined, + process.env.REPOSITORY_ROOT, + )));`, + ], + { + cwd: outsideCwd, + encoding: "utf8", + env: { + ...process.env, + REPOSITORY_ROOT: repositoryRoot, + SOURCE_SHA: sourceSha, + }, + }, + ); + expect(JSON.parse(output)).toMatchObject({ + script: SCRIPT, + sourceSha, + }); + } finally { + rmSync(repositoryRoot, { force: true, recursive: true }); + rmSync(outsideCwd, { force: true, recursive: true }); + } + }); + + it("binds the parent to the exact Full Release Validation REST run", () => { + const parentView = { + attempt: 2, + headBranch: "main", + headSha: "a".repeat(40), + }; + const parentRest = { + event: "workflow_dispatch", + head_branch: parentView.headBranch, + head_sha: parentView.headSha, + id: 29090000000, + path: ".github/workflows/full-release-validation.yml@refs/heads/main", + run_attempt: parentView.attempt, + }; + + expect(validateParentRunBinding(parentView, parentRest, "29090000000")).toBe(parentRest); + expect(() => + validateParentRunBinding( + parentView, + { ...parentRest, path: ".github/workflows/openclaw-release-checks.yml" }, + "29090000000", + ), + ).toThrow("full release parent run binding mismatch"); + }); + + it("derives every child title from the exact parent run and attempt", () => { + expect(expectedChildDispatches("29090000000", 3, "release/2026.7.1")).toEqual([ + { + displayTitle: "CI full-release-validation-29090000000-3-ci", + headBranch: "release/2026.7.1", + manifestKey: "normalCi", + name: "CI", + parentJobName: "Run normal full CI", + suffix: "-ci", + trustedRef: "parent", + workflow: "ci.yml", + }, + { + displayTitle: + "OpenClaw Release Checks full-release-validation-29090000000-3-release-checks", + headBranch: "release/2026.7.1", + manifestKey: "releaseChecks", + name: "OpenClaw Release Checks", + parentJobName: "Run release/live/Docker/QA validation", + suffix: "-release-checks", + trustedRef: "parent", + workflow: "openclaw-release-checks.yml", + }, + { + displayTitle: "Plugin Prerelease full-release-validation-29090000000-3-plugin-prerelease", + headBranch: "release/2026.7.1", + manifestKey: "pluginPrerelease", + name: "Plugin Prerelease", + parentJobName: "Run plugin prerelease validation", + suffix: "-plugin-prerelease", + trustedRef: "parent", + workflow: "plugin-prerelease.yml", + }, + { + displayTitle: "NPM Telegram Beta E2E full-release-validation-29090000000-3-npm-telegram", + headBranch: "release/2026.7.1", + manifestKey: "npmTelegram", + name: "NPM Telegram Beta E2E", + parentJobName: "Run package Telegram E2E", + suffix: "-npm-telegram", + trustedRef: "parent", + workflow: "npm-telegram-beta-e2e.yml", + }, + { + displayTitle: "OpenClaw Performance full-release-validation-29090000000-3", + headBranch: "release/2026.7.1", + manifestKey: "productPerformance", + name: "OpenClaw Performance", + parentJobName: "Run product performance evidence", + suffix: "", + trustedRef: "parent", + workflow: "openclaw-performance.yml", + }, + ]); + }); + + it("ignores same-SHA and nearby-name runs without the exact parent dispatch binding", () => { + const expected = "OpenClaw Performance full-release-validation-29090000000-3"; + const exact = { + display_title: expected, + event: "workflow_dispatch", + head_branch: "main", + head_sha: "a".repeat(40), + id: 303, + }; + expect( + selectExactChildRun( + [ + { + display_title: "OpenClaw Performance", + event: "workflow_dispatch", + head_branch: "main", + head_sha: exact.head_sha, + id: 101, + }, + { ...exact, event: "push", id: 202 }, + exact, + ], + expected, + "main", + ), + ).toBe(exact); + }); + + it("fails closed on duplicate exact dispatch bindings and ignores branch collisions", () => { + const expected = "CI full-release-validation-29090000000-3-ci"; + const exact = { + display_title: expected, + event: "workflow_dispatch", + head_branch: "main", + id: 1, + }; + expect( + selectExactChildRun( + [{ ...exact, head_branch: "release/2026.7.1", id: 0 }, exact], + expected, + "main", + ), + ).toBe(exact); + expect(() => selectExactChildRun([exact, { ...exact, id: 2 }], expected, "main")).toThrow( + "multiple child runs have exact dispatch title and branch", + ); + + const source = readFileSync(SCRIPT, "utf8"); + expect(source).not.toContain("created_at >= since"); + expect(source).not.toContain("head_sha === parent.headSha"); + expect(source).not.toContain("created:"); + expect(source).toContain("workflow-sha:"); + expect(source).toContain("candidate-sha:"); + expect(source).not.toContain("console.log(`sha:"); + expect(source).toContain("actions/workflows/${child.workflow}/runs"); + }); + + it("returns one exact child after a full bounded pagination scan", () => { + const expected = "OpenClaw Performance full-release-validation-29090000000-3"; + const exact = { + display_title: expected, + event: "workflow_dispatch", + head_branch: "main", + id: 999, + }; + const pages = Array.from({ length: 10 }, (_, pageIndex) => + Array.from({ length: 100 }, (_, runIndex) => ({ + display_title: `decoy-${pageIndex}-${runIndex}`, + event: "workflow_dispatch", + head_branch: "main", + id: pageIndex * 100 + runIndex, + })), + ); + pages[9][99] = exact; + + expect(selectExactChildRunFromPages(pages, expected, "main")).toBe(exact); + pages[0][0] = { ...exact, id: 1001 }; + expect(() => selectExactChildRunFromPages(pages, expected, "main")).toThrow( + "multiple child runs have exact dispatch title and branch", + ); + }); + + it("validates candidate identity and selected child completeness from the parent manifest", () => { + const manifest = validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + }); + expect(manifest.targetSha).toBe("a".repeat(40)); + expect(manifest.rerunGroup).toBe("all"); + const children = expectedChildDispatches(manifest.runId, manifest.runAttempt, "main"); + const selected = requiredChildKeysForRerunGroup(manifest.rerunGroup); + expect(manifestChildEntries(manifest, children, selected).map((entry) => entry.runId)).toEqual([ + "101", + "404", + "202", + "303", + ]); + + const missing = { + ...manifest, + childRunIds: { ...manifest.childRunIds, normalCi: "" }, + }; + expect(() => manifestChildEntries(missing, children, selected)).toThrow( + "selected child is missing from manifest: CI", + ); + }); + + it("keeps historical non-reuse v2 manifests readable without validation inputs", () => { + const legacy = rawManifest({}); + delete (legacy as { validationInputs?: unknown }).validationInputs; + const manifest = validateParentManifest(legacy, { + runAttempt: 2, + runId: "29090000000", + }); + + expect(manifest.validationInputs).toBeUndefined(); + expect(manifest.rerunGroup).toBe("all"); + }); + + it("binds v3 manifests to their immutable producer workflow SHA", () => { + const workflowSha = "b".repeat(40); + const manifest = validateParentManifest(rawManifest({ version: 3, workflowSha }), { + runAttempt: 2, + runId: "29090000000", + workflowRef: "main", + workflowSha, + }); + expect(manifest).toMatchObject({ + version: 3, + workflowSha, + }); + expect(() => + validateParentManifest(rawManifest({ version: 3, workflowSha }), { + runAttempt: 2, + runId: "29090000000", + workflowSha: "c".repeat(40), + }), + ).toThrow("release validation manifest workflow SHA mismatch"); + }); + + it("requires v3 manifests to record artifact-only performance publication", () => { + const workflowSha = "b".repeat(40); + const missing = rawManifest({ version: 3, workflowSha }); + delete ( + missing.controls as { + performanceReportPublication?: string; + } + ).performanceReportPublication; + expect(() => + validateParentManifest(missing, { + runAttempt: 2, + runId: "29090000000", + workflowSha, + }), + ).toThrow("release validation manifest performance report publication mode is invalid"); + + const publishing = rawManifest({ version: 3, workflowSha }); + publishing.controls.performanceReportPublication = "publish"; + expect(() => + validateParentManifest(publishing, { + runAttempt: 2, + runId: "29090000000", + workflowSha, + }), + ).toThrow("release validation manifest performance report publication mode is invalid"); + }); + + it("requires a successful artifact-only performance guard for the current attempt", () => { + const guard = { + conclusion: "success", + name: "Verify artifact-only report mode", + run_attempt: 2, + status: "completed", + }; + const skippedPublisher = { + conclusion: "skipped", + name: "Publish mock provider report", + run_attempt: 2, + status: "completed", + }; + expect( + validatePerformanceArtifactOnlyJobs( + [{ ...guard, conclusion: "failure", run_attempt: 1 }, guard, skippedPublisher], + 2, + ), + ).toBe(guard); + expect(() => validatePerformanceArtifactOnlyJobs([skippedPublisher], 2)).toThrow( + "performance artifact-only guard is missing or unsuccessful", + ); + expect(() => + validatePerformanceArtifactOnlyJobs([{ ...guard, conclusion: "failure" }], 2), + ).toThrow("performance artifact-only guard is missing or unsuccessful"); + expect(() => + validatePerformanceArtifactOnlyJobs( + [guard, { ...skippedPublisher, conclusion: "success" }], + 2, + ), + ).toThrow("performance report publisher was not skipped"); + }); + + it("requires the child mapped by rerunGroup and scans only selected in-progress workflows", () => { + const focused = validateParentManifest( + { + ...rawManifest({ rerunGroup: "npm-telegram" }), + childRuns: { + normalCi: "", + npmTelegram: "", + pluginPrerelease: "", + productPerformance: { runId: "" }, + releaseChecks: "", + }, + }, + { runAttempt: 2, runId: "29090000000" }, + ); + const selected = requiredChildKeysForRerunGroup(focused.rerunGroup); + const children = expectedSelectedChildDispatches( + focused.runId, + focused.runAttempt, + focused.workflowRef, + selected, + ); + expect(children.map((child) => child.manifestKey)).toEqual(["npmTelegram"]); + expect(() => manifestChildEntries(focused, children, selected)).toThrow( + "selected child is missing from manifest: NPM Telegram Beta E2E", + ); + + const inProgress = selectedChildKeys([ + { conclusion: "skipped", name: "Run normal full CI" }, + { conclusion: "skipped", name: "Run plugin prerelease validation" }, + { conclusion: undefined, name: "Run product performance evidence" }, + { conclusion: "skipped", name: "Run release/live/Docker/QA validation" }, + ]); + expect( + expectedSelectedChildDispatches("29090000000", 2, "main", inProgress).map( + (child) => child.manifestKey, + ), + ).toEqual(["productPerformance"]); + }); + + it("authorizes only exact-target reuse through the selected root manifest", () => { + const root = validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + }); + const current = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: [], + evidenceSha: root.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: root.runId, + }, + runId: "29090000001", + targetSha: root.targetSha, + }), + { runAttempt: 2, runId: "29090000001" }, + ); + + expect(validateEvidenceReuseChain(current, root, root)).toBe(root.targetSha); + expect(current.targetSha).toBe(root.targetSha); + }); + + it("rejects changed paths and cross-SHA targets in Full Release reuse", () => { + const root = validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + }); + const changedPaths = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: ["CHANGELOG.md"], + evidenceSha: root.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: root.runId, + }, + runId: "29090000001", + targetSha: root.targetSha, + }), + { runAttempt: 2, runId: "29090000001" }, + ); + expect(() => validateEvidenceReuseChain(changedPaths, root, root)).toThrow( + "requires an exact target with no changed paths", + ); + + const changedTarget = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: [], + evidenceSha: root.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: root.runId, + }, + runId: "29090000001", + targetSha: "b".repeat(40), + }), + { runAttempt: 2, runId: "29090000001" }, + ); + expect(() => validateEvidenceReuseChain(changedTarget, root, root)).toThrow( + "full release evidence reuse target SHA mismatch", + ); + }); + + it("rejects exact-target reuse without matching root policy and authorization", () => { + const root = validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + }); + const current = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: [], + evidenceSha: root.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: root.runId, + }, + runId: "29090000001", + targetSha: root.targetSha, + }), + { runAttempt: 2, runId: "29090000001" }, + ); + const mismatchedRoot = { + ...root, + validationInputs: { ...root.validationInputs, provider: "anthropic" }, + }; + + expect(() => validateEvidenceReuseChain(current, mismatchedRoot, mismatchedRoot)).toThrow( + "evidence reuse current manifest policy differs from the chain root", + ); + expect(() => + validateEvidenceReuseChain({ ...current, evidenceReuse: undefined }, root, root), + ).toThrow("does not authorize evidence reuse"); + }); + + it("rejects any selected manifest that itself reuses evidence", () => { + const root = validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + }); + const intermediate = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: [], + evidenceSha: root.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: root.runId, + }, + runId: "29090000001", + targetSha: root.targetSha, + }), + { runAttempt: 2, runId: "29090000001" }, + ); + const current = validateParentManifest( + rawManifest({ + evidenceReuse: { + changedPaths: [], + evidenceSha: intermediate.targetSha, + policy: "exact-target-full-validation-v1", + runId: root.runId, + selectedRunId: intermediate.runId, + }, + runId: "29090000002", + targetSha: intermediate.targetSha, + }), + { runAttempt: 2, runId: "29090000002" }, + ); + + expect(() => validateEvidenceReuseChain(current, intermediate, root)).toThrow( + "evidence reuse must select a root execution manifest", + ); + }); + + it("binds each manifest workflow ref to the fetched parent branch", () => { + expect(() => + validateParentManifest(rawManifest({}), { + runAttempt: 2, + runId: "29090000000", + workflowRef: "release/2026.7.1", + }), + ).toThrow("release validation manifest workflow ref mismatch"); + }); + + it("validates manifest child workflow, dispatch tuple, branch, and attempt", () => { + const child = expectedChildDispatches("29090000000", 3, "main")[0]; + const parentManifest = { + runAttempt: 3, + runId: "29090000000", + targetSha: "a".repeat(40), + workflowSha: "b".repeat(40), + }; + const parentJobs = [ + { + completed_at: "2026-07-10T01:10:00Z", + conclusion: "success", + id: 901, + name: child.parentJobName, + run_attempt: 3, + started_at: "2026-07-10T01:00:00Z", + status: "completed", + steps: [], + }, + ]; + const parentLog = [ + `TARGET_SHA: ${parentManifest.targetSha}`, + "Dispatched ci.yml: https://github.com/openclaw/openclaw/actions/runs/101", + ].join("\n"); + const run = { + actor: { login: "github-actions[bot]" }, + display_title: child.displayTitle, + event: "workflow_dispatch", + head_branch: child.headBranch, + head_sha: parentManifest.workflowSha, + id: 101, + path: ".github/workflows/ci.yml@refs/heads/main", + run_attempt: 1, + triggering_actor: { login: "github-actions[bot]" }, + }; + expect(validateManifestChildRun(run, child, "101", parentManifest, parentJobs, parentLog)).toBe( + run, + ); + expect(() => + validateManifestChildRun( + { ...run, head_branch: "release/2026.7.1" }, + child, + "101", + parentManifest, + parentJobs, + parentLog, + ), + ).toThrow("manifest child dispatch tuple mismatch"); + expect(() => + validateParentManifest(rawManifest({}), { runAttempt: 3, runId: "29090000000" }), + ).toThrow("release validation manifest run attempt mismatch"); + }); + + it("accepts strongly bound legacy and correlated children across parent attempts", () => { + const parentManifest = { + runAttempt: 2, + runId: "28717729503", + targetSha: "a".repeat(40), + workflowSha: "b".repeat(40), + }; + const children = expectedChildDispatches( + parentManifest.runId, + parentManifest.runAttempt, + "main", + ); + const fixtures = new Map([ + ["normalCi", { originAttempt: 2, runId: 28718903263, title: "CI" }], + ["pluginPrerelease", { originAttempt: 1, runId: 28717802268, title: "Plugin Prerelease" }], + [ + "productPerformance", + { + originAttempt: 1, + runId: 28717802171, + title: "OpenClaw Performance full-release-validation-28717729503-1", + }, + ], + ["releaseChecks", { originAttempt: 1, runId: 28717802397, title: "OpenClaw Release Checks" }], + ]); + const fingerprint = { + completed_at: "2026-07-04T20:29:21Z", + conclusion: "success", + started_at: "2026-07-04T19:53:02Z", + status: "completed", + steps: [ + { + completed_at: "2026-07-04T20:29:20Z", + conclusion: "success", + name: "Dispatch and monitor child", + number: 1, + started_at: "2026-07-04T19:53:03Z", + status: "completed", + }, + ], + }; + + for (const child of children.filter((entry) => fixtures.has(entry.manifestKey))) { + const fixture = fixtures.get(child.manifestKey); + if (!fixture) { + throw new Error(`missing fixture for ${child.manifestKey}`); + } + const { originAttempt, runId, title } = fixture; + const parentJobs = [ + ...(originAttempt === 1 + ? [ + { + ...fingerprint, + id: 900, + name: child.parentJobName, + run_attempt: 1, + }, + ] + : []), + { + ...fingerprint, + id: 901, + name: child.parentJobName, + run_attempt: 2, + }, + ]; + const run = { + actor: { login: "github-actions[bot]" }, + display_title: title, + event: "workflow_dispatch", + head_branch: child.headBranch, + head_sha: parentManifest.workflowSha, + id: runId, + path: `.github/workflows/${child.workflow}@refs/heads/${child.headBranch}`, + run_attempt: 1, + triggering_actor: { login: "github-actions[bot]" }, + }; + const parentLog = [ + `TARGET_SHA: ${parentManifest.targetSha}`, + ...(child.manifestKey === "productPerformance" ? ["-f publish_reports=false"] : []), + `Dispatched ${child.workflow}: https://github.com/openclaw/openclaw/actions/runs/${runId}`, + ].join("\n"); + expect(resolveManifestChildOriginAttempt(run, child, parentManifest, parentJobs)).toBe( + originAttempt, + ); + expect( + validateManifestChildRun(run, child, String(runId), parentManifest, parentJobs, parentLog), + ).toBe(run); + if (child.manifestKey === "productPerformance") { + expect(() => + validateManifestChildRun( + run, + child, + String(runId), + parentManifest, + parentJobs, + parentLog.replace("-f publish_reports=false\n", ""), + ), + ).toThrow("manifest performance child is not dispatched in artifact-only mode"); + } + } + + const ci = children.find((child) => child.manifestKey === "normalCi"); + if (!ci) { + throw new Error("missing CI child fixture"); + } + const wrongParent = { + display_title: `CI full-release-validation-28717729504-1-ci`, + event: "workflow_dispatch", + head_branch: "main", + id: 101, + path: ".github/workflows/ci.yml@refs/heads/main", + }; + const ciJobs = [ + { + ...fingerprint, + id: 901, + name: ci.parentJobName, + run_attempt: 2, + }, + ]; + const ciLog = [ + `TARGET_SHA: ${parentManifest.targetSha}`, + "Dispatched ci.yml: https://github.com/openclaw/openclaw/actions/runs/101", + ].join("\n"); + expect(() => + validateManifestChildRun(wrongParent, ci, "101", parentManifest, ciJobs, ciLog), + ).toThrow("manifest child dispatch tuple mismatch"); + expect(() => + validateManifestChildRun( + { + ...wrongParent, + display_title: `CI full-release-validation-${parentManifest.runId}-3-ci`, + }, + ci, + "101", + parentManifest, + ciJobs, + ciLog, + ), + ).toThrow("manifest child dispatch tuple mismatch"); + expect( + resolveManifestChildOriginAttempt({ display_title: "CI nearby" }, ci, parentManifest, ciJobs), + ).toBeUndefined(); + }); + + it("rejects carried parent jobs whose selected-attempt execution fingerprint changed", () => { + const child = expectedChildDispatches("28717729503", 2, "main").find( + (entry) => entry.manifestKey === "pluginPrerelease", + ); + if (!child) { + throw new Error("missing plugin prerelease fixture"); + } + const parentManifest = { runAttempt: 2, runId: "28717729503" }; + const parentJobs = [ + { + completed_at: "2026-07-04T20:29:21Z", + conclusion: "success", + id: 900, + name: child.parentJobName, + run_attempt: 1, + started_at: "2026-07-04T19:53:02Z", + status: "completed", + steps: [], + }, + { + completed_at: "2026-07-04T20:30:21Z", + conclusion: "success", + id: 901, + name: child.parentJobName, + run_attempt: 2, + started_at: "2026-07-04T19:53:02Z", + status: "completed", + steps: [], + }, + ]; + + expect(() => selectManifestParentJob(parentJobs, child, parentManifest, 1)).toThrow( + "manifest parent job carry-forward fingerprint mismatch", + ); + }); +}); diff --git a/test/scripts/release-no-push-workflow.test.ts b/test/scripts/release-no-push-workflow.test.ts new file mode 100644 index 000000000000..4a467ffed371 --- /dev/null +++ b/test/scripts/release-no-push-workflow.test.ts @@ -0,0 +1,994 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const FULL_RELEASE = ".github/workflows/full-release-validation.yml"; +const RELEASE_CHECKS = ".github/workflows/openclaw-release-checks.yml"; +const PACKAGE_ACCEPTANCE = ".github/workflows/package-acceptance.yml"; +const PLUGIN_PRERELEASE = ".github/workflows/plugin-prerelease.yml"; +const LIVE_E2E = ".github/workflows/openclaw-live-and-e2e-checks-reusable.yml"; +const INSTALL_SMOKE = ".github/workflows/install-smoke.yml"; +const INSTALL_SMOKE_REUSABLE = ".github/workflows/install-smoke-reusable.yml"; +const SHARED_IMAGE_PUBLISHER = ".github/workflows/openclaw-shared-image-publish-reusable.yml"; +const SCHEDULED_LIVE = ".github/workflows/openclaw-scheduled-live-checks.yml"; +const DOCKER_RELEASE = ".github/workflows/docker-release.yml"; +const UPDATE_MIGRATION = ".github/workflows/update-migration.yml"; +const PERFORMANCE = ".github/workflows/openclaw-performance.yml"; +const LIVE_BUILD = "scripts/test-live-build-docker.sh"; +const DOCKER_E2E_IMAGE_HELPER = "scripts/lib/docker-e2e-image.sh"; + +type WorkflowInput = { + default?: boolean | number | string; + options?: string[]; + type?: string; +}; + +type WorkflowStep = { + env?: Record; + id?: string; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + if?: string; + needs?: string | string[]; + outputs?: Record; + permissions?: PermissionMap; + steps?: WorkflowStep[]; + uses?: string; + with?: Record; +}; + +type Workflow = { + jobs?: Record; + on?: { + workflow_call?: { + inputs?: Record; + outputs?: Record; + }; + workflow_dispatch?: { inputs?: Record }; + }; + permissions?: PermissionMap; +}; + +type PermissionLevel = "none" | "read" | "write"; +type PermissionMap = "read-all" | "write-all" | Record; + +const PERMISSION_RANK: Record = { none: 0, read: 1, write: 2 }; + +function permissionAt( + permissions: PermissionMap | undefined, + scope: string, + inherited: PermissionLevel, +): PermissionLevel { + if (permissions === undefined) { + return inherited; + } + if (permissions === "read-all") { + return "read"; + } + if (permissions === "write-all") { + return "write"; + } + return permissions[scope] ?? "none"; +} + +function permissionScopes(...permissions: Array): string[] { + const scopes = new Set(["actions", "contents", "packages", "pull-requests"]); + for (const value of permissions) { + if (value && typeof value === "object") { + for (const scope of Object.keys(value)) { + scopes.add(scope); + } + } + } + return [...scopes].sort(); +} + +function reusablePermissionViolations( + callerPath: string, + callerJobName: string, + seen = new Set(), +): string[] { + const caller = readWorkflow(callerPath); + const callerJob = job(caller, callerJobName); + if (!callerJob.uses?.startsWith("./.github/workflows/")) { + throw new Error(`${callerPath}:${callerJobName} is not a local reusable-workflow call`); + } + const ceiling = callerJob.permissions ?? caller.permissions; + return workflowPermissionViolations( + callerJob.uses.slice(2), + Object.fromEntries( + permissionScopes(ceiling).map((scope) => [scope, permissionAt(ceiling, scope, "none")]), + ), + `${callerPath}:${callerJobName}`, + seen, + ); +} + +function workflowPermissionViolations( + workflowPath: string, + ceiling: Record, + chain: string, + seen: Set, +): string[] { + const visitKey = `${chain}->${workflowPath}`; + if (seen.has(visitKey)) { + return []; + } + seen.add(visitKey); + const workflow = readWorkflow(workflowPath); + const violations: string[] = []; + for (const [jobName, workflowJob] of Object.entries(workflow.jobs ?? {})) { + const requested = workflowJob.permissions ?? workflow.permissions; + const scopes = permissionScopes(requested, ceiling); + const effective: Record = {}; + for (const scope of scopes) { + const cap = ceiling[scope] ?? "none"; + const level = permissionAt(requested, scope, cap); + effective[scope] = level; + if (PERMISSION_RANK[level] > PERMISSION_RANK[cap]) { + violations.push( + `${chain} -> ${workflowPath}:${jobName} requests ${scope}:${level} above caller ${scope}:${cap}`, + ); + } + } + if (workflowJob.uses?.startsWith("./.github/workflows/")) { + violations.push( + ...workflowPermissionViolations( + workflowJob.uses.slice(2), + effective, + `${chain} -> ${workflowPath}:${jobName}`, + seen, + ), + ); + } + } + return violations; +} + +function readWorkflow(path: string): Workflow { + return parse(readFileSync(path, "utf8")) as Workflow; +} + +function job(workflow: Workflow, name: string): WorkflowJob { + const value = workflow.jobs?.[name]; + if (!value) { + throw new Error(`missing workflow job ${name}`); + } + return value; +} + +function step(workflowJob: WorkflowJob, name: string): WorkflowStep { + const value = workflowJob.steps?.find((candidate) => candidate.name === name); + if (!value) { + throw new Error(`missing workflow step ${name}`); + } + return value; +} + +function expectReadOnlyPackagePermission(workflowJob: WorkflowJob): void { + expect(permissionAt(workflowJob.permissions, "packages", "none")).toBe("read"); +} + +describe("release validation no-push transport", () => { + it("keeps every local reusable-workflow permission request within its caller ceiling", () => { + const readOnlyCalls = [ + [PLUGIN_PRERELEASE, "plugin-prerelease-docker-suite"], + [RELEASE_CHECKS, "live_repo_e2e_release_checks"], + [RELEASE_CHECKS, "docker_e2e_release_checks"], + [RELEASE_CHECKS, "package_acceptance_release_checks"], + [RELEASE_CHECKS, "install_smoke_release_checks"], + [PACKAGE_ACCEPTANCE, "docker_acceptance"], + [PACKAGE_ACCEPTANCE, "docker_acceptance_registry"], + [INSTALL_SMOKE, "install_smoke"], + [SCHEDULED_LIVE, "live_and_openwebui_checks"], + [UPDATE_MIGRATION, "update_migration"], + ] as const; + for (const [workflowPath, jobName] of readOnlyCalls) { + const callerWorkflow = readWorkflow(workflowPath); + const caller = job(callerWorkflow, jobName); + const callerPermissions = caller.permissions ?? callerWorkflow.permissions; + expect( + permissionAt(callerPermissions, "packages", "none"), + `${workflowPath}:${jobName}`, + ).toBe("read"); + expect( + reusablePermissionViolations(workflowPath, jobName), + `${workflowPath}:${jobName}`, + ).toEqual([]); + } + }); + + it("models conditional reusable jobs as permission requests before scheduling", () => { + const root = mkdtempSync(join(tmpdir(), "openclaw-permission-graph-")); + const fixture = join(root, "callee.yml"); + try { + writeFileSync( + fixture, + `permissions:\n packages: read\njobs:\n safe:\n runs-on: ubuntu-latest\n skippedWriter:\n if: false\n runs-on: ubuntu-latest\n permissions:\n packages: write\n`, + ); + const violations = workflowPermissionViolations( + fixture, + { actions: "none", contents: "none", packages: "read", "pull-requests": "none" }, + "fixture:caller", + new Set(), + ); + expect(violations).toEqual([ + `fixture:caller -> ${fixture}:skippedWriter requests packages:write above caller packages:read`, + ]); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("does not persist Git credentials in validation checkouts", () => { + for (const workflowPath of [PLUGIN_PRERELEASE, RELEASE_CHECKS]) { + const workflow = readWorkflow(workflowPath); + const checkoutSteps = Object.values(workflow.jobs ?? {}).flatMap( + (workflowJob) => + workflowJob.steps?.filter((candidate) => + candidate.uses?.startsWith("actions/checkout@"), + ) ?? [], + ); + expect(checkoutSteps, workflowPath).not.toHaveLength(0); + for (const checkout of checkoutSteps) { + expect(checkout.with?.["persist-credentials"], `${workflowPath}:${checkout.name}`).toBe( + false, + ); + } + } + }); + + it("runs evidence reuse from an immutable trusted-main workflow checkout", () => { + const full = readWorkflow(FULL_RELEASE); + for (const jobName of ["resolve_target", "evidence_reuse"]) { + const checkout = step(job(full, jobName), "Checkout trusted workflow helper"); + expect(checkout.with?.ref, jobName).toBe("${{ github.sha }}"); + expect(checkout.with?.ref, jobName).not.toBe("${{ github.ref_name }}"); + expect(checkout.with?.["persist-credentials"], jobName).toBe(false); + } + + const evidenceReuse = job(full, "evidence_reuse"); + expect(step(evidenceReuse, "Checkout target SHA").with?.["persist-credentials"]).toBe(false); + const dockerAssets = job(full, "docker_runtime_assets_preflight"); + expect(step(dockerAssets, "Checkout target SHA").with?.["persist-credentials"]).toBe(false); + expect(evidenceReuse.if).toContain("github.ref == 'refs/heads/main'"); + expect(evidenceReuse.if).toContain("startsWith(github.ref, 'refs/heads/release-ci/')"); + expect( + evidenceReuse.steps?.find( + (candidate) => candidate.name === "Require trusted main workflow ref", + ), + ).toBeUndefined(); + + const releaseChecks = readWorkflow(RELEASE_CHECKS); + const releaseHelper = step( + job(releaseChecks, "resolve_target"), + "Checkout trusted workflow helper", + ); + expect(releaseHelper.with?.ref).toBe("${{ github.sha }}"); + expect(releaseHelper.with?.ref).not.toBe("${{ github.ref_name }}"); + expect(releaseHelper.with?.["persist-credentials"]).toBe(false); + }); + + it("rejects every child whose workflow SHA differs from the parent workflow SHA", () => { + const full = readWorkflow(FULL_RELEASE); + for (const [jobName, stepName] of [ + ["normal_ci", "Dispatch and monitor CI"], + ["plugin_prerelease", "Dispatch and monitor plugin prerelease"], + ["release_checks", "Dispatch and monitor release checks"], + ["npm_telegram", "Dispatch and monitor npm Telegram E2E"], + ["performance", "Dispatch and monitor OpenClaw Performance"], + ] as const) { + const dispatch = step(job(full, jobName), stepName); + expect(dispatch.env?.PARENT_WORKFLOW_SHA, jobName).toBe("${{ github.sha }}"); + expect(dispatch.run, jobName).toContain('"$child_head_sha" != "$PARENT_WORKFLOW_SHA"'); + expect(dispatch.run, jobName).toContain("expected parent workflow SHA"); + } + + const verify = step(job(full, "summary"), "Verify child workflow results"); + expect(verify.env?.PARENT_WORKFLOW_SHA).toBe("${{ github.sha }}"); + expect(verify.run).toContain('"$head_sha" != "$PARENT_WORKFLOW_SHA"'); + expect(verify.run).not.toContain('"$head_sha" != "$TARGET_SHA"'); + }); + + it("publishes an attempt-qualified canonical manifest plus a temporary legacy alias", () => { + const summary = job(readWorkflow(FULL_RELEASE), "summary"); + expect(step(summary, "Upload release validation manifest").with).toMatchObject({ + name: "full-release-validation-${{ github.run_id }}-${{ github.run_attempt }}", + }); + expect(step(summary, "Upload legacy release validation manifest alias").with).toMatchObject({ + name: "full-release-validation-${{ github.run_id }}", + overwrite: true, + }); + }); + + it("pins every Full Release Docker caller to artifact-only transport", () => { + const fullText = readFileSync(FULL_RELEASE, "utf8"); + const release = readWorkflow(RELEASE_CHECKS); + const packageAcceptance = readWorkflow(PACKAGE_ACCEPTANCE); + const pluginPrerelease = readWorkflow(PLUGIN_PRERELEASE); + + expect(fullText).toContain("dispatch_and_wait plugin-prerelease.yml"); + expect(fullText).toContain("dispatch_and_wait openclaw-release-checks.yml"); + expect(fullText).toContain("gh workflow run openclaw-performance.yml"); + + const preparePackage = job(release, "prepare_release_package"); + const live = job(release, "live_repo_e2e_release_checks"); + const docker = job(release, "docker_e2e_release_checks"); + const acceptance = job(release, "package_acceptance_release_checks"); + expectReadOnlyPackagePermission(preparePackage); + expectReadOnlyPackagePermission(live); + expectReadOnlyPackagePermission(docker); + expectReadOnlyPackagePermission(acceptance); + expect(step(preparePackage, "Resolve release package artifact").run).toContain( + 'if [[ "$source_sha" != "$PACKAGE_REF" ]]', + ); + expect(live.with).toMatchObject({ + shared_image_artifact_namespace: "release-live", + shared_image_policy: "no-push-artifact", + }); + expect(docker.with).toMatchObject({ + package_artifact_digest: "${{ needs.prepare_release_package.outputs.artifact_digest }}", + package_artifact_id: "${{ needs.prepare_release_package.outputs.artifact_id }}", + package_artifact_name: "${{ needs.prepare_release_package.outputs.artifact_name }}", + package_artifact_run_attempt: + "${{ needs.prepare_release_package.outputs.artifact_run_attempt }}", + package_artifact_run_id: "${{ needs.prepare_release_package.outputs.artifact_run_id }}", + package_file_name: "${{ needs.prepare_release_package.outputs.package_file_name }}", + package_sha256: "${{ needs.prepare_release_package.outputs.package_sha256 }}", + package_source_sha: "${{ needs.prepare_release_package.outputs.source_sha }}", + package_version: "${{ needs.prepare_release_package.outputs.package_version }}", + shared_image_artifact_namespace: "release-docker", + shared_image_policy: "no-push-artifact", + }); + expect(acceptance.with).toMatchObject({ + artifact_digest: "${{ needs.prepare_release_package.outputs.artifact_digest }}", + artifact_id: "${{ needs.prepare_release_package.outputs.artifact_id }}", + artifact_name: "${{ needs.prepare_release_package.outputs.artifact_name }}", + artifact_run_attempt: "${{ needs.prepare_release_package.outputs.artifact_run_attempt }}", + artifact_run_id: "${{ needs.prepare_release_package.outputs.artifact_run_id }}", + package_file_name: "${{ needs.prepare_release_package.outputs.package_file_name }}", + package_source_sha: "${{ needs.prepare_release_package.outputs.source_sha }}", + package_version: "${{ needs.prepare_release_package.outputs.package_version }}", + shared_image_artifact_namespace: "release-package", + shared_image_policy: "no-push-artifact", + }); + + const standardAcceptance = job(packageAcceptance, "docker_acceptance"); + const registryAcceptance = job(packageAcceptance, "docker_acceptance_registry"); + expect(permissionAt(packageAcceptance.permissions, "packages", "none")).toBe("read"); + expect(packageAcceptance.on?.workflow_dispatch?.inputs?.shared_image_policy).toMatchObject({ + default: "no-push-artifact", + options: ["existing-only", "no-push-artifact"], + type: "choice", + }); + expect(packageAcceptance.on?.workflow_call?.inputs?.shared_image_policy).toMatchObject({ + default: "no-push-artifact", + type: "string", + }); + expect(standardAcceptance.with?.shared_image_policy).toBe("${{ inputs.shared_image_policy }}"); + expect(standardAcceptance.with?.shared_image_artifact_namespace).toBe( + "${{ inputs.shared_image_artifact_namespace }}", + ); + expect(standardAcceptance.with).toMatchObject({ + package_artifact_digest: "${{ needs.resolve_package.outputs.package_artifact_digest }}", + package_artifact_id: "${{ needs.resolve_package.outputs.package_artifact_id }}", + package_artifact_run_attempt: + "${{ needs.resolve_package.outputs.package_artifact_run_attempt }}", + package_artifact_run_id: "${{ needs.resolve_package.outputs.package_artifact_run_id }}", + package_file_name: "${{ needs.resolve_package.outputs.package_file_name }}", + package_sha256: "${{ needs.resolve_package.outputs.package_sha256 }}", + package_source_sha: "${{ needs.resolve_package.outputs.package_source_sha }}", + package_version: "${{ needs.resolve_package.outputs.package_version }}", + }); + expect(standardAcceptance.if).toContain("shared_image_policy == 'no-push-artifact'"); + expectReadOnlyPackagePermission(standardAcceptance); + expect(registryAcceptance.if).toContain("shared_image_policy == 'existing-only'"); + expectReadOnlyPackagePermission(registryAcceptance); + + const pluginDocker = job(pluginPrerelease, "plugin-prerelease-docker-suite"); + expectReadOnlyPackagePermission(pluginDocker); + expect(pluginDocker.with).toMatchObject({ + shared_image_artifact_namespace: "plugin-prerelease", + shared_image_policy: "no-push-artifact", + }); + expect( + new Set([ + live.with?.shared_image_artifact_namespace, + docker.with?.shared_image_artifact_namespace, + acceptance.with?.shared_image_artifact_namespace, + pluginDocker.with?.shared_image_artifact_namespace, + ]).size, + ).toBe(4); + }); + + it("builds shared images locally, verifies artifacts, and cannot fall back to a registry", () => { + const workflow = readWorkflow(LIVE_E2E); + const dispatchPolicy = workflow.on?.workflow_dispatch?.inputs?.shared_image_policy; + const callPolicy = workflow.on?.workflow_call?.inputs?.shared_image_policy; + expect(dispatchPolicy).toMatchObject({ + default: "no-push-artifact", + options: ["existing-only", "no-push-artifact"], + }); + expect(callPolicy).toMatchObject({ default: "no-push-artifact", type: "string" }); + + const validation = job(workflow, "validate_selected_ref"); + expect(validation.outputs?.workflow_repository).toBe( + "${{ steps.workflow.outputs.workflow_repository }}", + ); + expect(validation.outputs?.workflow_sha).toBe("${{ steps.workflow.outputs.workflow_sha }}"); + const workflowIdentity = step(validation, "Resolve job workflow identity"); + expect(workflowIdentity.env?.JOB_CONTEXT).toBe("${{ toJSON(job) }}"); + expect(workflowIdentity.run).toContain( + "job.workflow_repository must be an owner/repository slug", + ); + expect(workflowIdentity.run).toContain("job.workflow_sha must be a full lowercase commit SHA"); + const trustedCheckouts = Object.entries(workflow.jobs ?? {}).flatMap(([jobName, workflowJob]) => + (workflowJob.steps ?? []) + .filter((candidate) => candidate.name?.startsWith("Checkout trusted ")) + .map((candidate) => ({ candidate, jobName })), + ); + expect(trustedCheckouts).toHaveLength(12); + for (const { candidate, jobName } of trustedCheckouts) { + expect(candidate.with, jobName).toMatchObject({ + repository: "${{ needs.validate_selected_ref.outputs.workflow_repository }}", + ref: "${{ needs.validate_selected_ref.outputs.workflow_sha }}", + "persist-credentials": false, + }); + } + + const dockerProducer = job(workflow, "prepare_docker_e2e_image"); + const liveProducer = job(workflow, "prepare_live_test_image"); + expect(permissionAt(workflow.permissions, "actions", "none")).toBe("read"); + expect(permissionAt(workflow.permissions, "packages", "none")).toBe("read"); + expectReadOnlyPackagePermission(dockerProducer); + expectReadOnlyPackagePermission(liveProducer); + expect(workflow.jobs?.push_docker_e2e_images).toBeUndefined(); + expect(workflow.jobs?.push_live_test_image).toBeUndefined(); + expect( + permissionAt(job(workflow, "docker_e2e_image_ready").permissions, "packages", "none"), + ).toBe("none"); + expect( + permissionAt(job(workflow, "live_test_image_ready").permissions, "packages", "none"), + ).toBe("none"); + const packageWriters = Object.entries(workflow.jobs ?? {}).filter( + ([, workflowJob]) => permissionAt(workflowJob.permissions, "packages", "none") === "write", + ); + expect(packageWriters).toEqual([]); + expect(workflow.on?.workflow_call?.outputs?.publication_manifest).toBeUndefined(); + expect(workflow.jobs?.collect_shared_image_publication).toBeUndefined(); + const validateSelectedRef = step( + job(workflow, "validate_selected_ref"), + "Validate selected ref", + ); + const dispatchInputs = workflow.on?.workflow_dispatch?.inputs ?? {}; + for (const inputName of [ + "package_artifact_digest", + "package_artifact_id", + "package_artifact_name", + "package_artifact_run_attempt", + "package_artifact_run_id", + "package_file_name", + "package_sha256", + "package_source_sha", + "package_version", + ]) { + expect(dispatchInputs[inputName], inputName).toBeUndefined(); + expect(workflow.on?.workflow_call?.inputs?.[inputName], inputName).toBeDefined(); + } + expect(validateSelectedRef.env?.PACKAGE_ARTIFACT_DIGEST).toBe( + "${{ inputs.package_artifact_digest }}", + ); + expect(validateSelectedRef.env?.PACKAGE_ARTIFACT_RUN_ATTEMPT).toBe( + "${{ inputs.package_artifact_run_attempt }}", + ); + expect(validateSelectedRef.env?.PACKAGE_ARTIFACT_RUN_ID).toBe( + "${{ inputs.package_artifact_run_id }}", + ); + expect(validateSelectedRef.env?.PACKAGE_ARTIFACT_ID).toBe("${{ inputs.package_artifact_id }}"); + expect(validateSelectedRef.env?.PACKAGE_FILE_NAME).toBe("${{ inputs.package_file_name }}"); + expect(validateSelectedRef.env?.PACKAGE_SOURCE_SHA).toBe("${{ inputs.package_source_sha }}"); + expect(validateSelectedRef.run).toContain( + "Package artifact selection requires the complete immutable artifact and package identity tuple.", + ); + expect(validateSelectedRef.run).toContain('"$PACKAGE_SOURCE_SHA" == "$selected_sha"'); + for (const name of [ + "prepare_docker_e2e_image", + "prepare_live_test_image", + "validate_live_models_docker", + "validate_live_models_docker_targeted", + "validate_live_docker_provider_suites", + ]) { + const checkoutSteps = job(workflow, name).steps?.filter((candidate) => + candidate.uses?.startsWith("actions/checkout@"), + ); + expect(checkoutSteps, name).not.toHaveLength(0); + for (const checkout of checkoutSteps ?? []) { + expect(checkout.with?.["persist-credentials"], `${name}:${checkout.name}`).toBe(false); + } + } + expect(dockerProducer.outputs?.image_artifact_name).toContain("image_artifact"); + expect(liveProducer.outputs?.image_artifact_name).toContain("image_artifact"); + for (const producer of [dockerProducer, liveProducer]) { + expect(producer.outputs?.image_archive_sha256).toContain("archive_sha256"); + expect(producer.outputs?.image_artifact_id).toContain("artifact-id"); + expect(producer.outputs?.image_artifact_digest).toContain("artifact-digest"); + expect(producer.outputs?.image_artifact_run_id).toBe("${{ github.run_id }}"); + expect(producer.outputs?.image_artifact_run_attempt).toBe("${{ github.run_attempt }}"); + } + expect(dockerProducer.outputs?.package_artifact_id).toContain("artifact-id"); + expect(dockerProducer.outputs?.package_artifact_digest).toContain("artifact-digest"); + expect(dockerProducer.outputs?.package_artifact_run_attempt).toContain("run_attempt"); + expect(dockerProducer.outputs?.package_artifact_run_id).toContain("run_id"); + expect(dockerProducer.outputs?.package_file_name).toContain("file_name"); + expect(dockerProducer.outputs?.package_source_sha).toContain("source_sha"); + + const packageIdentity = step(dockerProducer, "Validate OpenClaw package artifact identity"); + expect(packageIdentity.env).toMatchObject({ + ARTIFACT_DIGEST: "${{ inputs.package_artifact_digest }}", + ARTIFACT_ID: "${{ inputs.package_artifact_id }}", + ARTIFACT_NAME: "${{ inputs.package_artifact_name }}", + ARTIFACT_RUN_ATTEMPT: "${{ inputs.package_artifact_run_attempt }}", + ARTIFACT_RUN_ID: "${{ inputs.package_artifact_run_id }}", + }); + expect(packageIdentity.run).toContain('--arg digest "sha256:${ARTIFACT_DIGEST}"'); + expect(packageIdentity.run).toContain( + "actions/runs/${ARTIFACT_RUN_ID}/attempts/${ARTIFACT_RUN_ATTEMPT}", + ); + expect(packageIdentity.run).toContain("artifact_digest=$ARTIFACT_DIGEST"); + for (const [name, condition] of [ + [ + "Download current-run OpenClaw Docker E2E package", + "inputs.package_artifact_run_id == github.run_id", + ], + [ + "Download previous-run OpenClaw Docker E2E package", + "inputs.package_artifact_run_id != github.run_id", + ], + ] as const) { + const packageDownload = step(dockerProducer, name); + expect(packageDownload.if).toContain(condition); + expect(packageDownload.with).toMatchObject({ + "artifact-ids": "${{ inputs.package_artifact_id }}", + "github-token": "${{ github.token }}", + "run-id": "${{ inputs.package_artifact_run_id }}", + }); + } + + for (const name of [ + "Build bare Docker E2E image artifact", + "Build functional Docker E2E image artifact", + ]) { + const build = step(dockerProducer, name); + expect(build.if).toContain("shared_image_policy == 'no-push-artifact'"); + expect(build.run).toContain("--load"); + expect(build.run).toContain("--sbom=false"); + expect(build.run).toContain("--provenance=false"); + expect(build.run).not.toContain("--push"); + expect(build.run).not.toContain("--sbom=true"); + expect(build.run).not.toContain("--provenance=mode=max"); + } + const packDockerArtifact = step(dockerProducer, "Pack Docker E2E image artifact"); + expect(packDockerArtifact.env?.PACKAGE_SHA256).toBe("${{ steps.package.outputs.sha256 }}"); + expect(packDockerArtifact.run).toContain("shared-image-artifact.sh"); + expect(packDockerArtifact.run).toContain( + "docker-e2e-shared-images-${SHARED_IMAGE_ARTIFACT_NAMESPACE}-${TARGET_SHA:0:12}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(packDockerArtifact.run).toContain( + 'OPENCLAW_SHARED_IMAGE_PACKAGE_SHA256="$PACKAGE_SHA256"', + ); + expect(packDockerArtifact.run).toContain("archive_sha256="); + const validatePackage = step(dockerProducer, "Validate OpenClaw Docker E2E package"); + expect(validatePackage.env).toMatchObject({ + EXPECTED_PACKAGE_FILE_NAME: "${{ inputs.package_file_name }}", + EXPECTED_PACKAGE_SHA256: "${{ inputs.package_sha256 }}", + EXPECTED_PACKAGE_SOURCE_SHA: "${{ inputs.package_source_sha }}", + EXPECTED_PACKAGE_VERSION: "${{ inputs.package_version }}", + }); + expect(validatePackage.run).toContain('"$SHARED_IMAGE_POLICY" == "no-push-artifact"'); + expect(validatePackage.run).toContain( + "Resolved package identity differs from the declared immutable tuple.", + ); + expect(validatePackage.run).toContain("package/dist/build-info.json"); + expect(validatePackage.run).toContain('[[ "$package_source_sha" == "$SELECTED_SHA" ]]'); + const targetedRun = step( + job(workflow, "validate_docker_lanes"), + "Run targeted Docker E2E lanes", + ); + expect(targetedRun.env).toMatchObject({ + ARTIFACT_SUFFIX: "${{ steps.plan.outputs.artifact_suffix }}", + INCLUDE_RELEASE_PATH_SUITES: "${{ inputs.include_release_path_suites }}", + }); + expect(targetedRun.run).toContain('if [[ "$INCLUDE_RELEASE_PATH_SUITES" == "true" ]]'); + expect(targetedRun.run).not.toContain("${{ inputs."); + for (const workflowJob of Object.values(workflow.jobs ?? {})) { + for (const workflowStep of workflowJob.steps ?? []) { + for (const inputName of ["shared_image_policy", "package_sha256", "package_version"]) { + expect(workflowStep.run ?? "", `${workflowStep.name}:${inputName}`).not.toContain( + `\${{ inputs.${inputName} }}`, + ); + } + } + } + expect(readFileSync(LIVE_E2E, "utf8")).not.toContain("fromJSON(toJSON(job)).workflow_"); + expect(readFileSync(LIVE_E2E, "utf8")).not.toContain("${{ github.workflow_sha }}"); + const artifactPackAndLoadSteps = Object.values(workflow.jobs ?? {}).flatMap((workflowJob) => + (workflowJob.steps ?? []).filter((candidate) => candidate.env?.WORKFLOW_SHA !== undefined), + ); + expect(artifactPackAndLoadSteps).toHaveLength(8); + for (const artifactStep of artifactPackAndLoadSteps) { + expect(artifactStep.env?.WORKFLOW_SHA, artifactStep.name).toBe( + "${{ needs.validate_selected_ref.outputs.workflow_sha }}", + ); + } + expect(step(dockerProducer, "Upload Docker E2E image artifact")).toMatchObject({ + id: "upload_image_artifact", + if: "inputs.shared_image_policy == 'no-push-artifact' && steps.plan.outputs.needs_e2e_image == '1'", + with: { "if-no-files-found": "error" }, + }); + expect(step(liveProducer, "Pack live-test image artifact").run).toContain( + "shared-image-artifact.sh", + ); + expect(step(liveProducer, "Pack live-test image artifact").run).toContain( + "live-test-shared-image-${SHARED_IMAGE_ARTIFACT_NAMESPACE}-${TARGET_SHA:0:12}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", + ); + expect(step(liveProducer, "Upload live-test image artifact")).toMatchObject({ + id: "upload_image_artifact", + if: "inputs.shared_image_policy == 'no-push-artifact'", + with: { "if-no-files-found": "error" }, + }); + expect(step(liveProducer, "Build shared live-test image").with).toMatchObject({ + load: true, + provenance: false, + push: false, + sbom: false, + }); + const dockerLoginCondition = step(dockerProducer, "Log in to GHCR").if; + expect(dockerLoginCondition).toContain("shared_image_policy == 'existing-only'"); + expect(dockerLoginCondition).not.toContain("allow-push"); + expect(step(liveProducer, "Log in to GHCR").if).toContain( + "shared_image_policy != 'no-push-artifact'", + ); + expect(step(dockerProducer, "Check existing shared Docker E2E images").if).toContain( + "shared_image_policy == 'existing-only'", + ); + expect(step(liveProducer, "Check existing shared live-test image").if).toContain( + "shared_image_policy != 'no-push-artifact'", + ); + + const shellPushSteps = Object.entries(workflow.jobs ?? {}).flatMap(([jobName, workflowJob]) => + (workflowJob.steps ?? []) + .filter((candidate) => candidate.run?.includes("--push")) + .map((candidate) => ({ candidate, jobName })), + ); + expect(shellPushSteps).toEqual([]); + + for (const name of [ + "validate_docker_e2e", + "validate_docker_lanes", + "validate_docker_openwebui", + ]) { + const consumer = job(workflow, name); + expect(consumer.needs).toContain("docker_e2e_image_ready"); + expect(consumer.env?.OPENCLAW_DOCKER_E2E_REQUIRE_LOCAL_IMAGE).toContain("no-push-artifact"); + expect(step(consumer, "Download OpenClaw Docker E2E package").with).toMatchObject({ + "artifact-ids": "${{ needs.prepare_docker_e2e_image.outputs.package_artifact_id }}", + "github-token": "${{ github.token }}", + "run-id": "${{ needs.prepare_docker_e2e_image.outputs.package_artifact_run_id }}", + }); + const binding = step(consumer, "Validate Docker E2E image artifact binding"); + expect(binding.if).toContain("shared_image_policy == 'no-push-artifact'"); + expect(binding.env).toMatchObject({ + ARTIFACT_DIGEST: "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_digest }}", + ARTIFACT_ID: "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_id }}", + ARTIFACT_NAME: "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_name }}", + ARTIFACT_RUN_ATTEMPT: + "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_run_attempt }}", + ARTIFACT_RUN_ID: "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_run_id }}", + GH_TOKEN: "${{ github.token }}", + }); + expect(binding.run).toContain('verify-upload "Docker E2E image"'); + expect(binding.run).toContain('"$ARTIFACT_ID" "$ARTIFACT_NAME" "$ARTIFACT_DIGEST"'); + expect(binding.run).toContain('"$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT"'); + const download = step(consumer, "Download Docker E2E image artifact"); + expect(download.if).toContain("shared_image_policy == 'no-push-artifact'"); + expect(download.with).toMatchObject({ + "artifact-ids": "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_id }}", + "github-token": "${{ github.token }}", + "run-id": "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_run_id }}", + }); + expect(consumer.steps?.indexOf(binding) ?? -1).toBeLessThan( + consumer.steps?.indexOf(download) ?? -1, + ); + const loadArtifact = step(consumer, "Verify and load Docker E2E image artifact"); + expect(loadArtifact.env?.ARCHIVE_SHA256).toBe( + "${{ needs.prepare_docker_e2e_image.outputs.image_archive_sha256 }}", + ); + expect(loadArtifact.env?.PACKAGE_SHA256).toBe( + "${{ needs.prepare_docker_e2e_image.outputs.package_sha256 }}", + ); + expect(loadArtifact.env?.OPENCLAW_SHARED_IMAGE_RUN_ATTEMPT).toBe( + "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_run_attempt }}", + ); + expect(loadArtifact.env?.OPENCLAW_SHARED_IMAGE_RUN_ID).toBe( + "${{ needs.prepare_docker_e2e_image.outputs.image_artifact_run_id }}", + ); + expect(loadArtifact.run).toContain("shared-image-artifact.sh"); + expect(loadArtifact.run).toContain('OPENCLAW_SHARED_IMAGE_ARCHIVE_SHA256="$ARCHIVE_SHA256"'); + expect(loadArtifact.run).toContain('OPENCLAW_SHARED_IMAGE_PACKAGE_SHA256="$PACKAGE_SHA256"'); + expect(step(consumer, "Log in to GHCR for shared Docker E2E image").if).toContain( + "shared_image_policy != 'no-push-artifact'", + ); + for (const pullName of [ + "Pull shared bare Docker E2E image", + "Pull shared functional Docker E2E image", + ]) { + expect(step(consumer, pullName).if).toContain("shared_image_policy != 'no-push-artifact'"); + } + } + + for (const name of [ + "validate_live_models_docker", + "validate_live_models_docker_targeted", + "validate_live_docker_provider_suites", + ]) { + const consumer = job(workflow, name); + expect(consumer.needs).toContain("live_test_image_ready"); + expect(consumer.env?.OPENCLAW_LIVE_REQUIRE_LOCAL_IMAGE).toContain("no-push-artifact"); + const binding = step(consumer, "Validate live-test image artifact binding"); + expect(binding.if).toContain("shared_image_policy == 'no-push-artifact'"); + expect(binding.env).toMatchObject({ + ARTIFACT_DIGEST: "${{ needs.prepare_live_test_image.outputs.image_artifact_digest }}", + ARTIFACT_ID: "${{ needs.prepare_live_test_image.outputs.image_artifact_id }}", + ARTIFACT_NAME: "${{ needs.prepare_live_test_image.outputs.image_artifact_name }}", + ARTIFACT_RUN_ATTEMPT: + "${{ needs.prepare_live_test_image.outputs.image_artifact_run_attempt }}", + ARTIFACT_RUN_ID: "${{ needs.prepare_live_test_image.outputs.image_artifact_run_id }}", + GH_TOKEN: "${{ github.token }}", + }); + expect(binding.run).toContain('verify-upload "live-test image"'); + expect(binding.run).toContain('"$ARTIFACT_ID" "$ARTIFACT_NAME" "$ARTIFACT_DIGEST"'); + expect(binding.run).toContain('"$ARTIFACT_RUN_ID" "$ARTIFACT_RUN_ATTEMPT"'); + const download = step(consumer, "Download live-test image artifact"); + expect(download.if).toContain("shared_image_policy == 'no-push-artifact'"); + expect(download.with).toMatchObject({ + "artifact-ids": "${{ needs.prepare_live_test_image.outputs.image_artifact_id }}", + "github-token": "${{ github.token }}", + "run-id": "${{ needs.prepare_live_test_image.outputs.image_artifact_run_id }}", + }); + expect(consumer.steps?.indexOf(binding) ?? -1).toBeLessThan( + consumer.steps?.indexOf(download) ?? -1, + ); + const loadArtifact = step(consumer, "Verify and load live-test image artifact"); + expect(loadArtifact.env?.ARCHIVE_SHA256).toBe( + "${{ needs.prepare_live_test_image.outputs.image_archive_sha256 }}", + ); + expect(loadArtifact.env?.OPENCLAW_SHARED_IMAGE_RUN_ATTEMPT).toBe( + "${{ needs.prepare_live_test_image.outputs.image_artifact_run_attempt }}", + ); + expect(loadArtifact.env?.OPENCLAW_SHARED_IMAGE_RUN_ID).toBe( + "${{ needs.prepare_live_test_image.outputs.image_artifact_run_id }}", + ); + expect(loadArtifact.run).toContain("shared-image-artifact.sh"); + expect(loadArtifact.run).toContain('OPENCLAW_SHARED_IMAGE_ARCHIVE_SHA256="$ARCHIVE_SHA256"'); + expect(step(consumer, "Log in to GHCR").if).toContain( + "shared_image_policy != 'no-push-artifact'", + ); + } + + const liveBuild = readFileSync(LIVE_BUILD, "utf8"); + const requireLocalIndex = liveBuild.indexOf("OPENCLAW_LIVE_REQUIRE_LOCAL_IMAGE"); + const pullIndex = liveBuild.indexOf("Live-test image not found locally; pulling"); + expect(requireLocalIndex).toBeGreaterThanOrEqual(0); + expect(pullIndex).toBeGreaterThan(requireLocalIndex); + expect(liveBuild).toContain("Required local live-test image not found"); + }); + + it("keeps Docker-save validation artifacts unreachable from package writers", () => { + const liveWorkflow = readWorkflow(LIVE_E2E); + const scheduled = readWorkflow(SCHEDULED_LIVE); + expect(existsSync(SHARED_IMAGE_PUBLISHER)).toBe(false); + expect(liveWorkflow.on?.workflow_call?.outputs?.publication_manifest).toBeUndefined(); + expect(liveWorkflow.jobs?.collect_shared_image_publication).toBeUndefined(); + expect(scheduled.jobs?.publish_shared_images).toBeUndefined(); + + const scheduledWriters = Object.entries(scheduled.jobs ?? {}) + .filter( + ([, workflowJob]) => permissionAt(workflowJob.permissions, "packages", "none") === "write", + ) + .map(([name]) => name); + expect(scheduledWriters).toEqual([]); + + const publisherCallers = readdirSync(".github/workflows") + .filter((name) => name.endsWith(".yml") || name.endsWith(".yaml")) + .filter((name) => + readFileSync(join(".github/workflows", name), "utf8").includes( + "openclaw-shared-image-publish-reusable.yml", + ), + ); + expect(publisherCallers).toEqual([]); + expect(JSON.stringify(liveWorkflow.jobs)).not.toContain("docker image push"); + expect(JSON.stringify(scheduled.jobs)).not.toContain("docker image push"); + + const scheduledValidation = job(scheduled, "live_and_openwebui_checks"); + expect(permissionAt(scheduled.permissions, "packages", "none")).toBe("read"); + expectReadOnlyPackagePermission(scheduledValidation); + expect(scheduledValidation.with).toMatchObject({ + shared_image_artifact_namespace: "scheduled-live", + shared_image_policy: "no-push-artifact", + }); + + const dockerRelease = readWorkflow(DOCKER_RELEASE); + const attestedBuilds = Object.values(dockerRelease.jobs ?? {}).flatMap((workflowJob) => + (workflowJob.steps ?? []).filter( + (candidate) => + candidate.uses?.startsWith("docker/build-push-action@") && candidate.with?.push === true, + ), + ); + expect(attestedBuilds).toHaveLength(4); + for (const build of attestedBuilds) { + expect(build.with).toMatchObject({ + provenance: "mode=max", + push: true, + sbom: true, + }); + } + }); + + it("keeps performance evidence artifact-only when dispatched by Full Release", () => { + const fullText = readFileSync(FULL_RELEASE, "utf8"); + const performance = readWorkflow(PERFORMANCE); + const publisher = job(performance, "publish"); + const dangerousSteps = [ + "Prepare clawgrit report commit", + "Create clawgrit reports app token", + "Publish to clawgrit reports", + ]; + + expect(performance.on?.workflow_dispatch?.inputs?.publish_reports).toMatchObject({ + default: true, + type: "boolean", + }); + expect(fullText).toContain("-f publish_reports=false"); + expect(fullText).toContain("Report publication: disabled (artifacts only)"); + expect(fullText).toContain('performanceReportPublication: "artifact-only"'); + expect(publisher.if).toContain("inputs.publish_reports == true"); + const guard = job(performance, "artifact_only_guard"); + expect(guard.if).toContain("inputs.publish_reports != true"); + expect(step(guard, "Verify report publisher stayed disabled").run).toContain( + '[[ "$PUBLISH_RESULT" != "skipped" ]]', + ); + for (const name of dangerousSteps) { + expect(step(publisher, name)).toBeDefined(); + } + + for (const [name, workflowJob] of Object.entries(performance.jobs ?? {})) { + if (name === "publish") { + continue; + } + const text = JSON.stringify(workflowJob); + expect(text).not.toContain("CLAWGRIT_REPORTS_APP_TOKEN"); + expect(text).not.toContain("create-github-app-token"); + expect(text).not.toContain("git push"); + } + }); + + it("fails a missing required local live image before any registry pull", () => { + const root = mkdtempSync(join(tmpdir(), "openclaw-live-local-image-")); + const bin = join(root, "bin"); + const calls = join(root, "docker.log"); + try { + mkdirSync(bin); + writeFileSync(calls, ""); + const docker = join(bin, "docker"); + writeFileSync( + docker, + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +if [[ "$1" == "image" && "$2" == "inspect" ]]; then + exit 1 +fi +if [[ "$1" == "pull" ]]; then + exit 0 +fi +exit 2 +`, + ); + chmodSync(docker, 0o755); + + const result = spawnSync("bash", [resolve(LIVE_BUILD)], { + encoding: "utf8", + env: { + ...process.env, + DOCKER_COMMAND_TIMEOUT: "5s", + FAKE_DOCKER_LOG: calls, + OPENCLAW_LIVE_IMAGE: "openclaw-live-test:required-local", + OPENCLAW_LIVE_REQUIRE_LOCAL_IMAGE: "1", + OPENCLAW_SKIP_DOCKER_BUILD: "1", + PATH: `${bin}:${process.env.PATH ?? ""}`, + }, + }); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Required local live-test image not found: openclaw-live-test:required-local", + ); + expect(readFileSync(calls, "utf8")).toBe("image inspect openclaw-live-test:required-local\n"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + + it("fails a missing required local Docker E2E image before pull or build fallback", () => { + const root = mkdtempSync(join(tmpdir(), "openclaw-docker-e2e-local-image-")); + const bin = join(root, "bin"); + const calls = join(root, "docker.log"); + try { + mkdirSync(bin); + writeFileSync(calls, ""); + const docker = join(bin, "docker"); + writeFileSync( + docker, + `#!/usr/bin/env bash +set -euo pipefail +printf '%s\\n' "$*" >> "$FAKE_DOCKER_LOG" +if [[ "$1" == "image" && "$2" == "inspect" ]]; then + exit 1 +fi +if [[ "$1" == "pull" ]]; then + exit 0 +fi +exit 2 +`, + ); + chmodSync(docker, 0o755); + + const result = spawnSync( + "bash", + [ + "-c", + `source "$1" +docker_e2e_build_or_reuse "openclaw-e2e:required-local" "required local image test"`, + "bash", + resolve(DOCKER_E2E_IMAGE_HELPER), + ], + { + encoding: "utf8", + env: { + ...process.env, + FAKE_DOCKER_LOG: calls, + OPENCLAW_DOCKER_BUILD_ON_MISSING: "1", + OPENCLAW_DOCKER_E2E_REQUIRE_LOCAL_IMAGE: "1", + OPENCLAW_SKIP_DOCKER_BUILD: "1", + PATH: `${bin}:${process.env.PATH ?? ""}`, + }, + }, + ); + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Required local Docker E2E image not found: openclaw-e2e:required-local", + ); + expect(readFileSync(calls, "utf8")).toBe("image inspect openclaw-e2e:required-local\n"); + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); +}); diff --git a/test/scripts/testbox-lease-freshness.test.ts b/test/scripts/testbox-lease-freshness.test.ts new file mode 100644 index 000000000000..12574b671cef --- /dev/null +++ b/test/scripts/testbox-lease-freshness.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { testboxLeaseStaleReasons } from "../../scripts/testbox-lease-freshness.mjs"; + +const fingerprint = { + version: 1, + baseSha: "a".repeat(40), + headSha: "d".repeat(40), + workingTreeClean: true, + dependencyDigest: "b".repeat(64), + environmentDigest: "c".repeat(64), + workflow: ".github/workflows/ci-check-testbox.yml", + job: "check", + ref: "main", +}; + +describe("Testbox lease freshness", () => { + it("reuses a lease when hydrated inputs still match", () => { + expect(testboxLeaseStaleReasons(fingerprint, { ...fingerprint })).toEqual([]); + }); + + it("rotates a lease when base, dependency, or workflow inputs drift", () => { + expect( + testboxLeaseStaleReasons(fingerprint, { + ...fingerprint, + baseSha: "d".repeat(40), + dependencyDigest: "e".repeat(64), + workflow: "other.yml", + }), + ).toEqual(["baseSha", "dependencyDigest", "workflow"]); + }); + + it("rejects unknown provenance schemas", () => { + expect(testboxLeaseStaleReasons({ ...fingerprint, version: 2 }, fingerprint)).toEqual([ + "state schema", + ]); + }); +}); diff --git a/test/scripts/validate-full-release-validation-evidence.test.ts b/test/scripts/validate-full-release-validation-evidence.test.ts index 212da64e679a..660ceb8f22a0 100644 --- a/test/scripts/validate-full-release-validation-evidence.test.ts +++ b/test/scripts/validate-full-release-validation-evidence.test.ts @@ -43,6 +43,31 @@ function releaseManifest(overrides: Record = {}) { }; } +function exactTargetEvidenceReuse() { + return { + changedPaths: [], + evidenceSha: targetSha, + policy: "exact-target-full-validation-v1", + runId: "122", + selectedRunId: "122", + }; +} + +function strictEvidenceReuse() { + return { + schema: "openclaw.release-validation-evidence/v3", + valid: true, + current: { runId: "123", targetSha }, + root: { runId: "122", targetSha }, + evidenceReuse: { + evidenceSha: targetSha, + rootRunId: "122", + selectedRunId: "122", + }, + conclusions: { allRequiredSucceeded: true }, + }; +} + function validate( runOverrides: Record = {}, manifestOverrides: Record = {}, @@ -57,6 +82,7 @@ function validate( expectedTargetSha: targetSha, expectedWorkflowBranch: "release/2026.7.1", isTrustedMainAncestor, + validateEvidenceReuseStrictly: () => strictEvidenceReuse(), }); return { isTrustedMainAncestor, result }; } @@ -176,24 +202,72 @@ describe("full release validation evidence", () => { expect(() => validate({}, {}, false)).toThrow("not reachable from current main"); }); - it("rejects evidence reuse on the SHA-pinned path", () => { - expect(() => validate({}, { evidenceReuse: { runId: "122" } })).toThrow( - "must not reuse another validation run", + it("accepts exact-target evidence reuse on the SHA-pinned path", () => { + expect(validate({}, { evidenceReuse: exactTargetEvidenceReuse() }).result.source).toBe( + "sha-pinned-main", ); }); + it("requires strict root and child validation for reused evidence", () => { + expect(() => + validateFullReleaseValidationEvidence({ + run: releaseRun(), + manifest: releaseManifest({ evidenceReuse: exactTargetEvidenceReuse() }), + expectedRepository: "openclaw/openclaw", + expectedRunId: "123", + expectedTargetSha: targetSha, + expectedWorkflowBranch: "release/2026.7.1", + isTrustedMainAncestor: () => true, + }), + ).toThrow("requires strict chain validation"); + + expect(() => + validateFullReleaseValidationEvidence({ + run: releaseRun(), + manifest: releaseManifest({ evidenceReuse: exactTargetEvidenceReuse() }), + expectedRepository: "openclaw/openclaw", + expectedRunId: "123", + expectedTargetSha: targetSha, + expectedWorkflowBranch: "release/2026.7.1", + isTrustedMainAncestor: () => true, + validateEvidenceReuseStrictly: () => ({ + ...strictEvidenceReuse(), + conclusions: { allRequiredSucceeded: false }, + }), + }), + ).toThrow("failed strict chain validation"); + }); + + it("rejects malformed evidence reuse on the SHA-pinned path", () => { + expect(() => + validate( + {}, + { evidenceReuse: { ...exactTargetEvidenceReuse(), changedPaths: ["src/a.ts"] } }, + ), + ).toThrow("evidence reuse is invalid"); + expect(() => + validate( + {}, + { evidenceReuse: { ...exactTargetEvidenceReuse(), evidenceSha: "c".repeat(40) } }, + ), + ).toThrow("evidence reuse is invalid"); + }); + it("keeps a pinned-shaped expected branch on the pinned trust path", () => { expect(() => validateFullReleaseValidationEvidence({ run: releaseRun(), - manifest: releaseManifest({ evidenceReuse: { runId: "122" } }), + manifest: releaseManifest({ + evidenceReuse: { ...exactTargetEvidenceReuse(), selectedRunId: "" }, + }), expectedRepository: "openclaw/openclaw", expectedRunId: "123", expectedTargetSha: targetSha, expectedWorkflowBranch: pinnedBranch, - isTrustedMainAncestor: () => false, + isTrustedMainAncestor: () => true, + validateEvidenceReuseStrictly: () => strictEvidenceReuse(), }), - ).toThrow("must not reuse another validation run"); + ).toThrow("evidence reuse is invalid"); }); it("does not treat a malformed release-ci expected branch as direct", () => { diff --git a/test/scripts/verify-release-notes.test.ts b/test/scripts/verify-release-notes.test.ts index f8eda7c462cb..ccc9decc3a2b 100644 --- a/test/scripts/verify-release-notes.test.ts +++ b/test/scripts/verify-release-notes.test.ts @@ -10,6 +10,7 @@ import { countTopLevelSectionBullets, createGithubSnapshotState, cumulativeShippedPullRequests, + defaultGithubSnapshotPath, githubApiWithSnapshot, highlightCountError, persistGithubSnapshot, @@ -89,6 +90,17 @@ describe("release-note verification", () => { expect(canonicalPullRequests([456], [])).toEqual([456]); }); + it("stores default GitHub snapshots in the shared Git common directory", () => { + const commonDir = resolve("/tmp/openclaw-shared-git"); + expect(defaultGithubSnapshotPath("a".repeat(40), "b".repeat(40), commonDir)).toBe( + join( + commonDir, + "openclaw-release-cache", + `verify-release-notes-${"a".repeat(40)}-${"b".repeat(40)}.json`, + ), + ); + }); + it("reuses exact-range GitHub GraphQL snapshots without caching REST reads", () => { const cwd = mkdtempSync(join(tmpdir(), "openclaw-release-notes-snapshot-")); try { @@ -139,6 +151,33 @@ describe("release-note verification", () => { } }); + it("checkpoints successful GraphQL responses during long verification runs", () => { + const cwd = mkdtempSync(join(tmpdir(), "openclaw-release-notes-snapshot-")); + try { + const filePath = join(cwd, "snapshot.json"); + const state = createGithubSnapshotState({ + base: "a".repeat(40), + checkpointEvery: 2, + filePath, + target: "b".repeat(40), + }); + const fetchApi = (args: string[]) => ({ data: { request: args } }); + + githubApiWithSnapshot(["graphql", "-f", "query=one"], fetchApi, state); + expect(state.dirty).toBe(true); + expect(state.writesSincePersist).toBe(1); + githubApiWithSnapshot(["graphql", "-f", "query=two"], fetchApi, state); + + expect(state.dirty).toBe(false); + expect(state.writesSincePersist).toBe(0); + expect(JSON.parse(readFileSync(filePath, "utf8")).responses).toHaveProperty( + JSON.stringify(["graphql", "-f", "query=two"]), + ); + } finally { + rmSync(cwd, { recursive: true, force: true }); + } + }); + it("does not cache transient GraphQL errors", () => { const cwd = mkdtempSync(join(tmpdir(), "openclaw-release-notes-snapshot-")); try {