From bb243bcffdcaf221bc137eebff155336da52de5c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Fri, 21 Aug 2026 12:13:34 -0700 Subject: [PATCH] fix(release): isolate checkpoint journal failures --- scripts/full-release-validation-state.mjs | 29 ++-- ...full-release-validation-log-checkpoint.mjs | 11 +- .../full-release-validation-state.test.ts | 153 ++++++++++++++++++ 3 files changed, 173 insertions(+), 20 deletions(-) diff --git a/scripts/full-release-validation-state.mjs b/scripts/full-release-validation-state.mjs index ea2d949614bb..910641095150 100644 --- a/scripts/full-release-validation-state.mjs +++ b/scripts/full-release-validation-state.mjs @@ -179,6 +179,15 @@ function emitCheckpoint(kind, payload, provenance) { } } +async function emitOptionalCheckpoint(kind, payload, signal) { + try { + const boundedSignal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]); + emitCheckpoint(kind, payload, await checkpointProvenance(boundedSignal)); + } catch (error) { + console.error(`[frv] ${kind} checkpoint unavailable: ${String(error?.message ?? error)}`); + } +} + function issue(kind, child, message, extra = {}) { return { child: child.key, @@ -568,9 +577,7 @@ async function planMode() { trustedWorkflow: trustedWorkflowFromInputs(planInputs), }); const stop = () => { - if (finished) { - return; - } + if (finished) return abortController.abort(new Error("execution plan checkpoint cancelled")); abortController.abort(new Error("execution plan collection cancelled")); plan = buildReleaseExecutionPlanArtifact({ blockers: plan.blockers, @@ -614,10 +621,10 @@ async function planMode() { }); writeExecutionPlan(outputPath, plan); validateReleaseExecutionPlanArtifact(plan, expected); - if (expected.targetSha) { - emitCheckpoint("plan", plan, await checkpointProvenance(abortController.signal)); - } finished = true; + if (expected.targetSha) { + await emitOptionalCheckpoint("plan", plan, abortController.signal); + } if ((reuse.blockers?.length ?? 0) > 0 || (reuse.errors?.length ?? 0) > 0) { throw new Error("release execution plan could not bind reusable evidence"); } @@ -681,9 +688,7 @@ async function collectMode(mode) { return payload; }; const stop = () => { - if (finished) { - return; - } + if (finished) return abortController.abort(new Error(`${mode} checkpoint cancelled`)); abortController.abort(new Error(`${mode} collector cancelled`)); const decision = classifyReleaseSnapshot({ cancelled: true, @@ -811,12 +816,12 @@ async function collectMode(mode) { (decision.state !== "qualifying" && decision.activeRunIds.length === 0); if (done) { const payload = writePayload(decision, { cancelledRunIds, requested: false }); - if (expected.targetSha) { - emitCheckpoint(mode, payload, await checkpointProvenance(abortController.signal)); - } finished = true; process.exitCode = payload.state === "passed" ? 0 : payload.state === "orchestration_error" ? 2 : 1; + if (expected.targetSha) { + await emitOptionalCheckpoint(mode, payload, abortController.signal); + } return; } await abortableSleep(pollIntervalMs, abortController.signal); diff --git a/scripts/lib/full-release-validation-log-checkpoint.mjs b/scripts/lib/full-release-validation-log-checkpoint.mjs index 0b2d6543c2f9..de36f38832d0 100644 --- a/scripts/lib/full-release-validation-log-checkpoint.mjs +++ b/scripts/lib/full-release-validation-log-checkpoint.mjs @@ -45,21 +45,16 @@ function encodeBase64url(bytes) { } function decodeBase64url(value, label) { - if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/u.test(value)) { + if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/u.test(value)) throw new Error(`${label} is not canonical base64url`); - } const decoded = Buffer.from(value, "base64url"); - if (encodeBase64url(decoded) !== value) { - throw new Error(`${label} is not canonical base64url`); - } + if (encodeBase64url(decoded) !== value) throw new Error(`${label} is not canonical base64url`); return decoded; } function normalizeKind(value) { const kind = requiredString(value, "checkpoint kind"); - if (!CHECKPOINT_KINDS.has(kind)) { - throw new Error(`checkpoint kind is invalid: ${kind}`); - } + if (!CHECKPOINT_KINDS.has(kind)) throw new Error(`checkpoint kind is invalid: ${kind}`); return kind; } diff --git a/test/scripts/full-release-validation-state.test.ts b/test/scripts/full-release-validation-state.test.ts index f9a775da8c99..4a145ef026fb 100644 --- a/test/scripts/full-release-validation-state.test.ts +++ b/test/scripts/full-release-validation-state.test.ts @@ -1090,6 +1090,159 @@ if (endpoint.endsWith("/actions/runs/77")) { }); }); + it("keeps a canonical plan when optional checkpoint provenance fails", () => { + const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-failure-")); + const gh = join(root, "gh"); + const output = join(root, "full-release-execution-plan.json"); + writeFileSync(gh, '#!/bin/sh\necho "Bad credentials" >&2\nexit 1\n'); + chmodSync(gh, 0o755); + const result = spawnSync(process.execPath, [SCRIPT, "plan"], { + encoding: "utf8", + env: { + ...process.env, + FULL_RELEASE_EXECUTION_PLAN_PATH: output, + FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({ + children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } }, + dockerPreflightResult: "skipped", + evidenceReuse: false, + parentRunAttempt: 1, + parentRunId: "77", + prepareCandidateResult: "skipped", + rerunGroup: "ci", + resolveTargetResult: "success", + trustedWorkflow: TRUSTED_MAIN, + workflowRef: "release-ci/tooling", + workflowSha: SHA, + }), + GITHUB_REF_NAME: "release-ci/tooling", + GITHUB_REPOSITORY: "openclaw/openclaw", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "77", + GITHUB_SHA: SHA, + PATH: `${root}:${process.env.PATH}`, + RELEASE_PROFILE: "stable", + RERUN_GROUP: "ci", + TARGET_SHA, + }, + timeout: 10_000, + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain("plan checkpoint unavailable:"); + expect(result.stderr).toContain("Bad credentials"); + expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({ + errors: [], + parentRunAttempt: 1, + }); + }); + + it.each(["decision", "drain"] as const)( + "keeps canonical %s state when optional checkpoint provenance fails", + (mode) => { + const root = mkdtempSync(join(tmpdir(), `frv-${mode}-checkpoint-failure-`)); + const gh = join(root, "gh"); + const output = join(root, `${mode}.json`); + const executionPlanPath = join(root, "full-release-execution-plan.json"); + writeFileSync( + executionPlanPath, + JSON.stringify( + executionPlan({ + children: { normalCi: { result: "success", runAttempt: 1, runId: "101" } }, + dockerPreflightResult: "skipped", + prepareCandidateResult: "skipped", + rerunGroup: "ci", + resolveTargetResult: "success", + }), + ), + ); + writeFileSync( + gh, + `#!/bin/sh +case "$*" in + *"/actions/runs/77") echo "Bad credentials" >&2; exit 1 ;; + *"/jobs?"*) exit 0 ;; +esac +printf '%s\\n' '{"id":101,"event":"workflow_dispatch","path":".github/workflows/ci.yml@refs/heads/release-ci/tooling","display_title":"CI full-release-validation-77-1-ci","head_branch":"release-ci/tooling","head_sha":"${SHA}","run_attempt":1,"status":"completed","conclusion":"success","created_at":"2026-08-21T00:00:00Z","updated_at":"2026-08-21T00:01:00Z","html_url":"https://example.invalid/runs/101"}' +`, + ); + chmodSync(gh, 0o755); + const result = spawnSync(process.execPath, [SCRIPT, mode], { + encoding: "utf8", + env: { + ...process.env, + FAIL_FAST: "false", + FULL_RELEASE_EXECUTION_PLAN_PATH: executionPlanPath, + FULL_RELEASE_STATE_PATH: output, + GITHUB_REF_NAME: "release-ci/tooling", + GITHUB_REPOSITORY: "openclaw/openclaw", + GITHUB_RUN_ATTEMPT: "2", + GITHUB_RUN_ID: "77", + GITHUB_SHA: SHA, + PATH: `${root}:${process.env.PATH}`, + RELEASE_PROFILE: "stable", + RERUN_GROUP: "ci", + TARGET_SHA, + }, + timeout: 10_000, + }); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toContain(`${mode} checkpoint unavailable:`); + expect(result.stderr).toContain("Bad credentials"); + expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({ + cancellation: { requested: false }, + state: "passed", + }); + }, + ); + + it("does not replace a completed plan when SIGTERM interrupts checkpoint provenance", async () => { + const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-signal-")); + const gh = join(root, "gh"); + const ghReady = join(root, "gh-ready"); + const output = join(root, "full-release-execution-plan.json"); + writeFileSync(gh, '#!/bin/sh\nprintf ready > "$FRV_GH_READY"\nsleep 30\n'); + chmodSync(gh, 0o755); + const childProcess = spawn(process.execPath, [SCRIPT, "plan"], { + env: { + ...process.env, + FRV_GH_READY: ghReady, + FULL_RELEASE_EXECUTION_PLAN_PATH: output, + FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({ + children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } }, + dockerPreflightResult: "skipped", + evidenceReuse: false, + parentRunAttempt: 1, + parentRunId: "77", + prepareCandidateResult: "skipped", + rerunGroup: "ci", + resolveTargetResult: "success", + trustedWorkflow: TRUSTED_MAIN, + workflowRef: "release-ci/tooling", + workflowSha: SHA, + }), + GITHUB_REF_NAME: "release-ci/tooling", + GITHUB_REPOSITORY: "openclaw/openclaw", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "77", + GITHUB_SHA: SHA, + PATH: `${root}:${process.env.PATH}`, + RELEASE_PROFILE: "stable", + RERUN_GROUP: "ci", + TARGET_SHA, + }, + stdio: "ignore", + }); + await waitForFile(ghReady, 5_000); + const exitPromise = waitForChildClose(childProcess); + const started = Date.now(); + childProcess.kill("SIGTERM"); + await exitPromise; + expect(Date.now() - started).toBeLessThan(2_000); + expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({ + errors: [], + parentRunAttempt: 1, + }); + }); + it("writes the execution plan immediately when SIGTERM interrupts a stalled reuse API", async () => { const root = mkdtempSync(join(tmpdir(), "frv-plan-signal-")); const gh = join(root, "gh");