ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-24 11:25:50 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(release): isolate checkpoint journal failures

Browse Source
This commit is contained in:
Vincent Koc
2026-08-21 12:13:34 -07:00
parent 16f6839fb6
commit bb243bcffd
3 changed files with 173 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/full-release-validation-state.mjs
+17 -12
View File
@@ -179,6 +179,15 @@ function emitCheckpoint(kind, payload, provenance) {
}
}
async function emitOptionalCheckpoint(kind, payload, signal) {
try {
const boundedSignal = AbortSignal.any([signal, AbortSignal.timeout(15_000)]);
emitCheckpoint(kind, payload, await checkpointProvenance(boundedSignal));
} catch (error) {
console.error(`[frv] ${kind} checkpoint unavailable: ${String(error?.message ?? error)}`);
}
}
function issue(kind, child, message, extra = {}) {
return {
child: child.key,
@@ -568,9 +577,7 @@ async function planMode() {
trustedWorkflow: trustedWorkflowFromInputs(planInputs),
});
const stop = () => {
if (finished) {
return;
}
if (finished) return abortController.abort(new Error("execution plan checkpoint cancelled"));
abortController.abort(new Error("execution plan collection cancelled"));
plan = buildReleaseExecutionPlanArtifact({
blockers: plan.blockers,
@@ -614,10 +621,10 @@ async function planMode() {
});
writeExecutionPlan(outputPath, plan);
validateReleaseExecutionPlanArtifact(plan, expected);
if (expected.targetSha) {
emitCheckpoint("plan", plan, await checkpointProvenance(abortController.signal));
}
finished = true;
if (expected.targetSha) {
await emitOptionalCheckpoint("plan", plan, abortController.signal);
}
if ((reuse.blockers?.length ?? 0) > 0 || (reuse.errors?.length ?? 0) > 0) {
throw new Error("release execution plan could not bind reusable evidence");
}
@@ -681,9 +688,7 @@ async function collectMode(mode) {
return payload;
};
const stop = () => {
if (finished) {
return;
}
if (finished) return abortController.abort(new Error(`${mode} checkpoint cancelled`));
abortController.abort(new Error(`${mode} collector cancelled`));
const decision = classifyReleaseSnapshot({
cancelled: true,
@@ -811,12 +816,12 @@ async function collectMode(mode) {
(decision.state !== "qualifying" && decision.activeRunIds.length === 0);
if (done) {
const payload = writePayload(decision, { cancelledRunIds, requested: false });
if (expected.targetSha) {
emitCheckpoint(mode, payload, await checkpointProvenance(abortController.signal));
}
finished = true;
process.exitCode =
payload.state === "passed" ? 0 : payload.state === "orchestration_error" ? 2 : 1;
if (expected.targetSha) {
await emitOptionalCheckpoint(mode, payload, abortController.signal);
}
return;
}
await abortableSleep(pollIntervalMs, abortController.signal);
scripts/lib/full-release-validation-log-checkpoint.mjs
+3 -8
View File
@@ -45,21 +45,16 @@ function encodeBase64url(bytes) {
}
function decodeBase64url(value, label) {
if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/u.test(value)) {
if (typeof value !== "string" || !/^[A-Za-z0-9_-]+$/u.test(value))
throw new Error(`${label} is not canonical base64url`);
}
const decoded = Buffer.from(value, "base64url");
if (encodeBase64url(decoded) !== value) {
throw new Error(`${label} is not canonical base64url`);
}
if (encodeBase64url(decoded) !== value) throw new Error(`${label} is not canonical base64url`);
return decoded;
}
function normalizeKind(value) {
const kind = requiredString(value, "checkpoint kind");
if (!CHECKPOINT_KINDS.has(kind)) {
throw new Error(`checkpoint kind is invalid: ${kind}`);
}
if (!CHECKPOINT_KINDS.has(kind)) throw new Error(`checkpoint kind is invalid: ${kind}`);
return kind;
}
test/scripts/full-release-validation-state.test.ts
+153
View File
@@ -1090,6 +1090,159 @@ if (endpoint.endsWith("/actions/runs/77")) {
});
});
it("keeps a canonical plan when optional checkpoint provenance fails", () => {
const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-failure-"));
const gh = join(root, "gh");
const output = join(root, "full-release-execution-plan.json");
writeFileSync(gh, '#!/bin/sh\necho "Bad credentials" >&2\nexit 1\n');
chmodSync(gh, 0o755);
const result = spawnSync(process.execPath, [SCRIPT, "plan"], {
encoding: "utf8",
env: {
...process.env,
FULL_RELEASE_EXECUTION_PLAN_PATH: output,
FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({
children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } },
dockerPreflightResult: "skipped",
evidenceReuse: false,
parentRunAttempt: 1,
parentRunId: "77",
prepareCandidateResult: "skipped",
rerunGroup: "ci",
resolveTargetResult: "success",
trustedWorkflow: TRUSTED_MAIN,
workflowRef: "release-ci/tooling",
workflowSha: SHA,
}),
GITHUB_REF_NAME: "release-ci/tooling",
GITHUB_REPOSITORY: "openclaw/openclaw",
GITHUB_RUN_ATTEMPT: "1",
GITHUB_RUN_ID: "77",
GITHUB_SHA: SHA,
PATH: `${root}:${process.env.PATH}`,
RELEASE_PROFILE: "stable",
RERUN_GROUP: "ci",
TARGET_SHA,
},
timeout: 10_000,
});
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toContain("plan checkpoint unavailable:");
expect(result.stderr).toContain("Bad credentials");
expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
errors: [],
parentRunAttempt: 1,
});
});
it.each(["decision", "drain"] as const)(
"keeps canonical %s state when optional checkpoint provenance fails",
(mode) => {
const root = mkdtempSync(join(tmpdir(), `frv-${mode}-checkpoint-failure-`));
const gh = join(root, "gh");
const output = join(root, `${mode}.json`);
const executionPlanPath = join(root, "full-release-execution-plan.json");
writeFileSync(
executionPlanPath,
JSON.stringify(
executionPlan({
children: { normalCi: { result: "success", runAttempt: 1, runId: "101" } },
dockerPreflightResult: "skipped",
prepareCandidateResult: "skipped",
rerunGroup: "ci",
resolveTargetResult: "success",
}),
),
);
writeFileSync(
gh,
`#!/bin/sh
case "$*" in
*"/actions/runs/77") echo "Bad credentials" >&2; exit 1 ;;
*"/jobs?"*) exit 0 ;;
esac
printf '%s\\n' '{"id":101,"event":"workflow_dispatch","path":".github/workflows/ci.yml@refs/heads/release-ci/tooling","display_title":"CI full-release-validation-77-1-ci","head_branch":"release-ci/tooling","head_sha":"${SHA}","run_attempt":1,"status":"completed","conclusion":"success","created_at":"2026-08-21T00:00:00Z","updated_at":"2026-08-21T00:01:00Z","html_url":"https://example.invalid/runs/101"}'
`,
);
chmodSync(gh, 0o755);
const result = spawnSync(process.execPath, [SCRIPT, mode], {
encoding: "utf8",
env: {
...process.env,
FAIL_FAST: "false",
FULL_RELEASE_EXECUTION_PLAN_PATH: executionPlanPath,
FULL_RELEASE_STATE_PATH: output,
GITHUB_REF_NAME: "release-ci/tooling",
GITHUB_REPOSITORY: "openclaw/openclaw",
GITHUB_RUN_ATTEMPT: "2",
GITHUB_RUN_ID: "77",
GITHUB_SHA: SHA,
PATH: `${root}:${process.env.PATH}`,
RELEASE_PROFILE: "stable",
RERUN_GROUP: "ci",
TARGET_SHA,
},
timeout: 10_000,
});
expect(result.status, result.stderr).toBe(0);
expect(result.stderr).toContain(`${mode} checkpoint unavailable:`);
expect(result.stderr).toContain("Bad credentials");
expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
cancellation: { requested: false },
state: "passed",
});
},
);
it("does not replace a completed plan when SIGTERM interrupts checkpoint provenance", async () => {
const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-signal-"));
const gh = join(root, "gh");
const ghReady = join(root, "gh-ready");
const output = join(root, "full-release-execution-plan.json");
writeFileSync(gh, '#!/bin/sh\nprintf ready > "$FRV_GH_READY"\nsleep 30\n');
chmodSync(gh, 0o755);
const childProcess = spawn(process.execPath, [SCRIPT, "plan"], {
env: {
...process.env,
FRV_GH_READY: ghReady,
FULL_RELEASE_EXECUTION_PLAN_PATH: output,
FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({
children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } },
dockerPreflightResult: "skipped",
evidenceReuse: false,
parentRunAttempt: 1,
parentRunId: "77",
prepareCandidateResult: "skipped",
rerunGroup: "ci",
resolveTargetResult: "success",
trustedWorkflow: TRUSTED_MAIN,
workflowRef: "release-ci/tooling",
workflowSha: SHA,
}),
GITHUB_REF_NAME: "release-ci/tooling",
GITHUB_REPOSITORY: "openclaw/openclaw",
GITHUB_RUN_ATTEMPT: "1",
GITHUB_RUN_ID: "77",
GITHUB_SHA: SHA,
PATH: `${root}:${process.env.PATH}`,
RELEASE_PROFILE: "stable",
RERUN_GROUP: "ci",
TARGET_SHA,
},
stdio: "ignore",
});
await waitForFile(ghReady, 5_000);
const exitPromise = waitForChildClose(childProcess);
const started = Date.now();
childProcess.kill("SIGTERM");
await exitPromise;
expect(Date.now() - started).toBeLessThan(2_000);
expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
errors: [],
parentRunAttempt: 1,
});
});
it("writes the execution plan immediately when SIGTERM interrupts a stalled reuse API", async () => {
const root = mkdtempSync(join(tmpdir(), "frv-plan-signal-"));
const gh = join(root, "gh");