fix(release): isolate checkpoint journal failures

2026-08-25 03:45:46 -06:00 · 2026-08-21 12:13:34 -07:00
parent 16f6839fb6
commit bb243bcffd
3 changed files with 173 additions and 20 deletions
@@ -1090,6 +1090,159 @@ if (endpoint.endsWith("/actions/runs/77")) {
    });
  });

+  it("keeps a canonical plan when optional checkpoint provenance fails", () => {
+    const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-failure-"));
+    const gh = join(root, "gh");
+    const output = join(root, "full-release-execution-plan.json");
+    writeFileSync(gh, '#!/bin/sh\necho "Bad credentials" >&2\nexit 1\n');
+    chmodSync(gh, 0o755);
+    const result = spawnSync(process.execPath, [SCRIPT, "plan"], {
+      encoding: "utf8",
+      env: {
+        ...process.env,
+        FULL_RELEASE_EXECUTION_PLAN_PATH: output,
+        FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({
+          children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } },
+          dockerPreflightResult: "skipped",
+          evidenceReuse: false,
+          parentRunAttempt: 1,
+          parentRunId: "77",
+          prepareCandidateResult: "skipped",
+          rerunGroup: "ci",
+          resolveTargetResult: "success",
+          trustedWorkflow: TRUSTED_MAIN,
+          workflowRef: "release-ci/tooling",
+          workflowSha: SHA,
+        }),
+        GITHUB_REF_NAME: "release-ci/tooling",
+        GITHUB_REPOSITORY: "openclaw/openclaw",
+        GITHUB_RUN_ATTEMPT: "1",
+        GITHUB_RUN_ID: "77",
+        GITHUB_SHA: SHA,
+        PATH: `${root}:${process.env.PATH}`,
+        RELEASE_PROFILE: "stable",
+        RERUN_GROUP: "ci",
+        TARGET_SHA,
+      },
+      timeout: 10_000,
+    });
+    expect(result.status, result.stderr).toBe(0);
+    expect(result.stderr).toContain("plan checkpoint unavailable:");
+    expect(result.stderr).toContain("Bad credentials");
+    expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
+      errors: [],
+      parentRunAttempt: 1,
+    });
+  });
+
+  it.each(["decision", "drain"] as const)(
+    "keeps canonical %s state when optional checkpoint provenance fails",
+    (mode) => {
+      const root = mkdtempSync(join(tmpdir(), `frv-${mode}-checkpoint-failure-`));
+      const gh = join(root, "gh");
+      const output = join(root, `${mode}.json`);
+      const executionPlanPath = join(root, "full-release-execution-plan.json");
+      writeFileSync(
+        executionPlanPath,
+        JSON.stringify(
+          executionPlan({
+            children: { normalCi: { result: "success", runAttempt: 1, runId: "101" } },
+            dockerPreflightResult: "skipped",
+            prepareCandidateResult: "skipped",
+            rerunGroup: "ci",
+            resolveTargetResult: "success",
+          }),
+        ),
+      );
+      writeFileSync(
+        gh,
+        `#!/bin/sh
+case "$*" in
+  *"/actions/runs/77") echo "Bad credentials" >&2; exit 1 ;;
+  *"/jobs?"*) exit 0 ;;
+esac
+printf '%s\\n' '{"id":101,"event":"workflow_dispatch","path":".github/workflows/ci.yml@refs/heads/release-ci/tooling","display_title":"CI full-release-validation-77-1-ci","head_branch":"release-ci/tooling","head_sha":"${SHA}","run_attempt":1,"status":"completed","conclusion":"success","created_at":"2026-08-21T00:00:00Z","updated_at":"2026-08-21T00:01:00Z","html_url":"https://example.invalid/runs/101"}'
+`,
+      );
+      chmodSync(gh, 0o755);
+      const result = spawnSync(process.execPath, [SCRIPT, mode], {
+        encoding: "utf8",
+        env: {
+          ...process.env,
+          FAIL_FAST: "false",
+          FULL_RELEASE_EXECUTION_PLAN_PATH: executionPlanPath,
+          FULL_RELEASE_STATE_PATH: output,
+          GITHUB_REF_NAME: "release-ci/tooling",
+          GITHUB_REPOSITORY: "openclaw/openclaw",
+          GITHUB_RUN_ATTEMPT: "2",
+          GITHUB_RUN_ID: "77",
+          GITHUB_SHA: SHA,
+          PATH: `${root}:${process.env.PATH}`,
+          RELEASE_PROFILE: "stable",
+          RERUN_GROUP: "ci",
+          TARGET_SHA,
+        },
+        timeout: 10_000,
+      });
+      expect(result.status, result.stderr).toBe(0);
+      expect(result.stderr).toContain(`${mode} checkpoint unavailable:`);
+      expect(result.stderr).toContain("Bad credentials");
+      expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
+        cancellation: { requested: false },
+        state: "passed",
+      });
+    },
+  );
+
+  it("does not replace a completed plan when SIGTERM interrupts checkpoint provenance", async () => {
+    const root = mkdtempSync(join(tmpdir(), "frv-plan-checkpoint-signal-"));
+    const gh = join(root, "gh");
+    const ghReady = join(root, "gh-ready");
+    const output = join(root, "full-release-execution-plan.json");
+    writeFileSync(gh, '#!/bin/sh\nprintf ready > "$FRV_GH_READY"\nsleep 30\n');
+    chmodSync(gh, 0o755);
+    const childProcess = spawn(process.execPath, [SCRIPT, "plan"], {
+      env: {
+        ...process.env,
+        FRV_GH_READY: ghReady,
+        FULL_RELEASE_EXECUTION_PLAN_PATH: output,
+        FULL_RELEASE_PLAN_INPUTS_JSON: JSON.stringify({
+          children: { normalCi: { result: "skipped", runAttempt: "", runId: "" } },
+          dockerPreflightResult: "skipped",
+          evidenceReuse: false,
+          parentRunAttempt: 1,
+          parentRunId: "77",
+          prepareCandidateResult: "skipped",
+          rerunGroup: "ci",
+          resolveTargetResult: "success",
+          trustedWorkflow: TRUSTED_MAIN,
+          workflowRef: "release-ci/tooling",
+          workflowSha: SHA,
+        }),
+        GITHUB_REF_NAME: "release-ci/tooling",
+        GITHUB_REPOSITORY: "openclaw/openclaw",
+        GITHUB_RUN_ATTEMPT: "1",
+        GITHUB_RUN_ID: "77",
+        GITHUB_SHA: SHA,
+        PATH: `${root}:${process.env.PATH}`,
+        RELEASE_PROFILE: "stable",
+        RERUN_GROUP: "ci",
+        TARGET_SHA,
+      },
+      stdio: "ignore",
+    });
+    await waitForFile(ghReady, 5_000);
+    const exitPromise = waitForChildClose(childProcess);
+    const started = Date.now();
+    childProcess.kill("SIGTERM");
+    await exitPromise;
+    expect(Date.now() - started).toBeLessThan(2_000);
+    expect(JSON.parse(readFileSync(output, "utf8"))).toMatchObject({
+      errors: [],
+      parentRunAttempt: 1,
+    });
+  });
+
  it("writes the execution plan immediately when SIGTERM interrupts a stalled reuse API", async () => {
    const root = mkdtempSync(join(tmpdir(), "frv-plan-signal-"));
    const gh = join(root, "gh");