fix(state): release leases on process exit (#113839)

This commit is contained in:
Peter Steinberger
2026-07-25 13:13:28 -07:00
committed by GitHub
parent a7d194ac74
commit 3e4fbe8329
2 changed files with 95 additions and 0 deletions
+56
View File
@@ -1,3 +1,6 @@
import { spawn } from "node:child_process";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { afterEach, describe, expect, it } from "vitest";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
@@ -17,6 +20,59 @@ afterEach(() => {
});
describe("OpenClaw state lease", () => {
it("releases ownership when a CLI exits from inside the leased operation", async () => {
await withOpenClawTestState({ label: "core-state-lease-process-exit" }, async (state) => {
const leaseModuleUrl = pathToFileURL(path.resolve("src/state/openclaw-state-lease.ts")).href;
const childScript = await state.writeText(
"lease-process-exit-child.mts",
`
import { withOpenClawStateLease } from ${JSON.stringify(leaseModuleUrl)};
const stateDir = process.argv[2];
await withOpenClawStateLease({
scope: "core:test",
key: "process-exit",
database: { scope: "shared", options: { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } } },
leaseMs: 300_000,
waitMs: 0,
}, async () => process.exit(23));
`,
);
const exitCode = await new Promise<number | null>((resolve, reject) => {
const child = spawn(process.execPath, ["--import", "tsx", childScript, state.stateDir], {
stdio: ["ignore", "pipe", "pipe"],
});
let output = "";
child.stdout.on("data", (chunk) => (output += chunk));
child.stderr.on("data", (chunk) => (output += chunk));
child.on("error", reject);
child.on("close", (code) => {
if (code !== 23) {
reject(new Error(`lease child exited ${code}: ${output}`));
return;
}
resolve(code);
});
});
expect(exitCode).toBe(23);
let reacquired = false;
await withOpenClawStateLease(
{
scope: "core:test",
key: "process-exit",
database: { scope: "shared", options: { env: state.env } },
leaseMs: 1_000,
waitMs: 0,
},
async () => {
reacquired = true;
},
);
expect(reacquired).toBe(true);
});
});
it("rechecks exact ownership inside the caller's write transaction", async () => {
await withOpenClawTestState({ label: "core-state-lease" }, async () => {
await expect(
+39
View File
@@ -76,6 +76,35 @@ const ACQUIRE_BACKOFF = {
const MIN_LEASE_MS = 1_000;
const LEASE_DB_BUSY_TIMEOUT_MS = 0;
const RELEASE_RETRY_TIMEOUT_MS = 2_000;
const processExitLeaseCleanups = new Set<() => void>();
let processExitListenerInstalled = false;
function runProcessExitLeaseCleanups(): void {
processExitListenerInstalled = false;
for (const cleanup of processExitLeaseCleanups) {
try {
cleanup();
} catch {
// Expiry still recovers a lease when synchronous process-exit cleanup loses a DB race.
}
}
processExitLeaseCleanups.clear();
}
function registerProcessExitLeaseCleanup(cleanup: () => void): () => void {
processExitLeaseCleanups.add(cleanup);
if (!processExitListenerInstalled) {
process.once("exit", runProcessExitLeaseCleanups);
processExitListenerInstalled = true;
}
return () => {
processExitLeaseCleanups.delete(cleanup);
if (processExitLeaseCleanups.size === 0 && processExitListenerInstalled) {
process.removeListener("exit", runProcessExitLeaseCleanups);
processExitListenerInstalled = false;
}
};
}
function leaseError(
code: OpenClawStateLeaseErrorCode,
@@ -480,6 +509,15 @@ export async function withOpenClawStateLease<T>(
owner,
leaseLabel: validated.leaseLabel,
};
// `process.exit()` skips async `finally` blocks. Release synchronously so a normal CLI error
// cannot strand the lease until its TTL and block the next lifecycle command.
const unregisterProcessExitCleanup = registerProcessExitLeaseCleanup(() => {
release({
...identity,
database: validated.database,
operationLabel: validated.operationLabel,
});
});
const leaseLost = new AbortController();
const operationSignal = validated.signal
? AbortSignal.any([validated.signal, leaseLost.signal])
@@ -580,6 +618,7 @@ export async function withOpenClawStateLease<T>(
verifyLeaseOwnership({ ...identity, database: validated.database });
return result;
} finally {
unregisterProcessExitCleanup();
clearInterval(heartbeat);
if (expiryTimer) {
clearTimeout(expiryTimer);