mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(state): release leases on process exit (#113839)
This commit is contained in:
committed by
GitHub
parent
a7d194ac74
commit
3e4fbe8329
@@ -1,3 +1,6 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
@@ -17,6 +20,59 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("OpenClaw state lease", () => {
|
||||
it("releases ownership when a CLI exits from inside the leased operation", async () => {
|
||||
await withOpenClawTestState({ label: "core-state-lease-process-exit" }, async (state) => {
|
||||
const leaseModuleUrl = pathToFileURL(path.resolve("src/state/openclaw-state-lease.ts")).href;
|
||||
const childScript = await state.writeText(
|
||||
"lease-process-exit-child.mts",
|
||||
`
|
||||
import { withOpenClawStateLease } from ${JSON.stringify(leaseModuleUrl)};
|
||||
const stateDir = process.argv[2];
|
||||
await withOpenClawStateLease({
|
||||
scope: "core:test",
|
||||
key: "process-exit",
|
||||
database: { scope: "shared", options: { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } } },
|
||||
leaseMs: 300_000,
|
||||
waitMs: 0,
|
||||
}, async () => process.exit(23));
|
||||
`,
|
||||
);
|
||||
|
||||
const exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
const child = spawn(process.execPath, ["--import", "tsx", childScript, state.stateDir], {
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let output = "";
|
||||
child.stdout.on("data", (chunk) => (output += chunk));
|
||||
child.stderr.on("data", (chunk) => (output += chunk));
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => {
|
||||
if (code !== 23) {
|
||||
reject(new Error(`lease child exited ${code}: ${output}`));
|
||||
return;
|
||||
}
|
||||
resolve(code);
|
||||
});
|
||||
});
|
||||
expect(exitCode).toBe(23);
|
||||
|
||||
let reacquired = false;
|
||||
await withOpenClawStateLease(
|
||||
{
|
||||
scope: "core:test",
|
||||
key: "process-exit",
|
||||
database: { scope: "shared", options: { env: state.env } },
|
||||
leaseMs: 1_000,
|
||||
waitMs: 0,
|
||||
},
|
||||
async () => {
|
||||
reacquired = true;
|
||||
},
|
||||
);
|
||||
expect(reacquired).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it("rechecks exact ownership inside the caller's write transaction", async () => {
|
||||
await withOpenClawTestState({ label: "core-state-lease" }, async () => {
|
||||
await expect(
|
||||
|
||||
@@ -76,6 +76,35 @@ const ACQUIRE_BACKOFF = {
|
||||
const MIN_LEASE_MS = 1_000;
|
||||
const LEASE_DB_BUSY_TIMEOUT_MS = 0;
|
||||
const RELEASE_RETRY_TIMEOUT_MS = 2_000;
|
||||
const processExitLeaseCleanups = new Set<() => void>();
|
||||
let processExitListenerInstalled = false;
|
||||
|
||||
function runProcessExitLeaseCleanups(): void {
|
||||
processExitListenerInstalled = false;
|
||||
for (const cleanup of processExitLeaseCleanups) {
|
||||
try {
|
||||
cleanup();
|
||||
} catch {
|
||||
// Expiry still recovers a lease when synchronous process-exit cleanup loses a DB race.
|
||||
}
|
||||
}
|
||||
processExitLeaseCleanups.clear();
|
||||
}
|
||||
|
||||
function registerProcessExitLeaseCleanup(cleanup: () => void): () => void {
|
||||
processExitLeaseCleanups.add(cleanup);
|
||||
if (!processExitListenerInstalled) {
|
||||
process.once("exit", runProcessExitLeaseCleanups);
|
||||
processExitListenerInstalled = true;
|
||||
}
|
||||
return () => {
|
||||
processExitLeaseCleanups.delete(cleanup);
|
||||
if (processExitLeaseCleanups.size === 0 && processExitListenerInstalled) {
|
||||
process.removeListener("exit", runProcessExitLeaseCleanups);
|
||||
processExitListenerInstalled = false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function leaseError(
|
||||
code: OpenClawStateLeaseErrorCode,
|
||||
@@ -480,6 +509,15 @@ export async function withOpenClawStateLease<T>(
|
||||
owner,
|
||||
leaseLabel: validated.leaseLabel,
|
||||
};
|
||||
// `process.exit()` skips async `finally` blocks. Release synchronously so a normal CLI error
|
||||
// cannot strand the lease until its TTL and block the next lifecycle command.
|
||||
const unregisterProcessExitCleanup = registerProcessExitLeaseCleanup(() => {
|
||||
release({
|
||||
...identity,
|
||||
database: validated.database,
|
||||
operationLabel: validated.operationLabel,
|
||||
});
|
||||
});
|
||||
const leaseLost = new AbortController();
|
||||
const operationSignal = validated.signal
|
||||
? AbortSignal.any([validated.signal, leaseLost.signal])
|
||||
@@ -580,6 +618,7 @@ export async function withOpenClawStateLease<T>(
|
||||
verifyLeaseOwnership({ ...identity, database: validated.database });
|
||||
return result;
|
||||
} finally {
|
||||
unregisterProcessExitCleanup();
|
||||
clearInterval(heartbeat);
|
||||
if (expiryTimer) {
|
||||
clearTimeout(expiryTimer);
|
||||
|
||||
Reference in New Issue
Block a user