From 3e4fbe83297c48dcbfa52cfc828b9d768466f5cc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Jul 2026 13:13:28 -0700 Subject: [PATCH] fix(state): release leases on process exit (#113839) --- src/state/openclaw-state-lease.test.ts | 56 ++++++++++++++++++++++++++ src/state/openclaw-state-lease.ts | 39 ++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/state/openclaw-state-lease.test.ts b/src/state/openclaw-state-lease.test.ts index 1eacf705c31f..400b32143432 100644 --- a/src/state/openclaw-state-lease.test.ts +++ b/src/state/openclaw-state-lease.test.ts @@ -1,3 +1,6 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; @@ -17,6 +20,59 @@ afterEach(() => { }); describe("OpenClaw state lease", () => { + it("releases ownership when a CLI exits from inside the leased operation", async () => { + await withOpenClawTestState({ label: "core-state-lease-process-exit" }, async (state) => { + const leaseModuleUrl = pathToFileURL(path.resolve("src/state/openclaw-state-lease.ts")).href; + const childScript = await state.writeText( + "lease-process-exit-child.mts", + ` + import { withOpenClawStateLease } from ${JSON.stringify(leaseModuleUrl)}; + const stateDir = process.argv[2]; + await withOpenClawStateLease({ + scope: "core:test", + key: "process-exit", + database: { scope: "shared", options: { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } } }, + leaseMs: 300_000, + waitMs: 0, + }, async () => process.exit(23)); + `, + ); + + const exitCode = await new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["--import", "tsx", childScript, state.stateDir], { + stdio: ["ignore", "pipe", "pipe"], + }); + let output = ""; + child.stdout.on("data", (chunk) => (output += chunk)); + child.stderr.on("data", (chunk) => (output += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code !== 23) { + reject(new Error(`lease child exited ${code}: ${output}`)); + return; + } + resolve(code); + }); + }); + expect(exitCode).toBe(23); + + let reacquired = false; + await withOpenClawStateLease( + { + scope: "core:test", + key: "process-exit", + database: { scope: "shared", options: { env: state.env } }, + leaseMs: 1_000, + waitMs: 0, + }, + async () => { + reacquired = true; + }, + ); + expect(reacquired).toBe(true); + }); + }); + it("rechecks exact ownership inside the caller's write transaction", async () => { await withOpenClawTestState({ label: "core-state-lease" }, async () => { await expect( diff --git a/src/state/openclaw-state-lease.ts b/src/state/openclaw-state-lease.ts index 310bf2bd8947..f576712d8d74 100644 --- a/src/state/openclaw-state-lease.ts +++ b/src/state/openclaw-state-lease.ts @@ -76,6 +76,35 @@ const ACQUIRE_BACKOFF = { const MIN_LEASE_MS = 1_000; const LEASE_DB_BUSY_TIMEOUT_MS = 0; const RELEASE_RETRY_TIMEOUT_MS = 2_000; +const processExitLeaseCleanups = new Set<() => void>(); +let processExitListenerInstalled = false; + +function runProcessExitLeaseCleanups(): void { + processExitListenerInstalled = false; + for (const cleanup of processExitLeaseCleanups) { + try { + cleanup(); + } catch { + // Expiry still recovers a lease when synchronous process-exit cleanup loses a DB race. + } + } + processExitLeaseCleanups.clear(); +} + +function registerProcessExitLeaseCleanup(cleanup: () => void): () => void { + processExitLeaseCleanups.add(cleanup); + if (!processExitListenerInstalled) { + process.once("exit", runProcessExitLeaseCleanups); + processExitListenerInstalled = true; + } + return () => { + processExitLeaseCleanups.delete(cleanup); + if (processExitLeaseCleanups.size === 0 && processExitListenerInstalled) { + process.removeListener("exit", runProcessExitLeaseCleanups); + processExitListenerInstalled = false; + } + }; +} function leaseError( code: OpenClawStateLeaseErrorCode, @@ -480,6 +509,15 @@ export async function withOpenClawStateLease( owner, leaseLabel: validated.leaseLabel, }; + // `process.exit()` skips async `finally` blocks. Release synchronously so a normal CLI error + // cannot strand the lease until its TTL and block the next lifecycle command. + const unregisterProcessExitCleanup = registerProcessExitLeaseCleanup(() => { + release({ + ...identity, + database: validated.database, + operationLabel: validated.operationLabel, + }); + }); const leaseLost = new AbortController(); const operationSignal = validated.signal ? AbortSignal.any([validated.signal, leaseLost.signal]) @@ -580,6 +618,7 @@ export async function withOpenClawStateLease( verifyLeaseOwnership({ ...identity, database: validated.database }); return result; } finally { + unregisterProcessExitCleanup(); clearInterval(heartbeat); if (expiryTimer) { clearTimeout(expiryTimer);