From 7dc4dc83fe8b5f602bfc52a64cfbfaa3133cb834 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 8 Aug 2026 23:56:15 -0700 Subject: [PATCH] fix(infra): wait for concurrent startup-migration lease instead of failing gateway startup (#120959) --- .../doctor-config-preflight-checkpoint.ts | 39 +++++++++ ...r-config-preflight.state-migration.test.ts | 68 ++++++++------- src/commands/doctor-config-preflight.ts | 79 ++++++++--------- .../startup-migration-checkpoint.test.ts | 76 ++++++++++++++++- src/infra/startup-migration-checkpoint.ts | 84 +++++++++++++++++-- 5 files changed, 262 insertions(+), 84 deletions(-) create mode 100644 src/commands/doctor-config-preflight-checkpoint.ts diff --git a/src/commands/doctor-config-preflight-checkpoint.ts b/src/commands/doctor-config-preflight-checkpoint.ts new file mode 100644 index 000000000000..a96cfed89e35 --- /dev/null +++ b/src/commands/doctor-config-preflight-checkpoint.ts @@ -0,0 +1,39 @@ +import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; +import type { ConfigFileSnapshot } from "../config/types.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { MigrationCheckpointIdentity } from "../infra/startup-migration-checkpoint.js"; +import { resolveStateMigrationConfigInput } from "./doctor/shared/legacy-config-state-migration-input.js"; + +export function resolveMigrationCheckpointIdentity(params: { + snapshot: ConfigFileSnapshot; + baseConfig: OpenClawConfig; + pluginMigrationFingerprint: string | null; +}): MigrationCheckpointIdentity | null { + if (!params.snapshot.valid || !params.pluginMigrationFingerprint) { + return null; + } + const stateMigrationInput = resolveStateMigrationConfigInput({ + snapshot: params.snapshot, + baseConfig: params.baseConfig, + }); + const effectiveConfig = stateMigrationInput?.cfg ?? params.baseConfig; + const pluginDoctorConfig = stateMigrationInput?.pluginDoctorConfig ?? effectiveConfig; + return { + effectiveConfigFingerprint: hashRuntimeConfigValue(effectiveConfig), + pluginDoctorConfigFingerprint: hashRuntimeConfigValue(pluginDoctorConfig), + pluginMigrationFingerprint: params.pluginMigrationFingerprint, + }; +} + +export function migrationCheckpointIdentitiesMatch( + left: MigrationCheckpointIdentity | null, + right: MigrationCheckpointIdentity | null, +): boolean { + return ( + left !== null && + right !== null && + left.effectiveConfigFingerprint === right.effectiveConfigFingerprint && + left.pluginDoctorConfigFingerprint === right.pluginDoctorConfigFingerprint && + left.pluginMigrationFingerprint === right.pluginMigrationFingerprint + ); +} diff --git a/src/commands/doctor-config-preflight.state-migration.test.ts b/src/commands/doctor-config-preflight.state-migration.test.ts index 118969de67ea..56f279ae15cf 100644 --- a/src/commands/doctor-config-preflight.state-migration.test.ts +++ b/src/commands/doctor-config-preflight.state-migration.test.ts @@ -81,8 +81,8 @@ const startupMigrationLease = vi.hoisted(() => ({ owner: "startup-test-owner", release: startupMigrationLeaseRelease, })); -const acquireStartupMigrationLease = vi.hoisted(() => - vi.fn((_params: { env: NodeJS.ProcessEnv }) => startupMigrationLease), +const acquireStartupMigrationLeaseWithWait = vi.hoisted(() => + vi.fn(async (_params: { env: NodeJS.ProcessEnv }) => startupMigrationLease), ); const recordSuccessfulStateMigrations = vi.hoisted(() => vi.fn()); const recordSuccessfulStartupMigrations = vi.hoisted(() => vi.fn()); @@ -184,7 +184,7 @@ vi.mock("./doctor/cron/legacy-repair.js", () => ({ })); vi.mock("../infra/startup-migration-checkpoint.js", () => ({ - acquireStartupMigrationLease, + acquireStartupMigrationLeaseWithWait, needsStateMigrationCheckpoint, needsStartupMigrationCheckpoint, recordSuccessfulStateMigrations, @@ -237,6 +237,7 @@ const { runDoctorConfigPreflight } = await import("./doctor-config-preflight.js" describe("runDoctorConfigPreflight state migration", () => { beforeEach(() => { vi.clearAllMocks(); + acquireStartupMigrationLeaseWithWait.mockResolvedValue(startupMigrationLease); pluginMigrationFingerprint.mockReset(); pluginMigrationFingerprint.mockReturnValue("plugin-migrations"); findDoctorLegacyConfigIssues.mockReset(); @@ -367,7 +368,7 @@ describe("runDoctorConfigPreflight state migration", () => { expect(planStartupPluginConvergence).not.toHaveBeenCalled(); if (needed && warnings.length === 0) { expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({ - env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env, + env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env, identity: expectMigrationIdentity(), lease: startupMigrationLease, }); @@ -399,6 +400,14 @@ describe("runDoctorConfigPreflight state migration", () => { registrySource: "derived", }, }) + .mockResolvedValueOnce({ + snapshot, + pluginMetadataSnapshot: { + configFingerprint: "plugin-migrations", + index, + registrySource: "derived", + }, + }) .mockResolvedValueOnce({ snapshot, pluginMetadataSnapshot: { @@ -411,7 +420,7 @@ describe("runDoctorConfigPreflight state migration", () => { await runDoctorConfigPreflight(stateCheckpointOptions); - const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env; + const pinnedEnv = acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env; expect(writePersistedInstalledPluginIndexWithLeaseSync).toHaveBeenCalledWith(index, { env: pinnedEnv, lease: startupMigrationLease, @@ -419,9 +428,9 @@ describe("runDoctorConfigPreflight state migration", () => { const writeOrder = writePersistedInstalledPluginIndexWithLeaseSync.mock.invocationCallOrder[0] ?? 0; const verificationReadOrder = - readConfigFileSnapshotWithPluginMetadata.mock.invocationCallOrder[2] ?? 0; + readConfigFileSnapshotWithPluginMetadata.mock.invocationCallOrder[3] ?? 0; expect(verificationReadOrder).toBeGreaterThan(writeOrder); - expect(readConfigFileSnapshotWithPluginMetadata.mock.calls[2]?.[0]).toEqual({ + expect(readConfigFileSnapshotWithPluginMetadata.mock.calls[3]?.[0]).toEqual({ allowCurrentPluginMetadata: false, }); const checkpointOrder = recordSuccessfulStateMigrations.mock.invocationCallOrder[0] ?? 0; @@ -486,7 +495,7 @@ describe("runDoctorConfigPreflight state migration", () => { ).rejects.toThrow("selected config changed during startup"); expect(needsStartupMigrationCheckpoint).not.toHaveBeenCalled(); - expect(acquireStartupMigrationLease).not.toHaveBeenCalled(); + expect(acquireStartupMigrationLeaseWithWait).not.toHaveBeenCalled(); expect(readConfigFileSnapshot).not.toHaveBeenCalled(); }); @@ -495,7 +504,7 @@ describe("runDoctorConfigPreflight state migration", () => { const previousStateDir = process.env.OPENCLAW_STATE_DIR; process.env.OPENCLAW_STATE_DIR = "/tmp/openclaw-original-state"; let leaseEnv: NodeJS.ProcessEnv | undefined; - acquireStartupMigrationLease.mockImplementationOnce(({ env }) => { + acquireStartupMigrationLeaseWithWait.mockImplementationOnce(async ({ env }) => { leaseEnv = env; return { ...startupMigrationLease, @@ -620,8 +629,9 @@ describe("runDoctorConfigPreflight state migration", () => { expect(result.cronCodexRuntimePolicyTargets).toEqual([{ modelRef: "openai/gpt-5.6-sol" }]); }); - it("records the startup migration checkpoint after clean startup migrations", async () => { - needsStartupMigrationCheckpoint.mockReturnValue(true); + it("rechecks the checkpoint after acquisition before running migrations", async () => { + needsStateMigrationCheckpoint.mockReturnValueOnce(true).mockReturnValue(false); + needsStartupMigrationCheckpoint.mockReturnValueOnce(true).mockReturnValue(false); await runDoctorConfigPreflight({ migrateLegacyConfig: false, @@ -629,24 +639,11 @@ describe("runDoctorConfigPreflight state migration", () => { requireStartupMigrationCheckpoint: true, }); - const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env; - expect(pinnedEnv).toBeDefined(); - expect(pinnedEnv).not.toBe(process.env); - expect(needsStartupMigrationCheckpoint).toHaveBeenCalledWith({ - env: pinnedEnv, - identity: expectMigrationIdentity(), - }); - expect(runPostCorePluginConvergence).toHaveBeenCalledWith({ - cfg: { gateway: { mode: "local", port: 19091 } }, - env: process.env, - compatibilityHostVersion: expect.any(String), - baselineInstallRecords: {}, - }); - expect(recordSuccessfulStartupMigrations).toHaveBeenCalledWith({ - env: pinnedEnv, - identity: expectMigrationIdentity(), - lease: startupMigrationLease, - }); + expect(autoMigrateLegacyStateDir).not.toHaveBeenCalled(); + expect(autoMigrateLegacyState).not.toHaveBeenCalled(); + expect(recordSuccessfulStateMigrations).not.toHaveBeenCalled(); + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + expect(readConfigFileSnapshotWithPluginMetadata).toHaveBeenCalledTimes(2); expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); }); @@ -680,6 +677,7 @@ describe("runDoctorConfigPreflight state migration", () => { it("refuses startup when plugin migration inputs change during convergence", async () => { needsStartupMigrationCheckpoint.mockReturnValue(true); pluginMigrationFingerprint + .mockReturnValueOnce("plugin-migrations-before") .mockReturnValueOnce("plugin-migrations-before") .mockReturnValueOnce("plugin-migrations-before") .mockReturnValueOnce("plugin-migrations-after"); @@ -693,7 +691,7 @@ describe("runDoctorConfigPreflight state migration", () => { ).rejects.toThrow("plugin migration inputs changed during startup convergence"); expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({ - env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env, + env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env, identity: expect.objectContaining({ pluginMigrationFingerprint: "plugin-migrations-before", }), @@ -719,7 +717,7 @@ describe("runDoctorConfigPreflight state migration", () => { requireStartupMigrationCheckpoint: true, }); - const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env; + const pinnedEnv = acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env; expect(recordSuccessfulStartupMigrations).toHaveBeenCalledWith({ env: pinnedEnv, identity: expectMigrationIdentity(), @@ -811,7 +809,7 @@ describe("runDoctorConfigPreflight state migration", () => { warnings: [], issues: [], }, - 2, + 3, ); runPostCorePluginConvergence.mockResolvedValueOnce( makeStartupConvergenceResult({ @@ -977,7 +975,7 @@ describe("runDoctorConfigPreflight state migration", () => { ).rejects.toThrow("Configured plugin discord is not installed"); expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({ - env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env, + env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env, identity: expectMigrationIdentity(), lease: startupMigrationLease, }); @@ -1011,7 +1009,7 @@ describe("runDoctorConfigPreflight state migration", () => { warnings: [], issues: [], }, - 3, + 4, ); runPostCorePluginConvergence.mockResolvedValueOnce( makeStartupConvergenceResult({ @@ -1080,7 +1078,7 @@ describe("runDoctorConfigPreflight state migration", () => { warnings: [], issues: [{ path: "gateway.port", message: "invalid" }], }, - 2, + 3, ); await expect( diff --git a/src/commands/doctor-config-preflight.ts b/src/commands/doctor-config-preflight.ts index 95f4e6d5942c..ff934f6518d8 100644 --- a/src/commands/doctor-config-preflight.ts +++ b/src/commands/doctor-config-preflight.ts @@ -13,7 +13,6 @@ import { import type { ConfigSnapshotReadMeasure } from "../config/io.js"; import { formatConfigIssueLines } from "../config/issue-format.js"; import { resolveCanonicalConfigPath } from "../config/paths.js"; -import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js"; import type { ConfigFileSnapshot } from "../config/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isTruthyEnvValue } from "../infra/env.js"; @@ -27,6 +26,10 @@ import { ExitError } from "../runtime.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; import { resolveHomeDir } from "../utils.js"; import { noteIncludeConfinementWarning } from "./doctor-config-analysis.js"; +import { + migrationCheckpointIdentitiesMatch, + resolveMigrationCheckpointIdentity, +} from "./doctor-config-preflight-checkpoint.js"; import { measureDoctorConfigPreflightStep } from "./doctor-config-preflight-measure.js"; import { needsRefreshedPluginIndexPersistence, @@ -172,40 +175,6 @@ function throwStartupMigrationIdentityChanged(): never { ); } -function resolveMigrationCheckpointIdentity(params: { - snapshot: ConfigFileSnapshot; - baseConfig: OpenClawConfig; - pluginMigrationFingerprint: string | null; -}): MigrationCheckpointIdentity | null { - if (!params.snapshot.valid || !params.pluginMigrationFingerprint) { - return null; - } - const stateMigrationInput = resolveStateMigrationConfigInput({ - snapshot: params.snapshot, - baseConfig: params.baseConfig, - }); - const effectiveConfig = stateMigrationInput?.cfg ?? params.baseConfig; - const pluginDoctorConfig = stateMigrationInput?.pluginDoctorConfig ?? effectiveConfig; - return { - effectiveConfigFingerprint: hashRuntimeConfigValue(effectiveConfig), - pluginDoctorConfigFingerprint: hashRuntimeConfigValue(pluginDoctorConfig), - pluginMigrationFingerprint: params.pluginMigrationFingerprint, - }; -} - -function migrationCheckpointIdentitiesMatch( - left: MigrationCheckpointIdentity | null, - right: MigrationCheckpointIdentity | null, -): boolean { - return ( - left !== null && - right !== null && - left.effectiveConfigFingerprint === right.effectiveConfigFingerprint && - left.pluginDoctorConfigFingerprint === right.pluginDoctorConfigFingerprint && - left.pluginMigrationFingerprint === right.pluginMigrationFingerprint - ); -} - /** * Runs early doctor config checks before the main config repair flow. * @@ -268,13 +237,45 @@ export async function runDoctorConfigPreflight( getBaseSnapshot: () => configSnapshotRead?.pluginMetadataSnapshot, env: process.env, }); - const ensureStartupMigrationLease = () => { + const ensureStartupMigrationLease = async () => { if (startupMigrationLease || !migrationCheckpoint) { return; } - startupMigrationLease = migrationCheckpoint.acquireStartupMigrationLease({ + startupMigrationLease = await migrationCheckpoint.acquireStartupMigrationLeaseWithWait({ env: startupMigrationEnv, }); + // Another process may have completed the same work between our pre-lease read and acquisition. + // Refresh every checkpoint input under the lease so only work still missing from state runs. + configSnapshotRead = await readConfigSnapshotForPreflight(); + const latestBaseConfig = + configSnapshotRead.snapshot.sourceConfig ?? configSnapshotRead.snapshot.config ?? {}; + migrationCheckpointIdentity = resolveMigrationCheckpointIdentity({ + snapshot: configSnapshotRead.snapshot, + baseConfig: latestBaseConfig, + pluginMigrationFingerprint: configSnapshotRead.pluginMigrationFingerprint, + }); + shouldRecordStateCheckpoint = + stateMigrationsRequested && + migrationCheckpoint.needsStateMigrationCheckpoint({ + env: startupMigrationEnv, + identity: migrationCheckpointIdentity, + }); + shouldRecordStartupCheckpoint = + gatewayStartupCheckpointRequired && + migrationCheckpoint.needsStartupMigrationCheckpoint({ + env: startupMigrationEnv, + identity: migrationCheckpointIdentity, + }); + shouldPersistRefreshedPluginIndex = needsRefreshedPluginIndexPersistence(configSnapshotRead); + if ( + !shouldRecordStateCheckpoint && + !shouldRecordStartupCheckpoint && + !shouldPersistRefreshedPluginIndex + ) { + startupMigrationLease.release(); + startupMigrationLease = undefined; + return; + } startupMigrationHeartbeat = setInterval(() => { try { startupMigrationLease?.heartbeat(); @@ -374,7 +375,7 @@ export async function runDoctorConfigPreflight( shouldRecordStartupCheckpoint || shouldPersistRefreshedPluginIndex ) { - ensureStartupMigrationLease(); + await ensureStartupMigrationLease(); } } // A current state checkpoint proves this root already completed every automatic migration. @@ -464,7 +465,7 @@ export async function runDoctorConfigPreflight( shouldPersistRefreshedPluginIndex = migrationCheckpoint !== undefined && needsRefreshedPluginIndexPersistence(configSnapshotRead); if (shouldPersistRefreshedPluginIndex) { - ensureStartupMigrationLease(); + await ensureStartupMigrationLease(); } const freshConfigGuardRequired = stateMigrations !== undefined || diff --git a/src/infra/startup-migration-checkpoint.test.ts b/src/infra/startup-migration-checkpoint.test.ts index 33ba54f4651c..896a1ef2c2f8 100644 --- a/src/infra/startup-migration-checkpoint.test.ts +++ b/src/infra/startup-migration-checkpoint.test.ts @@ -19,6 +19,7 @@ import { import { requireNodeSqlite } from "./node-sqlite.js"; import { acquireStartupMigrationLease, + acquireStartupMigrationLeaseWithWait, hasActiveStartupMigrationLease, needsStateMigrationCheckpoint, needsStartupMigrationCheckpoint, @@ -267,7 +268,74 @@ describe("startup migration checkpoint", () => { next.release(); }); - it("reclaims an active startup migration lease whose owner process is gone", () => { + it("waits for a live same-host startup migration lease to be released", async () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + let nowMs = 1001; + let elapsedMs = 0; + let sleepCount = 0; + const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + const checkpoint = { + env, + version: "2026.7.1", + buildIdentity: "2026-07-11T00:00:00.000Z", + identity: migrationIdentity, + }; + + expect(needsStartupMigrationCheckpoint(checkpoint)).toBe(true); + + const acquired = await acquireStartupMigrationLeaseWithWait({ + env, + owner: "second", + timeoutMs: 1000, + pollIntervalMs: 250, + now: () => nowMs, + monotonicNow: () => elapsedMs, + sleep: async (ms) => { + sleepCount += 1; + recordSuccessfulStartupMigrations({ ...checkpoint, lease, nowMs }); + lease.release(); + nowMs += ms; + elapsedMs += ms; + }, + }); + + expect(sleepCount).toBe(1); + expect(acquired.owner).toBe("second"); + expect(needsStartupMigrationCheckpoint(checkpoint)).toBe(false); + acquired.release(); + }); + + it("preserves the existing lease error when the wait bound expires", async () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + let nowMs = 1001; + let elapsedMs = 0; + const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + + await expect( + acquireStartupMigrationLeaseWithWait({ + env, + owner: "second", + timeoutMs: 500, + pollIntervalMs: 250, + now: () => nowMs, + monotonicNow: () => elapsedMs, + sleep: async (ms) => { + nowMs += ms; + elapsedMs += ms; + }, + }), + ).rejects.toThrow( + `OpenClaw startup migrations are already running for this state directory; retry after the other OpenClaw process finishes or after 1970-01-01T00:05:01.000Z. (held by pid ${process.pid})`, + ); + + lease.release(); + }); + + it("reclaims an active startup migration lease whose owner process is gone", async () => { const env = { OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), }; @@ -281,7 +349,11 @@ describe("startup migration checkpoint", () => { expect(hasActiveStartupMigrationLease({ env, nowMs: 1001 })).toBe(false); - const replacement = acquireStartupMigrationLease({ env, nowMs: 1001, owner: "replacement" }); + const replacement = await acquireStartupMigrationLeaseWithWait({ + env, + owner: "replacement", + now: () => 1001, + }); stale.release(); expect(hasActiveStartupMigrationLease({ env, nowMs: 1002 })).toBe(true); replacement.release(); diff --git a/src/infra/startup-migration-checkpoint.ts b/src/infra/startup-migration-checkpoint.ts index 01f3793f8db8..64421094d655 100644 --- a/src/infra/startup-migration-checkpoint.ts +++ b/src/infra/startup-migration-checkpoint.ts @@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; import { createRequire } from "node:module"; import { hostname } from "node:os"; +import { performance } from "node:perf_hooks"; import type { DatabaseSync } from "node:sqlite"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../shared/pid-alive.js"; @@ -29,6 +30,7 @@ const STARTUP_MIGRATION_BUILD_SEPARATOR = "\n"; const STARTUP_MIGRATION_CHECKPOINT_FORMAT = "3"; const STARTUP_MIGRATION_LEASE_SCOPE = "startup-migrations"; const STARTUP_MIGRATION_LEASE_KEY = "global"; +const STARTUP_MIGRATION_LEASE_POLL_INTERVAL_MS = 250; export const STARTUP_MIGRATION_LEASE_TTL_MS = 5 * 60_000; export type StartupMigrationLease = { @@ -38,6 +40,31 @@ export type StartupMigrationLease = { readonly owner: string; }; +type StartupMigrationLeaseParams = { + env?: NodeJS.ProcessEnv; + nowMs?: number; + owner?: string; + /** Process id that owns the startup migration work. */ + ownerPid?: number; +}; + +type StartupMigrationLeaseWaitParams = Omit & { + timeoutMs?: number; + pollIntervalMs?: number; + now?: () => number; + monotonicNow?: () => number; + sleep?: (ms: number) => Promise; +}; + +class StartupMigrationLeaseConflictError extends Error { + readonly canWaitForSameHostOwner: boolean; + + constructor(message: string, canWaitForSameHostOwner: boolean) { + super(message); + this.canWaitForSameHostOwner = canWaitForSameHostOwner; + } +} + type StartupMigrationLeaseOwner = { pid: number; host: string; @@ -297,13 +324,7 @@ export function needsStateMigrationCheckpoint(params: MigrationCheckpointParams } export function acquireStartupMigrationLease( - params: { - env?: NodeJS.ProcessEnv; - nowMs?: number; - owner?: string; - /** Process id that owns the startup migration work. */ - ownerPid?: number; - } = {}, + params: StartupMigrationLeaseParams = {}, ): StartupMigrationLease { const env = params.env ?? process.env; const nowMs = params.nowMs ?? Date.now(); @@ -346,8 +367,9 @@ export function acquireStartupMigrationLease( ); } else if (existing) { const ownerHint = existingOwner ? ` (held by pid ${existingOwner.pid})` : ""; - throw new Error( + throw new StartupMigrationLeaseConflictError( `OpenClaw startup migrations are already running for this state directory; retry after the other OpenClaw process finishes or after ${new Date(existing.expiresAt ?? expiresAt).toISOString()}.${ownerHint}`, + existingOwner?.host === hostname(), ); } executeSqliteQuerySync( @@ -416,6 +438,52 @@ export function acquireStartupMigrationLease( }; } +export async function acquireStartupMigrationLeaseWithWait( + params: StartupMigrationLeaseWaitParams = {}, +): Promise { + const now = params.now ?? Date.now; + const monotonicNow = params.monotonicNow ?? performance.now.bind(performance); + const sleep = + params.sleep ?? + (async (ms: number) => + await new Promise((resolve) => { + setTimeout(resolve, ms); + })); + const timeoutMs = Math.max( + 0, + Math.min(params.timeoutMs ?? STARTUP_MIGRATION_LEASE_TTL_MS, STARTUP_MIGRATION_LEASE_TTL_MS), + ); + const pollIntervalMs = Math.max( + 1, + params.pollIntervalMs ?? STARTUP_MIGRATION_LEASE_POLL_INTERVAL_MS, + ); + const owner = params.owner ?? randomUUID(); + const deadlineMs = monotonicNow() + timeoutMs; + + while (true) { + try { + return acquireStartupMigrationLease({ + env: params.env, + nowMs: now(), + owner, + ownerPid: params.ownerPid, + }); + } catch (error) { + if ( + !(error instanceof StartupMigrationLeaseConflictError) || + !error.canWaitForSameHostOwner + ) { + throw error; + } + const remainingMs = deadlineMs - monotonicNow(); + if (remainingMs <= 0) { + throw error; + } + await sleep(Math.min(pollIntervalMs, remainingMs)); + } + } +} + function recordSuccessfulMigrationCheckpoints( metaKeys: MigrationCheckpointMetaKey[], params: RecordMigrationCheckpointParams = {},