mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(infra): wait for concurrent startup-migration lease instead of failing gateway startup (#120959)
This commit is contained in:
committed by
GitHub
parent
1b404a1755
commit
7dc4dc83fe
@@ -0,0 +1,39 @@
|
||||
import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js";
|
||||
import type { ConfigFileSnapshot } from "../config/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { MigrationCheckpointIdentity } from "../infra/startup-migration-checkpoint.js";
|
||||
import { resolveStateMigrationConfigInput } from "./doctor/shared/legacy-config-state-migration-input.js";
|
||||
|
||||
export function resolveMigrationCheckpointIdentity(params: {
|
||||
snapshot: ConfigFileSnapshot;
|
||||
baseConfig: OpenClawConfig;
|
||||
pluginMigrationFingerprint: string | null;
|
||||
}): MigrationCheckpointIdentity | null {
|
||||
if (!params.snapshot.valid || !params.pluginMigrationFingerprint) {
|
||||
return null;
|
||||
}
|
||||
const stateMigrationInput = resolveStateMigrationConfigInput({
|
||||
snapshot: params.snapshot,
|
||||
baseConfig: params.baseConfig,
|
||||
});
|
||||
const effectiveConfig = stateMigrationInput?.cfg ?? params.baseConfig;
|
||||
const pluginDoctorConfig = stateMigrationInput?.pluginDoctorConfig ?? effectiveConfig;
|
||||
return {
|
||||
effectiveConfigFingerprint: hashRuntimeConfigValue(effectiveConfig),
|
||||
pluginDoctorConfigFingerprint: hashRuntimeConfigValue(pluginDoctorConfig),
|
||||
pluginMigrationFingerprint: params.pluginMigrationFingerprint,
|
||||
};
|
||||
}
|
||||
|
||||
export function migrationCheckpointIdentitiesMatch(
|
||||
left: MigrationCheckpointIdentity | null,
|
||||
right: MigrationCheckpointIdentity | null,
|
||||
): boolean {
|
||||
return (
|
||||
left !== null &&
|
||||
right !== null &&
|
||||
left.effectiveConfigFingerprint === right.effectiveConfigFingerprint &&
|
||||
left.pluginDoctorConfigFingerprint === right.pluginDoctorConfigFingerprint &&
|
||||
left.pluginMigrationFingerprint === right.pluginMigrationFingerprint
|
||||
);
|
||||
}
|
||||
@@ -81,8 +81,8 @@ const startupMigrationLease = vi.hoisted(() => ({
|
||||
owner: "startup-test-owner",
|
||||
release: startupMigrationLeaseRelease,
|
||||
}));
|
||||
const acquireStartupMigrationLease = vi.hoisted(() =>
|
||||
vi.fn((_params: { env: NodeJS.ProcessEnv }) => startupMigrationLease),
|
||||
const acquireStartupMigrationLeaseWithWait = vi.hoisted(() =>
|
||||
vi.fn(async (_params: { env: NodeJS.ProcessEnv }) => startupMigrationLease),
|
||||
);
|
||||
const recordSuccessfulStateMigrations = vi.hoisted(() => vi.fn());
|
||||
const recordSuccessfulStartupMigrations = vi.hoisted(() => vi.fn());
|
||||
@@ -184,7 +184,7 @@ vi.mock("./doctor/cron/legacy-repair.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("../infra/startup-migration-checkpoint.js", () => ({
|
||||
acquireStartupMigrationLease,
|
||||
acquireStartupMigrationLeaseWithWait,
|
||||
needsStateMigrationCheckpoint,
|
||||
needsStartupMigrationCheckpoint,
|
||||
recordSuccessfulStateMigrations,
|
||||
@@ -237,6 +237,7 @@ const { runDoctorConfigPreflight } = await import("./doctor-config-preflight.js"
|
||||
describe("runDoctorConfigPreflight state migration", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
acquireStartupMigrationLeaseWithWait.mockResolvedValue(startupMigrationLease);
|
||||
pluginMigrationFingerprint.mockReset();
|
||||
pluginMigrationFingerprint.mockReturnValue("plugin-migrations");
|
||||
findDoctorLegacyConfigIssues.mockReset();
|
||||
@@ -367,7 +368,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
expect(planStartupPluginConvergence).not.toHaveBeenCalled();
|
||||
if (needed && warnings.length === 0) {
|
||||
expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({
|
||||
env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env,
|
||||
env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env,
|
||||
identity: expectMigrationIdentity(),
|
||||
lease: startupMigrationLease,
|
||||
});
|
||||
@@ -399,6 +400,14 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
registrySource: "derived",
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
snapshot,
|
||||
pluginMetadataSnapshot: {
|
||||
configFingerprint: "plugin-migrations",
|
||||
index,
|
||||
registrySource: "derived",
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
snapshot,
|
||||
pluginMetadataSnapshot: {
|
||||
@@ -411,7 +420,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
|
||||
await runDoctorConfigPreflight(stateCheckpointOptions);
|
||||
|
||||
const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env;
|
||||
const pinnedEnv = acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env;
|
||||
expect(writePersistedInstalledPluginIndexWithLeaseSync).toHaveBeenCalledWith(index, {
|
||||
env: pinnedEnv,
|
||||
lease: startupMigrationLease,
|
||||
@@ -419,9 +428,9 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
const writeOrder =
|
||||
writePersistedInstalledPluginIndexWithLeaseSync.mock.invocationCallOrder[0] ?? 0;
|
||||
const verificationReadOrder =
|
||||
readConfigFileSnapshotWithPluginMetadata.mock.invocationCallOrder[2] ?? 0;
|
||||
readConfigFileSnapshotWithPluginMetadata.mock.invocationCallOrder[3] ?? 0;
|
||||
expect(verificationReadOrder).toBeGreaterThan(writeOrder);
|
||||
expect(readConfigFileSnapshotWithPluginMetadata.mock.calls[2]?.[0]).toEqual({
|
||||
expect(readConfigFileSnapshotWithPluginMetadata.mock.calls[3]?.[0]).toEqual({
|
||||
allowCurrentPluginMetadata: false,
|
||||
});
|
||||
const checkpointOrder = recordSuccessfulStateMigrations.mock.invocationCallOrder[0] ?? 0;
|
||||
@@ -486,7 +495,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
).rejects.toThrow("selected config changed during startup");
|
||||
|
||||
expect(needsStartupMigrationCheckpoint).not.toHaveBeenCalled();
|
||||
expect(acquireStartupMigrationLease).not.toHaveBeenCalled();
|
||||
expect(acquireStartupMigrationLeaseWithWait).not.toHaveBeenCalled();
|
||||
expect(readConfigFileSnapshot).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -495,7 +504,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
process.env.OPENCLAW_STATE_DIR = "/tmp/openclaw-original-state";
|
||||
let leaseEnv: NodeJS.ProcessEnv | undefined;
|
||||
acquireStartupMigrationLease.mockImplementationOnce(({ env }) => {
|
||||
acquireStartupMigrationLeaseWithWait.mockImplementationOnce(async ({ env }) => {
|
||||
leaseEnv = env;
|
||||
return {
|
||||
...startupMigrationLease,
|
||||
@@ -620,8 +629,9 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
expect(result.cronCodexRuntimePolicyTargets).toEqual([{ modelRef: "openai/gpt-5.6-sol" }]);
|
||||
});
|
||||
|
||||
it("records the startup migration checkpoint after clean startup migrations", async () => {
|
||||
needsStartupMigrationCheckpoint.mockReturnValue(true);
|
||||
it("rechecks the checkpoint after acquisition before running migrations", async () => {
|
||||
needsStateMigrationCheckpoint.mockReturnValueOnce(true).mockReturnValue(false);
|
||||
needsStartupMigrationCheckpoint.mockReturnValueOnce(true).mockReturnValue(false);
|
||||
|
||||
await runDoctorConfigPreflight({
|
||||
migrateLegacyConfig: false,
|
||||
@@ -629,24 +639,11 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
requireStartupMigrationCheckpoint: true,
|
||||
});
|
||||
|
||||
const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env;
|
||||
expect(pinnedEnv).toBeDefined();
|
||||
expect(pinnedEnv).not.toBe(process.env);
|
||||
expect(needsStartupMigrationCheckpoint).toHaveBeenCalledWith({
|
||||
env: pinnedEnv,
|
||||
identity: expectMigrationIdentity(),
|
||||
});
|
||||
expect(runPostCorePluginConvergence).toHaveBeenCalledWith({
|
||||
cfg: { gateway: { mode: "local", port: 19091 } },
|
||||
env: process.env,
|
||||
compatibilityHostVersion: expect.any(String),
|
||||
baselineInstallRecords: {},
|
||||
});
|
||||
expect(recordSuccessfulStartupMigrations).toHaveBeenCalledWith({
|
||||
env: pinnedEnv,
|
||||
identity: expectMigrationIdentity(),
|
||||
lease: startupMigrationLease,
|
||||
});
|
||||
expect(autoMigrateLegacyStateDir).not.toHaveBeenCalled();
|
||||
expect(autoMigrateLegacyState).not.toHaveBeenCalled();
|
||||
expect(recordSuccessfulStateMigrations).not.toHaveBeenCalled();
|
||||
expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled();
|
||||
expect(readConfigFileSnapshotWithPluginMetadata).toHaveBeenCalledTimes(2);
|
||||
expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -680,6 +677,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
it("refuses startup when plugin migration inputs change during convergence", async () => {
|
||||
needsStartupMigrationCheckpoint.mockReturnValue(true);
|
||||
pluginMigrationFingerprint
|
||||
.mockReturnValueOnce("plugin-migrations-before")
|
||||
.mockReturnValueOnce("plugin-migrations-before")
|
||||
.mockReturnValueOnce("plugin-migrations-before")
|
||||
.mockReturnValueOnce("plugin-migrations-after");
|
||||
@@ -693,7 +691,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
).rejects.toThrow("plugin migration inputs changed during startup convergence");
|
||||
|
||||
expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({
|
||||
env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env,
|
||||
env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env,
|
||||
identity: expect.objectContaining({
|
||||
pluginMigrationFingerprint: "plugin-migrations-before",
|
||||
}),
|
||||
@@ -719,7 +717,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
requireStartupMigrationCheckpoint: true,
|
||||
});
|
||||
|
||||
const pinnedEnv = acquireStartupMigrationLease.mock.calls[0]?.[0]?.env;
|
||||
const pinnedEnv = acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env;
|
||||
expect(recordSuccessfulStartupMigrations).toHaveBeenCalledWith({
|
||||
env: pinnedEnv,
|
||||
identity: expectMigrationIdentity(),
|
||||
@@ -811,7 +809,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
warnings: [],
|
||||
issues: [],
|
||||
},
|
||||
2,
|
||||
3,
|
||||
);
|
||||
runPostCorePluginConvergence.mockResolvedValueOnce(
|
||||
makeStartupConvergenceResult({
|
||||
@@ -977,7 +975,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
).rejects.toThrow("Configured plugin discord is not installed");
|
||||
|
||||
expect(recordSuccessfulStateMigrations).toHaveBeenCalledWith({
|
||||
env: acquireStartupMigrationLease.mock.calls[0]?.[0]?.env,
|
||||
env: acquireStartupMigrationLeaseWithWait.mock.calls[0]?.[0]?.env,
|
||||
identity: expectMigrationIdentity(),
|
||||
lease: startupMigrationLease,
|
||||
});
|
||||
@@ -1011,7 +1009,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
warnings: [],
|
||||
issues: [],
|
||||
},
|
||||
3,
|
||||
4,
|
||||
);
|
||||
runPostCorePluginConvergence.mockResolvedValueOnce(
|
||||
makeStartupConvergenceResult({
|
||||
@@ -1080,7 +1078,7 @@ describe("runDoctorConfigPreflight state migration", () => {
|
||||
warnings: [],
|
||||
issues: [{ path: "gateway.port", message: "invalid" }],
|
||||
},
|
||||
2,
|
||||
3,
|
||||
);
|
||||
|
||||
await expect(
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
import type { ConfigSnapshotReadMeasure } from "../config/io.js";
|
||||
import { formatConfigIssueLines } from "../config/issue-format.js";
|
||||
import { resolveCanonicalConfigPath } from "../config/paths.js";
|
||||
import { hashRuntimeConfigValue } from "../config/runtime-snapshot.js";
|
||||
import type { ConfigFileSnapshot } from "../config/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isTruthyEnvValue } from "../infra/env.js";
|
||||
@@ -27,6 +26,10 @@ import { ExitError } from "../runtime.js";
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import { resolveHomeDir } from "../utils.js";
|
||||
import { noteIncludeConfinementWarning } from "./doctor-config-analysis.js";
|
||||
import {
|
||||
migrationCheckpointIdentitiesMatch,
|
||||
resolveMigrationCheckpointIdentity,
|
||||
} from "./doctor-config-preflight-checkpoint.js";
|
||||
import { measureDoctorConfigPreflightStep } from "./doctor-config-preflight-measure.js";
|
||||
import {
|
||||
needsRefreshedPluginIndexPersistence,
|
||||
@@ -172,40 +175,6 @@ function throwStartupMigrationIdentityChanged(): never {
|
||||
);
|
||||
}
|
||||
|
||||
function resolveMigrationCheckpointIdentity(params: {
|
||||
snapshot: ConfigFileSnapshot;
|
||||
baseConfig: OpenClawConfig;
|
||||
pluginMigrationFingerprint: string | null;
|
||||
}): MigrationCheckpointIdentity | null {
|
||||
if (!params.snapshot.valid || !params.pluginMigrationFingerprint) {
|
||||
return null;
|
||||
}
|
||||
const stateMigrationInput = resolveStateMigrationConfigInput({
|
||||
snapshot: params.snapshot,
|
||||
baseConfig: params.baseConfig,
|
||||
});
|
||||
const effectiveConfig = stateMigrationInput?.cfg ?? params.baseConfig;
|
||||
const pluginDoctorConfig = stateMigrationInput?.pluginDoctorConfig ?? effectiveConfig;
|
||||
return {
|
||||
effectiveConfigFingerprint: hashRuntimeConfigValue(effectiveConfig),
|
||||
pluginDoctorConfigFingerprint: hashRuntimeConfigValue(pluginDoctorConfig),
|
||||
pluginMigrationFingerprint: params.pluginMigrationFingerprint,
|
||||
};
|
||||
}
|
||||
|
||||
function migrationCheckpointIdentitiesMatch(
|
||||
left: MigrationCheckpointIdentity | null,
|
||||
right: MigrationCheckpointIdentity | null,
|
||||
): boolean {
|
||||
return (
|
||||
left !== null &&
|
||||
right !== null &&
|
||||
left.effectiveConfigFingerprint === right.effectiveConfigFingerprint &&
|
||||
left.pluginDoctorConfigFingerprint === right.pluginDoctorConfigFingerprint &&
|
||||
left.pluginMigrationFingerprint === right.pluginMigrationFingerprint
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs early doctor config checks before the main config repair flow.
|
||||
*
|
||||
@@ -268,13 +237,45 @@ export async function runDoctorConfigPreflight(
|
||||
getBaseSnapshot: () => configSnapshotRead?.pluginMetadataSnapshot,
|
||||
env: process.env,
|
||||
});
|
||||
const ensureStartupMigrationLease = () => {
|
||||
const ensureStartupMigrationLease = async () => {
|
||||
if (startupMigrationLease || !migrationCheckpoint) {
|
||||
return;
|
||||
}
|
||||
startupMigrationLease = migrationCheckpoint.acquireStartupMigrationLease({
|
||||
startupMigrationLease = await migrationCheckpoint.acquireStartupMigrationLeaseWithWait({
|
||||
env: startupMigrationEnv,
|
||||
});
|
||||
// Another process may have completed the same work between our pre-lease read and acquisition.
|
||||
// Refresh every checkpoint input under the lease so only work still missing from state runs.
|
||||
configSnapshotRead = await readConfigSnapshotForPreflight();
|
||||
const latestBaseConfig =
|
||||
configSnapshotRead.snapshot.sourceConfig ?? configSnapshotRead.snapshot.config ?? {};
|
||||
migrationCheckpointIdentity = resolveMigrationCheckpointIdentity({
|
||||
snapshot: configSnapshotRead.snapshot,
|
||||
baseConfig: latestBaseConfig,
|
||||
pluginMigrationFingerprint: configSnapshotRead.pluginMigrationFingerprint,
|
||||
});
|
||||
shouldRecordStateCheckpoint =
|
||||
stateMigrationsRequested &&
|
||||
migrationCheckpoint.needsStateMigrationCheckpoint({
|
||||
env: startupMigrationEnv,
|
||||
identity: migrationCheckpointIdentity,
|
||||
});
|
||||
shouldRecordStartupCheckpoint =
|
||||
gatewayStartupCheckpointRequired &&
|
||||
migrationCheckpoint.needsStartupMigrationCheckpoint({
|
||||
env: startupMigrationEnv,
|
||||
identity: migrationCheckpointIdentity,
|
||||
});
|
||||
shouldPersistRefreshedPluginIndex = needsRefreshedPluginIndexPersistence(configSnapshotRead);
|
||||
if (
|
||||
!shouldRecordStateCheckpoint &&
|
||||
!shouldRecordStartupCheckpoint &&
|
||||
!shouldPersistRefreshedPluginIndex
|
||||
) {
|
||||
startupMigrationLease.release();
|
||||
startupMigrationLease = undefined;
|
||||
return;
|
||||
}
|
||||
startupMigrationHeartbeat = setInterval(() => {
|
||||
try {
|
||||
startupMigrationLease?.heartbeat();
|
||||
@@ -374,7 +375,7 @@ export async function runDoctorConfigPreflight(
|
||||
shouldRecordStartupCheckpoint ||
|
||||
shouldPersistRefreshedPluginIndex
|
||||
) {
|
||||
ensureStartupMigrationLease();
|
||||
await ensureStartupMigrationLease();
|
||||
}
|
||||
}
|
||||
// A current state checkpoint proves this root already completed every automatic migration.
|
||||
@@ -464,7 +465,7 @@ export async function runDoctorConfigPreflight(
|
||||
shouldPersistRefreshedPluginIndex =
|
||||
migrationCheckpoint !== undefined && needsRefreshedPluginIndexPersistence(configSnapshotRead);
|
||||
if (shouldPersistRefreshedPluginIndex) {
|
||||
ensureStartupMigrationLease();
|
||||
await ensureStartupMigrationLease();
|
||||
}
|
||||
const freshConfigGuardRequired =
|
||||
stateMigrations !== undefined ||
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
import { requireNodeSqlite } from "./node-sqlite.js";
|
||||
import {
|
||||
acquireStartupMigrationLease,
|
||||
acquireStartupMigrationLeaseWithWait,
|
||||
hasActiveStartupMigrationLease,
|
||||
needsStateMigrationCheckpoint,
|
||||
needsStartupMigrationCheckpoint,
|
||||
@@ -267,7 +268,74 @@ describe("startup migration checkpoint", () => {
|
||||
next.release();
|
||||
});
|
||||
|
||||
it("reclaims an active startup migration lease whose owner process is gone", () => {
|
||||
it("waits for a live same-host startup migration lease to be released", async () => {
|
||||
const env = {
|
||||
OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"),
|
||||
};
|
||||
let nowMs = 1001;
|
||||
let elapsedMs = 0;
|
||||
let sleepCount = 0;
|
||||
const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" });
|
||||
const checkpoint = {
|
||||
env,
|
||||
version: "2026.7.1",
|
||||
buildIdentity: "2026-07-11T00:00:00.000Z",
|
||||
identity: migrationIdentity,
|
||||
};
|
||||
|
||||
expect(needsStartupMigrationCheckpoint(checkpoint)).toBe(true);
|
||||
|
||||
const acquired = await acquireStartupMigrationLeaseWithWait({
|
||||
env,
|
||||
owner: "second",
|
||||
timeoutMs: 1000,
|
||||
pollIntervalMs: 250,
|
||||
now: () => nowMs,
|
||||
monotonicNow: () => elapsedMs,
|
||||
sleep: async (ms) => {
|
||||
sleepCount += 1;
|
||||
recordSuccessfulStartupMigrations({ ...checkpoint, lease, nowMs });
|
||||
lease.release();
|
||||
nowMs += ms;
|
||||
elapsedMs += ms;
|
||||
},
|
||||
});
|
||||
|
||||
expect(sleepCount).toBe(1);
|
||||
expect(acquired.owner).toBe("second");
|
||||
expect(needsStartupMigrationCheckpoint(checkpoint)).toBe(false);
|
||||
acquired.release();
|
||||
});
|
||||
|
||||
it("preserves the existing lease error when the wait bound expires", async () => {
|
||||
const env = {
|
||||
OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"),
|
||||
};
|
||||
let nowMs = 1001;
|
||||
let elapsedMs = 0;
|
||||
const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" });
|
||||
|
||||
await expect(
|
||||
acquireStartupMigrationLeaseWithWait({
|
||||
env,
|
||||
owner: "second",
|
||||
timeoutMs: 500,
|
||||
pollIntervalMs: 250,
|
||||
now: () => nowMs,
|
||||
monotonicNow: () => elapsedMs,
|
||||
sleep: async (ms) => {
|
||||
nowMs += ms;
|
||||
elapsedMs += ms;
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
`OpenClaw startup migrations are already running for this state directory; retry after the other OpenClaw process finishes or after 1970-01-01T00:05:01.000Z. (held by pid ${process.pid})`,
|
||||
);
|
||||
|
||||
lease.release();
|
||||
});
|
||||
|
||||
it("reclaims an active startup migration lease whose owner process is gone", async () => {
|
||||
const env = {
|
||||
OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"),
|
||||
};
|
||||
@@ -281,7 +349,11 @@ describe("startup migration checkpoint", () => {
|
||||
|
||||
expect(hasActiveStartupMigrationLease({ env, nowMs: 1001 })).toBe(false);
|
||||
|
||||
const replacement = acquireStartupMigrationLease({ env, nowMs: 1001, owner: "replacement" });
|
||||
const replacement = await acquireStartupMigrationLeaseWithWait({
|
||||
env,
|
||||
owner: "replacement",
|
||||
now: () => 1001,
|
||||
});
|
||||
stale.release();
|
||||
expect(hasActiveStartupMigrationLease({ env, nowMs: 1002 })).toBe(true);
|
||||
replacement.release();
|
||||
|
||||
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { hostname } from "node:os";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
||||
import { getFileLockProcessStartTime, isPidDefinitelyDead } from "../shared/pid-alive.js";
|
||||
@@ -29,6 +30,7 @@ const STARTUP_MIGRATION_BUILD_SEPARATOR = "\n";
|
||||
const STARTUP_MIGRATION_CHECKPOINT_FORMAT = "3";
|
||||
const STARTUP_MIGRATION_LEASE_SCOPE = "startup-migrations";
|
||||
const STARTUP_MIGRATION_LEASE_KEY = "global";
|
||||
const STARTUP_MIGRATION_LEASE_POLL_INTERVAL_MS = 250;
|
||||
export const STARTUP_MIGRATION_LEASE_TTL_MS = 5 * 60_000;
|
||||
|
||||
export type StartupMigrationLease = {
|
||||
@@ -38,6 +40,31 @@ export type StartupMigrationLease = {
|
||||
readonly owner: string;
|
||||
};
|
||||
|
||||
type StartupMigrationLeaseParams = {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
nowMs?: number;
|
||||
owner?: string;
|
||||
/** Process id that owns the startup migration work. */
|
||||
ownerPid?: number;
|
||||
};
|
||||
|
||||
type StartupMigrationLeaseWaitParams = Omit<StartupMigrationLeaseParams, "nowMs"> & {
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
now?: () => number;
|
||||
monotonicNow?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
};
|
||||
|
||||
class StartupMigrationLeaseConflictError extends Error {
|
||||
readonly canWaitForSameHostOwner: boolean;
|
||||
|
||||
constructor(message: string, canWaitForSameHostOwner: boolean) {
|
||||
super(message);
|
||||
this.canWaitForSameHostOwner = canWaitForSameHostOwner;
|
||||
}
|
||||
}
|
||||
|
||||
type StartupMigrationLeaseOwner = {
|
||||
pid: number;
|
||||
host: string;
|
||||
@@ -297,13 +324,7 @@ export function needsStateMigrationCheckpoint(params: MigrationCheckpointParams
|
||||
}
|
||||
|
||||
export function acquireStartupMigrationLease(
|
||||
params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
nowMs?: number;
|
||||
owner?: string;
|
||||
/** Process id that owns the startup migration work. */
|
||||
ownerPid?: number;
|
||||
} = {},
|
||||
params: StartupMigrationLeaseParams = {},
|
||||
): StartupMigrationLease {
|
||||
const env = params.env ?? process.env;
|
||||
const nowMs = params.nowMs ?? Date.now();
|
||||
@@ -346,8 +367,9 @@ export function acquireStartupMigrationLease(
|
||||
);
|
||||
} else if (existing) {
|
||||
const ownerHint = existingOwner ? ` (held by pid ${existingOwner.pid})` : "";
|
||||
throw new Error(
|
||||
throw new StartupMigrationLeaseConflictError(
|
||||
`OpenClaw startup migrations are already running for this state directory; retry after the other OpenClaw process finishes or after ${new Date(existing.expiresAt ?? expiresAt).toISOString()}.${ownerHint}`,
|
||||
existingOwner?.host === hostname(),
|
||||
);
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
@@ -416,6 +438,52 @@ export function acquireStartupMigrationLease(
|
||||
};
|
||||
}
|
||||
|
||||
export async function acquireStartupMigrationLeaseWithWait(
|
||||
params: StartupMigrationLeaseWaitParams = {},
|
||||
): Promise<StartupMigrationLease> {
|
||||
const now = params.now ?? Date.now;
|
||||
const monotonicNow = params.monotonicNow ?? performance.now.bind(performance);
|
||||
const sleep =
|
||||
params.sleep ??
|
||||
(async (ms: number) =>
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, ms);
|
||||
}));
|
||||
const timeoutMs = Math.max(
|
||||
0,
|
||||
Math.min(params.timeoutMs ?? STARTUP_MIGRATION_LEASE_TTL_MS, STARTUP_MIGRATION_LEASE_TTL_MS),
|
||||
);
|
||||
const pollIntervalMs = Math.max(
|
||||
1,
|
||||
params.pollIntervalMs ?? STARTUP_MIGRATION_LEASE_POLL_INTERVAL_MS,
|
||||
);
|
||||
const owner = params.owner ?? randomUUID();
|
||||
const deadlineMs = monotonicNow() + timeoutMs;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
return acquireStartupMigrationLease({
|
||||
env: params.env,
|
||||
nowMs: now(),
|
||||
owner,
|
||||
ownerPid: params.ownerPid,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
!(error instanceof StartupMigrationLeaseConflictError) ||
|
||||
!error.canWaitForSameHostOwner
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
const remainingMs = deadlineMs - monotonicNow();
|
||||
if (remainingMs <= 0) {
|
||||
throw error;
|
||||
}
|
||||
await sleep(Math.min(pollIntervalMs, remainingMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function recordSuccessfulMigrationCheckpoints(
|
||||
metaKeys: MigrationCheckpointMetaKey[],
|
||||
params: RecordMigrationCheckpointParams = {},
|
||||
|
||||
Reference in New Issue
Block a user