From 4bc37cd5976166ff2dbd34e5baa92d800c567837 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 16 Aug 2026 13:31:32 -0700 Subject: [PATCH] fix(state): copy-safe agent database registry via state-relative paths (schema v9) (#124728) * fix(state): store in-root agent database registry paths relative to the state dir (schema v9) Copied state directories retained stale absolute agent database registry rows. The combined gateway store then merged old and new copies, causing every sessions.list request to fail with SessionCanonicalKeyMigrationRequiredError. Schema v9 stores in-root registry paths relative to the state directory; migration rewrites eligible rows, deletes stale duplicates, and preserves external paths. * fix(voice-call): describe state-relative registry migration * fix(state): preserve registry locator traversal * test(macos): move PortGuardian schema-version boundary to v9 The store opens any state database up to maximumSupportedSchemaVersion, which the schema v9 bump moved from 8 to 9. Shift the supported/newer partition in the boundary test accordingly. * fix(state): re-anchor copied default-layout registry rows instead of deleting them Resolve the ClawSweeper P1 by preserving genuine external default-layout registrations, re-anchoring copied rows when an in-root counterpart exists, and deleting only dual default-layout conflicts. --- .../PortGuardianRecordStoreTests.swift | 4 +- .../OpenClawNativeStateSQLite.swift | 2 +- docs/reference/database-schemas.md | 12 +++ extensions/voice-call/doctor-contract-api.ts | 2 + package.json | 2 +- ...tartup-migration.registry-recovery.test.ts | 45 ++++++++- src/infra/state-migrations.doctor.ts | 2 + src/state/agent-deletion-journal.ts | 11 ++- .../openclaw-agent-db-registry-listing.ts | 7 +- src/state/openclaw-agent-db-registry.ts | 8 +- src/state/openclaw-agent-db.test.ts | 52 +++++++++- src/state/openclaw-database-preflight.ts | 19 +++- src/state/openclaw-state-db-contract.ts | 4 +- src/state/openclaw-state-db-maintenance.ts | 9 ++ src/state/openclaw-state-db-schema-repair.ts | 92 ++++++++++++++++++ src/state/openclaw-state-db.paths.ts | 27 ++++++ src/state/openclaw-state-db.test.ts | 97 ++++++++++++++++++- src/state/openclaw-state-db.ts | 21 +++- src/state/user-profiles.test.ts | 2 +- .../check-native-state-schema-version.test.ts | 6 +- 20 files changed, 396 insertions(+), 28 deletions(-) diff --git a/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift index 6e22760df771..f85295ed9cad 100644 --- a/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift @@ -532,7 +532,7 @@ struct PortGuardianRecordStoreTests { let fixture = try Self.fixture() defer { fixture.cleanup() } - for version in [4, 5, 6, 7, 8] { + for version in [4, 5, 6, 7, 8, 9] { let databaseURL = fixture.root.appendingPathComponent("supported-v\(version).sqlite") try Self.seedVersionedPortGuardianDatabase(databaseURL, schemaVersion: version) let store = try PortGuardianRecordStore(databaseURL: databaseURL) @@ -544,7 +544,7 @@ struct PortGuardianRecordStoreTests { #expect(try store.records() == [record]) } - for version in [9, 99] { + for version in [10, 99] { let databaseURL = fixture.root.appendingPathComponent("newer-v\(version).sqlite") try Self.seedVersionedPortGuardianDatabase(databaseURL, schemaVersion: version) #expect(throws: PortGuardianStoreError.self) { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift b/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift index 8c51fbc64864..3f98dfb67cef 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift @@ -37,7 +37,7 @@ public enum OpenClawNativeStateSQLiteValueType: Equatable, Sendable { /// One recursive connection lock serializes transactions and statement access. public final class OpenClawNativeStateSQLite: @unchecked Sendable { // Keep aligned with OPENCLAW_STATE_SCHEMA_VERSION. Native clients never upgrade this database. - private static let maximumSupportedSchemaVersion: Int64 = 8 + private static let maximumSupportedSchemaVersion: Int64 = 9 private static let defaultBusyTimeoutMilliseconds: Int32 = 5000 private struct SchemaObject: Hashable { diff --git a/docs/reference/database-schemas.md b/docs/reference/database-schemas.md index b9e846a21c13..85e8dfcc10a9 100644 --- a/docs/reference/database-schemas.md +++ b/docs/reference/database-schemas.md @@ -86,6 +86,12 @@ Version 3 was an unshipped development step folded into version 4. | 5 | Durable cloud-worker result references on pending workspace fences ([`7a7d6bb`](https://github.com/openclaw/openclaw/commit/7a7d6bb51f42bd896de2b8a4df2ee66f3dce0a21), [#110952](https://github.com/openclaw/openclaw/pull/110952)) | `v2026.7.2-beta.4` | | 6 | Every committed shared-state table becomes part of the canonical runtime schema ([`509a5f0`](https://github.com/openclaw/openclaw/commit/509a5f03737642fec4a940e6d605887f7957ddc8), [#113473](https://github.com/openclaw/openclaw/pull/113473)) | `v2026.7.2-beta.5` | | 7 | Retired inferred-commitment storage removed | Unreleased | +| 8 | Cloud-worker placement execution modes and mode-aware turn claims | Unreleased | +| 9 | In-root agent database registry paths stored relative to the state directory | Unreleased | + +### State schema 9 + +Schema 9 stores an `agent_databases.path` value relative to the state directory when the registered agent database is inside that directory. During migration, a foreign default-layout row is re-anchored to the in-root counterpart when that file exists. It is deleted only when the same agent already holds its in-root registration, because dual default-layout registrations cannot produce a valid combined session list. Otherwise, the absolute row is preserved, so genuine external registrations are never deleted. This keeps a copied state directory self-contained without dropping supported external database paths. ## Integrity checks @@ -138,6 +144,12 @@ The general procedure is: 3. Set `PRAGMA user_version` and `schema_meta.schema_version` to the target version. 4. Run the target release's full database verification before starting the Gateway. +### Example: state schema 9 to 8 + +Schema 8 expects every `agent_databases.path` value to be absolute. Before lowering `user_version`, inspect each registry row on the same platform that wrote it. Leave absolute external paths unchanged; replace every relative path with its platform-native absolute form by resolving it against the state directory that owns `state/openclaw.sqlite`. Then set both `PRAGMA user_version` and `schema_meta.schema_version` to 8 in the same transaction. + +Do not lower the version while relative registry rows remain. A schema 8 build interprets them relative to its process working directory rather than the copied state directory. + ### Example: state schema 7 to 6 Schema 7 removed the retired shared commitments table. A schema 6 build still requires that canonical table, so a manual downgrade must recreate its exact empty schema before lowering the version. diff --git a/extensions/voice-call/doctor-contract-api.ts b/extensions/voice-call/doctor-contract-api.ts index e3331041d040..dff745908b1b 100644 --- a/extensions/voice-call/doctor-contract-api.ts +++ b/extensions/voice-call/doctor-contract-api.ts @@ -138,6 +138,8 @@ function describeVoiceCallSchemaMigration(migration: OpenClawStateDatabaseSchema switch (migration.kind) { case "agent-databases-composite-primary-key": return "agent database registry primary key -> agent_id,path"; + case "agent-databases-relative-paths-v9": + return "agent database registry paths -> state-relative paths"; case "audit-events-v2": return "audit event ledger -> versioned message lifecycle schema"; case "commitments-retirement-v7": diff --git a/package.json b/package.json index 4d3643e47804..e02110c5f07a 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2026.8.1", "openclaw": { "schemaVersions": { - "state": 8, + "state": 9, "agent": 17 } }, diff --git a/src/config/sessions/startup-migration.registry-recovery.test.ts b/src/config/sessions/startup-migration.registry-recovery.test.ts index a5c99d9ced33..db644229faad 100644 --- a/src/config/sessions/startup-migration.registry-recovery.test.ts +++ b/src/config/sessions/startup-migration.registry-recovery.test.ts @@ -4,12 +4,16 @@ import { afterEach, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; import * as sessionDirs from "../../agents/session-dirs.js"; import { EMPTY_LEGACY_SESSION_SURFACES } from "../../plugins/legacy-session-surfaces.types.js"; +import { invalidateRegisteredAgentDatabasesMemo } from "../../state/openclaw-agent-db-registry-listing.js"; import { unregisterOpenClawAgentDatabase } from "../../state/openclaw-agent-db-registry.js"; import { closeOpenClawAgentDatabasesForTest, listOpenClawRegisteredAgentDatabases, } from "../../state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; +import { + closeOpenClawStateDatabaseForTest, + repairOpenClawStateDatabaseSchemaIfNeeded, +} from "../../state/openclaw-state-db.js"; import { withEnvAsync } from "../../test-utils/env.js"; import type { OpenClawConfig } from "../types.openclaw.js"; import { loadCombinedSessionStoreForGatewayCore } from "./combined-store-gateway.js"; @@ -109,3 +113,42 @@ it("re-registers durable lineage children before configured-only runtime reads", } }); }); + +it("keeps copied state directories self-contained for combined gateway reads", async () => { + const root = fs.realpathSync.native(tempDirs.make("openclaw-copied-state-registry-")); + const sourceStateDir = path.join(root, "source"); + fs.mkdirSync(sourceStateDir); + const canonicalSourceStateDir = fs.realpathSync.native(sourceStateDir); + const copiedStateDir = path.join(root, "copy"); + const cfg: OpenClawConfig = { + agents: { entries: { main: { default: true } } }, + }; + const sessionKey = "agent:main:copied-state"; + + await withEnvAsync({ OPENCLAW_STATE_DIR: canonicalSourceStateDir }, async () => { + const env = { ...process.env }; + await replaceSessionEntry( + { agentId: "main", env, sessionKey }, + { sessionId: "copied-session", updatedAt: 1 }, + ); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + invalidateRegisteredAgentDatabasesMemo({ env }); + }); + + fs.cpSync(canonicalSourceStateDir, copiedStateDir, { recursive: true }); + const canonicalCopiedStateDir = fs.realpathSync.native(copiedStateDir); + await withEnvAsync({ OPENCLAW_STATE_DIR: canonicalCopiedStateDir }, async () => { + const env = { ...process.env }; + expect(repairOpenClawStateDatabaseSchemaIfNeeded({ env }).warnings).toEqual([]); + const combined = loadCombinedSessionStoreForGatewayCore(cfg, { + configuredAgentsOnly: true, + }); + + expect(combined.store[sessionKey]?.sessionId).toBe("copied-session"); + expect(Object.keys(combined.store).filter((key) => key === sessionKey)).toHaveLength(1); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + invalidateRegisteredAgentDatabasesMemo({ env }); + }); +}); diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index 76d53eb70e8e..d1590c6615e4 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -194,6 +194,8 @@ function describeStateSchemaMigration(migration: OpenClawStateDatabaseSchemaMigr return "retired commitments storage → removed table and indexes"; case "worker-placement-execution-mode-v8": return "cloud worker placements → execution-mode claims"; + case "agent-databases-relative-paths-v9": + return "agent database registry paths → state-relative storage"; case "operator-approvals-system-agent": return "operator approvals → OpenClaw system changes"; case "session-watch-cursor-provenance-v4": diff --git a/src/state/agent-deletion-journal.ts b/src/state/agent-deletion-journal.ts index 1fdf5aec09d2..0b9e69c0e70f 100644 --- a/src/state/agent-deletion-journal.ts +++ b/src/state/agent-deletion-journal.ts @@ -16,7 +16,10 @@ import type { import { ensureAgentDeletionJournalSchema } from "./openclaw-state-db-schema-additive.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { runOpenClawStateWriteTransaction } from "./openclaw-state-db.js"; -import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; +import { + resolveOpenClawRegisteredAgentDatabasePath, + resolveOpenClawStateSqlitePath, +} from "./openclaw-state-db.paths.js"; type AgentDeletionDatabase = Pick< OpenClawStateKyselyDatabase, @@ -379,7 +382,11 @@ export function beginAgentDeletionJournal( const registeredDatabasePaths = executeSqliteQuerySync( database.db, db.selectFrom("agent_databases").select("path").where("agent_id", "=", normalized.agentId), - ).rows.flatMap((row) => resolveSqliteDatabaseFilePaths(row.path)); + ).rows.flatMap((row) => + resolveSqliteDatabaseFilePaths( + resolveOpenClawRegisteredAgentDatabasePath(database.path, row.path), + ), + ); const databasePaths = [ ...new Set( [ diff --git a/src/state/openclaw-agent-db-registry-listing.ts b/src/state/openclaw-agent-db-registry-listing.ts index bd36ae637fec..f8d71dab3a43 100644 --- a/src/state/openclaw-agent-db-registry-listing.ts +++ b/src/state/openclaw-agent-db-registry-listing.ts @@ -11,7 +11,10 @@ import { withOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly. import { detectOpenClawStateDatabaseSchemaMigrationsFromDatabase } from "./openclaw-state-db-schema-repair.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db.js"; -import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; +import { + resolveOpenClawRegisteredAgentDatabasePath, + resolveOpenClawStateSqlitePath, +} from "./openclaw-state-db.paths.js"; type OpenClawAgentRegistryDatabase = Pick; @@ -150,7 +153,7 @@ export function listOpenClawRegisteredAgentDatabases( ).rows; return rows.map((row) => ({ agentId: normalizeAgentId(row.agent_id), - path: row.path, + path: resolveOpenClawRegisteredAgentDatabasePath(pathname, row.path), schemaVersion: row.schema_version, lastSeenAt: row.last_seen_at, sizeBytes: row.size_bytes, diff --git a/src/state/openclaw-agent-db-registry.ts b/src/state/openclaw-agent-db-registry.ts index a23a32232c11..5b6e47b379a4 100644 --- a/src/state/openclaw-agent-db-registry.ts +++ b/src/state/openclaw-agent-db-registry.ts @@ -12,6 +12,7 @@ import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js"; import { invalidateRegisteredAgentDatabasesMemo } from "./openclaw-agent-db-registry-listing.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { runOpenClawStateWriteTransaction } from "./openclaw-state-db.js"; +import { resolveOpenClawAgentDatabaseStoredPath } from "./openclaw-state-db.paths.js"; export { listOpenClawRegisteredAgentDatabases, @@ -620,6 +621,7 @@ export function registerOpenClawAgentDatabase(params: { runOpenClawStateWriteTransaction( (database) => { assertAgentDeletionPathFence(database.db, deletionFence); + const storedPath = resolveOpenClawAgentDatabaseStoredPath(database.path, params.path); const db = getNodeSqliteKysely(database.db); executeSqliteQuerySync( database.db, @@ -627,7 +629,7 @@ export function registerOpenClawAgentDatabase(params: { .insertInto("agent_databases") .values({ agent_id: params.agentId, - path: params.path, + path: storedPath, schema_version: params.schemaVersion ?? OPENCLAW_AGENT_SCHEMA_VERSION, last_seen_at: lastSeenAt, size_bytes: sizeBytes, @@ -686,13 +688,15 @@ export function unregisterOpenClawAgentDatabase(params: { }): void { runOpenClawStateWriteTransaction( (database) => { + const storedPath = resolveOpenClawAgentDatabaseStoredPath(database.path, params.path); + const matchingPaths = [...new Set([storedPath, params.path, path.resolve(params.path)])]; const db = getNodeSqliteKysely(database.db); executeSqliteQuerySync( database.db, db .deleteFrom("agent_databases") .where("agent_id", "=", params.agentId) - .where("path", "=", params.path), + .where("path", "in", matchingPaths), ); }, { env: params.env }, diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 2f4924be0eeb..7b63934c6d0c 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -421,11 +421,10 @@ function downgradeCurrentAgentDatabaseToV13(databasePath: string): void { function readRegisteredAgentDatabaseLastSeenAt(params: { agentId: string; env?: NodeJS.ProcessEnv; - path: string; }): number | undefined { const row = openOpenClawStateDatabase({ env: params.env }) - .db.prepare("SELECT last_seen_at FROM agent_databases WHERE agent_id = ? AND path = ?") - .get(params.agentId, params.path) as { last_seen_at?: unknown } | undefined; + .db.prepare("SELECT last_seen_at FROM agent_databases WHERE agent_id = ?") + .get(params.agentId) as { last_seen_at?: unknown } | undefined; return typeof row?.last_seen_at === "number" ? row.last_seen_at : undefined; } @@ -1108,6 +1107,51 @@ describe("openclaw agent database", () => { expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([]); }); + it("stores state-owned registrations relative while preserving external paths", () => { + const stateDir = createTempStateDir(); + const externalStateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const inRootPath = path.join(stateDir, "agents", "worker-1", "agent", "openclaw-agent.sqlite"); + const externalPath = path.join(externalStateDir, "external.sqlite"); + + registerOpenClawAgentDatabase({ agentId: "worker-1", path: inRootPath, env }); + registerOpenClawAgentDatabase({ agentId: "external", path: externalPath, env }); + + const stateDatabase = openOpenClawStateDatabase({ env }); + expect( + stateDatabase.db + .prepare("SELECT agent_id, path FROM agent_databases ORDER BY agent_id") + .all(), + ).toEqual([ + { agent_id: "external", path: externalPath }, + { + agent_id: "worker-1", + path: path.join("agents", "worker-1", "agent", "openclaw-agent.sqlite"), + }, + ]); + expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([ + expect.objectContaining({ agentId: "external", path: externalPath }), + expect.objectContaining({ agentId: "worker-1", path: inRootPath }), + ]); + + stateDatabase.db + .prepare( + `INSERT INTO agent_databases ( + agent_id, path, schema_version, last_seen_at, size_bytes + ) VALUES (?, ?, ?, 0, NULL)`, + ) + .run("worker-1", inRootPath, OPENCLAW_AGENT_SCHEMA_VERSION); + unregisterOpenClawAgentDatabase({ agentId: "worker-1", path: inRootPath, env }); + expect( + stateDatabase.db + .prepare("SELECT path FROM agent_databases WHERE agent_id = ?") + .all("worker-1"), + ).toEqual([]); + + unregisterOpenClawAgentDatabase({ agentId: "external", path: externalPath, env }); + expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([]); + }); + it("keeps incompatible schema versions maintenance-only", () => { const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -2617,7 +2661,6 @@ describe("openclaw agent database", () => { readRegisteredAgentDatabaseLastSeenAt({ agentId: "worker-1", env, - path: first.path, }), ).toBe(1_000); @@ -2632,7 +2675,6 @@ describe("openclaw agent database", () => { readRegisteredAgentDatabaseLastSeenAt({ agentId: "worker-1", env, - path: first.path, }), ).toBe(1_000); } finally { diff --git a/src/state/openclaw-database-preflight.ts b/src/state/openclaw-database-preflight.ts index 52ef339519a2..5e0b6158f09b 100644 --- a/src/state/openclaw-database-preflight.ts +++ b/src/state/openclaw-database-preflight.ts @@ -30,7 +30,10 @@ import { assertOpenClawStateDatabaseForMaintenance, } from "./openclaw-state-db-maintenance.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; -import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; +import { + resolveOpenClawRegisteredAgentDatabasePath, + resolveOpenClawStateSqlitePath, +} from "./openclaw-state-db.paths.js"; import { inspectOpenClawStateOwnershipFromDatabase, type OpenClawExternalStateOwnership, @@ -153,7 +156,10 @@ function readWriterAppVersion(database: DatabaseSync): string | undefined { } } -function readRegisteredAgentDatabases(database: DatabaseSync): Array<{ +function readRegisteredAgentDatabases( + database: DatabaseSync, + registryPath: string, +): Array<{ agentId: string; path: string; }> { @@ -169,7 +175,12 @@ function readRegisteredAgentDatabases(database: DatabaseSync): Array<{ db.selectFrom("agent_databases").select(["agent_id", "path"]), ).rows.flatMap((row) => typeof row.agent_id === "string" && typeof row.path === "string" - ? [{ agentId: row.agent_id, path: row.path }] + ? [ + { + agentId: row.agent_id, + path: resolveOpenClawRegisteredAgentDatabasePath(registryPath, row.path), + }, + ] : [], ); } @@ -340,7 +351,7 @@ export function preflightOpenClawDatabaseSchemas(options: { let registeredDatabases: ReturnType; try { - registeredDatabases = readRegisteredAgentDatabases(stateDatabase); + registeredDatabases = readRegisteredAgentDatabases(stateDatabase, statePath); } catch (error) { result.indeterminate.push({ kind: "state", diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index 98fadfc81cf6..4226e9856eea 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -1,11 +1,12 @@ import type { DatabaseSync } from "node:sqlite"; import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; +// v9 stores in-root agent database registry paths relative to the state dir. // v8 records cloud-placement execution mode and mode-aware turn claims. // v7 retires the inert shared commitments table. // v6 makes every committed shared-state table part of the canonical runtime schema. // v5 records durable cloud-worker result refs on pending workspace fences. -export const OPENCLAW_STATE_SCHEMA_VERSION = 8; +export const OPENCLAW_STATE_SCHEMA_VERSION = 9; export const OPENCLAW_STATE_STRICT_SCHEMA_VERSION = 3; // Privacy-sensitive feature tables remain absent even in fresh databases until // their feature-local first write. The canonical SQL still owns their shape. @@ -74,6 +75,7 @@ export type OpenClawStateDatabaseSchemaMigration = { | "audit-events-v2" | "commitments-retirement-v7" | "worker-placement-execution-mode-v8" + | "agent-databases-relative-paths-v9" | "operator-approvals-system-agent" | "session-watch-cursor-provenance-v4" | "strict-tables-v3"; diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index d45824206435..1e86fa676305 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -46,6 +46,7 @@ const STATE_MIGRATION_ALLOWED_MISSING_TABLES = { 5: STATE_V5_ADDITIVE_TABLES, 6: STATE_V6_ADDITIVE_TABLES, 7: STATE_V6_ADDITIVE_TABLES, + 8: STATE_V6_ADDITIVE_TABLES, } as const satisfies Record; type OpenClawStateMigrationVersion = keyof typeof STATE_MIGRATION_ALLOWED_MISSING_TABLES; @@ -187,6 +188,14 @@ export function assertOpenClawStateDatabaseV7ForMigration( assertOpenClawStateDatabaseVersionForMigration(database, { ...options, version: 7 }); } +/** Require every stable v8 table before the v9 registry migration can run. */ +export function assertOpenClawStateDatabaseV8ForMigration( + database: DatabaseSync, + options: { pathname: string }, +): void { + assertOpenClawStateDatabaseVersionForMigration(database, { ...options, version: 8 }); +} + export function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string { return path.resolve(options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env)); } diff --git a/src/state/openclaw-state-db-schema-repair.ts b/src/state/openclaw-state-db-schema-repair.ts index 380b0661a4d7..4a527434aaef 100644 --- a/src/state/openclaw-state-db-schema-repair.ts +++ b/src/state/openclaw-state-db-schema-repair.ts @@ -1,4 +1,5 @@ import { existsSync } from "node:fs"; +import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { @@ -29,6 +30,10 @@ import { } from "./openclaw-state-db-schema-helpers.js"; import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; +import { + resolveOpenClawAgentDatabaseStoredPath, + resolveOpenClawStateDirForDatabasePath, +} from "./openclaw-state-db.paths.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; export function dropLegacyStateTables(db: DatabaseSync): void { @@ -449,6 +454,90 @@ export function migrateWorkerPlacementExecutionModeSchema( return true; } +function isDefaultAgentDatabasePath(pathname: string, agentId: string): boolean { + const agentDir = path.dirname(pathname); + const agentIdDir = path.dirname(agentDir); + return ( + path.basename(pathname) === "openclaw-agent.sqlite" && + path.basename(agentDir) === "agent" && + path.basename(agentIdDir) === agentId && + path.basename(path.dirname(agentIdDir)) === "agents" + ); +} + +export function migrateAgentDatabaseRelativePaths( + db: DatabaseSync, + previousVersion: number, + databasePath: string, +): boolean { + if (previousVersion >= 9 || !tableExists(db, "agent_databases")) { + return false; + } + const rows = db.prepare("SELECT agent_id, path FROM agent_databases").all(); + const updatePath = db.prepare( + "UPDATE agent_databases SET path = ? WHERE agent_id = ? AND path = ?", + ); + const deletePath = db.prepare("DELETE FROM agent_databases WHERE agent_id = ? AND path = ?"); + const hasPath = db.prepare( + "SELECT 1 FROM agent_databases WHERE agent_id = ? AND path = ? LIMIT 1", + ); + let changed = false; + for (const row of rows) { + const agentId = row.agent_id; + const registeredPath = row.path; + if (typeof agentId !== "string" || typeof registeredPath !== "string") { + throw new Error("OpenClaw v8 agent database registry paths are not canonical"); + } + if (!path.isAbsolute(registeredPath)) { + continue; + } + const storedPath = resolveOpenClawAgentDatabaseStoredPath(databasePath, registeredPath); + if (!path.isAbsolute(storedPath)) { + updatePath.run(storedPath, agentId, registeredPath); + changed = true; + } + } + const stateDir = resolveOpenClawStateDirForDatabasePath(databasePath); + for (const row of rows) { + const agentId = row.agent_id; + const registeredPath = row.path; + if ( + typeof agentId !== "string" || + typeof registeredPath !== "string" || + !path.isAbsolute(registeredPath) || + !path.isAbsolute(resolveOpenClawAgentDatabaseStoredPath(databasePath, registeredPath)) + ) { + continue; + } + const absolutePath = path.resolve(registeredPath); + if (isDefaultAgentDatabasePath(absolutePath, agentId)) { + const counterpartAbsolute = path.join( + stateDir, + "agents", + agentId, + "agent", + "openclaw-agent.sqlite", + ); + const counterpartStored = resolveOpenClawAgentDatabaseStoredPath( + databasePath, + counterpartAbsolute, + ); + if (hasPath.get(agentId, counterpartStored)) { + // The same agent already owns its in-root canonical registration. Keeping a second + // default-layout registration guarantees duplicate canonical session keys on every list. + deletePath.run(agentId, registeredPath); + changed = true; + } else if (existsSync(counterpartAbsolute)) { + // Re-anchor a copied or moved state directory onto its copied database instead of + // deleting the registration or leaving it dangling at the source root. + updatePath.run(counterpartStored, agentId, registeredPath); + changed = true; + } + } + } + return changed; +} + function hasCanonicalAgentDatabasesPrimaryKey(db: DatabaseSync): boolean { if (!tableExists(db, "agent_databases")) { return true; @@ -623,6 +712,9 @@ export function detectOpenClawStateDatabaseSchemaMigrationsFromDatabase( if (userVersion === 7 && tableExists(db, "worker_session_placements")) { migrations.push({ kind: "worker-placement-execution-mode-v8", path: pathname }); } + if (userVersion === 8 && tableExists(db, "agent_databases")) { + migrations.push({ kind: "agent-databases-relative-paths-v9", path: pathname }); + } if (!hasCanonicalAgentDatabasesPrimaryKey(db)) { migrations.push({ kind: "agent-databases-composite-primary-key", path: pathname }); } diff --git a/src/state/openclaw-state-db.paths.ts b/src/state/openclaw-state-db.paths.ts index 482aa7b68c3d..33031a098051 100644 --- a/src/state/openclaw-state-db.paths.ts +++ b/src/state/openclaw-state-db.paths.ts @@ -45,3 +45,30 @@ export function resolveOpenClawStateDirForDatabasePath(databasePath: string): st const databaseDir = path.dirname(path.resolve(databasePath)); return path.basename(databaseDir) === "state" ? path.dirname(databaseDir) : databaseDir; } + +/** Resolve the durable registry form for one agent database path. */ +export function resolveOpenClawAgentDatabaseStoredPath( + registryDatabasePath: string, + agentDatabasePath: string, +): string { + const stateDir = resolveOpenClawStateDirForDatabasePath(registryDatabasePath); + const absolutePath = path.resolve(agentDatabasePath); + const relativePath = path.relative(stateDir, absolutePath); + if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) { + return absolutePath; + } + const statePrefix = `${stateDir}${stateDir.endsWith(path.sep) ? "" : path.sep}`; + return path.isAbsolute(agentDatabasePath) && agentDatabasePath.startsWith(statePrefix) + ? agentDatabasePath.slice(statePrefix.length) + : relativePath; +} + +/** Resolve one stored agent database registry path for runtime consumers. */ +export function resolveOpenClawRegisteredAgentDatabasePath( + registryDatabasePath: string, + storedPath: string, +): string { + return path.isAbsolute(storedPath) + ? storedPath + : `${resolveOpenClawStateDirForDatabasePath(registryDatabasePath)}${path.sep}${storedPath}`; +} diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 449a15af3dea..03d48d72ec2b 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -25,6 +25,7 @@ import { assertSqliteSchemaContains } from "../infra/sqlite-schema-contract.js"; import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { VERSION } from "../version.js"; +import { listOpenClawRegisteredAgentDatabases } from "./openclaw-agent-db-registry.js"; import { FIRST_USE_STATE_TABLES } from "./openclaw-state-db-contract.js"; import { findOpenClawStateDatabaseSchemaMigrationRequiredError, @@ -1559,7 +1560,9 @@ describe("openclaw state database", () => { }); } const migrated = openOpenClawStateDatabase(options); - expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(8); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe( + OPENCLAW_STATE_SCHEMA_VERSION, + ); expect( migrated.db .prepare( @@ -1574,6 +1577,98 @@ describe("openclaw state database", () => { }, ); + it("migrates v8 agent database registrations to state-relative paths", () => { + const stateDir = createTempStateDir(); + const foreignStateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = materializeCurrentStateDatabase(stateDir); + const inRootPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + const dualInRootPath = path.join(stateDir, "agents", "dual", "agent", "openclaw-agent.sqlite"); + const dualForeignPath = path.join( + foreignStateDir, + "agents", + "dual", + "agent", + "openclaw-agent.sqlite", + ); + const copiedForeignPath = path.join( + foreignStateDir, + "agents", + "copied", + "agent", + "openclaw-agent.sqlite", + ); + const copiedInRootPath = path.join( + stateDir, + "agents", + "copied", + "agent", + "openclaw-agent.sqlite", + ); + const preservedDefaultPath = path.join( + foreignStateDir, + "agents", + "preserved", + "agent", + "openclaw-agent.sqlite", + ); + const externalPath = path.join(foreignStateDir, "explicit", "external.sqlite"); + fs.mkdirSync(path.dirname(dualInRootPath), { recursive: true }); + fs.writeFileSync(dualInRootPath, ""); + fs.mkdirSync(path.dirname(copiedInRootPath), { recursive: true }); + fs.writeFileSync(copiedInRootPath, ""); + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + const insert = legacy.prepare( + `INSERT INTO agent_databases ( + agent_id, path, schema_version, last_seen_at, size_bytes + ) VALUES (?, ?, 17, 1, NULL)`, + ); + insert.run("main", inRootPath); + insert.run("dual", dualInRootPath); + insert.run("dual", dualForeignPath); + insert.run("copied", copiedForeignPath); + insert.run("preserved", preservedDefaultPath); + insert.run("external", externalPath); + legacy.exec(` + PRAGMA user_version = 8; + UPDATE schema_meta SET schema_version = 8 WHERE meta_key = 'primary'; + `); + legacy.close(); + + expect(detectOpenClawStateDatabaseSchemaMigrations({ env })).toContainEqual({ + kind: "agent-databases-relative-paths-v9", + path: databasePath, + }); + const migrated = openOpenClawStateDatabase({ env }); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(9); + expect( + migrated.db.prepare("SELECT agent_id, path FROM agent_databases ORDER BY agent_id").all(), + ).toEqual([ + { + agent_id: "copied", + path: path.join("agents", "copied", "agent", "openclaw-agent.sqlite"), + }, + { + agent_id: "dual", + path: path.join("agents", "dual", "agent", "openclaw-agent.sqlite"), + }, + { agent_id: "external", path: externalPath }, + { + agent_id: "main", + path: path.join("agents", "main", "agent", "openclaw-agent.sqlite"), + }, + { agent_id: "preserved", path: preservedDefaultPath }, + ]); + expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([ + expect.objectContaining({ agentId: "copied", path: copiedInRootPath }), + expect.objectContaining({ agentId: "dual", path: dualInRootPath }), + expect.objectContaining({ agentId: "external", path: externalPath }), + expect.objectContaining({ agentId: "main", path: inRootPath }), + expect.objectContaining({ agentId: "preserved", path: preservedDefaultPath }), + ]); + }); + it.each(["runtime open", "doctor repair"] as const)( "retires v6 commitments through %s while preserving shared leases", (migrationPath) => { diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 2a6e5bfdd9dd..1cadf840887e 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -55,6 +55,7 @@ import { assertOpenClawStateDatabaseV5ForMigration, assertOpenClawStateDatabaseV6ForMigration, assertOpenClawStateDatabaseV7ForMigration, + assertOpenClawStateDatabaseV8ForMigration, assertSupportedSchemaVersion, resolveDatabasePath, } from "./openclaw-state-db-maintenance.js"; @@ -70,6 +71,7 @@ import { detectOpenClawStateDatabaseSchemaMigrationsFromDatabase, dropLegacyStateTables, markCurrentStateSchemaVersion, + migrateAgentDatabaseRelativePaths, migrateRetiredCommitmentsSchema, migrateWorkerPlacementExecutionModeSchema, repairAgentDatabasesCompositePrimaryKey, @@ -90,6 +92,7 @@ const STATE_MIGRATION_ASSERTIONS = { 5: assertOpenClawStateDatabaseV5ForMigration, 6: assertOpenClawStateDatabaseV6ForMigration, 7: assertOpenClawStateDatabaseV7ForMigration, + 8: assertOpenClawStateDatabaseV8ForMigration, } as const; export { @@ -183,7 +186,12 @@ function repairOpenClawStateDatabaseSchemaWithWriteAccess( assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, }); - } else if (previousVersion === 5 || previousVersion === 6 || previousVersion === 7) { + } else if ( + previousVersion === 5 || + previousVersion === 6 || + previousVersion === 7 || + previousVersion === 8 + ) { STATE_MIGRATION_ASSERTIONS[previousVersion](db, { pathname }); } if (rebuiltIndexNames.size === 0) { @@ -196,6 +204,9 @@ function repairOpenClawStateDatabaseSchemaWithWriteAccess( if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { applied.push("Migrated cloud worker placements to execution modes"); } + if (migrateAgentDatabaseRelativePaths(db, previousVersion, pathname)) { + applied.push("Migrated agent database registry paths to state-relative storage"); + } if (repairAgentDatabasesCompositePrimaryKey(db)) { applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); } @@ -375,12 +386,18 @@ function ensureSchema(db: DatabaseSync, pathname: string, env: NodeJS.ProcessEnv }); ensureAdditiveStateColumns(db); assertCurrentStateRuntimeSchema(db, pathname); - } else if (previousVersion === 5 || previousVersion === 6 || previousVersion === 7) { + } else if ( + previousVersion === 5 || + previousVersion === 6 || + previousVersion === 7 || + previousVersion === 8 + ) { STATE_MIGRATION_ASSERTIONS[previousVersion](db, { pathname }); } dropLegacyStateTables(db); migrateRetiredCommitmentsSchema(db, previousVersion); migrateWorkerPlacementExecutionModeSchema(db, previousVersion); + migrateAgentDatabaseRelativePaths(db, previousVersion, pathname); ensureAdditiveStateColumns(db); sessionWatchMigration.migrateSessionWatchCursorProvenance(db); assertCanonicalStateSchemaShape(db, pathname); diff --git a/src/state/user-profiles.test.ts b/src/state/user-profiles.test.ts index a448da7b6046..39d58105bcae 100644 --- a/src/state/user-profiles.test.ts +++ b/src/state/user-profiles.test.ts @@ -74,7 +74,7 @@ describe("user profiles", () => { expect( openOpenClawStateDatabase(options).db.prepare("PRAGMA user_version").get()?.user_version, ).toBe(versionBefore); - expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(8); + expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(9); expect(second).toEqual(first); expect(ensureProfileForEmail("ADA@example.com", options)).toEqual(first); expect(listProfiles(options)).toEqual([ diff --git a/test/scripts/check-native-state-schema-version.test.ts b/test/scripts/check-native-state-schema-version.test.ts index 4ed9eaefc4b5..a6b6bf3b92ec 100644 --- a/test/scripts/check-native-state-schema-version.test.ts +++ b/test/scripts/check-native-state-schema-version.test.ts @@ -6,15 +6,15 @@ import { describe("native state schema version guard", () => { it("keeps the checked-in Swift and TypeScript contracts aligned", () => { - expect(checkNativeStateSchemaVersion()).toBe(8); + expect(checkNativeStateSchemaVersion()).toBe(9); }); it("fails when a deliberate Swift fixture drifts behind TypeScript", () => { expect(() => compareNativeStateSchemaVersions({ swiftSource: "private static let maximumSupportedSchemaVersion: Int64 = 5\n", - typescriptSource: "export const OPENCLAW_STATE_SCHEMA_VERSION = 8;\n", + typescriptSource: "export const OPENCLAW_STATE_SCHEMA_VERSION = 9;\n", }), - ).toThrow("Native state schema version drift: Swift supports 5, TypeScript owns 8"); + ).toThrow("Native state schema version drift: Swift supports 5, TypeScript owns 9"); }); });