mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(state): repair mis-stamped v17 agent databases (#122171)
* fix(state): repair mis-stamped v17 agent databases * refactor(state): keep lease repair helpers internal
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
2d0c30fd65
commit
0771787da2
@@ -0,0 +1,274 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
|
||||
import { migrateLegacyMediaPersistence } from "../infra/state-migrations.media-persistence.js";
|
||||
import { withOpenClawAgentDatabaseReadOnly } from "./openclaw-agent-db-readonly.js";
|
||||
import {
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
migrateOpenClawAgentDatabaseForMaintenance,
|
||||
OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
openOpenClawAgentDatabase,
|
||||
} from "./openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "./openclaw-state-db.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function createCurrentAgentDatabase(): { databasePath: string; env: NodeJS.ProcessEnv } {
|
||||
const stateDir = makeTempDir(tempDirs, "agent-db-retired-lease-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
return { databasePath, env };
|
||||
}
|
||||
|
||||
function installRetiredLeaseSchema(databasePath: string): void {
|
||||
const database = openNodeSqliteDatabase(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
CREATE TABLE state_leases (
|
||||
scope TEXT NOT NULL,
|
||||
lease_key TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,
|
||||
expires_at INTEGER,
|
||||
heartbeat_at INTEGER,
|
||||
payload_json TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (scope, lease_key)
|
||||
) STRICT;
|
||||
CREATE INDEX idx_agent_state_leases_expiry
|
||||
ON state_leases(expires_at, scope, lease_key)
|
||||
WHERE expires_at IS NOT NULL;
|
||||
CREATE INDEX idx_agent_state_leases_owner
|
||||
ON state_leases(owner, updated_at DESC);
|
||||
INSERT INTO state_leases (
|
||||
scope, lease_key, owner, expires_at, heartbeat_at, payload_json, created_at, updated_at
|
||||
) VALUES ('retired', 'orphan', 'nobody', NULL, NULL, NULL, 1, 1);
|
||||
ANALYZE state_leases;
|
||||
`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
function readPrimarySchemaMetadata(databasePath: string): unknown {
|
||||
const database = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
try {
|
||||
return database.prepare("SELECT * FROM schema_meta WHERE meta_key = 'primary'").get();
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
cleanupTempDirs(tempDirs);
|
||||
});
|
||||
|
||||
describe("retired agent state lease repair", () => {
|
||||
it("repairs a mis-stamped v17 database while preserving auth and ownership", () => {
|
||||
const { databasePath, env } = createCurrentAgentDatabase();
|
||||
installRetiredLeaseSchema(databasePath);
|
||||
const beforeMetadata = readPrimarySchemaMetadata(databasePath);
|
||||
const database = openNodeSqliteDatabase(databasePath);
|
||||
try {
|
||||
database
|
||||
.prepare(
|
||||
"INSERT INTO auth_profile_state (state_key, state_json, updated_at) VALUES (?, ?, ?)",
|
||||
)
|
||||
.run("last-good", '{"profile":"primary"}', 10);
|
||||
database
|
||||
.prepare(
|
||||
"INSERT INTO auth_profile_store (store_key, store_json, updated_at) VALUES (?, ?, ?)",
|
||||
)
|
||||
.run("primary", '{"profiles":{"primary":{"provider":"openai"}}}', 10);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
expect(migrateLegacyMediaPersistence({ env }).warnings).toEqual([]);
|
||||
expect(migrateLegacyMediaPersistence({ env }).warnings).toEqual([]);
|
||||
|
||||
const repaired = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
try {
|
||||
expect(repaired.prepare("PRAGMA user_version").get()).toEqual({
|
||||
user_version: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
});
|
||||
expect(
|
||||
repaired.prepare("SELECT * FROM schema_meta WHERE meta_key = 'primary'").get(),
|
||||
).toEqual(beforeMetadata);
|
||||
expect(
|
||||
repaired
|
||||
.prepare("SELECT state_json FROM auth_profile_state WHERE state_key = 'last-good'")
|
||||
.get(),
|
||||
).toEqual({ state_json: '{"profile":"primary"}' });
|
||||
expect(
|
||||
repaired
|
||||
.prepare("SELECT store_json FROM auth_profile_store WHERE store_key = 'primary'")
|
||||
.get(),
|
||||
).toEqual({ store_json: '{"profiles":{"primary":{"provider":"openai"}}}' });
|
||||
expect(
|
||||
repaired
|
||||
.prepare(
|
||||
`SELECT type, name FROM sqlite_schema
|
||||
WHERE name IN (
|
||||
'state_leases',
|
||||
'idx_agent_state_leases_expiry',
|
||||
'idx_agent_state_leases_owner'
|
||||
)`,
|
||||
)
|
||||
.all(),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
repaired.prepare("SELECT tbl, idx FROM sqlite_stat1 WHERE tbl = 'state_leases'").all(),
|
||||
).toEqual([]);
|
||||
expect(repaired.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" });
|
||||
expect(repaired.prepare("PRAGMA foreign_key_check").all()).toEqual([]);
|
||||
} finally {
|
||||
repaired.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves a clean v17 database unchanged", () => {
|
||||
const { databasePath } = createCurrentAgentDatabase();
|
||||
const beforeMetadata = readPrimarySchemaMetadata(databasePath);
|
||||
const before = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
const beforeSchemaVersion = before.prepare("PRAGMA schema_version").get();
|
||||
before.close();
|
||||
|
||||
migrateOpenClawAgentDatabaseForMaintenance({
|
||||
agentId: "worker-1",
|
||||
pathname: databasePath,
|
||||
});
|
||||
|
||||
const after = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
try {
|
||||
expect(after.prepare("PRAGMA user_version").get()).toEqual({
|
||||
user_version: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
});
|
||||
expect(after.prepare("SELECT * FROM schema_meta WHERE meta_key = 'primary'").get()).toEqual(
|
||||
beforeMetadata,
|
||||
);
|
||||
expect(after.prepare("PRAGMA schema_version").get()).toEqual(beforeSchemaVersion);
|
||||
expect(
|
||||
after.prepare("SELECT name FROM sqlite_schema WHERE name = 'state_leases'").get(),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
after.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a foreign state_leases structure without changing it", () => {
|
||||
const { databasePath } = createCurrentAgentDatabase();
|
||||
const beforeMetadata = readPrimarySchemaMetadata(databasePath);
|
||||
const database = openNodeSqliteDatabase(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
CREATE TABLE state_leases (
|
||||
foreign_id TEXT NOT NULL PRIMARY KEY,
|
||||
foreign_payload TEXT NOT NULL
|
||||
) STRICT;
|
||||
INSERT INTO state_leases VALUES ('foreign', 'preserve-me');
|
||||
`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
migrateOpenClawAgentDatabaseForMaintenance({
|
||||
agentId: "worker-1",
|
||||
pathname: databasePath,
|
||||
}),
|
||||
).toThrow(/state_leases.*noncanonical|column definitions differ for state_leases/iu);
|
||||
|
||||
const after = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
try {
|
||||
expect(after.prepare("SELECT * FROM state_leases").all()).toEqual([
|
||||
{ foreign_id: "foreign", foreign_payload: "preserve-me" },
|
||||
]);
|
||||
expect(after.prepare("SELECT * FROM schema_meta WHERE meta_key = 'primary'").get()).toEqual(
|
||||
beforeMetadata,
|
||||
);
|
||||
expect(after.prepare("PRAGMA user_version").get()).toEqual({
|
||||
user_version: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
});
|
||||
} finally {
|
||||
after.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rolls back the lease drop when canonical validation fails", () => {
|
||||
const { databasePath } = createCurrentAgentDatabase();
|
||||
installRetiredLeaseSchema(databasePath);
|
||||
const beforeMetadata = readPrimarySchemaMetadata(databasePath);
|
||||
const database = openNodeSqliteDatabase(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
DROP TABLE session_key_contract;
|
||||
CREATE VIEW session_key_contract AS SELECT 1 AS id, 'main' AS main_key, 0 AS updated_at;
|
||||
`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
expect(() =>
|
||||
migrateOpenClawAgentDatabaseForMaintenance({
|
||||
agentId: "worker-1",
|
||||
pathname: databasePath,
|
||||
}),
|
||||
).toThrow(/session_key_contract/iu);
|
||||
|
||||
const after = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
try {
|
||||
expect(after.prepare("SELECT scope, lease_key FROM state_leases").all()).toEqual([
|
||||
{ lease_key: "orphan", scope: "retired" },
|
||||
]);
|
||||
expect(
|
||||
after
|
||||
.prepare(
|
||||
`SELECT name FROM sqlite_schema
|
||||
WHERE type = 'index' AND name LIKE 'idx_agent_state_leases_%'
|
||||
ORDER BY name`,
|
||||
)
|
||||
.all(),
|
||||
).toEqual([
|
||||
{ name: "idx_agent_state_leases_expiry" },
|
||||
{ name: "idx_agent_state_leases_owner" },
|
||||
]);
|
||||
expect(after.prepare("SELECT * FROM schema_meta WHERE meta_key = 'primary'").get()).toEqual(
|
||||
beforeMetadata,
|
||||
);
|
||||
expect(after.prepare("PRAGMA user_version").get()).toEqual({
|
||||
user_version: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
});
|
||||
} finally {
|
||||
after.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("repairs the same canonical drift through the writable schema owner", () => {
|
||||
const { databasePath, env } = createCurrentAgentDatabase();
|
||||
installRetiredLeaseSchema(databasePath);
|
||||
expect(
|
||||
withOpenClawAgentDatabaseReadOnly(
|
||||
({ db }) => db.prepare("SELECT COUNT(*) AS count FROM state_leases").get(),
|
||||
{ agentId: "worker-1", env },
|
||||
),
|
||||
).toEqual({ found: true, value: { count: 1 } });
|
||||
const beforeWritableOpen = openNodeSqliteDatabase(databasePath, { readOnly: true });
|
||||
expect(
|
||||
beforeWritableOpen
|
||||
.prepare("SELECT name FROM sqlite_schema WHERE name = 'state_leases'")
|
||||
.get(),
|
||||
).toEqual({ name: "state_leases" });
|
||||
beforeWritableOpen.close();
|
||||
|
||||
const repaired = openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
|
||||
expect(
|
||||
repaired.db.prepare("SELECT name FROM sqlite_schema WHERE name = 'state_leases'").get(),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,12 @@ const AGENT_SCHEMA_COMPATIBILITY = {
|
||||
],
|
||||
} satisfies SqliteSchemaCompatibility;
|
||||
|
||||
function hasRetiredAgentStateLeaseSchema(database: DatabaseSync): boolean {
|
||||
return Boolean(
|
||||
database.prepare("SELECT 1 FROM main.sqlite_schema WHERE name = 'state_leases'").get(),
|
||||
);
|
||||
}
|
||||
|
||||
export function assertOpenClawAgentSchemaContains(
|
||||
database: DatabaseSync,
|
||||
pathname: string,
|
||||
@@ -90,6 +96,11 @@ export function assertOpenClawAgentCurrentRuntimeSchema(
|
||||
`OpenClaw agent database ${options.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${OPENCLAW_AGENT_SCHEMA_VERSION}; run openclaw doctor --fix before using it.`,
|
||||
);
|
||||
}
|
||||
if (hasRetiredAgentStateLeaseSchema(database)) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${options.pathname} retains retired state_leases storage; run openclaw doctor --fix before using it.`,
|
||||
);
|
||||
}
|
||||
assertOpenClawAgentSchemaContains(database, options.pathname, OPENCLAW_AGENT_SCHEMA_SQL);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
verifyAndRepairCanonicalSqliteIndexes,
|
||||
} from "../infra/sqlite-index-schema.js";
|
||||
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
|
||||
import { assertSqliteSchemaContains } from "../infra/sqlite-schema-contract.js";
|
||||
import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js";
|
||||
import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js";
|
||||
import { readSqliteUserVersion } from "../infra/sqlite-user-version.js";
|
||||
@@ -292,21 +293,37 @@ function migrateOpenClawAgentSchema(db: DatabaseSync): void {
|
||||
backfillTranscriptMutationWatermarks(db);
|
||||
}
|
||||
|
||||
const RETIRED_AGENT_STATE_LEASE_SCHEMA_SQL = `
|
||||
CREATE TABLE state_leases (
|
||||
scope TEXT NOT NULL,
|
||||
lease_key TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,
|
||||
expires_at INTEGER,
|
||||
heartbeat_at INTEGER,
|
||||
payload_json TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (scope, lease_key)
|
||||
) STRICT;
|
||||
`;
|
||||
|
||||
function hasRetiredAgentStateLeaseSchema(db: DatabaseSync): boolean {
|
||||
return Boolean(db.prepare("SELECT 1 FROM main.sqlite_schema WHERE name = 'state_leases'").get());
|
||||
}
|
||||
|
||||
function migrateRetiredAgentStateLeaseSchema(
|
||||
db: DatabaseSync,
|
||||
previousVersion: number,
|
||||
pathname: string,
|
||||
targetVersion: number,
|
||||
): void {
|
||||
if (previousVersion >= 17 || targetVersion < 17) {
|
||||
if (targetVersion < 17 || !hasRetiredAgentStateLeaseSchema(db)) {
|
||||
return;
|
||||
}
|
||||
// The 2026-08-10 tenant audit found no agent-DB lease writers after #121113;
|
||||
// #121615 removed the unreachable routing arm, so v17 retires this table.
|
||||
db.exec(`
|
||||
DROP INDEX IF EXISTS idx_agent_state_leases_owner;
|
||||
DROP INDEX IF EXISTS idx_agent_state_leases_expiry;
|
||||
DROP TABLE IF EXISTS state_leases;
|
||||
`);
|
||||
assertSqliteSchemaContains(db, pathname, RETIRED_AGENT_STATE_LEASE_SCHEMA_SQL);
|
||||
// DROP TABLE also removes the retired indexes and sqlite_stat rows atomically.
|
||||
db.exec("DROP TABLE state_leases;");
|
||||
}
|
||||
|
||||
/** Backfill one generation token without copying or rewriting transcript rows. */
|
||||
@@ -555,9 +572,13 @@ export function assertAgentDatabaseIntegrityBeforeMutation(
|
||||
const hasPendingSessionContractMigration =
|
||||
userVersion === OPENCLAW_AGENT_SCHEMA_VERSION &&
|
||||
hasPendingSessionKeyContractSchemaMigration(database);
|
||||
const hasPendingAdditiveMigration =
|
||||
hasPendingMemoryMigration || hasPendingSessionContractMigration;
|
||||
if (userVersion === OPENCLAW_AGENT_SCHEMA_VERSION && !hasPendingAdditiveMigration) {
|
||||
const hasPendingRetiredLeaseMigration =
|
||||
userVersion === OPENCLAW_AGENT_SCHEMA_VERSION && hasRetiredAgentStateLeaseSchema(database);
|
||||
const hasPendingCurrentVersionMigration =
|
||||
hasPendingMemoryMigration ||
|
||||
hasPendingSessionContractMigration ||
|
||||
hasPendingRetiredLeaseMigration;
|
||||
if (userVersion === OPENCLAW_AGENT_SCHEMA_VERSION && !hasPendingCurrentVersionMigration) {
|
||||
verifyAndRepairCanonicalSqliteIndexes(database, pathname, OPENCLAW_AGENT_SCHEMA_SQL, {
|
||||
allowMissingColumns: true,
|
||||
validateAfterRepair: () =>
|
||||
@@ -567,9 +588,9 @@ export function assertAgentDatabaseIntegrityBeforeMutation(
|
||||
// Every physical open proves the full file before schema mutation or exposure.
|
||||
assertSqliteIntegrity(database, pathname);
|
||||
}
|
||||
// Current-version additive surfaces are installed atomically by ensureAgentSchema below.
|
||||
// Validating them here would make the same-version repair path unreachable after an update.
|
||||
if (userVersion === OPENCLAW_AGENT_SCHEMA_VERSION && !hasPendingAdditiveMigration) {
|
||||
// Current-version convergence runs atomically in ensureAgentSchema below.
|
||||
// Validating here would make same-version repair unreachable after an update.
|
||||
if (userVersion === OPENCLAW_AGENT_SCHEMA_VERSION && !hasPendingCurrentVersionMigration) {
|
||||
assertOpenClawAgentCurrentRuntimeSchema(database, { agentId, pathname });
|
||||
}
|
||||
}
|
||||
@@ -613,6 +634,7 @@ function ensureAgentSchema(
|
||||
`OpenClaw agent database ${pathname} uses schema version ${previousVersion}; expected at most ${targetVersion} for this migration.`,
|
||||
);
|
||||
}
|
||||
migrateRetiredAgentStateLeaseSchema(db, pathname, targetVersion);
|
||||
if (previousVersion === targetVersion) {
|
||||
ensureSessionEntryValidityProjection(db);
|
||||
ensureSessionKeyContractSchemaInTransaction(db);
|
||||
@@ -640,7 +662,6 @@ function ensureAgentSchema(
|
||||
dropLegacyRuntimeJournalSchemas(db);
|
||||
migrateMemoryIndexSourcesIdentity(db);
|
||||
migrateOpenClawAgentSchema(db);
|
||||
migrateRetiredAgentStateLeaseSchema(db, previousVersion, targetVersion);
|
||||
migrateConversationDeliveryTargetColumn(db);
|
||||
backfillOpenClawAgentSchema(db, previousVersion);
|
||||
// Remove after 2026-10-01: drop the pre-v11 conversation backfill once schema 11 is the support floor.
|
||||
|
||||
Reference in New Issue
Block a user