diff --git a/extensions/matrix/doctor-contract-api.test.ts b/extensions/matrix/doctor-contract-api.test.ts index 0d6d68177607..a8a3d986a5db 100644 --- a/extensions/matrix/doctor-contract-api.test.ts +++ b/extensions/matrix/doctor-contract-api.test.ts @@ -1,4 +1,5 @@ // Matrix tests cover doctor contract state migrations. +import "fake-indexeddb/auto"; import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -25,10 +26,14 @@ import { type MatrixStoredCredentialRecord, } from "./src/matrix/credentials-read.js"; import { + MATRIX_IDB_SNAPSHOT_FILENAME, MATRIX_RECOVERY_KEY_FILENAME, + openMatrixIdbSnapshotStoreOptions, readMatrixIdbSnapshotJson, readMatrixRecoveryKeyStateForPath, scoreMatrixCryptoStateInStore, + writeMatrixIdbSnapshotJson, + type MatrixIdbSnapshotRecord, } from "./src/matrix/crypto-state-store.js"; import { importNewestInboundDedupeMarkers } from "./src/matrix/monitor/inbound-dedupe-migration.js"; import { @@ -36,8 +41,15 @@ import { MATRIX_INBOUND_DEDUPE_TTL_MS, resolveMatrixInboundDedupeStateNamespace, } from "./src/matrix/monitor/inbound-dedupe.js"; +import { restoreIdbFromDisk } from "./src/matrix/sdk/idb-persistence.js"; +import { + clearAllIndexedDbState, + readDatabaseRecords, +} from "./src/matrix/sdk/idb-persistence.test-helpers.js"; import { installMatrixTestRuntime } from "./src/test-runtime.js"; +const DOCTOR_IDB_DATABASE_PREFIX = "openclaw-matrix-doctor-test"; + function createContext(): PluginDoctorStateMigrationContext { return { openPluginStateKeyedStore: (options: OpenKeyedStoreOptions): PluginStateKeyedStore => @@ -71,7 +83,8 @@ describe("matrix doctor contract state migrations", () => { installMatrixTestRuntime(); }); - afterEach(() => { + afterEach(async () => { + await clearAllIndexedDbState({ databasePrefix: DOCTOR_IDB_DATABASE_PREFIX }); resetPluginStateStoreForTests(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); @@ -364,21 +377,16 @@ describe("matrix doctor contract state migrations", () => { expect(fs.existsSync(path.join(storageRootDir, "recovery-key.json"))).toBe(false); }); - it("migrates Matrix IndexedDB snapshot JSON to SQLite plugin state", async () => { + it("migrates legacy Matrix crypto state and restores the snapshot from SQLite", async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); tempDirs.push(stateDir); - const storageRootDir = path.join( - stateDir, - "matrix", - "accounts", - "default", - "matrix.example.org__bot", - "token-hash", - ); + const storageRootDir = path.join(stateDir, "matrix"); fs.mkdirSync(storageRootDir, { recursive: true }); + const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); + const snapshotDatabaseName = `${DOCTOR_IDB_DATABASE_PREFIX}::matrix-sdk-crypto`; const snapshot = [ { - name: "openclaw-matrix::matrix-sdk-crypto", + name: snapshotDatabaseName, version: 1, stores: [ { @@ -391,40 +399,7 @@ describe("matrix doctor contract state migrations", () => { ], }, ]; - fs.writeFileSync( - path.join(storageRootDir, "crypto-idb-snapshot.json"), - JSON.stringify(snapshot), - ); - - const migration = migrationById("matrix-idb-snapshot-json-to-plugin-state"); - await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ - preview: [`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`], - }); - - await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ - changes: [ - `Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`, - `Archived Matrix IndexedDB snapshot legacy source -> ${path.join(storageRootDir, "crypto-idb-snapshot.json")}.migrated`, - ], - warnings: [], - }); - - expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot); - expect(fs.existsSync(path.join(storageRootDir, "crypto-idb-snapshot.json"))).toBe(false); - }); - - it("migrates Matrix legacy crypto migration JSON to SQLite plugin state", async () => { - const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); - tempDirs.push(stateDir); - const storageRootDir = path.join( - stateDir, - "matrix", - "accounts", - "default", - "matrix.example.org__bot", - "token-hash", - ); - fs.mkdirSync(storageRootDir, { recursive: true }); + fs.writeFileSync(snapshotPath, JSON.stringify(snapshot)); fs.writeFileSync( path.join(storageRootDir, "legacy-crypto-migration.json"), JSON.stringify({ @@ -443,19 +418,149 @@ describe("matrix doctor contract state migrations", () => { const migration = migrationById("matrix-legacy-crypto-migration-json-to-plugin-state"); await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ - preview: [`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`], - }); - - await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ - changes: [ - `Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`, - `Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`, + preview: [ + `Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`, + `Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`, ], - warnings: [], }); - expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(3); + const result = await migration.migrateLegacyState(createMigrationParams(stateDir)); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + `Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`, + `Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`, + `Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`, + expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u), + ]); + const archivePath = result.changes[3]?.split(" -> ")[1]; + expect(archivePath).toMatch(/crypto-idb-snapshot\.json\.migrated-\d{4}-/u); + expect(JSON.parse(fs.readFileSync(archivePath ?? "", "utf8"))).toEqual(snapshot); + + expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(5); + expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot); expect(fs.existsSync(path.join(storageRootDir, "legacy-crypto-migration.json"))).toBe(false); + expect(fs.existsSync(snapshotPath)).toBe(false); + + await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true); + await expect( + readDatabaseRecords({ + name: snapshotDatabaseName, + storeName: "sessions", + }), + ).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]); + await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toBeNull(); + }); + + it("archives an invalid legacy snapshot for recovery and unblocks runtime", async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); + tempDirs.push(stateDir); + const storageRootDir = path.join(stateDir, "matrix"); + const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); + fs.mkdirSync(storageRootDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "not-json"); + + const result = await migrationById( + "matrix-legacy-crypto-migration-json-to-plugin-state", + ).migrateLegacyState(createMigrationParams(stateDir)); + + expect(result.warnings).toEqual([ + `Matrix IndexedDB snapshot legacy source is invalid for ${storageRootDir}; archived without import`, + ]); + expect(result.changes).toEqual([ + expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u), + ]); + const archivePath = result.changes[0]?.split(" -> ")[1]; + expect(fs.readFileSync(archivePath ?? "", "utf8")).toBe("not-json"); + expect(fs.existsSync(snapshotPath)).toBe(false); + await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(false); + }); + + it("repairs invalid snapshots, archives equivalents, and preserves conflicts", async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); + tempDirs.push(stateDir); + const partialRoot = path.join(stateDir, "matrix", "accounts", "partial"); + const conflictRoot = path.join(stateDir, "matrix", "accounts", "conflict"); + const equivalentRoot = path.join(stateDir, "matrix", "accounts", "equivalent"); + const invalidRoot = path.join(stateDir, "matrix", "accounts", "invalid"); + fs.mkdirSync(partialRoot, { recursive: true }); + fs.mkdirSync(conflictRoot, { recursive: true }); + fs.mkdirSync(equivalentRoot, { recursive: true }); + fs.mkdirSync(invalidRoot, { recursive: true }); + const partialSnapshot = [{ name: "partial-source", version: 1, stores: [] }]; + const conflictSnapshot = [{ name: "conflicting-source", version: 1, stores: [] }]; + const equivalentSnapshot = [{ name: "equivalent", version: 1, stores: [] }]; + const invalidReplacement = [{ name: "invalid-replacement", version: 1, stores: [] }]; + fs.writeFileSync( + path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME), + JSON.stringify(partialSnapshot), + ); + fs.writeFileSync( + path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME), + JSON.stringify(conflictSnapshot), + ); + fs.writeFileSync( + path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME), + JSON.stringify(equivalentSnapshot, null, 2), + ); + fs.writeFileSync( + path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME), + JSON.stringify(invalidReplacement), + ); + const partialStore = createContext().openPluginStateKeyedStore( + openMatrixIdbSnapshotStoreOptions(partialRoot), + ); + await partialStore.register("current:snapshot:interrupted:0", { + kind: "snapshot-chunk", + index: 0, + data: "[", + }); + const currentSnapshot = JSON.stringify([{ name: "current", version: 1, stores: [] }]); + writeMatrixIdbSnapshotJson({ + storageRootDir: conflictRoot, + snapshotJson: currentSnapshot, + databaseCount: 1, + }); + writeMatrixIdbSnapshotJson({ + storageRootDir: equivalentRoot, + snapshotJson: JSON.stringify(equivalentSnapshot), + databaseCount: 1, + }); + writeMatrixIdbSnapshotJson({ + storageRootDir: invalidRoot, + snapshotJson: JSON.stringify({ malformed: true }), + databaseCount: 1, + }); + + const result = await migrationById( + "matrix-legacy-crypto-migration-json-to-plugin-state", + ).migrateLegacyState(createMigrationParams(stateDir)); + + expect(result.changes).toEqual([ + expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"), + expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"), + `Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${invalidRoot}`, + expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"), + `Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${partialRoot}`, + expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"), + ]); + expect(result.warnings).toEqual([]); + expect(result.notices).toEqual([ + `Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${conflictRoot}`, + ]); + const conflictArchivePath = result.changes[0]?.split(" -> ")[1]; + expect(JSON.parse(fs.readFileSync(conflictArchivePath ?? "", "utf8"))).toEqual( + conflictSnapshot, + ); + expect(JSON.parse(readMatrixIdbSnapshotJson(partialRoot) ?? "null")).toEqual(partialSnapshot); + expect(JSON.parse(readMatrixIdbSnapshotJson(invalidRoot) ?? "null")).toEqual( + invalidReplacement, + ); + expect(readMatrixIdbSnapshotJson(conflictRoot)).toBe(currentSnapshot); + expect(fs.existsSync(path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false); + expect(fs.existsSync(path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false); + expect(fs.existsSync(path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false); + expect(fs.existsSync(path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false); }); it("migrates legacy inbound dedupe markers into the claimable dedupe store", async () => { diff --git a/extensions/matrix/doctor-contract-api.ts b/extensions/matrix/doctor-contract-api.ts index df99822a725c..7174b768caab 100644 --- a/extensions/matrix/doctor-contract-api.ts +++ b/extensions/matrix/doctor-contract-api.ts @@ -35,22 +35,19 @@ import { type MatrixCredentialStateRecord, type MatrixStoredCredentialRecord, } from "./src/matrix/credentials-read.js"; +import { migrateLegacyMatrixIdbSnapshot } from "./src/matrix/crypto-snapshot-doctor.js"; import { MATRIX_IDB_SNAPSHOT_FILENAME, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, MATRIX_RECOVERY_KEY_FILENAME, - hasMatrixIdbSnapshotStateInStore, hasMatrixLegacyCryptoMigrationStateInStore, hasMatrixRecoveryKeyStateInStore, - openMatrixIdbSnapshotStoreOptions, openMatrixLegacyCryptoMigrationStoreOptions, openMatrixRecoveryKeyStoreOptions, readLegacyMatrixLegacyCryptoMigrationState, readLegacyMatrixRecoveryKeyState, - writeMatrixIdbSnapshotJsonToStore, writeMatrixLegacyCryptoMigrationStateToStore, writeMatrixRecoveryKeyStateToStore, - type MatrixIdbSnapshotRecord, type MatrixLegacyCryptoMigrationState, } from "./src/matrix/crypto-state-store.js"; import { @@ -63,7 +60,6 @@ import { type LegacyInboundDedupeMarker, type MatrixInboundDedupeMigrationIo, } from "./src/matrix/monitor/inbound-dedupe-migration.js"; -import { readLegacyMatrixIdbSnapshotState } from "./src/matrix/sdk/idb-persistence.js"; import type { MatrixStoredRecoveryKey } from "./src/matrix/sdk/types.js"; import { resolveMatrixCredentialsDir } from "./src/storage-paths.js"; @@ -136,6 +132,7 @@ async function readLegacyMatrixCredentials( async function collectLegacyMatrixStateRoots( stateDir: string, filename: string, + options?: { includeMatrixRoot?: boolean }, ): Promise { const matrixRoot = path.join(stateDir, "matrix"); const roots: string[] = []; @@ -158,7 +155,9 @@ async function collectLegacyMatrixStateRoots( } } await visit(matrixRoot); - return roots.filter((root) => path.resolve(root) !== path.resolve(matrixRoot)).toSorted(); + return roots + .filter((root) => options?.includeMatrixRoot || path.resolve(root) !== path.resolve(matrixRoot)) + .toSorted(); } async function collectLegacySyncCacheRoots(stateDir: string): Promise { @@ -575,84 +574,31 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ }, }, { - id: "matrix-idb-snapshot-json-to-plugin-state", - label: "Matrix IndexedDB snapshot", + id: "matrix-legacy-crypto-migration-json-to-plugin-state", + label: "Matrix legacy crypto state", async detectLegacyState(params) { const previews: string[] = []; for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, - MATRIX_IDB_SNAPSHOT_FILENAME, + MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, + { includeMatrixRoot: true }, )) { - const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir); - if (!snapshot) { + if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) { continue; } + previews.push( + `Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`, + ); + } + for (const storageRootDir of await collectLegacyMatrixStateRoots( + params.stateDir, + MATRIX_IDB_SNAPSHOT_FILENAME, + { includeMatrixRoot: true }, + )) { previews.push(`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`); } return previews.length > 0 ? { preview: previews } : null; }, - async migrateLegacyState(params) { - const changes: string[] = []; - const warnings: string[] = []; - const notices: string[] = []; - for (const storageRootDir of await collectLegacyMatrixStateRoots( - params.stateDir, - MATRIX_IDB_SNAPSHOT_FILENAME, - )) { - const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir); - if (!snapshot) { - continue; - } - const store = params.context.openPluginStateKeyedStore( - openMatrixIdbSnapshotStoreOptions(storageRootDir), - ); - if (await hasMatrixIdbSnapshotStateInStore({ store })) { - await archiveLegacyMatrixStateFile({ - storageRootDir, - filename: MATRIX_IDB_SNAPSHOT_FILENAME, - label: "Matrix IndexedDB snapshot", - changes, - warnings, - notices, - notice: `Kept existing Matrix IndexedDB snapshot in SQLite and archived the legacy source for ${storageRootDir}`, - }); - continue; - } - await writeMatrixIdbSnapshotJsonToStore({ - snapshotJson: JSON.stringify(snapshot), - databaseCount: snapshot.length, - store, - }); - changes.push(`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`); - await archiveLegacyMatrixStateFile({ - storageRootDir, - filename: MATRIX_IDB_SNAPSHOT_FILENAME, - label: "Matrix IndexedDB snapshot", - changes, - warnings, - }); - } - return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; - }, - }, - { - id: "matrix-legacy-crypto-migration-json-to-plugin-state", - label: "Matrix legacy crypto migration", - async detectLegacyState(params) { - const previews: string[] = []; - for (const storageRootDir of await collectLegacyMatrixStateRoots( - params.stateDir, - MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, - )) { - if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) { - continue; - } - previews.push( - `Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`, - ); - } - return previews.length > 0 ? { preview: previews } : null; - }, async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; @@ -660,6 +606,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, + { includeMatrixRoot: true }, )) { const state = readLegacyMatrixLegacyCryptoMigrationState(storageRootDir); if (!state) { @@ -692,6 +639,19 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } + for (const storageRootDir of await collectLegacyMatrixStateRoots( + params.stateDir, + MATRIX_IDB_SNAPSHOT_FILENAME, + { includeMatrixRoot: true }, + )) { + await migrateLegacyMatrixIdbSnapshot({ + storageRootDir, + context: params.context, + changes, + notices, + warnings, + }); + } return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, diff --git a/extensions/matrix/src/matrix/client/storage.test.ts b/extensions/matrix/src/matrix/client/storage.test.ts index 2b91cf195db6..697819095c60 100644 --- a/extensions/matrix/src/matrix/client/storage.test.ts +++ b/extensions/matrix/src/matrix/client/storage.test.ts @@ -9,7 +9,6 @@ import { import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resolveMatrixAccountStorageRoot } from "../../storage-paths.js"; import { installMatrixTestRuntime } from "../../test-runtime.js"; -import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js"; import { SqliteBackedMatrixSyncStore } from "./file-sync-store.js"; import { claimCurrentTokenStorageState, @@ -372,38 +371,6 @@ describe("matrix client storage paths", () => { await expect(syncStore.getSavedSyncToken()).resolves.toBe("account-token"); }); - it("does not overwrite existing SQLite IDB snapshot state with a stale legacy sidecar", async () => { - const stateDir = setupStateDir(); - const storagePaths = resolveDefaultStoragePaths(); - fs.mkdirSync(storagePaths.rootDir, { recursive: true }); - const currentSnapshot = JSON.stringify([ - { - name: "current", - version: 1, - stores: [], - }, - ]); - writeMatrixIdbSnapshotJson({ - storageRootDir: storagePaths.rootDir, - snapshotJson: currentSnapshot, - databaseCount: 1, - }); - fs.writeFileSync( - storagePaths.idbSnapshotPath, - JSON.stringify([{ name: "stale", version: 1, stores: [] }]), - ); - const env = createMigrationEnv(stateDir); - - await maybeMigrateLegacyStorage({ - storagePaths, - env, - }); - - expect(readMatrixIdbSnapshotJson(storagePaths.rootDir)).toBe(currentSnapshot); - expect(fs.existsSync(storagePaths.idbSnapshotPath)).toBe(false); - expect(fs.existsSync(`${storagePaths.idbSnapshotPath}.migrated`)).toBe(true); - }); - it("ignores unrecognized account-scoped sync cache files without a migration snapshot", async () => { const stateDir = setupStateDir(); const storagePaths = resolveDefaultStoragePaths(); diff --git a/extensions/matrix/src/matrix/client/storage.ts b/extensions/matrix/src/matrix/client/storage.ts index 7889b504081a..75965d0f1783 100644 --- a/extensions/matrix/src/matrix/client/storage.ts +++ b/extensions/matrix/src/matrix/client/storage.ts @@ -17,9 +17,7 @@ import { MATRIX_RECOVERY_KEY_FILENAME, migrateLegacyMatrixLegacyCryptoMigrationFileToStore, migrateLegacyMatrixRecoveryKeyFileToStore, - readMatrixIdbSnapshotJson, scoreMatrixCryptoStateInStore, - writeMatrixIdbSnapshotJson, } from "../crypto-state-store.js"; import { resolveMatrixSqliteStateEnv } from "../sqlite-state.js"; import type { MatrixAuth } from "./types.js"; @@ -394,14 +392,12 @@ export async function maybeMigrateLegacyStorage(params: { hasAccountScopedLegacyStorageFile && (await syncCache?.readLegacyMatrixSyncCacheState(params.storagePaths.rootDir)) !== null; const hasAccountScopedRecoveryKey = fs.existsSync(params.storagePaths.recoveryKeyPath); - const hasAccountScopedIdbSnapshot = fs.existsSync(params.storagePaths.idbSnapshotPath); const hasAccountScopedLegacyCryptoMigration = fs.existsSync( path.join(params.storagePaths.rootDir, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME), ); if ( !hasAccountScopedLegacyStorage && !hasAccountScopedRecoveryKey && - !hasAccountScopedIdbSnapshot && !hasAccountScopedLegacyCryptoMigration ) { return; @@ -439,14 +435,6 @@ export async function maybeMigrateLegacyStorage(params: { label: "legacy crypto migration", }); } - if (hasAccountScopedIdbSnapshot) { - await migrateLegacyIdbSnapshotToSqlite({ - storageRootDir: params.storagePaths.rootDir, - snapshotPath: params.storagePaths.idbSnapshotPath, - moved, - pendingArchives, - }); - } } catch (err) { const rollbackError = rollbackLegacyMoves(moved); throw new Error( @@ -476,40 +464,6 @@ export async function maybeMigrateLegacyStorage(params: { } } -async function migrateLegacyIdbSnapshotToSqlite(params: { - storageRootDir: string; - snapshotPath: string; - moved: LegacyMoveRecord[]; - pendingArchives: LegacyArchiveRecord[]; -}): Promise { - if (readMatrixIdbSnapshotJson(params.storageRootDir)) { - params.pendingArchives.push({ - sourcePath: params.snapshotPath, - label: "IndexedDB snapshot", - }); - return; - } - const { readLegacyMatrixIdbSnapshotState } = await import("../sdk/idb-persistence.js"); - const snapshot = await readLegacyMatrixIdbSnapshotState(params.storageRootDir); - if (!snapshot) { - return; - } - writeMatrixIdbSnapshotJson({ - storageRootDir: params.storageRootDir, - snapshotJson: JSON.stringify(snapshot), - databaseCount: snapshot.length, - }); - params.moved.push({ - sourcePath: params.snapshotPath, - targetPath: `${params.storageRootDir} SQLite IndexedDB snapshot state`, - label: "IndexedDB snapshot", - }); - params.pendingArchives.push({ - sourcePath: params.snapshotPath, - label: "IndexedDB snapshot", - }); -} - async function migrateLegacySyncCacheToSqlite(params: { sourceRootDir: string; sourcePath: string; diff --git a/extensions/matrix/src/matrix/crypto-snapshot-doctor.ts b/extensions/matrix/src/matrix/crypto-snapshot-doctor.ts new file mode 100644 index 000000000000..479604fe6985 --- /dev/null +++ b/extensions/matrix/src/matrix/crypto-snapshot-doctor.ts @@ -0,0 +1,153 @@ +// Matrix plugin module owns the doctor-only crypto snapshot import. +import { randomUUID } from "node:crypto"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { withFileLock } from "openclaw/plugin-sdk/file-lock"; +import type { PluginDoctorStateMigrationContext } from "openclaw/plugin-sdk/runtime-doctor"; +import { + MATRIX_IDB_SNAPSHOT_FILENAME, + openMatrixIdbSnapshotStoreOptions, + readMatrixIdbSnapshotJsonFromStore, + writeMatrixIdbSnapshotJsonToStore, + type MatrixIdbSnapshotRecord, +} from "./crypto-state-store.js"; +import { MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS } from "./sdk/idb-persistence-lock.js"; +import { + isValidMatrixIdbSnapshotJson, + readLegacyMatrixIdbSnapshotStateUnlocked, +} from "./sdk/idb-persistence.js"; + +type SnapshotMigrationParams = { + storageRootDir: string; + context: PluginDoctorStateMigrationContext; + changes: string[]; + notices: string[]; + warnings: string[]; +}; + +export async function migrateLegacyMatrixIdbSnapshot( + params: SnapshotMigrationParams, +): Promise { + const snapshotPath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); + try { + // withFileLock is acquire-or-throw; it never skips the callback on contention. + await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, () => + migrateLegacyMatrixIdbSnapshotLocked(params), + ); + } catch (err) { + params.warnings.push( + `Failed locking Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`, + ); + } +} + +async function migrateLegacyMatrixIdbSnapshotLocked( + params: SnapshotMigrationParams, +): Promise { + const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); + let snapshot: ReturnType; + try { + snapshot = readLegacyMatrixIdbSnapshotStateUnlocked(params.storageRootDir); + } catch (err) { + params.warnings.push( + `Failed reading Matrix IndexedDB snapshot legacy source for ${params.storageRootDir}: ${String(err)}; left source in place`, + ); + return; + } + if (!snapshot) { + if (!fsSync.existsSync(sourcePath)) { + return; + } + const archived = await archiveLegacyMatrixIdbSnapshot(params); + params.warnings.push( + archived + ? `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; archived without import` + : `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; left active because archival failed`, + ); + return; + } + const snapshotJson = JSON.stringify(snapshot); + const store = params.context.openPluginStateKeyedStore( + openMatrixIdbSnapshotStoreOptions(params.storageRootDir), + ); + let persisted: string | null; + let hadPartialState: boolean; + try { + persisted = await readMatrixIdbSnapshotJsonFromStore({ store }); + const persistedIsValid = persisted ? isValidMatrixIdbSnapshotJson(persisted) : false; + hadPartialState = !persistedIsValid && (await store.entries()).length > 0; + if (!persistedIsValid) { + persisted = null; + } + } catch (err) { + params.warnings.push( + `Failed inspecting Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}: ${String(err)}; left legacy source in place`, + ); + return; + } + if (persisted && !snapshotContentMatches(persisted, snapshot)) { + if (await archiveLegacyMatrixIdbSnapshot(params)) { + params.notices.push( + `Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${params.storageRootDir}`, + ); + } + return; + } + if (!persisted) { + try { + await writeMatrixIdbSnapshotJsonToStore({ + snapshotJson, + databaseCount: snapshot.length, + store, + }); + persisted = await readMatrixIdbSnapshotJsonFromStore({ store }); + } catch (err) { + params.warnings.push( + `Failed importing Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`, + ); + return; + } + if (!persisted || !snapshotContentMatches(persisted, snapshot)) { + params.warnings.push( + `Failed verifying Matrix IndexedDB snapshot for ${params.storageRootDir}; left legacy source in place`, + ); + return; + } + params.changes.push( + hadPartialState + ? `Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}` + : `Migrated Matrix IndexedDB snapshot JSON to SQLite for ${params.storageRootDir}`, + ); + } + await archiveLegacyMatrixIdbSnapshot(params); +} + +function snapshotContentMatches(persistedJson: string, snapshot: unknown): boolean { + try { + return isDeepStrictEqual(JSON.parse(persistedJson), snapshot); + } catch { + return false; + } +} + +async function archiveLegacyMatrixIdbSnapshot(params: { + storageRootDir: string; + changes: string[]; + warnings: string[]; +}): Promise { + const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); + const timestamp = new Date().toISOString().replaceAll(":", "-"); + const archivePath = `${sourcePath}.migrated-${timestamp}-${randomUUID()}`; + try { + await fs.rename(sourcePath, archivePath); + params.changes.push(`Archived Matrix IndexedDB snapshot legacy source -> ${archivePath}`); + return true; + } catch (err) { + params.warnings.push( + `Failed archiving Matrix IndexedDB snapshot legacy source: ${String(err)}`, + ); + return false; + } +} diff --git a/extensions/matrix/src/matrix/crypto-state-store.ts b/extensions/matrix/src/matrix/crypto-state-store.ts index 98022f614f14..1d50762e5ab9 100644 --- a/extensions/matrix/src/matrix/crypto-state-store.ts +++ b/extensions/matrix/src/matrix/crypto-state-store.ts @@ -230,10 +230,10 @@ export function writeMatrixIdbSnapshotJson(params: { }); } -export async function hasMatrixIdbSnapshotStateInStore(params: { +export async function readMatrixIdbSnapshotJsonFromStore(params: { store: Pick, "lookup">; -}): Promise { - return (await readIdbSnapshotJsonFromAsyncStore(params.store)) !== null; +}): Promise { + return await readIdbSnapshotJsonFromAsyncStore(params.store); } export async function writeMatrixIdbSnapshotJsonToStore(params: { diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts b/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts deleted file mode 100644 index 6e1fdf87eaa1..000000000000 --- a/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -// Matrix tests cover idb persistence.lock order plugin behavior. -import "fake-indexeddb/auto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; -import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { installMatrixTestRuntime } from "../../test-runtime.js"; -import { MATRIX_IDB_PERSIST_INTERVAL_MS } from "./idb-persistence-lock.js"; -import { clearAllIndexedDbState, seedDatabase } from "./idb-persistence.test-helpers.js"; - -const { withFileLockMock } = vi.hoisted(() => ({ - withFileLockMock: vi.fn( - async (_filePath: string, _options: unknown, fn: () => Promise) => await fn(), - ), -})); - -vi.mock("openclaw/plugin-sdk/file-lock", async () => { - const actual = await vi.importActual( - "openclaw/plugin-sdk/file-lock", - ); - return { - ...actual, - withFileLock: withFileLockMock, - }; -}); - -let persistIdbToDisk: typeof import("./idb-persistence.js").persistIdbToDisk; -let restoreIdbFromDisk: typeof import("./idb-persistence.js").restoreIdbFromDisk; -type CapturedLockOptions = - typeof import("./idb-persistence-lock.js").MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS; -const DATABASE_PREFIX = "openclaw-matrix-lock-order-test"; -const cryptoDatabaseName = `${DATABASE_PREFIX}::matrix-sdk-crypto`; - -function minimumRetryWindowMs(options: CapturedLockOptions): number { - let total = 0; - for (let attempt = 0; attempt < options.retries.retries; attempt += 1) { - total += Math.min( - options.retries.maxTimeout, - Math.max( - options.retries.minTimeout, - options.retries.minTimeout * options.retries.factor ** attempt, - ), - ); - } - return total; -} - -beforeAll(async () => { - ({ persistIdbToDisk, restoreIdbFromDisk } = await import("./idb-persistence.js")); -}); - -describe("Matrix IndexedDB persistence lock ordering", () => { - let tmpDir: string; - - beforeEach(async () => { - resetPluginStateStoreForTests(); - installMatrixTestRuntime(); - tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "matrix-idb-lock-order-")); - withFileLockMock.mockReset(); - withFileLockMock.mockImplementation( - async (_filePath: string, _options: unknown, fn: () => Promise) => await fn(), - ); - await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX }); - }); - - afterEach(async () => { - await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX }); - resetPluginStateStoreForTests(); - fs.rmSync(tmpDir, { recursive: true, force: true }); - }); - - it("captures the current snapshot into SQLite state", async () => { - const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); - await seedDatabase({ - name: cryptoDatabaseName, - storeName: "sessions", - records: [{ key: "room-1", value: { session: "old-session" } }], - }); - - await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX }); - await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX }); - - await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true); - const dbs = await indexedDB.databases(); - expect(dbs.map((entry) => entry.name)).toContain(cryptoDatabaseName); - }); - - it("uses the long snapshot lock options when importing a legacy file", async () => { - const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); - const capturedOptions: CapturedLockOptions[] = []; - - withFileLockMock.mockImplementationOnce(async (_filePath, options) => { - capturedOptions.push(options as CapturedLockOptions); - return false; - }); - fs.writeFileSync(snapshotPath, "[]", "utf8"); - await restoreIdbFromDisk(snapshotPath); - - expect(capturedOptions).toHaveLength(1); - for (const options of capturedOptions) { - expect(minimumRetryWindowMs(options)).toBeGreaterThanOrEqual(MATRIX_IDB_PERSIST_INTERVAL_MS); - expect(options.stale).toBe(5 * 60_000); - } - }); -}); diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence.test.ts b/extensions/matrix/src/matrix/sdk/idb-persistence.test.ts index d33619a445f9..4dcd719c880f 100644 --- a/extensions/matrix/src/matrix/sdk/idb-persistence.test.ts +++ b/extensions/matrix/src/matrix/sdk/idb-persistence.test.ts @@ -8,6 +8,7 @@ import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state- import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getMatrixRuntime } from "../../runtime.js"; import { installMatrixTestRuntime } from "../../test-runtime.js"; +import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js"; import { persistIdbToDisk, restoreIdbFromDisk } from "./idb-persistence.js"; import { clearAllIndexedDbState, @@ -80,113 +81,70 @@ describe("Matrix IndexedDB persistence", () => { expect(dbs.map((entry) => entry.name)).not.toContain(otherCryptoDatabaseName); }); - it("imports and archives a legacy JSON snapshot during restore", async () => { + it("blocks runtime restore and persistence until doctor migrates the legacy snapshot", async () => { const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); - fs.writeFileSync( - snapshotPath, - JSON.stringify([ - { - name: cryptoDatabaseName, - version: 1, - stores: [ - { - name: "sessions", - keyPath: null, - autoIncrement: false, - indexes: [], - records: [{ key: "room-1", value: { session: "legacy" } }], - }, - ], - }, - ]), - "utf8", - ); + const snapshot = JSON.stringify([{ name: cryptoDatabaseName, version: 1, stores: [] }]); + fs.writeFileSync(snapshotPath, snapshot); - const restored = await restoreIdbFromDisk(snapshotPath); - expect(restored).toBe(true); - expect(fs.existsSync(snapshotPath)).toBe(false); - expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true); - - await clearTestIndexedDbState(); - await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true); - await expect( - readDatabaseRecords({ - name: cryptoDatabaseName, - storeName: "sessions", - }), - ).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]); - }); - - it("restores a valid legacy JSON snapshot when SQLite import fails", async () => { - const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); - fs.writeFileSync( - snapshotPath, - JSON.stringify([ - { - name: cryptoDatabaseName, - version: 1, - stores: [ - { - name: "sessions", - keyPath: null, - autoIncrement: false, - indexes: [], - records: [{ key: "room-1", value: { session: "legacy" } }], - }, - ], - }, - ]), - "utf8", - ); - vi.spyOn(getMatrixRuntime().state, "openSyncKeyedStore").mockImplementation(() => { - throw new Error("sqlite unavailable"); + await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({ + name: "MatrixIdbSnapshotMigrationRequiredError", + code: "matrix-idb-snapshot-requires-doctor", + remediation: "openclaw doctor --fix", }); - - const restored = await restoreIdbFromDisk(snapshotPath); - - expect(restored).toBe(true); - expect(fs.existsSync(snapshotPath)).toBe(true); - await expect( - readDatabaseRecords({ - name: cryptoDatabaseName, - storeName: "sessions", + expect(warnSpy).toHaveBeenCalledWith( + "IdbPersistence", + expect.objectContaining({ + code: "matrix-idb-snapshot-requires-doctor", + remediation: "openclaw doctor --fix", }), - ).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]); - }); + ); + expect(JSON.stringify(warnSpy.mock.calls)).not.toContain(snapshotPath); - it("returns false and logs a warning for malformed snapshots", async () => { - const snapshotPath = path.join(tmpDir, "bad-snapshot.json"); - fs.writeFileSync(snapshotPath, JSON.stringify([{ nope: true }]), "utf8"); + await seedDatabase({ + name: cryptoDatabaseName, + storeName: "sessions", + records: [{ key: "new-room", value: { session: "new" } }], + }); + await expect( + persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX }), + ).rejects.toMatchObject({ + code: "matrix-idb-snapshot-requires-doctor", + }); + expect(readMatrixIdbSnapshotJson(tmpDir)).toBeNull(); + expect(fs.existsSync(snapshotPath)).toBe(true); - const restored = await restoreIdbFromDisk(snapshotPath); - expect(restored).toBe(false); - expect(warnSpy).toHaveBeenCalledTimes(1); - const [scope, message, error] = warnSpy.mock.calls.at(0) ?? []; - expect(scope).toBe("IdbPersistence"); - expect(message).toBe(`Failed to restore IndexedDB snapshot from ${snapshotPath}:`); - expect(error).toBeInstanceOf(Error); - }); + writeMatrixIdbSnapshotJson({ + storageRootDir: tmpDir, + snapshotJson: JSON.stringify({ malformed: true }), + databaseCount: 1, + }); + await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({ + code: "matrix-idb-snapshot-requires-doctor", + }); + const storeSpy = vi + .spyOn(getMatrixRuntime().state, "openSyncKeyedStore") + .mockImplementation(() => { + throw new Error("sqlite unavailable"); + }); - it("returns false for empty snapshot payloads without restoring databases", async () => { - const snapshotPath = path.join(tmpDir, "empty-snapshot.json"); - fs.writeFileSync(snapshotPath, JSON.stringify([]), "utf8"); - - const restored = await restoreIdbFromDisk(snapshotPath); - expect(restored).toBe(false); - - const dbs = await indexedDB.databases(); - expect(dbs).toStrictEqual([]); + try { + await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({ + code: "matrix-idb-snapshot-requires-doctor", + }); + } finally { + storeSpy.mockRestore(); + } }); it("returns false without warning when the snapshot does not exist yet", async () => { - const restored = await restoreIdbFromDisk(path.join(tmpDir, "missing-snapshot.json")); + const restored = await restoreIdbFromDisk(path.join(tmpDir, "crypto-idb-snapshot.json")); expect(restored).toBe(false); expect(warnSpy).not.toHaveBeenCalled(); }); it("handles concurrent persist operations in SQLite state", async () => { - const snapshotPath = path.join(tmpDir, "concurrent-persist.json"); + const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); await seedDatabase({ name: cryptoDatabaseName, storeName: "sessions", @@ -208,19 +166,4 @@ describe("Matrix IndexedDB persistence", () => { }), ).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]); }); - - it("archives an existing legacy snapshot file after persist", async () => { - const snapshotPath = path.join(tmpDir, "persist-archives-legacy.json"); - fs.writeFileSync(snapshotPath, "[]", "utf8"); - await seedDatabase({ - name: cryptoDatabaseName, - storeName: "sessions", - records: [{ key: "room-1", value: { session: "abc123" } }], - }); - - await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX }); - - expect(fs.existsSync(snapshotPath)).toBe(false); - expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true); - }); }); diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence.ts b/extensions/matrix/src/matrix/sdk/idb-persistence.ts index 5e6970a4bdce..57818ee401af 100644 --- a/extensions/matrix/src/matrix/sdk/idb-persistence.ts +++ b/extensions/matrix/src/matrix/sdk/idb-persistence.ts @@ -31,6 +31,28 @@ type IdbDatabaseSnapshot = { stores: IdbStoreSnapshot[]; }; +type IdbPersistenceDiagnostic = { + code: "matrix-idb-snapshot-requires-doctor"; + message: string; + remediation: "openclaw doctor --fix"; +}; + +const LEGACY_SNAPSHOT_DIAGNOSTIC: IdbPersistenceDiagnostic = { + code: "matrix-idb-snapshot-requires-doctor", + message: "Matrix IndexedDB snapshot exists outside canonical SQLite state", + remediation: "openclaw doctor --fix", +}; + +class MatrixIdbSnapshotMigrationRequiredError extends Error { + readonly code = LEGACY_SNAPSHOT_DIAGNOSTIC.code; + readonly remediation = LEGACY_SNAPSHOT_DIAGNOSTIC.remediation; + + constructor() { + super(`${LEGACY_SNAPSHOT_DIAGNOSTIC.message}; run ${LEGACY_SNAPSHOT_DIAGNOSTIC.remediation}`); + this.name = "MatrixIdbSnapshotMigrationRequiredError"; + } +} + function isValidIdbIndexSnapshot(value: unknown): value is IdbStoreSnapshot["indexes"][number] { if (!value || typeof value !== "object") { return false; @@ -100,6 +122,14 @@ function parseSnapshotPayload(data: string): IdbDatabaseSnapshot[] | null { return parsed; } +export function isValidMatrixIdbSnapshotJson(data: string): boolean { + try { + return parseSnapshotPayload(data) !== null; + } catch { + return false; + } +} + function idbReq(req: IDBRequest): Promise { return new Promise((resolve, reject) => { req.addEventListener("success", () => resolve(req.result), { once: true }); @@ -228,106 +258,86 @@ function resolveDefaultIdbSnapshotPath(): string { return path.join(stateDir, "matrix", "crypto-idb-snapshot.json"); } +// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests. export async function restoreIdbFromDisk(snapshotPath?: string): Promise { - const candidatePaths = snapshotPath ? [snapshotPath] : [resolveDefaultIdbSnapshotPath()]; - for (const resolvedPath of candidatePaths) { - const storageRootDir = path.dirname(resolvedPath); - try { - const restored = await withFileLock( - resolvedPath, - MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, - async () => { - try { - const storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir); - if (storedSnapshotJson) { - const snapshot = parseSnapshotPayload(storedSnapshotJson); - if (snapshot) { - await restoreIndexedDatabases(snapshot); - LogService.info( - "IdbPersistence", - `Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`, - ); - return true; - } - } - } catch (err) { - LogService.warn( - "IdbPersistence", - "Failed to restore IndexedDB snapshot from SQLite:", - err, - ); - } - - if (!fs.existsSync(resolvedPath)) { - return false; - } - const data = fs.readFileSync(resolvedPath, "utf8"); - const snapshot = parseSnapshotPayload(data); - if (!snapshot) { - return false; - } - let migratedToSqlite = false; - try { - writeMatrixIdbSnapshotJson({ - storageRootDir, - snapshotJson: data, - databaseCount: snapshot.length, - }); - archiveLegacyIdbSnapshotFile(resolvedPath); - migratedToSqlite = true; - } catch (err) { - LogService.warn( - "IdbPersistence", - `Failed to migrate IndexedDB snapshot to SQLite from ${resolvedPath}:`, - err, - ); - } - await restoreIndexedDatabases(snapshot); - LogService.info( - "IdbPersistence", - migratedToSqlite - ? `Migrated and restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}` - : `Restored ${snapshot.length} IndexedDB database(s) from legacy snapshot ${resolvedPath}`, - ); - return true; - }, - ); - if (restored) { - return true; + const resolvedPath = snapshotPath ?? resolveDefaultIdbSnapshotPath(); + const storageRootDir = path.dirname(resolvedPath); + let callbackStarted = false; + try { + // withFileLock is acquire-or-throw; it never skips the callback on contention. + return await withFileLock(resolvedPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => { + callbackStarted = true; + let storedSnapshotJson: string | null; + try { + storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir); + } catch (err) { + if (fs.existsSync(resolvedPath)) { + throwLegacySnapshotMigrationRequired(); + } + throw err; } - } catch (err) { - LogService.warn( + throwIfLegacySnapshotNeedsDoctor(resolvedPath, storedSnapshotJson); + if (!storedSnapshotJson) { + return false; + } + const snapshot = parseSnapshotPayload(storedSnapshotJson); + if (!snapshot) { + return false; + } + await restoreIndexedDatabases(snapshot); + LogService.info( "IdbPersistence", - `Failed to restore IndexedDB snapshot from ${resolvedPath}:`, - err, + `Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`, ); - continue; + return true; + }); + } catch (err) { + if (err instanceof MatrixIdbSnapshotMigrationRequiredError) { + throw err; } + if (!callbackStarted && fs.existsSync(resolvedPath)) { + throwLegacySnapshotMigrationRequired(); + } + LogService.warn("IdbPersistence", "Failed to restore IndexedDB snapshot from SQLite:", err); + return false; } - return false; } export async function persistIdbToDisk(params?: { + // Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests. snapshotPath?: string; databasePrefix?: string; }): Promise { const snapshotPath = params?.snapshotPath ?? resolveDefaultIdbSnapshotPath(); + let callbackStarted = false; try { fs.mkdirSync(path.dirname(snapshotPath), { recursive: true }); + // withFileLock is acquire-or-throw; it never skips the callback on contention. const persistedCount = await withFileLock( snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => { + callbackStarted = true; + const storageRootDir = path.dirname(snapshotPath); + let storedSnapshotJson: string | null; + try { + storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir); + } catch (err) { + if (fs.existsSync(snapshotPath)) { + throwLegacySnapshotMigrationRequired(); + } + throw err; + } + throwIfLegacySnapshotNeedsDoctor(snapshotPath, storedSnapshotJson); const snapshot = await dumpIndexedDatabases(params?.databasePrefix); if (snapshot.length === 0) { return 0; } writeMatrixIdbSnapshotJson({ - storageRootDir: path.dirname(snapshotPath), + storageRootDir, snapshotJson: JSON.stringify(snapshot), databaseCount: snapshot.length, }); - archiveLegacyIdbSnapshotFile(snapshotPath); return snapshot.length; }, ); @@ -339,36 +349,46 @@ export async function persistIdbToDisk(params?: { `Persisted ${persistedCount} IndexedDB database(s) to Matrix SQLite state`, ); } catch (err) { + if (err instanceof MatrixIdbSnapshotMigrationRequiredError) { + throw err; + } + if (!callbackStarted && fs.existsSync(snapshotPath)) { + throwLegacySnapshotMigrationRequired(); + } LogService.warn("IdbPersistence", "Failed to persist IndexedDB snapshot:", err); } } -export async function readLegacyMatrixIdbSnapshotState( +export function readLegacyMatrixIdbSnapshotStateUnlocked( storageRootDir: string, -): Promise { +): IdbDatabaseSnapshot[] | null { const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME); if (!fs.existsSync(snapshotPath)) { return null; } + const data = fs.readFileSync(snapshotPath, "utf8"); try { - return await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => { - const snapshot = parseSnapshotPayload(fs.readFileSync(snapshotPath, "utf8")); - return snapshot; - }); + return parseSnapshotPayload(data); } catch { return null; } } -function archiveLegacyIdbSnapshotFile(snapshotPath: string): void { - if (!fs.existsSync(snapshotPath)) { - return; +function throwIfLegacySnapshotNeedsDoctor( + snapshotPath: string, + storedSnapshotJson: string | null, +): void { + if ( + fs.existsSync(snapshotPath) && + (!storedSnapshotJson || !isValidMatrixIdbSnapshotJson(storedSnapshotJson)) + ) { + throwLegacySnapshotMigrationRequired(); } - const archivedPath = `${snapshotPath}.migrated`; - if (fs.existsSync(archivedPath)) { - return; - } - fs.renameSync(snapshotPath, archivedPath); +} + +function throwLegacySnapshotMigrationRequired(): never { + LogService.warn("IdbPersistence", LEGACY_SNAPSHOT_DIAGNOSTIC); + throw new MatrixIdbSnapshotMigrationRequiredError(); } function toLintErrorObject(value: unknown, fallbackMessage: string): Error {