fix(matrix): preserve encryption keys across snapshot upgrades (#112919)

* fix(matrix): migrate legacy crypto snapshots via doctor

* fix(matrix): harden snapshot migration failures

* fix(matrix): preserve unreadable crypto snapshots

* test(matrix): consolidate crypto snapshot coverage
This commit is contained in:
Peter Steinberger
2026-07-23 02:25:52 -04:00
committed by GitHub
parent e76c9aac30
commit d893f79ccc
9 changed files with 504 additions and 508 deletions
+159 -54
View File
@@ -1,4 +1,5 @@
// Matrix tests cover doctor contract state migrations.
import "fake-indexeddb/auto";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
@@ -25,10 +26,14 @@ import {
type MatrixStoredCredentialRecord,
} from "./src/matrix/credentials-read.js";
import {
MATRIX_IDB_SNAPSHOT_FILENAME,
MATRIX_RECOVERY_KEY_FILENAME,
openMatrixIdbSnapshotStoreOptions,
readMatrixIdbSnapshotJson,
readMatrixRecoveryKeyStateForPath,
scoreMatrixCryptoStateInStore,
writeMatrixIdbSnapshotJson,
type MatrixIdbSnapshotRecord,
} from "./src/matrix/crypto-state-store.js";
import { importNewestInboundDedupeMarkers } from "./src/matrix/monitor/inbound-dedupe-migration.js";
import {
@@ -36,8 +41,15 @@ import {
MATRIX_INBOUND_DEDUPE_TTL_MS,
resolveMatrixInboundDedupeStateNamespace,
} from "./src/matrix/monitor/inbound-dedupe.js";
import { restoreIdbFromDisk } from "./src/matrix/sdk/idb-persistence.js";
import {
clearAllIndexedDbState,
readDatabaseRecords,
} from "./src/matrix/sdk/idb-persistence.test-helpers.js";
import { installMatrixTestRuntime } from "./src/test-runtime.js";
const DOCTOR_IDB_DATABASE_PREFIX = "openclaw-matrix-doctor-test";
function createContext(): PluginDoctorStateMigrationContext {
return {
openPluginStateKeyedStore: <T>(options: OpenKeyedStoreOptions): PluginStateKeyedStore<T> =>
@@ -71,7 +83,8 @@ describe("matrix doctor contract state migrations", () => {
installMatrixTestRuntime();
});
afterEach(() => {
afterEach(async () => {
await clearAllIndexedDbState({ databasePrefix: DOCTOR_IDB_DATABASE_PREFIX });
resetPluginStateStoreForTests();
for (const dir of tempDirs.splice(0)) {
fs.rmSync(dir, { recursive: true, force: true });
@@ -364,21 +377,16 @@ describe("matrix doctor contract state migrations", () => {
expect(fs.existsSync(path.join(storageRootDir, "recovery-key.json"))).toBe(false);
});
it("migrates Matrix IndexedDB snapshot JSON to SQLite plugin state", async () => {
it("migrates legacy Matrix crypto state and restores the snapshot from SQLite", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
tempDirs.push(stateDir);
const storageRootDir = path.join(
stateDir,
"matrix",
"accounts",
"default",
"matrix.example.org__bot",
"token-hash",
);
const storageRootDir = path.join(stateDir, "matrix");
fs.mkdirSync(storageRootDir, { recursive: true });
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
const snapshotDatabaseName = `${DOCTOR_IDB_DATABASE_PREFIX}::matrix-sdk-crypto`;
const snapshot = [
{
name: "openclaw-matrix::matrix-sdk-crypto",
name: snapshotDatabaseName,
version: 1,
stores: [
{
@@ -391,40 +399,7 @@ describe("matrix doctor contract state migrations", () => {
],
},
];
fs.writeFileSync(
path.join(storageRootDir, "crypto-idb-snapshot.json"),
JSON.stringify(snapshot),
);
const migration = migrationById("matrix-idb-snapshot-json-to-plugin-state");
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
preview: [`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`],
});
await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
changes: [
`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`,
`Archived Matrix IndexedDB snapshot legacy source -> ${path.join(storageRootDir, "crypto-idb-snapshot.json")}.migrated`,
],
warnings: [],
});
expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot);
expect(fs.existsSync(path.join(storageRootDir, "crypto-idb-snapshot.json"))).toBe(false);
});
it("migrates Matrix legacy crypto migration JSON to SQLite plugin state", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
tempDirs.push(stateDir);
const storageRootDir = path.join(
stateDir,
"matrix",
"accounts",
"default",
"matrix.example.org__bot",
"token-hash",
);
fs.mkdirSync(storageRootDir, { recursive: true });
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot));
fs.writeFileSync(
path.join(storageRootDir, "legacy-crypto-migration.json"),
JSON.stringify({
@@ -443,19 +418,149 @@ describe("matrix doctor contract state migrations", () => {
const migration = migrationById("matrix-legacy-crypto-migration-json-to-plugin-state");
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
preview: [`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`],
});
await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
changes: [
`Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`,
`Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`,
preview: [
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`,
],
warnings: [],
});
expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(3);
const result = await migration.migrateLegacyState(createMigrationParams(stateDir));
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
`Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`,
`Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`,
`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`,
expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u),
]);
const archivePath = result.changes[3]?.split(" -> ")[1];
expect(archivePath).toMatch(/crypto-idb-snapshot\.json\.migrated-\d{4}-/u);
expect(JSON.parse(fs.readFileSync(archivePath ?? "", "utf8"))).toEqual(snapshot);
expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(5);
expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot);
expect(fs.existsSync(path.join(storageRootDir, "legacy-crypto-migration.json"))).toBe(false);
expect(fs.existsSync(snapshotPath)).toBe(false);
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
await expect(
readDatabaseRecords({
name: snapshotDatabaseName,
storeName: "sessions",
}),
).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]);
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toBeNull();
});
it("archives an invalid legacy snapshot for recovery and unblocks runtime", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
tempDirs.push(stateDir);
const storageRootDir = path.join(stateDir, "matrix");
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
fs.mkdirSync(storageRootDir, { recursive: true });
fs.writeFileSync(snapshotPath, "not-json");
const result = await migrationById(
"matrix-legacy-crypto-migration-json-to-plugin-state",
).migrateLegacyState(createMigrationParams(stateDir));
expect(result.warnings).toEqual([
`Matrix IndexedDB snapshot legacy source is invalid for ${storageRootDir}; archived without import`,
]);
expect(result.changes).toEqual([
expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u),
]);
const archivePath = result.changes[0]?.split(" -> ")[1];
expect(fs.readFileSync(archivePath ?? "", "utf8")).toBe("not-json");
expect(fs.existsSync(snapshotPath)).toBe(false);
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(false);
});
it("repairs invalid snapshots, archives equivalents, and preserves conflicts", async () => {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
tempDirs.push(stateDir);
const partialRoot = path.join(stateDir, "matrix", "accounts", "partial");
const conflictRoot = path.join(stateDir, "matrix", "accounts", "conflict");
const equivalentRoot = path.join(stateDir, "matrix", "accounts", "equivalent");
const invalidRoot = path.join(stateDir, "matrix", "accounts", "invalid");
fs.mkdirSync(partialRoot, { recursive: true });
fs.mkdirSync(conflictRoot, { recursive: true });
fs.mkdirSync(equivalentRoot, { recursive: true });
fs.mkdirSync(invalidRoot, { recursive: true });
const partialSnapshot = [{ name: "partial-source", version: 1, stores: [] }];
const conflictSnapshot = [{ name: "conflicting-source", version: 1, stores: [] }];
const equivalentSnapshot = [{ name: "equivalent", version: 1, stores: [] }];
const invalidReplacement = [{ name: "invalid-replacement", version: 1, stores: [] }];
fs.writeFileSync(
path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
JSON.stringify(partialSnapshot),
);
fs.writeFileSync(
path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
JSON.stringify(conflictSnapshot),
);
fs.writeFileSync(
path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
JSON.stringify(equivalentSnapshot, null, 2),
);
fs.writeFileSync(
path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
JSON.stringify(invalidReplacement),
);
const partialStore = createContext().openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
openMatrixIdbSnapshotStoreOptions(partialRoot),
);
await partialStore.register("current:snapshot:interrupted:0", {
kind: "snapshot-chunk",
index: 0,
data: "[",
});
const currentSnapshot = JSON.stringify([{ name: "current", version: 1, stores: [] }]);
writeMatrixIdbSnapshotJson({
storageRootDir: conflictRoot,
snapshotJson: currentSnapshot,
databaseCount: 1,
});
writeMatrixIdbSnapshotJson({
storageRootDir: equivalentRoot,
snapshotJson: JSON.stringify(equivalentSnapshot),
databaseCount: 1,
});
writeMatrixIdbSnapshotJson({
storageRootDir: invalidRoot,
snapshotJson: JSON.stringify({ malformed: true }),
databaseCount: 1,
});
const result = await migrationById(
"matrix-legacy-crypto-migration-json-to-plugin-state",
).migrateLegacyState(createMigrationParams(stateDir));
expect(result.changes).toEqual([
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
`Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${invalidRoot}`,
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
`Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${partialRoot}`,
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
]);
expect(result.warnings).toEqual([]);
expect(result.notices).toEqual([
`Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${conflictRoot}`,
]);
const conflictArchivePath = result.changes[0]?.split(" -> ")[1];
expect(JSON.parse(fs.readFileSync(conflictArchivePath ?? "", "utf8"))).toEqual(
conflictSnapshot,
);
expect(JSON.parse(readMatrixIdbSnapshotJson(partialRoot) ?? "null")).toEqual(partialSnapshot);
expect(JSON.parse(readMatrixIdbSnapshotJson(invalidRoot) ?? "null")).toEqual(
invalidReplacement,
);
expect(readMatrixIdbSnapshotJson(conflictRoot)).toBe(currentSnapshot);
expect(fs.existsSync(path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
expect(fs.existsSync(path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
expect(fs.existsSync(path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
expect(fs.existsSync(path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
});
it("migrates legacy inbound dedupe markers into the claimable dedupe store", async () => {
+33 -73
View File
@@ -35,22 +35,19 @@ import {
type MatrixCredentialStateRecord,
type MatrixStoredCredentialRecord,
} from "./src/matrix/credentials-read.js";
import { migrateLegacyMatrixIdbSnapshot } from "./src/matrix/crypto-snapshot-doctor.js";
import {
MATRIX_IDB_SNAPSHOT_FILENAME,
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
MATRIX_RECOVERY_KEY_FILENAME,
hasMatrixIdbSnapshotStateInStore,
hasMatrixLegacyCryptoMigrationStateInStore,
hasMatrixRecoveryKeyStateInStore,
openMatrixIdbSnapshotStoreOptions,
openMatrixLegacyCryptoMigrationStoreOptions,
openMatrixRecoveryKeyStoreOptions,
readLegacyMatrixLegacyCryptoMigrationState,
readLegacyMatrixRecoveryKeyState,
writeMatrixIdbSnapshotJsonToStore,
writeMatrixLegacyCryptoMigrationStateToStore,
writeMatrixRecoveryKeyStateToStore,
type MatrixIdbSnapshotRecord,
type MatrixLegacyCryptoMigrationState,
} from "./src/matrix/crypto-state-store.js";
import {
@@ -63,7 +60,6 @@ import {
type LegacyInboundDedupeMarker,
type MatrixInboundDedupeMigrationIo,
} from "./src/matrix/monitor/inbound-dedupe-migration.js";
import { readLegacyMatrixIdbSnapshotState } from "./src/matrix/sdk/idb-persistence.js";
import type { MatrixStoredRecoveryKey } from "./src/matrix/sdk/types.js";
import { resolveMatrixCredentialsDir } from "./src/storage-paths.js";
@@ -136,6 +132,7 @@ async function readLegacyMatrixCredentials(
async function collectLegacyMatrixStateRoots(
stateDir: string,
filename: string,
options?: { includeMatrixRoot?: boolean },
): Promise<string[]> {
const matrixRoot = path.join(stateDir, "matrix");
const roots: string[] = [];
@@ -158,7 +155,9 @@ async function collectLegacyMatrixStateRoots(
}
}
await visit(matrixRoot);
return roots.filter((root) => path.resolve(root) !== path.resolve(matrixRoot)).toSorted();
return roots
.filter((root) => options?.includeMatrixRoot || path.resolve(root) !== path.resolve(matrixRoot))
.toSorted();
}
async function collectLegacySyncCacheRoots(stateDir: string): Promise<string[]> {
@@ -575,84 +574,31 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
},
},
{
id: "matrix-idb-snapshot-json-to-plugin-state",
label: "Matrix IndexedDB snapshot",
id: "matrix-legacy-crypto-migration-json-to-plugin-state",
label: "Matrix legacy crypto state",
async detectLegacyState(params) {
const previews: string[] = [];
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_IDB_SNAPSHOT_FILENAME,
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
{ includeMatrixRoot: true },
)) {
const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir);
if (!snapshot) {
if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) {
continue;
}
previews.push(
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
);
}
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_IDB_SNAPSHOT_FILENAME,
{ includeMatrixRoot: true },
)) {
previews.push(`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`);
}
return previews.length > 0 ? { preview: previews } : null;
},
async migrateLegacyState(params) {
const changes: string[] = [];
const warnings: string[] = [];
const notices: string[] = [];
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_IDB_SNAPSHOT_FILENAME,
)) {
const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir);
if (!snapshot) {
continue;
}
const store = params.context.openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
openMatrixIdbSnapshotStoreOptions(storageRootDir),
);
if (await hasMatrixIdbSnapshotStateInStore({ store })) {
await archiveLegacyMatrixStateFile({
storageRootDir,
filename: MATRIX_IDB_SNAPSHOT_FILENAME,
label: "Matrix IndexedDB snapshot",
changes,
warnings,
notices,
notice: `Kept existing Matrix IndexedDB snapshot in SQLite and archived the legacy source for ${storageRootDir}`,
});
continue;
}
await writeMatrixIdbSnapshotJsonToStore({
snapshotJson: JSON.stringify(snapshot),
databaseCount: snapshot.length,
store,
});
changes.push(`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`);
await archiveLegacyMatrixStateFile({
storageRootDir,
filename: MATRIX_IDB_SNAPSHOT_FILENAME,
label: "Matrix IndexedDB snapshot",
changes,
warnings,
});
}
return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) };
},
},
{
id: "matrix-legacy-crypto-migration-json-to-plugin-state",
label: "Matrix legacy crypto migration",
async detectLegacyState(params) {
const previews: string[] = [];
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
)) {
if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) {
continue;
}
previews.push(
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
);
}
return previews.length > 0 ? { preview: previews } : null;
},
async migrateLegacyState(params) {
const changes: string[] = [];
const warnings: string[] = [];
@@ -660,6 +606,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
{ includeMatrixRoot: true },
)) {
const state = readLegacyMatrixLegacyCryptoMigrationState(storageRootDir);
if (!state) {
@@ -692,6 +639,19 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
warnings,
});
}
for (const storageRootDir of await collectLegacyMatrixStateRoots(
params.stateDir,
MATRIX_IDB_SNAPSHOT_FILENAME,
{ includeMatrixRoot: true },
)) {
await migrateLegacyMatrixIdbSnapshot({
storageRootDir,
context: params.context,
changes,
notices,
warnings,
});
}
return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) };
},
},
@@ -9,7 +9,6 @@ import {
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { resolveMatrixAccountStorageRoot } from "../../storage-paths.js";
import { installMatrixTestRuntime } from "../../test-runtime.js";
import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js";
import { SqliteBackedMatrixSyncStore } from "./file-sync-store.js";
import {
claimCurrentTokenStorageState,
@@ -372,38 +371,6 @@ describe("matrix client storage paths", () => {
await expect(syncStore.getSavedSyncToken()).resolves.toBe("account-token");
});
it("does not overwrite existing SQLite IDB snapshot state with a stale legacy sidecar", async () => {
const stateDir = setupStateDir();
const storagePaths = resolveDefaultStoragePaths();
fs.mkdirSync(storagePaths.rootDir, { recursive: true });
const currentSnapshot = JSON.stringify([
{
name: "current",
version: 1,
stores: [],
},
]);
writeMatrixIdbSnapshotJson({
storageRootDir: storagePaths.rootDir,
snapshotJson: currentSnapshot,
databaseCount: 1,
});
fs.writeFileSync(
storagePaths.idbSnapshotPath,
JSON.stringify([{ name: "stale", version: 1, stores: [] }]),
);
const env = createMigrationEnv(stateDir);
await maybeMigrateLegacyStorage({
storagePaths,
env,
});
expect(readMatrixIdbSnapshotJson(storagePaths.rootDir)).toBe(currentSnapshot);
expect(fs.existsSync(storagePaths.idbSnapshotPath)).toBe(false);
expect(fs.existsSync(`${storagePaths.idbSnapshotPath}.migrated`)).toBe(true);
});
it("ignores unrecognized account-scoped sync cache files without a migration snapshot", async () => {
const stateDir = setupStateDir();
const storagePaths = resolveDefaultStoragePaths();
@@ -17,9 +17,7 @@ import {
MATRIX_RECOVERY_KEY_FILENAME,
migrateLegacyMatrixLegacyCryptoMigrationFileToStore,
migrateLegacyMatrixRecoveryKeyFileToStore,
readMatrixIdbSnapshotJson,
scoreMatrixCryptoStateInStore,
writeMatrixIdbSnapshotJson,
} from "../crypto-state-store.js";
import { resolveMatrixSqliteStateEnv } from "../sqlite-state.js";
import type { MatrixAuth } from "./types.js";
@@ -394,14 +392,12 @@ export async function maybeMigrateLegacyStorage(params: {
hasAccountScopedLegacyStorageFile &&
(await syncCache?.readLegacyMatrixSyncCacheState(params.storagePaths.rootDir)) !== null;
const hasAccountScopedRecoveryKey = fs.existsSync(params.storagePaths.recoveryKeyPath);
const hasAccountScopedIdbSnapshot = fs.existsSync(params.storagePaths.idbSnapshotPath);
const hasAccountScopedLegacyCryptoMigration = fs.existsSync(
path.join(params.storagePaths.rootDir, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME),
);
if (
!hasAccountScopedLegacyStorage &&
!hasAccountScopedRecoveryKey &&
!hasAccountScopedIdbSnapshot &&
!hasAccountScopedLegacyCryptoMigration
) {
return;
@@ -439,14 +435,6 @@ export async function maybeMigrateLegacyStorage(params: {
label: "legacy crypto migration",
});
}
if (hasAccountScopedIdbSnapshot) {
await migrateLegacyIdbSnapshotToSqlite({
storageRootDir: params.storagePaths.rootDir,
snapshotPath: params.storagePaths.idbSnapshotPath,
moved,
pendingArchives,
});
}
} catch (err) {
const rollbackError = rollbackLegacyMoves(moved);
throw new Error(
@@ -476,40 +464,6 @@ export async function maybeMigrateLegacyStorage(params: {
}
}
async function migrateLegacyIdbSnapshotToSqlite(params: {
storageRootDir: string;
snapshotPath: string;
moved: LegacyMoveRecord[];
pendingArchives: LegacyArchiveRecord[];
}): Promise<void> {
if (readMatrixIdbSnapshotJson(params.storageRootDir)) {
params.pendingArchives.push({
sourcePath: params.snapshotPath,
label: "IndexedDB snapshot",
});
return;
}
const { readLegacyMatrixIdbSnapshotState } = await import("../sdk/idb-persistence.js");
const snapshot = await readLegacyMatrixIdbSnapshotState(params.storageRootDir);
if (!snapshot) {
return;
}
writeMatrixIdbSnapshotJson({
storageRootDir: params.storageRootDir,
snapshotJson: JSON.stringify(snapshot),
databaseCount: snapshot.length,
});
params.moved.push({
sourcePath: params.snapshotPath,
targetPath: `${params.storageRootDir} SQLite IndexedDB snapshot state`,
label: "IndexedDB snapshot",
});
params.pendingArchives.push({
sourcePath: params.snapshotPath,
label: "IndexedDB snapshot",
});
}
async function migrateLegacySyncCacheToSqlite(params: {
sourceRootDir: string;
sourcePath: string;
@@ -0,0 +1,153 @@
// Matrix plugin module owns the doctor-only crypto snapshot import.
import { randomUUID } from "node:crypto";
import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { withFileLock } from "openclaw/plugin-sdk/file-lock";
import type { PluginDoctorStateMigrationContext } from "openclaw/plugin-sdk/runtime-doctor";
import {
MATRIX_IDB_SNAPSHOT_FILENAME,
openMatrixIdbSnapshotStoreOptions,
readMatrixIdbSnapshotJsonFromStore,
writeMatrixIdbSnapshotJsonToStore,
type MatrixIdbSnapshotRecord,
} from "./crypto-state-store.js";
import { MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS } from "./sdk/idb-persistence-lock.js";
import {
isValidMatrixIdbSnapshotJson,
readLegacyMatrixIdbSnapshotStateUnlocked,
} from "./sdk/idb-persistence.js";
type SnapshotMigrationParams = {
storageRootDir: string;
context: PluginDoctorStateMigrationContext;
changes: string[];
notices: string[];
warnings: string[];
};
export async function migrateLegacyMatrixIdbSnapshot(
params: SnapshotMigrationParams,
): Promise<void> {
const snapshotPath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
try {
// withFileLock is acquire-or-throw; it never skips the callback on contention.
await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, () =>
migrateLegacyMatrixIdbSnapshotLocked(params),
);
} catch (err) {
params.warnings.push(
`Failed locking Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
);
}
}
async function migrateLegacyMatrixIdbSnapshotLocked(
params: SnapshotMigrationParams,
): Promise<void> {
const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
let snapshot: ReturnType<typeof readLegacyMatrixIdbSnapshotStateUnlocked>;
try {
snapshot = readLegacyMatrixIdbSnapshotStateUnlocked(params.storageRootDir);
} catch (err) {
params.warnings.push(
`Failed reading Matrix IndexedDB snapshot legacy source for ${params.storageRootDir}: ${String(err)}; left source in place`,
);
return;
}
if (!snapshot) {
if (!fsSync.existsSync(sourcePath)) {
return;
}
const archived = await archiveLegacyMatrixIdbSnapshot(params);
params.warnings.push(
archived
? `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; archived without import`
: `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; left active because archival failed`,
);
return;
}
const snapshotJson = JSON.stringify(snapshot);
const store = params.context.openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
openMatrixIdbSnapshotStoreOptions(params.storageRootDir),
);
let persisted: string | null;
let hadPartialState: boolean;
try {
persisted = await readMatrixIdbSnapshotJsonFromStore({ store });
const persistedIsValid = persisted ? isValidMatrixIdbSnapshotJson(persisted) : false;
hadPartialState = !persistedIsValid && (await store.entries()).length > 0;
if (!persistedIsValid) {
persisted = null;
}
} catch (err) {
params.warnings.push(
`Failed inspecting Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
);
return;
}
if (persisted && !snapshotContentMatches(persisted, snapshot)) {
if (await archiveLegacyMatrixIdbSnapshot(params)) {
params.notices.push(
`Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${params.storageRootDir}`,
);
}
return;
}
if (!persisted) {
try {
await writeMatrixIdbSnapshotJsonToStore({
snapshotJson,
databaseCount: snapshot.length,
store,
});
persisted = await readMatrixIdbSnapshotJsonFromStore({ store });
} catch (err) {
params.warnings.push(
`Failed importing Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
);
return;
}
if (!persisted || !snapshotContentMatches(persisted, snapshot)) {
params.warnings.push(
`Failed verifying Matrix IndexedDB snapshot for ${params.storageRootDir}; left legacy source in place`,
);
return;
}
params.changes.push(
hadPartialState
? `Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}`
: `Migrated Matrix IndexedDB snapshot JSON to SQLite for ${params.storageRootDir}`,
);
}
await archiveLegacyMatrixIdbSnapshot(params);
}
function snapshotContentMatches(persistedJson: string, snapshot: unknown): boolean {
try {
return isDeepStrictEqual(JSON.parse(persistedJson), snapshot);
} catch {
return false;
}
}
async function archiveLegacyMatrixIdbSnapshot(params: {
storageRootDir: string;
changes: string[];
warnings: string[];
}): Promise<boolean> {
const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
const timestamp = new Date().toISOString().replaceAll(":", "-");
const archivePath = `${sourcePath}.migrated-${timestamp}-${randomUUID()}`;
try {
await fs.rename(sourcePath, archivePath);
params.changes.push(`Archived Matrix IndexedDB snapshot legacy source -> ${archivePath}`);
return true;
} catch (err) {
params.warnings.push(
`Failed archiving Matrix IndexedDB snapshot legacy source: ${String(err)}`,
);
return false;
}
}
@@ -230,10 +230,10 @@ export function writeMatrixIdbSnapshotJson(params: {
});
}
export async function hasMatrixIdbSnapshotStateInStore(params: {
export async function readMatrixIdbSnapshotJsonFromStore(params: {
store: Pick<PluginStateKeyedStore<MatrixIdbSnapshotRecord>, "lookup">;
}): Promise<boolean> {
return (await readIdbSnapshotJsonFromAsyncStore(params.store)) !== null;
}): Promise<string | null> {
return await readIdbSnapshotJsonFromAsyncStore(params.store);
}
export async function writeMatrixIdbSnapshotJsonToStore(params: {
@@ -1,106 +0,0 @@
// Matrix tests cover idb persistence.lock order plugin behavior.
import "fake-indexeddb/auto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { installMatrixTestRuntime } from "../../test-runtime.js";
import { MATRIX_IDB_PERSIST_INTERVAL_MS } from "./idb-persistence-lock.js";
import { clearAllIndexedDbState, seedDatabase } from "./idb-persistence.test-helpers.js";
const { withFileLockMock } = vi.hoisted(() => ({
withFileLockMock: vi.fn(
async <T>(_filePath: string, _options: unknown, fn: () => Promise<T>) => await fn(),
),
}));
vi.mock("openclaw/plugin-sdk/file-lock", async () => {
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/file-lock")>(
"openclaw/plugin-sdk/file-lock",
);
return {
...actual,
withFileLock: withFileLockMock,
};
});
let persistIdbToDisk: typeof import("./idb-persistence.js").persistIdbToDisk;
let restoreIdbFromDisk: typeof import("./idb-persistence.js").restoreIdbFromDisk;
type CapturedLockOptions =
typeof import("./idb-persistence-lock.js").MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS;
const DATABASE_PREFIX = "openclaw-matrix-lock-order-test";
const cryptoDatabaseName = `${DATABASE_PREFIX}::matrix-sdk-crypto`;
function minimumRetryWindowMs(options: CapturedLockOptions): number {
let total = 0;
for (let attempt = 0; attempt < options.retries.retries; attempt += 1) {
total += Math.min(
options.retries.maxTimeout,
Math.max(
options.retries.minTimeout,
options.retries.minTimeout * options.retries.factor ** attempt,
),
);
}
return total;
}
beforeAll(async () => {
({ persistIdbToDisk, restoreIdbFromDisk } = await import("./idb-persistence.js"));
});
describe("Matrix IndexedDB persistence lock ordering", () => {
let tmpDir: string;
beforeEach(async () => {
resetPluginStateStoreForTests();
installMatrixTestRuntime();
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "matrix-idb-lock-order-"));
withFileLockMock.mockReset();
withFileLockMock.mockImplementation(
async <T>(_filePath: string, _options: unknown, fn: () => Promise<T>) => await fn(),
);
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
});
afterEach(async () => {
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
resetPluginStateStoreForTests();
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it("captures the current snapshot into SQLite state", async () => {
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
await seedDatabase({
name: cryptoDatabaseName,
storeName: "sessions",
records: [{ key: "room-1", value: { session: "old-session" } }],
});
await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX });
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
const dbs = await indexedDB.databases();
expect(dbs.map((entry) => entry.name)).toContain(cryptoDatabaseName);
});
it("uses the long snapshot lock options when importing a legacy file", async () => {
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
const capturedOptions: CapturedLockOptions[] = [];
withFileLockMock.mockImplementationOnce(async (_filePath, options) => {
capturedOptions.push(options as CapturedLockOptions);
return false;
});
fs.writeFileSync(snapshotPath, "[]", "utf8");
await restoreIdbFromDisk(snapshotPath);
expect(capturedOptions).toHaveLength(1);
for (const options of capturedOptions) {
expect(minimumRetryWindowMs(options)).toBeGreaterThanOrEqual(MATRIX_IDB_PERSIST_INTERVAL_MS);
expect(options.stale).toBe(5 * 60_000);
}
});
});
@@ -8,6 +8,7 @@ import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { getMatrixRuntime } from "../../runtime.js";
import { installMatrixTestRuntime } from "../../test-runtime.js";
import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js";
import { persistIdbToDisk, restoreIdbFromDisk } from "./idb-persistence.js";
import {
clearAllIndexedDbState,
@@ -80,113 +81,70 @@ describe("Matrix IndexedDB persistence", () => {
expect(dbs.map((entry) => entry.name)).not.toContain(otherCryptoDatabaseName);
});
it("imports and archives a legacy JSON snapshot during restore", async () => {
it("blocks runtime restore and persistence until doctor migrates the legacy snapshot", async () => {
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
fs.writeFileSync(
snapshotPath,
JSON.stringify([
{
name: cryptoDatabaseName,
version: 1,
stores: [
{
name: "sessions",
keyPath: null,
autoIncrement: false,
indexes: [],
records: [{ key: "room-1", value: { session: "legacy" } }],
},
],
},
]),
"utf8",
);
const snapshot = JSON.stringify([{ name: cryptoDatabaseName, version: 1, stores: [] }]);
fs.writeFileSync(snapshotPath, snapshot);
const restored = await restoreIdbFromDisk(snapshotPath);
expect(restored).toBe(true);
expect(fs.existsSync(snapshotPath)).toBe(false);
expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true);
await clearTestIndexedDbState();
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
await expect(
readDatabaseRecords({
name: cryptoDatabaseName,
storeName: "sessions",
}),
).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]);
});
it("restores a valid legacy JSON snapshot when SQLite import fails", async () => {
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
fs.writeFileSync(
snapshotPath,
JSON.stringify([
{
name: cryptoDatabaseName,
version: 1,
stores: [
{
name: "sessions",
keyPath: null,
autoIncrement: false,
indexes: [],
records: [{ key: "room-1", value: { session: "legacy" } }],
},
],
},
]),
"utf8",
);
vi.spyOn(getMatrixRuntime().state, "openSyncKeyedStore").mockImplementation(() => {
throw new Error("sqlite unavailable");
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
name: "MatrixIdbSnapshotMigrationRequiredError",
code: "matrix-idb-snapshot-requires-doctor",
remediation: "openclaw doctor --fix",
});
const restored = await restoreIdbFromDisk(snapshotPath);
expect(restored).toBe(true);
expect(fs.existsSync(snapshotPath)).toBe(true);
await expect(
readDatabaseRecords({
name: cryptoDatabaseName,
storeName: "sessions",
expect(warnSpy).toHaveBeenCalledWith(
"IdbPersistence",
expect.objectContaining({
code: "matrix-idb-snapshot-requires-doctor",
remediation: "openclaw doctor --fix",
}),
).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]);
});
);
expect(JSON.stringify(warnSpy.mock.calls)).not.toContain(snapshotPath);
it("returns false and logs a warning for malformed snapshots", async () => {
const snapshotPath = path.join(tmpDir, "bad-snapshot.json");
fs.writeFileSync(snapshotPath, JSON.stringify([{ nope: true }]), "utf8");
await seedDatabase({
name: cryptoDatabaseName,
storeName: "sessions",
records: [{ key: "new-room", value: { session: "new" } }],
});
await expect(
persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX }),
).rejects.toMatchObject({
code: "matrix-idb-snapshot-requires-doctor",
});
expect(readMatrixIdbSnapshotJson(tmpDir)).toBeNull();
expect(fs.existsSync(snapshotPath)).toBe(true);
const restored = await restoreIdbFromDisk(snapshotPath);
expect(restored).toBe(false);
expect(warnSpy).toHaveBeenCalledTimes(1);
const [scope, message, error] = warnSpy.mock.calls.at(0) ?? [];
expect(scope).toBe("IdbPersistence");
expect(message).toBe(`Failed to restore IndexedDB snapshot from ${snapshotPath}:`);
expect(error).toBeInstanceOf(Error);
});
writeMatrixIdbSnapshotJson({
storageRootDir: tmpDir,
snapshotJson: JSON.stringify({ malformed: true }),
databaseCount: 1,
});
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
code: "matrix-idb-snapshot-requires-doctor",
});
const storeSpy = vi
.spyOn(getMatrixRuntime().state, "openSyncKeyedStore")
.mockImplementation(() => {
throw new Error("sqlite unavailable");
});
it("returns false for empty snapshot payloads without restoring databases", async () => {
const snapshotPath = path.join(tmpDir, "empty-snapshot.json");
fs.writeFileSync(snapshotPath, JSON.stringify([]), "utf8");
const restored = await restoreIdbFromDisk(snapshotPath);
expect(restored).toBe(false);
const dbs = await indexedDB.databases();
expect(dbs).toStrictEqual([]);
try {
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
code: "matrix-idb-snapshot-requires-doctor",
});
} finally {
storeSpy.mockRestore();
}
});
it("returns false without warning when the snapshot does not exist yet", async () => {
const restored = await restoreIdbFromDisk(path.join(tmpDir, "missing-snapshot.json"));
const restored = await restoreIdbFromDisk(path.join(tmpDir, "crypto-idb-snapshot.json"));
expect(restored).toBe(false);
expect(warnSpy).not.toHaveBeenCalled();
});
it("handles concurrent persist operations in SQLite state", async () => {
const snapshotPath = path.join(tmpDir, "concurrent-persist.json");
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
await seedDatabase({
name: cryptoDatabaseName,
storeName: "sessions",
@@ -208,19 +166,4 @@ describe("Matrix IndexedDB persistence", () => {
}),
).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]);
});
it("archives an existing legacy snapshot file after persist", async () => {
const snapshotPath = path.join(tmpDir, "persist-archives-legacy.json");
fs.writeFileSync(snapshotPath, "[]", "utf8");
await seedDatabase({
name: cryptoDatabaseName,
storeName: "sessions",
records: [{ key: "room-1", value: { session: "abc123" } }],
});
await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX });
expect(fs.existsSync(snapshotPath)).toBe(false);
expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true);
});
});
@@ -31,6 +31,28 @@ type IdbDatabaseSnapshot = {
stores: IdbStoreSnapshot[];
};
type IdbPersistenceDiagnostic = {
code: "matrix-idb-snapshot-requires-doctor";
message: string;
remediation: "openclaw doctor --fix";
};
const LEGACY_SNAPSHOT_DIAGNOSTIC: IdbPersistenceDiagnostic = {
code: "matrix-idb-snapshot-requires-doctor",
message: "Matrix IndexedDB snapshot exists outside canonical SQLite state",
remediation: "openclaw doctor --fix",
};
class MatrixIdbSnapshotMigrationRequiredError extends Error {
readonly code = LEGACY_SNAPSHOT_DIAGNOSTIC.code;
readonly remediation = LEGACY_SNAPSHOT_DIAGNOSTIC.remediation;
constructor() {
super(`${LEGACY_SNAPSHOT_DIAGNOSTIC.message}; run ${LEGACY_SNAPSHOT_DIAGNOSTIC.remediation}`);
this.name = "MatrixIdbSnapshotMigrationRequiredError";
}
}
function isValidIdbIndexSnapshot(value: unknown): value is IdbStoreSnapshot["indexes"][number] {
if (!value || typeof value !== "object") {
return false;
@@ -100,6 +122,14 @@ function parseSnapshotPayload(data: string): IdbDatabaseSnapshot[] | null {
return parsed;
}
export function isValidMatrixIdbSnapshotJson(data: string): boolean {
try {
return parseSnapshotPayload(data) !== null;
} catch {
return false;
}
}
function idbReq<T>(req: IDBRequest<T>): Promise<T> {
return new Promise((resolve, reject) => {
req.addEventListener("success", () => resolve(req.result), { once: true });
@@ -228,106 +258,86 @@ function resolveDefaultIdbSnapshotPath(): string {
return path.join(stateDir, "matrix", "crypto-idb-snapshot.json");
}
// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests.
export async function restoreIdbFromDisk(snapshotPath?: string): Promise<boolean> {
const candidatePaths = snapshotPath ? [snapshotPath] : [resolveDefaultIdbSnapshotPath()];
for (const resolvedPath of candidatePaths) {
const storageRootDir = path.dirname(resolvedPath);
try {
const restored = await withFileLock(
resolvedPath,
MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS,
async () => {
try {
const storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
if (storedSnapshotJson) {
const snapshot = parseSnapshotPayload(storedSnapshotJson);
if (snapshot) {
await restoreIndexedDatabases(snapshot);
LogService.info(
"IdbPersistence",
`Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`,
);
return true;
}
}
} catch (err) {
LogService.warn(
"IdbPersistence",
"Failed to restore IndexedDB snapshot from SQLite:",
err,
);
}
if (!fs.existsSync(resolvedPath)) {
return false;
}
const data = fs.readFileSync(resolvedPath, "utf8");
const snapshot = parseSnapshotPayload(data);
if (!snapshot) {
return false;
}
let migratedToSqlite = false;
try {
writeMatrixIdbSnapshotJson({
storageRootDir,
snapshotJson: data,
databaseCount: snapshot.length,
});
archiveLegacyIdbSnapshotFile(resolvedPath);
migratedToSqlite = true;
} catch (err) {
LogService.warn(
"IdbPersistence",
`Failed to migrate IndexedDB snapshot to SQLite from ${resolvedPath}:`,
err,
);
}
await restoreIndexedDatabases(snapshot);
LogService.info(
"IdbPersistence",
migratedToSqlite
? `Migrated and restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`
: `Restored ${snapshot.length} IndexedDB database(s) from legacy snapshot ${resolvedPath}`,
);
return true;
},
);
if (restored) {
return true;
const resolvedPath = snapshotPath ?? resolveDefaultIdbSnapshotPath();
const storageRootDir = path.dirname(resolvedPath);
let callbackStarted = false;
try {
// withFileLock is acquire-or-throw; it never skips the callback on contention.
return await withFileLock(resolvedPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => {
callbackStarted = true;
let storedSnapshotJson: string | null;
try {
storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
} catch (err) {
if (fs.existsSync(resolvedPath)) {
throwLegacySnapshotMigrationRequired();
}
throw err;
}
} catch (err) {
LogService.warn(
throwIfLegacySnapshotNeedsDoctor(resolvedPath, storedSnapshotJson);
if (!storedSnapshotJson) {
return false;
}
const snapshot = parseSnapshotPayload(storedSnapshotJson);
if (!snapshot) {
return false;
}
await restoreIndexedDatabases(snapshot);
LogService.info(
"IdbPersistence",
`Failed to restore IndexedDB snapshot from ${resolvedPath}:`,
err,
`Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`,
);
continue;
return true;
});
} catch (err) {
if (err instanceof MatrixIdbSnapshotMigrationRequiredError) {
throw err;
}
if (!callbackStarted && fs.existsSync(resolvedPath)) {
throwLegacySnapshotMigrationRequired();
}
LogService.warn("IdbPersistence", "Failed to restore IndexedDB snapshot from SQLite:", err);
return false;
}
return false;
}
export async function persistIdbToDisk(params?: {
// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests.
snapshotPath?: string;
databasePrefix?: string;
}): Promise<void> {
const snapshotPath = params?.snapshotPath ?? resolveDefaultIdbSnapshotPath();
let callbackStarted = false;
try {
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
// withFileLock is acquire-or-throw; it never skips the callback on contention.
const persistedCount = await withFileLock(
snapshotPath,
MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS,
async () => {
callbackStarted = true;
const storageRootDir = path.dirname(snapshotPath);
let storedSnapshotJson: string | null;
try {
storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
} catch (err) {
if (fs.existsSync(snapshotPath)) {
throwLegacySnapshotMigrationRequired();
}
throw err;
}
throwIfLegacySnapshotNeedsDoctor(snapshotPath, storedSnapshotJson);
const snapshot = await dumpIndexedDatabases(params?.databasePrefix);
if (snapshot.length === 0) {
return 0;
}
writeMatrixIdbSnapshotJson({
storageRootDir: path.dirname(snapshotPath),
storageRootDir,
snapshotJson: JSON.stringify(snapshot),
databaseCount: snapshot.length,
});
archiveLegacyIdbSnapshotFile(snapshotPath);
return snapshot.length;
},
);
@@ -339,36 +349,46 @@ export async function persistIdbToDisk(params?: {
`Persisted ${persistedCount} IndexedDB database(s) to Matrix SQLite state`,
);
} catch (err) {
if (err instanceof MatrixIdbSnapshotMigrationRequiredError) {
throw err;
}
if (!callbackStarted && fs.existsSync(snapshotPath)) {
throwLegacySnapshotMigrationRequired();
}
LogService.warn("IdbPersistence", "Failed to persist IndexedDB snapshot:", err);
}
}
export async function readLegacyMatrixIdbSnapshotState(
export function readLegacyMatrixIdbSnapshotStateUnlocked(
storageRootDir: string,
): Promise<IdbDatabaseSnapshot[] | null> {
): IdbDatabaseSnapshot[] | null {
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
if (!fs.existsSync(snapshotPath)) {
return null;
}
const data = fs.readFileSync(snapshotPath, "utf8");
try {
return await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => {
const snapshot = parseSnapshotPayload(fs.readFileSync(snapshotPath, "utf8"));
return snapshot;
});
return parseSnapshotPayload(data);
} catch {
return null;
}
}
function archiveLegacyIdbSnapshotFile(snapshotPath: string): void {
if (!fs.existsSync(snapshotPath)) {
return;
function throwIfLegacySnapshotNeedsDoctor(
snapshotPath: string,
storedSnapshotJson: string | null,
): void {
if (
fs.existsSync(snapshotPath) &&
(!storedSnapshotJson || !isValidMatrixIdbSnapshotJson(storedSnapshotJson))
) {
throwLegacySnapshotMigrationRequired();
}
const archivedPath = `${snapshotPath}.migrated`;
if (fs.existsSync(archivedPath)) {
return;
}
fs.renameSync(snapshotPath, archivedPath);
}
function throwLegacySnapshotMigrationRequired(): never {
LogService.warn("IdbPersistence", LEGACY_SNAPSHOT_DIAGNOSTIC);
throw new MatrixIdbSnapshotMigrationRequiredError();
}
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {