mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(matrix): preserve encryption keys across snapshot upgrades (#112919)
* fix(matrix): migrate legacy crypto snapshots via doctor * fix(matrix): harden snapshot migration failures * fix(matrix): preserve unreadable crypto snapshots * test(matrix): consolidate crypto snapshot coverage
This commit is contained in:
committed by
GitHub
parent
e76c9aac30
commit
d893f79ccc
@@ -1,4 +1,5 @@
|
||||
// Matrix tests cover doctor contract state migrations.
|
||||
import "fake-indexeddb/auto";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
@@ -25,10 +26,14 @@ import {
|
||||
type MatrixStoredCredentialRecord,
|
||||
} from "./src/matrix/credentials-read.js";
|
||||
import {
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
MATRIX_RECOVERY_KEY_FILENAME,
|
||||
openMatrixIdbSnapshotStoreOptions,
|
||||
readMatrixIdbSnapshotJson,
|
||||
readMatrixRecoveryKeyStateForPath,
|
||||
scoreMatrixCryptoStateInStore,
|
||||
writeMatrixIdbSnapshotJson,
|
||||
type MatrixIdbSnapshotRecord,
|
||||
} from "./src/matrix/crypto-state-store.js";
|
||||
import { importNewestInboundDedupeMarkers } from "./src/matrix/monitor/inbound-dedupe-migration.js";
|
||||
import {
|
||||
@@ -36,8 +41,15 @@ import {
|
||||
MATRIX_INBOUND_DEDUPE_TTL_MS,
|
||||
resolveMatrixInboundDedupeStateNamespace,
|
||||
} from "./src/matrix/monitor/inbound-dedupe.js";
|
||||
import { restoreIdbFromDisk } from "./src/matrix/sdk/idb-persistence.js";
|
||||
import {
|
||||
clearAllIndexedDbState,
|
||||
readDatabaseRecords,
|
||||
} from "./src/matrix/sdk/idb-persistence.test-helpers.js";
|
||||
import { installMatrixTestRuntime } from "./src/test-runtime.js";
|
||||
|
||||
const DOCTOR_IDB_DATABASE_PREFIX = "openclaw-matrix-doctor-test";
|
||||
|
||||
function createContext(): PluginDoctorStateMigrationContext {
|
||||
return {
|
||||
openPluginStateKeyedStore: <T>(options: OpenKeyedStoreOptions): PluginStateKeyedStore<T> =>
|
||||
@@ -71,7 +83,8 @@ describe("matrix doctor contract state migrations", () => {
|
||||
installMatrixTestRuntime();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
await clearAllIndexedDbState({ databasePrefix: DOCTOR_IDB_DATABASE_PREFIX });
|
||||
resetPluginStateStoreForTests();
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
@@ -364,21 +377,16 @@ describe("matrix doctor contract state migrations", () => {
|
||||
expect(fs.existsSync(path.join(storageRootDir, "recovery-key.json"))).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates Matrix IndexedDB snapshot JSON to SQLite plugin state", async () => {
|
||||
it("migrates legacy Matrix crypto state and restores the snapshot from SQLite", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
|
||||
tempDirs.push(stateDir);
|
||||
const storageRootDir = path.join(
|
||||
stateDir,
|
||||
"matrix",
|
||||
"accounts",
|
||||
"default",
|
||||
"matrix.example.org__bot",
|
||||
"token-hash",
|
||||
);
|
||||
const storageRootDir = path.join(stateDir, "matrix");
|
||||
fs.mkdirSync(storageRootDir, { recursive: true });
|
||||
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
const snapshotDatabaseName = `${DOCTOR_IDB_DATABASE_PREFIX}::matrix-sdk-crypto`;
|
||||
const snapshot = [
|
||||
{
|
||||
name: "openclaw-matrix::matrix-sdk-crypto",
|
||||
name: snapshotDatabaseName,
|
||||
version: 1,
|
||||
stores: [
|
||||
{
|
||||
@@ -391,40 +399,7 @@ describe("matrix doctor contract state migrations", () => {
|
||||
],
|
||||
},
|
||||
];
|
||||
fs.writeFileSync(
|
||||
path.join(storageRootDir, "crypto-idb-snapshot.json"),
|
||||
JSON.stringify(snapshot),
|
||||
);
|
||||
|
||||
const migration = migrationById("matrix-idb-snapshot-json-to-plugin-state");
|
||||
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
|
||||
preview: [`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`],
|
||||
});
|
||||
|
||||
await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
|
||||
changes: [
|
||||
`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`,
|
||||
`Archived Matrix IndexedDB snapshot legacy source -> ${path.join(storageRootDir, "crypto-idb-snapshot.json")}.migrated`,
|
||||
],
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot);
|
||||
expect(fs.existsSync(path.join(storageRootDir, "crypto-idb-snapshot.json"))).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates Matrix legacy crypto migration JSON to SQLite plugin state", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
|
||||
tempDirs.push(stateDir);
|
||||
const storageRootDir = path.join(
|
||||
stateDir,
|
||||
"matrix",
|
||||
"accounts",
|
||||
"default",
|
||||
"matrix.example.org__bot",
|
||||
"token-hash",
|
||||
);
|
||||
fs.mkdirSync(storageRootDir, { recursive: true });
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify(snapshot));
|
||||
fs.writeFileSync(
|
||||
path.join(storageRootDir, "legacy-crypto-migration.json"),
|
||||
JSON.stringify({
|
||||
@@ -443,19 +418,149 @@ describe("matrix doctor contract state migrations", () => {
|
||||
|
||||
const migration = migrationById("matrix-legacy-crypto-migration-json-to-plugin-state");
|
||||
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
|
||||
preview: [`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`],
|
||||
});
|
||||
|
||||
await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({
|
||||
changes: [
|
||||
`Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`,
|
||||
`Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`,
|
||||
preview: [
|
||||
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
|
||||
`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`,
|
||||
],
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(3);
|
||||
const result = await migration.migrateLegacyState(createMigrationParams(stateDir));
|
||||
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.changes).toEqual([
|
||||
`Migrated Matrix legacy crypto migration JSON to SQLite for ${storageRootDir}`,
|
||||
`Archived Matrix legacy crypto migration legacy source -> ${path.join(storageRootDir, "legacy-crypto-migration.json")}.migrated`,
|
||||
`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`,
|
||||
expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u),
|
||||
]);
|
||||
const archivePath = result.changes[3]?.split(" -> ")[1];
|
||||
expect(archivePath).toMatch(/crypto-idb-snapshot\.json\.migrated-\d{4}-/u);
|
||||
expect(JSON.parse(fs.readFileSync(archivePath ?? "", "utf8"))).toEqual(snapshot);
|
||||
|
||||
expect(scoreMatrixCryptoStateInStore(storageRootDir)).toBe(5);
|
||||
expect(JSON.parse(readMatrixIdbSnapshotJson(storageRootDir) ?? "null")).toEqual(snapshot);
|
||||
expect(fs.existsSync(path.join(storageRootDir, "legacy-crypto-migration.json"))).toBe(false);
|
||||
expect(fs.existsSync(snapshotPath)).toBe(false);
|
||||
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
|
||||
await expect(
|
||||
readDatabaseRecords({
|
||||
name: snapshotDatabaseName,
|
||||
storeName: "sessions",
|
||||
}),
|
||||
).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]);
|
||||
await expect(migration.detectLegacyState(createMigrationParams(stateDir))).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it("archives an invalid legacy snapshot for recovery and unblocks runtime", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
|
||||
tempDirs.push(stateDir);
|
||||
const storageRootDir = path.join(stateDir, "matrix");
|
||||
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
fs.mkdirSync(storageRootDir, { recursive: true });
|
||||
fs.writeFileSync(snapshotPath, "not-json");
|
||||
|
||||
const result = await migrationById(
|
||||
"matrix-legacy-crypto-migration-json-to-plugin-state",
|
||||
).migrateLegacyState(createMigrationParams(stateDir));
|
||||
|
||||
expect(result.warnings).toEqual([
|
||||
`Matrix IndexedDB snapshot legacy source is invalid for ${storageRootDir}; archived without import`,
|
||||
]);
|
||||
expect(result.changes).toEqual([
|
||||
expect.stringMatching(/^Archived Matrix IndexedDB snapshot legacy source -> /u),
|
||||
]);
|
||||
const archivePath = result.changes[0]?.split(" -> ")[1];
|
||||
expect(fs.readFileSync(archivePath ?? "", "utf8")).toBe("not-json");
|
||||
expect(fs.existsSync(snapshotPath)).toBe(false);
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("repairs invalid snapshots, archives equivalents, and preserves conflicts", async () => {
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-"));
|
||||
tempDirs.push(stateDir);
|
||||
const partialRoot = path.join(stateDir, "matrix", "accounts", "partial");
|
||||
const conflictRoot = path.join(stateDir, "matrix", "accounts", "conflict");
|
||||
const equivalentRoot = path.join(stateDir, "matrix", "accounts", "equivalent");
|
||||
const invalidRoot = path.join(stateDir, "matrix", "accounts", "invalid");
|
||||
fs.mkdirSync(partialRoot, { recursive: true });
|
||||
fs.mkdirSync(conflictRoot, { recursive: true });
|
||||
fs.mkdirSync(equivalentRoot, { recursive: true });
|
||||
fs.mkdirSync(invalidRoot, { recursive: true });
|
||||
const partialSnapshot = [{ name: "partial-source", version: 1, stores: [] }];
|
||||
const conflictSnapshot = [{ name: "conflicting-source", version: 1, stores: [] }];
|
||||
const equivalentSnapshot = [{ name: "equivalent", version: 1, stores: [] }];
|
||||
const invalidReplacement = [{ name: "invalid-replacement", version: 1, stores: [] }];
|
||||
fs.writeFileSync(
|
||||
path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
|
||||
JSON.stringify(partialSnapshot),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
|
||||
JSON.stringify(conflictSnapshot),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
|
||||
JSON.stringify(equivalentSnapshot, null, 2),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME),
|
||||
JSON.stringify(invalidReplacement),
|
||||
);
|
||||
const partialStore = createContext().openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
|
||||
openMatrixIdbSnapshotStoreOptions(partialRoot),
|
||||
);
|
||||
await partialStore.register("current:snapshot:interrupted:0", {
|
||||
kind: "snapshot-chunk",
|
||||
index: 0,
|
||||
data: "[",
|
||||
});
|
||||
const currentSnapshot = JSON.stringify([{ name: "current", version: 1, stores: [] }]);
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: conflictRoot,
|
||||
snapshotJson: currentSnapshot,
|
||||
databaseCount: 1,
|
||||
});
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: equivalentRoot,
|
||||
snapshotJson: JSON.stringify(equivalentSnapshot),
|
||||
databaseCount: 1,
|
||||
});
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: invalidRoot,
|
||||
snapshotJson: JSON.stringify({ malformed: true }),
|
||||
databaseCount: 1,
|
||||
});
|
||||
|
||||
const result = await migrationById(
|
||||
"matrix-legacy-crypto-migration-json-to-plugin-state",
|
||||
).migrateLegacyState(createMigrationParams(stateDir));
|
||||
|
||||
expect(result.changes).toEqual([
|
||||
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
|
||||
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
|
||||
`Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${invalidRoot}`,
|
||||
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
|
||||
`Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${partialRoot}`,
|
||||
expect.stringContaining("Archived Matrix IndexedDB snapshot legacy source"),
|
||||
]);
|
||||
expect(result.warnings).toEqual([]);
|
||||
expect(result.notices).toEqual([
|
||||
`Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${conflictRoot}`,
|
||||
]);
|
||||
const conflictArchivePath = result.changes[0]?.split(" -> ")[1];
|
||||
expect(JSON.parse(fs.readFileSync(conflictArchivePath ?? "", "utf8"))).toEqual(
|
||||
conflictSnapshot,
|
||||
);
|
||||
expect(JSON.parse(readMatrixIdbSnapshotJson(partialRoot) ?? "null")).toEqual(partialSnapshot);
|
||||
expect(JSON.parse(readMatrixIdbSnapshotJson(invalidRoot) ?? "null")).toEqual(
|
||||
invalidReplacement,
|
||||
);
|
||||
expect(readMatrixIdbSnapshotJson(conflictRoot)).toBe(currentSnapshot);
|
||||
expect(fs.existsSync(path.join(partialRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
|
||||
expect(fs.existsSync(path.join(conflictRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
|
||||
expect(fs.existsSync(path.join(equivalentRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
|
||||
expect(fs.existsSync(path.join(invalidRoot, MATRIX_IDB_SNAPSHOT_FILENAME))).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates legacy inbound dedupe markers into the claimable dedupe store", async () => {
|
||||
|
||||
@@ -35,22 +35,19 @@ import {
|
||||
type MatrixCredentialStateRecord,
|
||||
type MatrixStoredCredentialRecord,
|
||||
} from "./src/matrix/credentials-read.js";
|
||||
import { migrateLegacyMatrixIdbSnapshot } from "./src/matrix/crypto-snapshot-doctor.js";
|
||||
import {
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
|
||||
MATRIX_RECOVERY_KEY_FILENAME,
|
||||
hasMatrixIdbSnapshotStateInStore,
|
||||
hasMatrixLegacyCryptoMigrationStateInStore,
|
||||
hasMatrixRecoveryKeyStateInStore,
|
||||
openMatrixIdbSnapshotStoreOptions,
|
||||
openMatrixLegacyCryptoMigrationStoreOptions,
|
||||
openMatrixRecoveryKeyStoreOptions,
|
||||
readLegacyMatrixLegacyCryptoMigrationState,
|
||||
readLegacyMatrixRecoveryKeyState,
|
||||
writeMatrixIdbSnapshotJsonToStore,
|
||||
writeMatrixLegacyCryptoMigrationStateToStore,
|
||||
writeMatrixRecoveryKeyStateToStore,
|
||||
type MatrixIdbSnapshotRecord,
|
||||
type MatrixLegacyCryptoMigrationState,
|
||||
} from "./src/matrix/crypto-state-store.js";
|
||||
import {
|
||||
@@ -63,7 +60,6 @@ import {
|
||||
type LegacyInboundDedupeMarker,
|
||||
type MatrixInboundDedupeMigrationIo,
|
||||
} from "./src/matrix/monitor/inbound-dedupe-migration.js";
|
||||
import { readLegacyMatrixIdbSnapshotState } from "./src/matrix/sdk/idb-persistence.js";
|
||||
import type { MatrixStoredRecoveryKey } from "./src/matrix/sdk/types.js";
|
||||
import { resolveMatrixCredentialsDir } from "./src/storage-paths.js";
|
||||
|
||||
@@ -136,6 +132,7 @@ async function readLegacyMatrixCredentials(
|
||||
async function collectLegacyMatrixStateRoots(
|
||||
stateDir: string,
|
||||
filename: string,
|
||||
options?: { includeMatrixRoot?: boolean },
|
||||
): Promise<string[]> {
|
||||
const matrixRoot = path.join(stateDir, "matrix");
|
||||
const roots: string[] = [];
|
||||
@@ -158,7 +155,9 @@ async function collectLegacyMatrixStateRoots(
|
||||
}
|
||||
}
|
||||
await visit(matrixRoot);
|
||||
return roots.filter((root) => path.resolve(root) !== path.resolve(matrixRoot)).toSorted();
|
||||
return roots
|
||||
.filter((root) => options?.includeMatrixRoot || path.resolve(root) !== path.resolve(matrixRoot))
|
||||
.toSorted();
|
||||
}
|
||||
|
||||
async function collectLegacySyncCacheRoots(stateDir: string): Promise<string[]> {
|
||||
@@ -575,84 +574,31 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "matrix-idb-snapshot-json-to-plugin-state",
|
||||
label: "Matrix IndexedDB snapshot",
|
||||
id: "matrix-legacy-crypto-migration-json-to-plugin-state",
|
||||
label: "Matrix legacy crypto state",
|
||||
async detectLegacyState(params) {
|
||||
const previews: string[] = [];
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
|
||||
{ includeMatrixRoot: true },
|
||||
)) {
|
||||
const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir);
|
||||
if (!snapshot) {
|
||||
if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) {
|
||||
continue;
|
||||
}
|
||||
previews.push(
|
||||
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
|
||||
);
|
||||
}
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
{ includeMatrixRoot: true },
|
||||
)) {
|
||||
previews.push(`Matrix IndexedDB snapshot JSON can migrate to SQLite: ${storageRootDir}`);
|
||||
}
|
||||
return previews.length > 0 ? { preview: previews } : null;
|
||||
},
|
||||
async migrateLegacyState(params) {
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
const notices: string[] = [];
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
)) {
|
||||
const snapshot = await readLegacyMatrixIdbSnapshotState(storageRootDir);
|
||||
if (!snapshot) {
|
||||
continue;
|
||||
}
|
||||
const store = params.context.openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
|
||||
openMatrixIdbSnapshotStoreOptions(storageRootDir),
|
||||
);
|
||||
if (await hasMatrixIdbSnapshotStateInStore({ store })) {
|
||||
await archiveLegacyMatrixStateFile({
|
||||
storageRootDir,
|
||||
filename: MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
label: "Matrix IndexedDB snapshot",
|
||||
changes,
|
||||
warnings,
|
||||
notices,
|
||||
notice: `Kept existing Matrix IndexedDB snapshot in SQLite and archived the legacy source for ${storageRootDir}`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
await writeMatrixIdbSnapshotJsonToStore({
|
||||
snapshotJson: JSON.stringify(snapshot),
|
||||
databaseCount: snapshot.length,
|
||||
store,
|
||||
});
|
||||
changes.push(`Migrated Matrix IndexedDB snapshot JSON to SQLite for ${storageRootDir}`);
|
||||
await archiveLegacyMatrixStateFile({
|
||||
storageRootDir,
|
||||
filename: MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
label: "Matrix IndexedDB snapshot",
|
||||
changes,
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) };
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "matrix-legacy-crypto-migration-json-to-plugin-state",
|
||||
label: "Matrix legacy crypto migration",
|
||||
async detectLegacyState(params) {
|
||||
const previews: string[] = [];
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
|
||||
)) {
|
||||
if (!readLegacyMatrixLegacyCryptoMigrationState(storageRootDir)) {
|
||||
continue;
|
||||
}
|
||||
previews.push(
|
||||
`Matrix legacy crypto migration JSON can migrate to SQLite: ${storageRootDir}`,
|
||||
);
|
||||
}
|
||||
return previews.length > 0 ? { preview: previews } : null;
|
||||
},
|
||||
async migrateLegacyState(params) {
|
||||
const changes: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
@@ -660,6 +606,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME,
|
||||
{ includeMatrixRoot: true },
|
||||
)) {
|
||||
const state = readLegacyMatrixLegacyCryptoMigrationState(storageRootDir);
|
||||
if (!state) {
|
||||
@@ -692,6 +639,19 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
for (const storageRootDir of await collectLegacyMatrixStateRoots(
|
||||
params.stateDir,
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
{ includeMatrixRoot: true },
|
||||
)) {
|
||||
await migrateLegacyMatrixIdbSnapshot({
|
||||
storageRootDir,
|
||||
context: params.context,
|
||||
changes,
|
||||
notices,
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) };
|
||||
},
|
||||
},
|
||||
|
||||
@@ -9,7 +9,6 @@ import {
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveMatrixAccountStorageRoot } from "../../storage-paths.js";
|
||||
import { installMatrixTestRuntime } from "../../test-runtime.js";
|
||||
import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js";
|
||||
import { SqliteBackedMatrixSyncStore } from "./file-sync-store.js";
|
||||
import {
|
||||
claimCurrentTokenStorageState,
|
||||
@@ -372,38 +371,6 @@ describe("matrix client storage paths", () => {
|
||||
await expect(syncStore.getSavedSyncToken()).resolves.toBe("account-token");
|
||||
});
|
||||
|
||||
it("does not overwrite existing SQLite IDB snapshot state with a stale legacy sidecar", async () => {
|
||||
const stateDir = setupStateDir();
|
||||
const storagePaths = resolveDefaultStoragePaths();
|
||||
fs.mkdirSync(storagePaths.rootDir, { recursive: true });
|
||||
const currentSnapshot = JSON.stringify([
|
||||
{
|
||||
name: "current",
|
||||
version: 1,
|
||||
stores: [],
|
||||
},
|
||||
]);
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: storagePaths.rootDir,
|
||||
snapshotJson: currentSnapshot,
|
||||
databaseCount: 1,
|
||||
});
|
||||
fs.writeFileSync(
|
||||
storagePaths.idbSnapshotPath,
|
||||
JSON.stringify([{ name: "stale", version: 1, stores: [] }]),
|
||||
);
|
||||
const env = createMigrationEnv(stateDir);
|
||||
|
||||
await maybeMigrateLegacyStorage({
|
||||
storagePaths,
|
||||
env,
|
||||
});
|
||||
|
||||
expect(readMatrixIdbSnapshotJson(storagePaths.rootDir)).toBe(currentSnapshot);
|
||||
expect(fs.existsSync(storagePaths.idbSnapshotPath)).toBe(false);
|
||||
expect(fs.existsSync(`${storagePaths.idbSnapshotPath}.migrated`)).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores unrecognized account-scoped sync cache files without a migration snapshot", async () => {
|
||||
const stateDir = setupStateDir();
|
||||
const storagePaths = resolveDefaultStoragePaths();
|
||||
|
||||
@@ -17,9 +17,7 @@ import {
|
||||
MATRIX_RECOVERY_KEY_FILENAME,
|
||||
migrateLegacyMatrixLegacyCryptoMigrationFileToStore,
|
||||
migrateLegacyMatrixRecoveryKeyFileToStore,
|
||||
readMatrixIdbSnapshotJson,
|
||||
scoreMatrixCryptoStateInStore,
|
||||
writeMatrixIdbSnapshotJson,
|
||||
} from "../crypto-state-store.js";
|
||||
import { resolveMatrixSqliteStateEnv } from "../sqlite-state.js";
|
||||
import type { MatrixAuth } from "./types.js";
|
||||
@@ -394,14 +392,12 @@ export async function maybeMigrateLegacyStorage(params: {
|
||||
hasAccountScopedLegacyStorageFile &&
|
||||
(await syncCache?.readLegacyMatrixSyncCacheState(params.storagePaths.rootDir)) !== null;
|
||||
const hasAccountScopedRecoveryKey = fs.existsSync(params.storagePaths.recoveryKeyPath);
|
||||
const hasAccountScopedIdbSnapshot = fs.existsSync(params.storagePaths.idbSnapshotPath);
|
||||
const hasAccountScopedLegacyCryptoMigration = fs.existsSync(
|
||||
path.join(params.storagePaths.rootDir, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME),
|
||||
);
|
||||
if (
|
||||
!hasAccountScopedLegacyStorage &&
|
||||
!hasAccountScopedRecoveryKey &&
|
||||
!hasAccountScopedIdbSnapshot &&
|
||||
!hasAccountScopedLegacyCryptoMigration
|
||||
) {
|
||||
return;
|
||||
@@ -439,14 +435,6 @@ export async function maybeMigrateLegacyStorage(params: {
|
||||
label: "legacy crypto migration",
|
||||
});
|
||||
}
|
||||
if (hasAccountScopedIdbSnapshot) {
|
||||
await migrateLegacyIdbSnapshotToSqlite({
|
||||
storageRootDir: params.storagePaths.rootDir,
|
||||
snapshotPath: params.storagePaths.idbSnapshotPath,
|
||||
moved,
|
||||
pendingArchives,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
const rollbackError = rollbackLegacyMoves(moved);
|
||||
throw new Error(
|
||||
@@ -476,40 +464,6 @@ export async function maybeMigrateLegacyStorage(params: {
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateLegacyIdbSnapshotToSqlite(params: {
|
||||
storageRootDir: string;
|
||||
snapshotPath: string;
|
||||
moved: LegacyMoveRecord[];
|
||||
pendingArchives: LegacyArchiveRecord[];
|
||||
}): Promise<void> {
|
||||
if (readMatrixIdbSnapshotJson(params.storageRootDir)) {
|
||||
params.pendingArchives.push({
|
||||
sourcePath: params.snapshotPath,
|
||||
label: "IndexedDB snapshot",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { readLegacyMatrixIdbSnapshotState } = await import("../sdk/idb-persistence.js");
|
||||
const snapshot = await readLegacyMatrixIdbSnapshotState(params.storageRootDir);
|
||||
if (!snapshot) {
|
||||
return;
|
||||
}
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: params.storageRootDir,
|
||||
snapshotJson: JSON.stringify(snapshot),
|
||||
databaseCount: snapshot.length,
|
||||
});
|
||||
params.moved.push({
|
||||
sourcePath: params.snapshotPath,
|
||||
targetPath: `${params.storageRootDir} SQLite IndexedDB snapshot state`,
|
||||
label: "IndexedDB snapshot",
|
||||
});
|
||||
params.pendingArchives.push({
|
||||
sourcePath: params.snapshotPath,
|
||||
label: "IndexedDB snapshot",
|
||||
});
|
||||
}
|
||||
|
||||
async function migrateLegacySyncCacheToSqlite(params: {
|
||||
sourceRootDir: string;
|
||||
sourcePath: string;
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
// Matrix plugin module owns the doctor-only crypto snapshot import.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { withFileLock } from "openclaw/plugin-sdk/file-lock";
|
||||
import type { PluginDoctorStateMigrationContext } from "openclaw/plugin-sdk/runtime-doctor";
|
||||
import {
|
||||
MATRIX_IDB_SNAPSHOT_FILENAME,
|
||||
openMatrixIdbSnapshotStoreOptions,
|
||||
readMatrixIdbSnapshotJsonFromStore,
|
||||
writeMatrixIdbSnapshotJsonToStore,
|
||||
type MatrixIdbSnapshotRecord,
|
||||
} from "./crypto-state-store.js";
|
||||
import { MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS } from "./sdk/idb-persistence-lock.js";
|
||||
import {
|
||||
isValidMatrixIdbSnapshotJson,
|
||||
readLegacyMatrixIdbSnapshotStateUnlocked,
|
||||
} from "./sdk/idb-persistence.js";
|
||||
|
||||
type SnapshotMigrationParams = {
|
||||
storageRootDir: string;
|
||||
context: PluginDoctorStateMigrationContext;
|
||||
changes: string[];
|
||||
notices: string[];
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
export async function migrateLegacyMatrixIdbSnapshot(
|
||||
params: SnapshotMigrationParams,
|
||||
): Promise<void> {
|
||||
const snapshotPath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
try {
|
||||
// withFileLock is acquire-or-throw; it never skips the callback on contention.
|
||||
await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, () =>
|
||||
migrateLegacyMatrixIdbSnapshotLocked(params),
|
||||
);
|
||||
} catch (err) {
|
||||
params.warnings.push(
|
||||
`Failed locking Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function migrateLegacyMatrixIdbSnapshotLocked(
|
||||
params: SnapshotMigrationParams,
|
||||
): Promise<void> {
|
||||
const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
let snapshot: ReturnType<typeof readLegacyMatrixIdbSnapshotStateUnlocked>;
|
||||
try {
|
||||
snapshot = readLegacyMatrixIdbSnapshotStateUnlocked(params.storageRootDir);
|
||||
} catch (err) {
|
||||
params.warnings.push(
|
||||
`Failed reading Matrix IndexedDB snapshot legacy source for ${params.storageRootDir}: ${String(err)}; left source in place`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!snapshot) {
|
||||
if (!fsSync.existsSync(sourcePath)) {
|
||||
return;
|
||||
}
|
||||
const archived = await archiveLegacyMatrixIdbSnapshot(params);
|
||||
params.warnings.push(
|
||||
archived
|
||||
? `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; archived without import`
|
||||
: `Matrix IndexedDB snapshot legacy source is invalid for ${params.storageRootDir}; left active because archival failed`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const snapshotJson = JSON.stringify(snapshot);
|
||||
const store = params.context.openPluginStateKeyedStore<MatrixIdbSnapshotRecord>(
|
||||
openMatrixIdbSnapshotStoreOptions(params.storageRootDir),
|
||||
);
|
||||
let persisted: string | null;
|
||||
let hadPartialState: boolean;
|
||||
try {
|
||||
persisted = await readMatrixIdbSnapshotJsonFromStore({ store });
|
||||
const persistedIsValid = persisted ? isValidMatrixIdbSnapshotJson(persisted) : false;
|
||||
hadPartialState = !persistedIsValid && (await store.entries()).length > 0;
|
||||
if (!persistedIsValid) {
|
||||
persisted = null;
|
||||
}
|
||||
} catch (err) {
|
||||
params.warnings.push(
|
||||
`Failed inspecting Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (persisted && !snapshotContentMatches(persisted, snapshot)) {
|
||||
if (await archiveLegacyMatrixIdbSnapshot(params)) {
|
||||
params.notices.push(
|
||||
`Kept the canonical Matrix IndexedDB snapshot in SQLite and archived a differing legacy source for ${params.storageRootDir}`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!persisted) {
|
||||
try {
|
||||
await writeMatrixIdbSnapshotJsonToStore({
|
||||
snapshotJson,
|
||||
databaseCount: snapshot.length,
|
||||
store,
|
||||
});
|
||||
persisted = await readMatrixIdbSnapshotJsonFromStore({ store });
|
||||
} catch (err) {
|
||||
params.warnings.push(
|
||||
`Failed importing Matrix IndexedDB snapshot for ${params.storageRootDir}: ${String(err)}; left legacy source in place`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!persisted || !snapshotContentMatches(persisted, snapshot)) {
|
||||
params.warnings.push(
|
||||
`Failed verifying Matrix IndexedDB snapshot for ${params.storageRootDir}; left legacy source in place`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
params.changes.push(
|
||||
hadPartialState
|
||||
? `Repaired partial or invalid Matrix IndexedDB snapshot SQLite state for ${params.storageRootDir}`
|
||||
: `Migrated Matrix IndexedDB snapshot JSON to SQLite for ${params.storageRootDir}`,
|
||||
);
|
||||
}
|
||||
await archiveLegacyMatrixIdbSnapshot(params);
|
||||
}
|
||||
|
||||
function snapshotContentMatches(persistedJson: string, snapshot: unknown): boolean {
|
||||
try {
|
||||
return isDeepStrictEqual(JSON.parse(persistedJson), snapshot);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function archiveLegacyMatrixIdbSnapshot(params: {
|
||||
storageRootDir: string;
|
||||
changes: string[];
|
||||
warnings: string[];
|
||||
}): Promise<boolean> {
|
||||
const sourcePath = path.join(params.storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
const timestamp = new Date().toISOString().replaceAll(":", "-");
|
||||
const archivePath = `${sourcePath}.migrated-${timestamp}-${randomUUID()}`;
|
||||
try {
|
||||
await fs.rename(sourcePath, archivePath);
|
||||
params.changes.push(`Archived Matrix IndexedDB snapshot legacy source -> ${archivePath}`);
|
||||
return true;
|
||||
} catch (err) {
|
||||
params.warnings.push(
|
||||
`Failed archiving Matrix IndexedDB snapshot legacy source: ${String(err)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -230,10 +230,10 @@ export function writeMatrixIdbSnapshotJson(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function hasMatrixIdbSnapshotStateInStore(params: {
|
||||
export async function readMatrixIdbSnapshotJsonFromStore(params: {
|
||||
store: Pick<PluginStateKeyedStore<MatrixIdbSnapshotRecord>, "lookup">;
|
||||
}): Promise<boolean> {
|
||||
return (await readIdbSnapshotJsonFromAsyncStore(params.store)) !== null;
|
||||
}): Promise<string | null> {
|
||||
return await readIdbSnapshotJsonFromAsyncStore(params.store);
|
||||
}
|
||||
|
||||
export async function writeMatrixIdbSnapshotJsonToStore(params: {
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
// Matrix tests cover idb persistence.lock order plugin behavior.
|
||||
import "fake-indexeddb/auto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { installMatrixTestRuntime } from "../../test-runtime.js";
|
||||
import { MATRIX_IDB_PERSIST_INTERVAL_MS } from "./idb-persistence-lock.js";
|
||||
import { clearAllIndexedDbState, seedDatabase } from "./idb-persistence.test-helpers.js";
|
||||
|
||||
const { withFileLockMock } = vi.hoisted(() => ({
|
||||
withFileLockMock: vi.fn(
|
||||
async <T>(_filePath: string, _options: unknown, fn: () => Promise<T>) => await fn(),
|
||||
),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/file-lock", async () => {
|
||||
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/file-lock")>(
|
||||
"openclaw/plugin-sdk/file-lock",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
withFileLock: withFileLockMock,
|
||||
};
|
||||
});
|
||||
|
||||
let persistIdbToDisk: typeof import("./idb-persistence.js").persistIdbToDisk;
|
||||
let restoreIdbFromDisk: typeof import("./idb-persistence.js").restoreIdbFromDisk;
|
||||
type CapturedLockOptions =
|
||||
typeof import("./idb-persistence-lock.js").MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS;
|
||||
const DATABASE_PREFIX = "openclaw-matrix-lock-order-test";
|
||||
const cryptoDatabaseName = `${DATABASE_PREFIX}::matrix-sdk-crypto`;
|
||||
|
||||
function minimumRetryWindowMs(options: CapturedLockOptions): number {
|
||||
let total = 0;
|
||||
for (let attempt = 0; attempt < options.retries.retries; attempt += 1) {
|
||||
total += Math.min(
|
||||
options.retries.maxTimeout,
|
||||
Math.max(
|
||||
options.retries.minTimeout,
|
||||
options.retries.minTimeout * options.retries.factor ** attempt,
|
||||
),
|
||||
);
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
({ persistIdbToDisk, restoreIdbFromDisk } = await import("./idb-persistence.js"));
|
||||
});
|
||||
|
||||
describe("Matrix IndexedDB persistence lock ordering", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
resetPluginStateStoreForTests();
|
||||
installMatrixTestRuntime();
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "matrix-idb-lock-order-"));
|
||||
withFileLockMock.mockReset();
|
||||
withFileLockMock.mockImplementation(
|
||||
async <T>(_filePath: string, _options: unknown, fn: () => Promise<T>) => await fn(),
|
||||
);
|
||||
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
|
||||
resetPluginStateStoreForTests();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("captures the current snapshot into SQLite state", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
|
||||
await seedDatabase({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
records: [{ key: "room-1", value: { session: "old-session" } }],
|
||||
});
|
||||
|
||||
await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX });
|
||||
await clearAllIndexedDbState({ databasePrefix: DATABASE_PREFIX });
|
||||
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
|
||||
const dbs = await indexedDB.databases();
|
||||
expect(dbs.map((entry) => entry.name)).toContain(cryptoDatabaseName);
|
||||
});
|
||||
|
||||
it("uses the long snapshot lock options when importing a legacy file", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
|
||||
const capturedOptions: CapturedLockOptions[] = [];
|
||||
|
||||
withFileLockMock.mockImplementationOnce(async (_filePath, options) => {
|
||||
capturedOptions.push(options as CapturedLockOptions);
|
||||
return false;
|
||||
});
|
||||
fs.writeFileSync(snapshotPath, "[]", "utf8");
|
||||
await restoreIdbFromDisk(snapshotPath);
|
||||
|
||||
expect(capturedOptions).toHaveLength(1);
|
||||
for (const options of capturedOptions) {
|
||||
expect(minimumRetryWindowMs(options)).toBeGreaterThanOrEqual(MATRIX_IDB_PERSIST_INTERVAL_MS);
|
||||
expect(options.stale).toBe(5 * 60_000);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getMatrixRuntime } from "../../runtime.js";
|
||||
import { installMatrixTestRuntime } from "../../test-runtime.js";
|
||||
import { readMatrixIdbSnapshotJson, writeMatrixIdbSnapshotJson } from "../crypto-state-store.js";
|
||||
import { persistIdbToDisk, restoreIdbFromDisk } from "./idb-persistence.js";
|
||||
import {
|
||||
clearAllIndexedDbState,
|
||||
@@ -80,113 +81,70 @@ describe("Matrix IndexedDB persistence", () => {
|
||||
expect(dbs.map((entry) => entry.name)).not.toContain(otherCryptoDatabaseName);
|
||||
});
|
||||
|
||||
it("imports and archives a legacy JSON snapshot during restore", async () => {
|
||||
it("blocks runtime restore and persistence until doctor migrates the legacy snapshot", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
|
||||
fs.writeFileSync(
|
||||
snapshotPath,
|
||||
JSON.stringify([
|
||||
{
|
||||
name: cryptoDatabaseName,
|
||||
version: 1,
|
||||
stores: [
|
||||
{
|
||||
name: "sessions",
|
||||
keyPath: null,
|
||||
autoIncrement: false,
|
||||
indexes: [],
|
||||
records: [{ key: "room-1", value: { session: "legacy" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
"utf8",
|
||||
);
|
||||
const snapshot = JSON.stringify([{ name: cryptoDatabaseName, version: 1, stores: [] }]);
|
||||
fs.writeFileSync(snapshotPath, snapshot);
|
||||
|
||||
const restored = await restoreIdbFromDisk(snapshotPath);
|
||||
expect(restored).toBe(true);
|
||||
expect(fs.existsSync(snapshotPath)).toBe(false);
|
||||
expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true);
|
||||
|
||||
await clearTestIndexedDbState();
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).resolves.toBe(true);
|
||||
await expect(
|
||||
readDatabaseRecords({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
}),
|
||||
).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]);
|
||||
});
|
||||
|
||||
it("restores a valid legacy JSON snapshot when SQLite import fails", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
|
||||
fs.writeFileSync(
|
||||
snapshotPath,
|
||||
JSON.stringify([
|
||||
{
|
||||
name: cryptoDatabaseName,
|
||||
version: 1,
|
||||
stores: [
|
||||
{
|
||||
name: "sessions",
|
||||
keyPath: null,
|
||||
autoIncrement: false,
|
||||
indexes: [],
|
||||
records: [{ key: "room-1", value: { session: "legacy" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
"utf8",
|
||||
);
|
||||
vi.spyOn(getMatrixRuntime().state, "openSyncKeyedStore").mockImplementation(() => {
|
||||
throw new Error("sqlite unavailable");
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
|
||||
name: "MatrixIdbSnapshotMigrationRequiredError",
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
remediation: "openclaw doctor --fix",
|
||||
});
|
||||
|
||||
const restored = await restoreIdbFromDisk(snapshotPath);
|
||||
|
||||
expect(restored).toBe(true);
|
||||
expect(fs.existsSync(snapshotPath)).toBe(true);
|
||||
await expect(
|
||||
readDatabaseRecords({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
"IdbPersistence",
|
||||
expect.objectContaining({
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
remediation: "openclaw doctor --fix",
|
||||
}),
|
||||
).resolves.toEqual([{ key: "room-1", value: { session: "legacy" } }]);
|
||||
});
|
||||
);
|
||||
expect(JSON.stringify(warnSpy.mock.calls)).not.toContain(snapshotPath);
|
||||
|
||||
it("returns false and logs a warning for malformed snapshots", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "bad-snapshot.json");
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify([{ nope: true }]), "utf8");
|
||||
await seedDatabase({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
records: [{ key: "new-room", value: { session: "new" } }],
|
||||
});
|
||||
await expect(
|
||||
persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX }),
|
||||
).rejects.toMatchObject({
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
});
|
||||
expect(readMatrixIdbSnapshotJson(tmpDir)).toBeNull();
|
||||
expect(fs.existsSync(snapshotPath)).toBe(true);
|
||||
|
||||
const restored = await restoreIdbFromDisk(snapshotPath);
|
||||
expect(restored).toBe(false);
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
const [scope, message, error] = warnSpy.mock.calls.at(0) ?? [];
|
||||
expect(scope).toBe("IdbPersistence");
|
||||
expect(message).toBe(`Failed to restore IndexedDB snapshot from ${snapshotPath}:`);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
});
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: tmpDir,
|
||||
snapshotJson: JSON.stringify({ malformed: true }),
|
||||
databaseCount: 1,
|
||||
});
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
});
|
||||
const storeSpy = vi
|
||||
.spyOn(getMatrixRuntime().state, "openSyncKeyedStore")
|
||||
.mockImplementation(() => {
|
||||
throw new Error("sqlite unavailable");
|
||||
});
|
||||
|
||||
it("returns false for empty snapshot payloads without restoring databases", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "empty-snapshot.json");
|
||||
fs.writeFileSync(snapshotPath, JSON.stringify([]), "utf8");
|
||||
|
||||
const restored = await restoreIdbFromDisk(snapshotPath);
|
||||
expect(restored).toBe(false);
|
||||
|
||||
const dbs = await indexedDB.databases();
|
||||
expect(dbs).toStrictEqual([]);
|
||||
try {
|
||||
await expect(restoreIdbFromDisk(snapshotPath)).rejects.toMatchObject({
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
});
|
||||
} finally {
|
||||
storeSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns false without warning when the snapshot does not exist yet", async () => {
|
||||
const restored = await restoreIdbFromDisk(path.join(tmpDir, "missing-snapshot.json"));
|
||||
const restored = await restoreIdbFromDisk(path.join(tmpDir, "crypto-idb-snapshot.json"));
|
||||
|
||||
expect(restored).toBe(false);
|
||||
expect(warnSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("handles concurrent persist operations in SQLite state", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "concurrent-persist.json");
|
||||
const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json");
|
||||
await seedDatabase({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
@@ -208,19 +166,4 @@ describe("Matrix IndexedDB persistence", () => {
|
||||
}),
|
||||
).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]);
|
||||
});
|
||||
|
||||
it("archives an existing legacy snapshot file after persist", async () => {
|
||||
const snapshotPath = path.join(tmpDir, "persist-archives-legacy.json");
|
||||
fs.writeFileSync(snapshotPath, "[]", "utf8");
|
||||
await seedDatabase({
|
||||
name: cryptoDatabaseName,
|
||||
storeName: "sessions",
|
||||
records: [{ key: "room-1", value: { session: "abc123" } }],
|
||||
});
|
||||
|
||||
await persistIdbToDisk({ snapshotPath, databasePrefix: DATABASE_PREFIX });
|
||||
|
||||
expect(fs.existsSync(snapshotPath)).toBe(false);
|
||||
expect(fs.existsSync(`${snapshotPath}.migrated`)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -31,6 +31,28 @@ type IdbDatabaseSnapshot = {
|
||||
stores: IdbStoreSnapshot[];
|
||||
};
|
||||
|
||||
type IdbPersistenceDiagnostic = {
|
||||
code: "matrix-idb-snapshot-requires-doctor";
|
||||
message: string;
|
||||
remediation: "openclaw doctor --fix";
|
||||
};
|
||||
|
||||
const LEGACY_SNAPSHOT_DIAGNOSTIC: IdbPersistenceDiagnostic = {
|
||||
code: "matrix-idb-snapshot-requires-doctor",
|
||||
message: "Matrix IndexedDB snapshot exists outside canonical SQLite state",
|
||||
remediation: "openclaw doctor --fix",
|
||||
};
|
||||
|
||||
class MatrixIdbSnapshotMigrationRequiredError extends Error {
|
||||
readonly code = LEGACY_SNAPSHOT_DIAGNOSTIC.code;
|
||||
readonly remediation = LEGACY_SNAPSHOT_DIAGNOSTIC.remediation;
|
||||
|
||||
constructor() {
|
||||
super(`${LEGACY_SNAPSHOT_DIAGNOSTIC.message}; run ${LEGACY_SNAPSHOT_DIAGNOSTIC.remediation}`);
|
||||
this.name = "MatrixIdbSnapshotMigrationRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
function isValidIdbIndexSnapshot(value: unknown): value is IdbStoreSnapshot["indexes"][number] {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
@@ -100,6 +122,14 @@ function parseSnapshotPayload(data: string): IdbDatabaseSnapshot[] | null {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function isValidMatrixIdbSnapshotJson(data: string): boolean {
|
||||
try {
|
||||
return parseSnapshotPayload(data) !== null;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function idbReq<T>(req: IDBRequest<T>): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
req.addEventListener("success", () => resolve(req.result), { once: true });
|
||||
@@ -228,106 +258,86 @@ function resolveDefaultIdbSnapshotPath(): string {
|
||||
return path.join(stateDir, "matrix", "crypto-idb-snapshot.json");
|
||||
}
|
||||
|
||||
// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests.
|
||||
export async function restoreIdbFromDisk(snapshotPath?: string): Promise<boolean> {
|
||||
const candidatePaths = snapshotPath ? [snapshotPath] : [resolveDefaultIdbSnapshotPath()];
|
||||
for (const resolvedPath of candidatePaths) {
|
||||
const storageRootDir = path.dirname(resolvedPath);
|
||||
try {
|
||||
const restored = await withFileLock(
|
||||
resolvedPath,
|
||||
MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS,
|
||||
async () => {
|
||||
try {
|
||||
const storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
|
||||
if (storedSnapshotJson) {
|
||||
const snapshot = parseSnapshotPayload(storedSnapshotJson);
|
||||
if (snapshot) {
|
||||
await restoreIndexedDatabases(snapshot);
|
||||
LogService.info(
|
||||
"IdbPersistence",
|
||||
`Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
LogService.warn(
|
||||
"IdbPersistence",
|
||||
"Failed to restore IndexedDB snapshot from SQLite:",
|
||||
err,
|
||||
);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(resolvedPath)) {
|
||||
return false;
|
||||
}
|
||||
const data = fs.readFileSync(resolvedPath, "utf8");
|
||||
const snapshot = parseSnapshotPayload(data);
|
||||
if (!snapshot) {
|
||||
return false;
|
||||
}
|
||||
let migratedToSqlite = false;
|
||||
try {
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir,
|
||||
snapshotJson: data,
|
||||
databaseCount: snapshot.length,
|
||||
});
|
||||
archiveLegacyIdbSnapshotFile(resolvedPath);
|
||||
migratedToSqlite = true;
|
||||
} catch (err) {
|
||||
LogService.warn(
|
||||
"IdbPersistence",
|
||||
`Failed to migrate IndexedDB snapshot to SQLite from ${resolvedPath}:`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
await restoreIndexedDatabases(snapshot);
|
||||
LogService.info(
|
||||
"IdbPersistence",
|
||||
migratedToSqlite
|
||||
? `Migrated and restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`
|
||||
: `Restored ${snapshot.length} IndexedDB database(s) from legacy snapshot ${resolvedPath}`,
|
||||
);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
if (restored) {
|
||||
return true;
|
||||
const resolvedPath = snapshotPath ?? resolveDefaultIdbSnapshotPath();
|
||||
const storageRootDir = path.dirname(resolvedPath);
|
||||
let callbackStarted = false;
|
||||
try {
|
||||
// withFileLock is acquire-or-throw; it never skips the callback on contention.
|
||||
return await withFileLock(resolvedPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => {
|
||||
callbackStarted = true;
|
||||
let storedSnapshotJson: string | null;
|
||||
try {
|
||||
storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
|
||||
} catch (err) {
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
throwLegacySnapshotMigrationRequired();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
} catch (err) {
|
||||
LogService.warn(
|
||||
throwIfLegacySnapshotNeedsDoctor(resolvedPath, storedSnapshotJson);
|
||||
if (!storedSnapshotJson) {
|
||||
return false;
|
||||
}
|
||||
const snapshot = parseSnapshotPayload(storedSnapshotJson);
|
||||
if (!snapshot) {
|
||||
return false;
|
||||
}
|
||||
await restoreIndexedDatabases(snapshot);
|
||||
LogService.info(
|
||||
"IdbPersistence",
|
||||
`Failed to restore IndexedDB snapshot from ${resolvedPath}:`,
|
||||
err,
|
||||
`Restored ${snapshot.length} IndexedDB database(s) from Matrix SQLite state`,
|
||||
);
|
||||
continue;
|
||||
return true;
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof MatrixIdbSnapshotMigrationRequiredError) {
|
||||
throw err;
|
||||
}
|
||||
if (!callbackStarted && fs.existsSync(resolvedPath)) {
|
||||
throwLegacySnapshotMigrationRequired();
|
||||
}
|
||||
LogService.warn("IdbPersistence", "Failed to restore IndexedDB snapshot from SQLite:", err);
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function persistIdbToDisk(params?: {
|
||||
// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests.
|
||||
snapshotPath?: string;
|
||||
databasePrefix?: string;
|
||||
}): Promise<void> {
|
||||
const snapshotPath = params?.snapshotPath ?? resolveDefaultIdbSnapshotPath();
|
||||
let callbackStarted = false;
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
||||
// withFileLock is acquire-or-throw; it never skips the callback on contention.
|
||||
const persistedCount = await withFileLock(
|
||||
snapshotPath,
|
||||
MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS,
|
||||
async () => {
|
||||
callbackStarted = true;
|
||||
const storageRootDir = path.dirname(snapshotPath);
|
||||
let storedSnapshotJson: string | null;
|
||||
try {
|
||||
storedSnapshotJson = readMatrixIdbSnapshotJson(storageRootDir);
|
||||
} catch (err) {
|
||||
if (fs.existsSync(snapshotPath)) {
|
||||
throwLegacySnapshotMigrationRequired();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
throwIfLegacySnapshotNeedsDoctor(snapshotPath, storedSnapshotJson);
|
||||
const snapshot = await dumpIndexedDatabases(params?.databasePrefix);
|
||||
if (snapshot.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
writeMatrixIdbSnapshotJson({
|
||||
storageRootDir: path.dirname(snapshotPath),
|
||||
storageRootDir,
|
||||
snapshotJson: JSON.stringify(snapshot),
|
||||
databaseCount: snapshot.length,
|
||||
});
|
||||
archiveLegacyIdbSnapshotFile(snapshotPath);
|
||||
return snapshot.length;
|
||||
},
|
||||
);
|
||||
@@ -339,36 +349,46 @@ export async function persistIdbToDisk(params?: {
|
||||
`Persisted ${persistedCount} IndexedDB database(s) to Matrix SQLite state`,
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof MatrixIdbSnapshotMigrationRequiredError) {
|
||||
throw err;
|
||||
}
|
||||
if (!callbackStarted && fs.existsSync(snapshotPath)) {
|
||||
throwLegacySnapshotMigrationRequired();
|
||||
}
|
||||
LogService.warn("IdbPersistence", "Failed to persist IndexedDB snapshot:", err);
|
||||
}
|
||||
}
|
||||
|
||||
export async function readLegacyMatrixIdbSnapshotState(
|
||||
export function readLegacyMatrixIdbSnapshotStateUnlocked(
|
||||
storageRootDir: string,
|
||||
): Promise<IdbDatabaseSnapshot[] | null> {
|
||||
): IdbDatabaseSnapshot[] | null {
|
||||
const snapshotPath = path.join(storageRootDir, MATRIX_IDB_SNAPSHOT_FILENAME);
|
||||
if (!fs.existsSync(snapshotPath)) {
|
||||
return null;
|
||||
}
|
||||
const data = fs.readFileSync(snapshotPath, "utf8");
|
||||
try {
|
||||
return await withFileLock(snapshotPath, MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, async () => {
|
||||
const snapshot = parseSnapshotPayload(fs.readFileSync(snapshotPath, "utf8"));
|
||||
return snapshot;
|
||||
});
|
||||
return parseSnapshotPayload(data);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function archiveLegacyIdbSnapshotFile(snapshotPath: string): void {
|
||||
if (!fs.existsSync(snapshotPath)) {
|
||||
return;
|
||||
function throwIfLegacySnapshotNeedsDoctor(
|
||||
snapshotPath: string,
|
||||
storedSnapshotJson: string | null,
|
||||
): void {
|
||||
if (
|
||||
fs.existsSync(snapshotPath) &&
|
||||
(!storedSnapshotJson || !isValidMatrixIdbSnapshotJson(storedSnapshotJson))
|
||||
) {
|
||||
throwLegacySnapshotMigrationRequired();
|
||||
}
|
||||
const archivedPath = `${snapshotPath}.migrated`;
|
||||
if (fs.existsSync(archivedPath)) {
|
||||
return;
|
||||
}
|
||||
fs.renameSync(snapshotPath, archivedPath);
|
||||
}
|
||||
|
||||
function throwLegacySnapshotMigrationRequired(): never {
|
||||
LogService.warn("IdbPersistence", LEGACY_SNAPSHOT_DIAGNOSTIC);
|
||||
throw new MatrixIdbSnapshotMigrationRequiredError();
|
||||
}
|
||||
|
||||
function toLintErrorObject(value: unknown, fallbackMessage: string): Error {
|
||||
|
||||
Reference in New Issue
Block a user