diff --git a/docs/channels/matrix-migration.md b/docs/channels/matrix-migration.md index 93e798b0a504..4d028c7c36fb 100644 --- a/docs/channels/matrix-migration.md +++ b/docs/channels/matrix-migration.md @@ -13,7 +13,7 @@ For most users, the upgrade is in place: - the plugin stays `@openclaw/matrix` - the channel stays `matrix` - your config stays under `channels.matrix` -- cached credentials stay under `~/.openclaw/credentials/matrix/` +- cached credentials move into the shared `state/openclaw.sqlite` plugin state - runtime state stays under `~/.openclaw/matrix/` You do not need to rename config keys or reinstall the plugin under a new name. @@ -25,11 +25,11 @@ into the root OpenClaw package. ## What the migration does automatically -Matrix migration runs when you run [`openclaw doctor --fix`](/gateway/doctor), and as a fallback when the Matrix client starts and still finds file-based sidecar state next to its SQLite store. +Matrix migration runs when you run [`openclaw doctor --fix`](/gateway/doctor). File-based sidecars next to the dedicated Matrix store retain their client-start fallback, but credential-file import is Doctor-only; runtime reads only canonical SQLite credential state. -Automatic migration covers: +Doctor migration covers: -- reusing your cached Matrix credentials +- importing and verifying retired `~/.openclaw/credentials/matrix/credentials*.json` files before archiving them - keeping the same account selection and `channels.matrix` config - importing file-based sidecar state (`bot-storage.json` sync cache, `recovery-key.json`, `legacy-crypto-migration.json`, IndexedDB snapshots) into Matrix SQLite state; migrated files are archived with a `.migrated` suffix - reusing the most complete existing token-hash storage root for the same Matrix account, homeserver, user, and device when the access token changes later diff --git a/docs/channels/matrix.md b/docs/channels/matrix.md index ff3aa60f20a5..4d27bf7e112e 100644 --- a/docs/channels/matrix.md +++ b/docs/channels/matrix.md @@ -103,7 +103,7 @@ The wizard converts a friendly name into a normalized account ID (`Ops Bot` -> ` ### Cached credentials -Matrix caches credentials under `~/.openclaw/credentials/matrix/`: `credentials.json` for the default account, `credentials-.json` for named accounts. When cached credentials exist, OpenClaw treats Matrix as configured even without an `accessToken` in the config file - this covers setup, `openclaw doctor`, and channel-status probes. +Matrix caches account credentials in the shared `state/openclaw.sqlite` plugin state. When cached credentials exist, OpenClaw treats Matrix as configured even without an `accessToken` in the config file - this covers setup, `openclaw doctor`, and channel-status probes. Upgrades import the retired `~/.openclaw/credentials/matrix/credentials*.json` files through `openclaw doctor --fix`, verify the SQLite rows, then archive the files. ### Environment variables diff --git a/docs/channels/reef.md b/docs/channels/reef.md index 9ef991e6232e..494c1041fc32 100644 --- a/docs/channels/reef.md +++ b/docs/channels/reef.md @@ -18,7 +18,7 @@ Reef is a guarded, end-to-end-encrypted side channel between OpenClaw agents own openclaw channels add ``` -The wizard asks for the relay URL (default `https://reefwire.ai`), your email, the setup session, a unique unlisted handle, an inbound friend-request policy (`code-only` is recommended), a local state directory for your keys, and the guard model configuration. +The wizard asks for the relay URL (default `https://reefwire.ai`), your email, the setup session, a unique unlisted handle, an inbound friend-request policy (`code-only` is recommended), and the guard model configuration. 3. Restart the Gateway and confirm the channel connects: @@ -63,7 +63,6 @@ Reef lives under `channels.reef`: handle: "myclaw", email: "you@example.com", requestPolicy: "code-only", // code-only | friends-of-friends | open - stateDir: "~/.openclaw/data/reef", guard: { provider: "openai", // or "anthropic" pinnedModel: "gpt-5.6-terra", @@ -78,8 +77,8 @@ Reef lives under `channels.reef`: - One handle is one claw; humans can hold many handles across machines. - `relayUrl` is an HTTP(S) origin such as `https://reefwire.ai`; paths, queries, URL credentials, and fragments are rejected because Reef uses an origin-wide `/v1` API. -- Private Ed25519/X25519 keys are generated into `stateDir` and never leave the machine. -- Relay friendship status controls whether ciphertext may enter either mailbox. OpenClaw separately keeps each approved peer's public-key pins and autonomy tier in the shared `state/openclaw.sqlite` plugin state. `channels.reef` has no friendship allowlist to edit. +- Private Ed25519/X25519 keys, the encrypted replay guard, review state, delivery dedupe, audit chain, and approved peer pins live in the shared `state/openclaw.sqlite` plugin state and never leave the machine. `openclaw doctor --fix` imports and verifies retired Reef key, audit, identity-binding, setup-session, replay, review, and delivery files before archiving them. +- Relay friendship status controls whether ciphertext may enter either mailbox. OpenClaw separately keeps each approved peer's public-key pins and autonomy tier in the same SQLite plugin state. `channels.reef` has no friendship allowlist to edit. - A normal OpenClaw pairing approval becomes an identity-, key-, and revocation-bound one-time handoff. Reef consumes it before accepting the relay edge or writing the verified peer pins, and the relay activates only if that exact peer key snapshot is still current. A stale approval cannot authorize changed keys or undo a local removal. Removing a friend clears local trust first, then blocks the relay edge. - `pinnedModel` must be an immutable model id: a dated snapshot, or one of the documented undated ids (`gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`). Floating aliases are rejected, and every guard response must echo the exact configured id. - `apiKeyEnv` names an environment variable visible to the Gateway process. The guard fails closed: a missing key or provider error denies the message. diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index b8674a4655a8..c60f1f7a36cc 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -564,9 +564,9 @@ The branch already has a real shared SQLite base: an internal `installedPluginIndex.installRecords.*` diff namespace. Runtime reload decisions no longer wrap those rows in fake `plugins.installs` config objects. -- Matrix named-account credential upgrade no longer happens during runtime - reads. Doctor owns the old top-level `credentials/matrix/credentials.json` - rename when a single/default Matrix account can be resolved. +- Matrix account credentials now live in SQLite plugin state. Runtime reads + only that canonical store; Doctor imports, verifies, and archives retired + `credentials/matrix/credentials*.json` files when their account can be resolved. - Core pairing and cron runtime modules no longer use legacy JSON path builders. The deprecated pairing-path SDK helper remains migration-only compatibility; doctor state migration owns its file reads and imports. Doctor-owned legacy diff --git a/extensions/matrix/auth-presence.ts b/extensions/matrix/auth-presence.ts index bafa4ee8257f..5d445b49d94c 100644 --- a/extensions/matrix/auth-presence.ts +++ b/extensions/matrix/auth-presence.ts @@ -1,13 +1,12 @@ // Matrix plugin module implements auth presence behavior. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; +import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor"; import { - resolveMatrixCredentialsDir, - resolveMatrixCredentialsFilename, -} from "./src/storage-paths.js"; + MATRIX_CREDENTIALS_MAX_ENTRIES, + MATRIX_CREDENTIALS_NAMESPACE, + normalizeMatrixStoredCredentials, + type MatrixCredentialStateRecord, +} from "./src/matrix/credentials-read.js"; type MatrixAuthPresenceParams = | { @@ -16,42 +15,21 @@ type MatrixAuthPresenceParams = } | OpenClawConfig; -function listMatrixCredentialPaths( - _cfg: OpenClawConfig, - env: NodeJS.ProcessEnv = process.env, -): readonly string[] { - const credentialsDir = resolveMatrixCredentialsDir(resolveStateDir(env, os.homedir)); - const paths = new Set([ - resolveMatrixCredentialsFilename(), - resolveMatrixCredentialsFilename("default"), - ]); - - try { - const entries = fs.readdirSync(credentialsDir, { withFileTypes: true }); - for (const entry of entries) { - if (entry.isFile() && /^credentials(?:-[a-z0-9._-]+)?\.json$/i.test(entry.name)) { - paths.add(entry.name); - } - } - } catch { - // Missing credentials directories mean no persisted Matrix auth state. - } - - return [...paths].map((filename) => path.join(credentialsDir, filename)); -} - export function hasAnyMatrixAuth( params: MatrixAuthPresenceParams, env: NodeJS.ProcessEnv = process.env, ): boolean { - const cfg = params && typeof params === "object" && "cfg" in params ? params.cfg : params; const resolvedEnv = params && typeof params === "object" && "cfg" in params ? (params.env ?? env) : env; - return listMatrixCredentialPaths(cfg, resolvedEnv).some((filePath) => { - try { - return fs.existsSync(filePath); - } catch { - return false; - } - }); + try { + const store = createPluginStateSyncKeyedStore("matrix", { + namespace: MATRIX_CREDENTIALS_NAMESPACE, + maxEntries: MATRIX_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + env: resolvedEnv, + }); + return store.entries().some((entry) => normalizeMatrixStoredCredentials(entry.value) !== null); + } catch { + return false; + } } diff --git a/extensions/matrix/doctor-contract-api.test.ts b/extensions/matrix/doctor-contract-api.test.ts index 3950a5fc93cb..0d6d68177607 100644 --- a/extensions/matrix/doctor-contract-api.test.ts +++ b/extensions/matrix/doctor-contract-api.test.ts @@ -17,6 +17,13 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { stateMigrations } from "./doctor-contract-api.js"; import { SqliteBackedMatrixSyncStore } from "./src/matrix/client/file-sync-store.js"; import { openMatrixStorageMetaStoreOptions } from "./src/matrix/client/storage.js"; +import { + MATRIX_CREDENTIALS_MAX_ENTRIES, + MATRIX_CREDENTIALS_NAMESPACE, + matrixCredentialsStoreKey, + type MatrixCredentialStateRecord, + type MatrixStoredCredentialRecord, +} from "./src/matrix/credentials-read.js"; import { MATRIX_RECOVERY_KEY_FILENAME, readMatrixIdbSnapshotJson, @@ -71,6 +78,85 @@ describe("matrix doctor contract state migrations", () => { } }); + it("imports account credentials into SQLite before archiving the JSON", async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); + tempDirs.push(stateDir); + const credentialsDir = path.join(stateDir, "credentials", "matrix"); + const filePath = path.join(credentialsDir, "credentials-ops.json"); + const credentials = { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + deviceId: "DEVICE123", + createdAt: "2026-07-01T12:00:00.000Z", + lastUsedAt: "2026-07-02T12:00:00.000Z", + }; + fs.mkdirSync(credentialsDir, { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(credentials)); + const migration = migrationById("matrix-credentials-json-to-plugin-state"); + const params = createMigrationParams(stateDir); + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: ["Matrix credential JSON can migrate to SQLite (1 file)"], + }); + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Matrix credentials for account ops to SQLite", + expect.stringContaining("Archived Matrix credentials legacy source"), + ]); + const store = params.context.openPluginStateKeyedStore({ + namespace: MATRIX_CREDENTIALS_NAMESPACE, + maxEntries: MATRIX_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(store.lookup(matrixCredentialsStoreKey("ops"))).resolves.toEqual({ + accountId: "ops", + ...credentials, + }); + expect(fs.existsSync(`${filePath}.migrated`)).toBe(true); + }); + + it("archives legacy credentials without restoring an explicitly cleared account", async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); + tempDirs.push(stateDir); + const credentialsDir = path.join(stateDir, "credentials", "matrix"); + const filePath = path.join(credentialsDir, "credentials-ops.json"); + fs.mkdirSync(credentialsDir, { recursive: true }); + fs.writeFileSync( + filePath, + JSON.stringify({ + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "legacy-token", + createdAt: "2026-07-01T12:00:00.000Z", + }), + ); + const params = createMigrationParams(stateDir); + const credentialStore = params.context.openPluginStateKeyedStore({ + namespace: MATRIX_CREDENTIALS_NAMESPACE, + maxEntries: MATRIX_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await credentialStore.register(matrixCredentialsStoreKey("ops"), { + accountId: "ops", + kind: "revoked", + revokedAt: "2026-07-02T12:00:00.000Z", + }); + + const result = await migrationById( + "matrix-credentials-json-to-plugin-state", + ).migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Archived revoked Matrix credential legacy source for account ops", + expect.stringContaining("Archived Matrix credentials legacy source"), + ]); + expect(fs.existsSync(`${filePath}.migrated`)).toBe(true); + }); + it("migrates legacy sync cache JSON to SQLite plugin state", async () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-doctor-")); tempDirs.push(stateDir); diff --git a/extensions/matrix/doctor-contract-api.ts b/extensions/matrix/doctor-contract-api.ts index 6ca36941d2a4..df99822a725c 100644 --- a/extensions/matrix/doctor-contract-api.ts +++ b/extensions/matrix/doctor-contract-api.ts @@ -2,10 +2,16 @@ import type { Dirent } from "node:fs"; // Matrix API module exposes the plugin public contract. import fs from "node:fs/promises"; import path from "node:path"; +import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; import { archiveLegacyStateSource, type PluginDoctorStateMigration, } from "openclaw/plugin-sdk/runtime-doctor"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + requiresExplicitMatrixDefaultAccount, + resolveMatrixDefaultOrOnlyAccountId, +} from "./src/account-selection.js"; import { hasMatrixSyncCacheStateInStore, openMatrixSyncCacheStoreOptions, @@ -20,6 +26,15 @@ import { writeMatrixStorageMetaStateToStore, type MatrixStorageMetadata, } from "./src/matrix/client/storage.js"; +import { + MATRIX_CREDENTIALS_MAX_ENTRIES, + MATRIX_CREDENTIALS_NAMESPACE, + isMatrixCredentialRevocation, + matrixCredentialsStoreKey, + normalizeMatrixStoredCredentials, + type MatrixCredentialStateRecord, + type MatrixStoredCredentialRecord, +} from "./src/matrix/credentials-read.js"; import { MATRIX_IDB_SNAPSHOT_FILENAME, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, @@ -50,12 +65,74 @@ import { } from "./src/matrix/monitor/inbound-dedupe-migration.js"; import { readLegacyMatrixIdbSnapshotState } from "./src/matrix/sdk/idb-persistence.js"; import type { MatrixStoredRecoveryKey } from "./src/matrix/sdk/types.js"; +import { resolveMatrixCredentialsDir } from "./src/storage-paths.js"; export { normalizeCompatibilityConfig, legacyConfigRules } from "./src/doctor-contract.js"; const MATRIX_SYNC_CACHE_FILENAME = "bot-storage.json"; const MATRIX_STORAGE_META_FILENAME = "storage-meta.json"; +type LegacyMatrixCredentialSource = { + accountId: string | null; + filePath: string; +}; + +async function collectLegacyMatrixCredentialSources(params: { + config: Parameters[0]["config"]; + env: NodeJS.ProcessEnv; + stateDir: string; +}): Promise { + const credentialsDir = resolveMatrixCredentialsDir(params.stateDir); + let entries: Dirent[]; + try { + entries = await fs.readdir(credentialsDir, { withFileTypes: true }); + } catch { + return []; + } + const files = entries + .filter((entry) => entry.isFile() && /^credentials(?:-[a-z0-9._-]+)?\.json$/iu.test(entry.name)) + .toSorted((left, right) => { + if (left.name === "credentials.json") { + return 1; + } + if (right.name === "credentials.json") { + return -1; + } + return left.name.localeCompare(right.name); + }); + return files.map((entry) => { + const match = /^credentials(?:-([a-z0-9._-]+))?\.json$/iu.exec(entry.name); + const namedAccount = match?.[1]; + const accountId = namedAccount + ? normalizeAccountId(namedAccount) + : requiresExplicitMatrixDefaultAccount(params.config, params.env) + ? null + : normalizeAccountId(resolveMatrixDefaultOrOnlyAccountId(params.config, params.env)); + return { accountId, filePath: path.join(credentialsDir, entry.name) }; + }); +} + +async function readLegacyMatrixCredentials( + source: LegacyMatrixCredentialSource, +): Promise { + if (!source.accountId) { + return null; + } + try { + const raw = JSON.parse(await fs.readFile(source.filePath, "utf8")) as unknown; + const createdAt = + isRecord(raw) && typeof raw.createdAt === "string" && raw.createdAt + ? raw.createdAt + : (await fs.stat(source.filePath)).mtime.toISOString(); + return normalizeMatrixStoredCredentials( + isRecord(raw) ? { ...raw, createdAt } : raw, + source.accountId, + ); + } catch { + return null; + } +} + async function collectLegacyMatrixStateRoots( stateDir: string, filename: string, @@ -138,6 +215,94 @@ async function archiveLegacyMatrixStateFile(params: { } export const stateMigrations: PluginDoctorStateMigration[] = [ + { + id: "matrix-credentials-json-to-plugin-state", + label: "Matrix credentials", + async detectLegacyState(params) { + const sources = await collectLegacyMatrixCredentialSources(params); + return sources.length > 0 + ? { + preview: [ + `Matrix credential JSON can migrate to SQLite (${sources.length} ${sources.length === 1 ? "file" : "files"})`, + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const sources = await collectLegacyMatrixCredentialSources(params); + const store = params.context.openPluginStateKeyedStore({ + namespace: MATRIX_CREDENTIALS_NAMESPACE, + maxEntries: MATRIX_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + for (const source of sources) { + if (!source.accountId) { + warnings.push( + `Left ambiguous Matrix credential legacy source in place because no default account is selected: ${source.filePath}`, + ); + continue; + } + const credentials = await readLegacyMatrixCredentials(source); + if (!credentials) { + warnings.push( + `Left invalid Matrix credential legacy source in place: ${source.filePath}`, + ); + continue; + } + const key = matrixCredentialsStoreKey(source.accountId); + const stored = await store.lookup(key); + if (isMatrixCredentialRevocation(stored, source.accountId)) { + changes.push( + `Archived revoked Matrix credential legacy source for account ${source.accountId}`, + ); + await archiveLegacyStateSource({ + filePath: source.filePath, + label: "Matrix credentials", + changes, + warnings, + }); + continue; + } + const existing = normalizeMatrixStoredCredentials(stored, source.accountId); + if (existing && JSON.stringify(existing) !== JSON.stringify(credentials)) { + warnings.push( + `Kept existing Matrix credentials for account ${source.accountId}; left differing legacy source in place`, + ); + continue; + } + if (!existing) { + try { + await store.registerIfAbsent(key, credentials); + } catch (error) { + warnings.push( + `Failed importing Matrix credentials for account ${source.accountId}: ${String(error)}; left legacy source in place`, + ); + continue; + } + } + const persisted = normalizeMatrixStoredCredentials( + await store.lookup(key), + source.accountId, + ); + if (!persisted || JSON.stringify(persisted) !== JSON.stringify(credentials)) { + warnings.push( + `Failed verifying Matrix credentials for account ${source.accountId}; left legacy source in place`, + ); + continue; + } + changes.push(`Migrated Matrix credentials for account ${source.accountId} to SQLite`); + await archiveLegacyStateSource({ + filePath: source.filePath, + label: "Matrix credentials", + changes, + warnings, + }); + } + return { changes, warnings }; + }, + }, { id: "matrix-inbound-dedupe-to-claimable-dedupe", label: "Matrix inbound dedupe markers", diff --git a/extensions/matrix/src/matrix/async-lock.ts b/extensions/matrix/src/matrix/async-lock.ts index bbfe3d02ebf1..7ac0fc8ddf43 100644 --- a/extensions/matrix/src/matrix/async-lock.ts +++ b/extensions/matrix/src/matrix/async-lock.ts @@ -1,5 +1,5 @@ // Matrix plugin module implements async lock behavior. -export type AsyncLock = (fn: () => Promise) => Promise; +type AsyncLock = (fn: () => Promise) => Promise; export function createAsyncLock(): AsyncLock { let lock: Promise = Promise.resolve(); diff --git a/extensions/matrix/src/matrix/credentials-read.ts b/extensions/matrix/src/matrix/credentials-read.ts index dabcdb052ccb..451849804a1f 100644 --- a/extensions/matrix/src/matrix/credentials-read.ts +++ b/extensions/matrix/src/matrix/credentials-read.ts @@ -1,18 +1,10 @@ // Matrix plugin module implements credentials read behavior. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { DEFAULT_ACCOUNT_ID, normalizeAccountId } from "openclaw/plugin-sdk/account-id"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import { - requiresExplicitMatrixDefaultAccount, - resolveMatrixDefaultOrOnlyAccountId, -} from "../account-selection.js"; -import { getMatrixRuntime } from "../runtime.js"; -import { - resolveMatrixCredentialsDir as resolveSharedMatrixCredentialsDir, - resolveMatrixCredentialsPath as resolveSharedMatrixCredentialsPath, -} from "../storage-paths.js"; +import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { createPluginStateSyncKeyedStore } from "openclaw/plugin-sdk/runtime-doctor"; +import { getOptionalMatrixRuntime } from "../runtime.js"; + +export { resolveMatrixCredentialsDir, resolveMatrixCredentialsPath } from "../storage-paths.js"; export type MatrixStoredCredentials = { homeserver: string; @@ -23,175 +15,117 @@ export type MatrixStoredCredentials = { lastUsedAt?: string; }; -type MatrixCredentialsSource = "current" | "legacy"; +export type MatrixStoredCredentialRecord = MatrixStoredCredentials & { + accountId: string; +}; -type MatrixCredentialsFileLoadResult = - | { - kind: "loaded"; - source: MatrixCredentialsSource; - credentials: MatrixStoredCredentials | null; - } - | { - kind: "missing"; - }; +export type MatrixCredentialRevocationRecord = { + accountId: string; + kind: "revoked"; + revokedAt: string; +}; -function resolveStateDir(env: NodeJS.ProcessEnv): string { - try { - return getMatrixRuntime().state.resolveStateDir(env, os.homedir); - } catch { - // Some config-only helpers read stored credentials before the Matrix plugin - // runtime is installed. Fall back to the standard state-dir env contract. - const override = env.OPENCLAW_STATE_DIR?.trim(); - if (override) { - return path.resolve(override); - } - const homeDir = env.OPENCLAW_HOME?.trim() || env.HOME?.trim() || os.homedir(); - return path.join(homeDir, ".openclaw"); - } +export type MatrixCredentialStateRecord = + | MatrixStoredCredentialRecord + | MatrixCredentialRevocationRecord; + +export const MATRIX_CREDENTIALS_NAMESPACE = "credentials"; +export const MATRIX_CREDENTIALS_MAX_ENTRIES = 256; + +export function matrixCredentialsStoreKey(accountId?: string | null): string { + return `account:${normalizeAccountId(accountId)}`; } -function resolveLegacyMatrixCredentialsPath(env: NodeJS.ProcessEnv): string { - return path.join(resolveMatrixCredentialsDir(env), "credentials.json"); -} - -function shouldReadLegacyCredentialsForAccount(accountId?: string | null): boolean { - const normalizedAccountId = normalizeAccountId(accountId); - const cfg = getMatrixRuntime().config.current() as OpenClawConfig; - if (!cfg.channels?.matrix || typeof cfg.channels.matrix !== "object") { - return normalizedAccountId === DEFAULT_ACCOUNT_ID; - } - if (requiresExplicitMatrixDefaultAccount(cfg)) { - return false; - } - return normalizeAccountId(resolveMatrixDefaultOrOnlyAccountId(cfg)) === normalizedAccountId; -} - -function resolveLegacyMigrationSourcePath( - env: NodeJS.ProcessEnv, +export function normalizeMatrixStoredCredentials( + value: unknown, accountId?: string | null, -): string | null { - if (!shouldReadLegacyCredentialsForAccount(accountId)) { +): MatrixStoredCredentialRecord | null { + if (!value || typeof value !== "object") { return null; } - const legacyPath = resolveLegacyMatrixCredentialsPath(env); - return legacyPath === resolveMatrixCredentialsPath(env, accountId) ? null : legacyPath; -} - -function parseMatrixCredentialsFile(filePath: string): MatrixStoredCredentials | null { - const raw = fs.readFileSync(filePath, "utf-8"); - const parsed = JSON.parse(raw) as Partial; + const parsed = value as Partial; if ( typeof parsed.homeserver !== "string" || + !parsed.homeserver || typeof parsed.userId !== "string" || - typeof parsed.accessToken !== "string" + !parsed.userId || + typeof parsed.accessToken !== "string" || + !parsed.accessToken || + typeof parsed.createdAt !== "string" || + !parsed.createdAt ) { return null; } - return parsed as MatrixStoredCredentials; + const normalizedAccountId = normalizeAccountId(accountId ?? parsed.accountId); + return { + accountId: normalizedAccountId, + homeserver: parsed.homeserver, + userId: parsed.userId, + accessToken: parsed.accessToken, + ...(typeof parsed.deviceId === "string" ? { deviceId: parsed.deviceId } : {}), + createdAt: parsed.createdAt, + ...(typeof parsed.lastUsedAt === "string" ? { lastUsedAt: parsed.lastUsedAt } : {}), + }; } -function loadMatrixCredentialsFile( - filePath: string, - source: MatrixCredentialsSource, -): MatrixCredentialsFileLoadResult { - try { - return { - kind: "loaded", - source, - credentials: parseMatrixCredentialsFile(filePath), - }; - } catch (error) { - if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { - return { kind: "missing" }; - } - throw error; - } -} - -function loadLegacyMatrixCredentialsWithCurrentFallback(params: { - legacyPath: string; - currentPath: string; -}): MatrixCredentialsFileLoadResult { - const legacy = loadMatrixCredentialsFile(params.legacyPath, "legacy"); - if (legacy.kind === "loaded") { - return legacy; - } - return loadMatrixCredentialsFile(params.currentPath, "current"); -} - -export function resolveMatrixCredentialsDir( - env: NodeJS.ProcessEnv = process.env, - stateDir?: string, -): string { - const resolvedStateDir = stateDir ?? resolveStateDir(env); - return resolveSharedMatrixCredentialsDir(resolvedStateDir); -} - -export function resolveMatrixCredentialsPath( - env: NodeJS.ProcessEnv = process.env, +export function isMatrixCredentialRevocation( + value: unknown, accountId?: string | null, -): string { - const resolvedStateDir = resolveStateDir(env); - return resolveSharedMatrixCredentialsPath({ stateDir: resolvedStateDir, accountId }); +): value is MatrixCredentialRevocationRecord { + if (!value || typeof value !== "object") { + return false; + } + const parsed = value as Partial; + return ( + parsed.kind === "revoked" && + typeof parsed.revokedAt === "string" && + parsed.revokedAt.length > 0 && + normalizeAccountId(parsed.accountId) === normalizeAccountId(accountId ?? parsed.accountId) + ); +} + +export function openMatrixCredentialsStore( + env: NodeJS.ProcessEnv = process.env, +): PluginStateSyncKeyedStore { + const runtime = getOptionalMatrixRuntime(); + const resolvedEnv = + env.OPENCLAW_STATE_DIR?.trim() || !runtime + ? env + : { ...env, OPENCLAW_STATE_DIR: runtime.state.resolveStateDir(env) }; + return createPluginStateSyncKeyedStore("matrix", { + namespace: MATRIX_CREDENTIALS_NAMESPACE, + maxEntries: MATRIX_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + env: resolvedEnv, + }); } export function loadMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): MatrixStoredCredentials | null { - const currentPath = resolveMatrixCredentialsPath(env, accountId); - try { - const current = loadMatrixCredentialsFile(currentPath, "current"); - if (current.kind === "loaded") { - return current.credentials; - } - - const legacyPath = resolveLegacyMigrationSourcePath(env, accountId); - if (!legacyPath) { - return null; - } - - const loaded = loadLegacyMatrixCredentialsWithCurrentFallback({ - legacyPath, - currentPath, - }); - if (loaded.kind !== "loaded" || !loaded.credentials) { - return null; - } - - if (loaded.source === "legacy") { - try { - fs.mkdirSync(path.dirname(currentPath), { recursive: true }); - fs.renameSync(legacyPath, currentPath); - } catch { - // Keep returning the legacy credentials even if migration fails. - } - } - - return loaded.credentials; - } catch { + const normalizedAccountId = normalizeAccountId(accountId); + const stored = openMatrixCredentialsStore(env).lookup(matrixCredentialsStoreKey(accountId)); + const parsed = normalizeMatrixStoredCredentials(stored, normalizedAccountId); + if (!parsed || parsed.accountId !== normalizedAccountId) { return null; } + const { accountId: _accountId, ...credentials } = parsed; + return credentials; } export function clearMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): void { - const paths = [ - resolveMatrixCredentialsPath(env, accountId), - resolveLegacyMigrationSourcePath(env, accountId), - ]; - for (const filePath of paths) { - if (!filePath) { - continue; - } - try { - fs.unlinkSync(filePath); - } catch { - // ignore - } - } + const normalizedAccountId = normalizeAccountId(accountId); + // Keep a durable revocation marker so doctor cannot resurrect explicitly + // cleared credentials from a legacy file left by an interrupted migration. + openMatrixCredentialsStore(env).register(matrixCredentialsStoreKey(normalizedAccountId), { + accountId: normalizedAccountId, + kind: "revoked", + revokedAt: new Date().toISOString(), + }); } export function credentialsMatchConfig( diff --git a/extensions/matrix/src/matrix/credentials.test.ts b/extensions/matrix/src/matrix/credentials.test.ts index edaf854b9d31..784a0360bd57 100644 --- a/extensions/matrix/src/matrix/credentials.test.ts +++ b/extensions/matrix/src/matrix/credentials.test.ts @@ -1,29 +1,21 @@ -// Matrix tests cover credentials plugin behavior. +// Matrix tests cover SQLite-backed credentials behavior. import fs from "node:fs"; -import fsPromises from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { hasAnyMatrixAuth } from "../../auth-presence.js"; import { installMatrixTestRuntime } from "../test-runtime.js"; +import { openMatrixCredentialsStore } from "./credentials-read.js"; import { + clearMatrixCredentials, credentialsMatchConfig, loadMatrixCredentials, - clearMatrixCredentials, - resolveMatrixCredentialsPath, saveBackfilledMatrixDeviceId, saveMatrixCredentials, touchMatrixCredentials, } from "./credentials.js"; -const DEFAULT_LEGACY_CREDENTIALS = { - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "legacy-token", - createdAt: "2026-03-01T10:00:00.000Z", -}; - -const EXPECTS_POSIX_PRIVATE_FILE_MODE = process.platform !== "win32"; - type MatrixCredentials = NonNullable>; function expectMatrixCredentials( @@ -37,43 +29,21 @@ function expectMatrixCredentials( } describe("matrix credentials storage", () => { - const tempDirs: string[] = []; + let stateDir = ""; - afterEach(() => { - vi.restoreAllMocks(); - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); - } + beforeEach(() => { + resetPluginStateStoreForTests(); + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-creds-")); + installMatrixTestRuntime({ stateDir }); }); - function setupStateDir( - cfg: Record = { - channels: { - matrix: {}, - }, - }, - ): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-creds-")); - tempDirs.push(dir); - installMatrixTestRuntime({ cfg, stateDir: dir }); - return dir; - } + afterEach(() => { + vi.useRealTimers(); + resetPluginStateStoreForTests(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }); - function setupLegacyCredentialsFile(params: { - cfg: Record; - accountId: string; - credentials?: Record; - }) { - const stateDir = setupStateDir(params.cfg); - const legacyPath = path.join(stateDir, "credentials", "matrix", "credentials.json"); - const currentPath = resolveMatrixCredentialsPath({}, params.accountId); - fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); - fs.writeFileSync(legacyPath, JSON.stringify(params.credentials ?? DEFAULT_LEGACY_CREDENTIALS)); - return { stateDir, legacyPath, currentPath }; - } - - it("writes credentials atomically with secure file permissions", async () => { - const stateDir = setupStateDir(); + it("roundtrips account-scoped credentials through shared plugin-state SQLite", async () => { await saveMatrixCredentials( { homeserver: "https://matrix.example.org", @@ -85,51 +55,60 @@ describe("matrix credentials storage", () => { "ops", ); - const credPath = resolveMatrixCredentialsPath({}, "ops"); - expect(fs.existsSync(credPath)).toBe(true); - expect(credPath).toBe(path.join(stateDir, "credentials", "matrix", "credentials-ops.json")); - const mode = fs.statSync(credPath).mode & 0o777; - if (EXPECTS_POSIX_PRIVATE_FILE_MODE) { - expect(mode).toBe(0o600); - } + expect(loadMatrixCredentials({}, "ops")).toMatchObject({ + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + deviceId: "DEVICE123", + }); + expect(loadMatrixCredentials({}, "default")).toBeNull(); + expect(fs.existsSync(path.join(stateDir, "state", "openclaw.sqlite"))).toBe(true); + expect(fs.existsSync(path.join(stateDir, "credentials", "matrix"))).toBe(false); }); it("touch updates lastUsedAt while preserving createdAt", async () => { - setupStateDir(); vi.useFakeTimers(); - try { - vi.setSystemTime(new Date("2026-03-01T10:00:00.000Z")); - await saveMatrixCredentials( - { - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "secret-token", - }, - {}, - "default", - ); - const initial = loadMatrixCredentials({}, "default"); - const initialCredentials = expectMatrixCredentials(initial); - - vi.setSystemTime(new Date("2026-03-01T10:05:00.000Z")); - await touchMatrixCredentials({}, "default"); - const touched = loadMatrixCredentials({}, "default"); - const touchedCredentials = expectMatrixCredentials(touched); - - expect(touchedCredentials.createdAt).toBe(initialCredentials.createdAt); - expect(touchedCredentials.lastUsedAt).toBe("2026-03-01T10:05:00.000Z"); - } finally { - vi.useRealTimers(); - } - }); - - it("backfill updates deviceId when credentials still match the same auth lineage", async () => { - setupStateDir(); + vi.setSystemTime(new Date("2026-03-01T10:00:00.000Z")); await saveMatrixCredentials( { homeserver: "https://matrix.example.org", userId: "@bot:example.org", - accessToken: "tok-123", + accessToken: "secret-token", + }, + {}, + "default", + ); + const initial = expectMatrixCredentials(loadMatrixCredentials({}, "default")); + + vi.setSystemTime(new Date("2026-03-01T10:05:00.000Z")); + await touchMatrixCredentials({}, "default"); + const touched = expectMatrixCredentials(loadMatrixCredentials({}, "default")); + + expect(touched.createdAt).toBe(initial.createdAt); + expect(touched.lastUsedAt).toBe("2026-03-01T10:05:00.000Z"); + }); + + it("omits an explicitly undefined device id from persisted credentials", async () => { + const credentials = { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + deviceId: undefined, + }; + + await saveMatrixCredentials(credentials, {}, "default"); + await expect(saveBackfilledMatrixDeviceId(credentials, {}, "ops")).resolves.toBe("saved"); + + expect(openMatrixCredentialsStore({}).lookup("account:default")).not.toHaveProperty("deviceId"); + expect(openMatrixCredentialsStore({}).lookup("account:ops")).not.toHaveProperty("deviceId"); + }); + + it("backfills a matching device id but preserves newer auth lineage", async () => { + await saveMatrixCredentials( + { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "tok-new", }, {}, "default", @@ -140,363 +119,138 @@ describe("matrix credentials storage", () => { { homeserver: "https://matrix.example.org", userId: "@bot:example.org", - accessToken: "tok-123", + accessToken: "tok-new", deviceId: "DEVICE123", }, {}, "default", ), ).resolves.toBe("saved"); - - const credentials = expectMatrixCredentials(loadMatrixCredentials({}, "default")); - expect(credentials.accessToken).toBe("tok-123"); - expect(credentials.deviceId).toBe("DEVICE123"); - }); - - it("backfill skips when newer credentials already changed the token", async () => { - setupStateDir(); - await saveMatrixCredentials( - { - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "tok-new", - deviceId: "DEVICE999", - }, - {}, - "default", - ); - await expect( saveBackfilledMatrixDeviceId( { homeserver: "https://matrix.example.org", userId: "@bot:example.org", accessToken: "tok-old", - deviceId: "DEVICE123", + deviceId: "STALE", }, {}, "default", ), ).resolves.toBe("skipped"); - const credentials = expectMatrixCredentials(loadMatrixCredentials({}, "default")); - expect(credentials.accessToken).toBe("tok-new"); - expect(credentials.deviceId).toBe("DEVICE999"); + expect(loadMatrixCredentials({}, "default")).toMatchObject({ + accessToken: "tok-new", + deviceId: "DEVICE123", + }); }); - it("serializes stale backfill writes behind newer credential saves", async () => { - setupStateDir(); + it("does not let delayed background writes undo credential revocation", async () => { await saveMatrixCredentials( { homeserver: "https://matrix.example.org", userId: "@bot:example.org", - accessToken: "tok-old", + accessToken: "secret-token", }, {}, "default", ); + clearMatrixCredentials({}, "default"); - let releaseFirstWrite: (() => void) | undefined; - let resolveFirstWriteStarted: (() => void) | undefined; - const firstWriteStarted = new Promise((resolve) => { - resolveFirstWriteStarted = resolve; - }); - const originalRename = fsPromises.rename.bind(fsPromises); - const renameSpy = vi - .spyOn(fsPromises, "rename") - .mockImplementation(async (...args: Parameters) => { - if (resolveFirstWriteStarted) { - resolveFirstWriteStarted(); - resolveFirstWriteStarted = undefined; - await new Promise((resolve) => { - releaseFirstWrite = resolve; - }); - } - await originalRename(...args); - }); - - try { - const staleBackfillPromise = saveBackfilledMatrixDeviceId( + await expect( + saveBackfilledMatrixDeviceId( { homeserver: "https://matrix.example.org", userId: "@bot:example.org", - accessToken: "tok-old", - deviceId: "DEVICE123", + accessToken: "secret-token", + deviceId: "STALE", }, {}, "default", - ); + ), + ).resolves.toBe("skipped"); + await touchMatrixCredentials({}, "default"); - await firstWriteStarted; - - const newerSavePromise = saveMatrixCredentials( - { - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "tok-new", - deviceId: "DEVICE999", - }, - {}, - "default", - ); - - releaseFirstWrite?.(); - await Promise.all([staleBackfillPromise, newerSavePromise]); - - const credentials = expectMatrixCredentials(loadMatrixCredentials({}, "default")); - expect(credentials.accessToken).toBe("tok-new"); - expect(credentials.deviceId).toBe("DEVICE999"); - } finally { - renameSpy.mockRestore(); - } - }); - - it("migrates legacy matrix credential files on read", () => { - const { legacyPath, currentPath } = setupLegacyCredentialsFile({ - cfg: { - channels: { - matrix: { - accounts: { - ops: {}, - }, - }, - }, - }, - accountId: "ops", + expect(loadMatrixCredentials({}, "default")).toBeNull(); + expect(openMatrixCredentialsStore({}).lookup("account:default")).toMatchObject({ + kind: "revoked", }); - - const loaded = loadMatrixCredentials({}, "ops"); - - expect(loaded?.accessToken).toBe("legacy-token"); - expect(fs.existsSync(legacyPath)).toBe(false); - expect(fs.existsSync(currentPath)).toBe(true); }); - it("returns migrated credentials when another process moves the legacy file mid-read", () => { - const { legacyPath, currentPath } = setupLegacyCredentialsFile({ - cfg: { - channels: { - matrix: { - accounts: { - ops: {}, - }, - }, - }, - }, - accountId: "ops", - }); - - const originalReadFileSync = fs.readFileSync.bind(fs); - let moved = false; - const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation((( - filePath: fs.PathOrFileDescriptor, - options?: Parameters[1], - ) => { - if (!moved && filePath === legacyPath) { - fs.renameSync(legacyPath, currentPath); - moved = true; - } - return originalReadFileSync(filePath, options as never); - }) as typeof fs.readFileSync); - try { - const loaded = loadMatrixCredentials({}, "ops"); - - expect(loaded?.accessToken).toBe("legacy-token"); - expect(moved).toBe(true); - expect(fs.existsSync(legacyPath)).toBe(false); - expect(fs.existsSync(currentPath)).toBe(true); - } finally { - readFileSpy.mockRestore(); - } - }); - - it("does not rename the legacy path after falling back to already-migrated current credentials", () => { - const { legacyPath, currentPath } = setupLegacyCredentialsFile({ - cfg: { - channels: { - matrix: { - accounts: { - ops: {}, - }, - }, - }, - }, - accountId: "ops", - }); - - const originalReadFileSync = fs.readFileSync.bind(fs); - const originalRenameSync = fs.renameSync.bind(fs); - const renameSpy = vi.spyOn(fs, "renameSync"); - let migrated = false; - const readFileSpy = vi.spyOn(fs, "readFileSync").mockImplementation((( - filePath: fs.PathOrFileDescriptor, - options?: Parameters[1], - ) => { - if (!migrated && filePath === legacyPath && fs.existsSync(legacyPath)) { - originalRenameSync(legacyPath, currentPath); - fs.writeFileSync( - currentPath, - JSON.stringify({ - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "current-token", - createdAt: "2026-03-01T10:00:00.000Z", - }), - ); - migrated = true; - try { - return originalReadFileSync(filePath, options as never); - } finally { - fs.writeFileSync( - legacyPath, - JSON.stringify({ - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "recreated-stale-legacy-token", - createdAt: "2026-03-01T10:00:00.000Z", - }), - ); - } - } - return originalReadFileSync(filePath, options as never); - }) as typeof fs.readFileSync); - - try { - const loaded = loadMatrixCredentials({}, "ops"); - - expect(loaded?.accessToken).toBe("current-token"); - expect(renameSpy).not.toHaveBeenCalled(); - const currentFile = JSON.parse(fs.readFileSync(currentPath, "utf8")) as { - accessToken?: unknown; - }; - const legacyFile = JSON.parse(fs.readFileSync(legacyPath, "utf8")) as { - accessToken?: unknown; - }; - expect(currentFile.accessToken).toBe("current-token"); - expect(legacyFile.accessToken).toBe("recreated-stale-legacy-token"); - } finally { - readFileSpy.mockRestore(); - renameSpy.mockRestore(); - } - }); - - it("does not migrate legacy default credentials during a non-selected account read", () => { - const { legacyPath, currentPath } = setupLegacyCredentialsFile({ - cfg: { - channels: { - matrix: { - defaultAccount: "default", - accounts: { - default: { - homeserver: "https://matrix.default.example.org", - accessToken: "default-token", - }, - ops: {}, - }, - }, - }, - }, - accountId: "ops", - credentials: { - homeserver: "https://matrix.default.example.org", - userId: "@default:example.org", - accessToken: "default-token", - createdAt: "2026-03-01T10:00:00.000Z", - }, - }); - - const loaded = loadMatrixCredentials({}, "ops"); - - expect(loaded).toBeNull(); - expect(fs.existsSync(legacyPath)).toBe(true); - expect(fs.existsSync(currentPath)).toBe(false); - }); - - it("migrates legacy credentials to the named account when top-level auth is only a shared default", () => { - const { legacyPath, currentPath } = setupLegacyCredentialsFile({ - cfg: { - channels: { - matrix: { - accessToken: "shared-token", - accounts: { - ops: { - homeserver: "https://matrix.example.org", - accessToken: "ops-token", - }, - }, - }, - }, - }, - accountId: "ops", - credentials: { + it("does not read or remove legacy credential files at runtime", () => { + const legacyPath = path.join(stateDir, "credentials", "matrix", "credentials.json"); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync( + legacyPath, + JSON.stringify({ homeserver: "https://matrix.example.org", - userId: "@ops:example.org", + userId: "@bot:example.org", accessToken: "legacy-token", createdAt: "2026-03-01T10:00:00.000Z", - }, - }); + }), + ); - const loaded = loadMatrixCredentials({}, "ops"); - - expect(loaded?.accessToken).toBe("legacy-token"); - expect(fs.existsSync(legacyPath)).toBe(false); - expect(fs.existsSync(currentPath)).toBe(true); + expect(loadMatrixCredentials({}, "default")).toBeNull(); + clearMatrixCredentials({}, "default"); + expect(fs.existsSync(legacyPath)).toBe(true); }); - it("clears both current and legacy credential paths", () => { - const stateDir = setupStateDir({ - channels: { - matrix: { - accounts: { - ops: {}, - }, - }, - }, - }); - const currentPath = resolveMatrixCredentialsPath({}, "ops"); - const legacyPath = path.join(stateDir, "credentials", "matrix", "credentials.json"); - fs.mkdirSync(path.dirname(currentPath), { recursive: true }); - fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); - fs.writeFileSync(currentPath, "{}"); - fs.writeFileSync(legacyPath, "{}"); + it("clears only the requested canonical account", async () => { + const credentials = { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "token", + }; + await saveMatrixCredentials(credentials, {}, "default"); + await saveMatrixCredentials(credentials, {}, "ops"); clearMatrixCredentials({}, "ops"); - expect(fs.existsSync(currentPath)).toBe(false); - expect(fs.existsSync(legacyPath)).toBe(false); + expect(loadMatrixCredentials({}, "ops")).toBeNull(); + expect(openMatrixCredentialsStore({}).lookup("account:ops")).toMatchObject({ + kind: "revoked", + accountId: "ops", + }); + expect(loadMatrixCredentials({}, "default")).not.toBeNull(); + }); + + it("reports persisted auth from SQLite for package-state probes", async () => { + const env = { OPENCLAW_STATE_DIR: stateDir }; + expect(hasAnyMatrixAuth({ cfg: {}, env })).toBe(false); + + await saveMatrixCredentials( + { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "token", + }, + env, + "default", + ); + + expect(hasAnyMatrixAuth({ cfg: {}, env })).toBe(true); }); it("requires a token match when userId is absent", () => { + const stored = { + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "tok-123", + createdAt: "2026-01-01T00:00:00.000Z", + }; expect( - credentialsMatchConfig( - { - homeserver: "https://matrix.example.org", - userId: "@old:example.org", - accessToken: "tok-old", - createdAt: "2026-01-01T00:00:00.000Z", - }, - { - homeserver: "https://matrix.example.org", - userId: "", - accessToken: "tok-new", - }, - ), + credentialsMatchConfig(stored, { + homeserver: stored.homeserver, + userId: "", + accessToken: "tok-new", + }), ).toBe(false); - expect( - credentialsMatchConfig( - { - homeserver: "https://matrix.example.org", - userId: "@bot:example.org", - accessToken: "tok-123", - createdAt: "2026-01-01T00:00:00.000Z", - }, - { - homeserver: "https://matrix.example.org", - userId: "", - accessToken: "tok-123", - }, - ), + credentialsMatchConfig(stored, { + homeserver: stored.homeserver, + userId: "", + accessToken: "tok-123", + }), ).toBe(true); }); }); diff --git a/extensions/matrix/src/matrix/credentials.ts b/extensions/matrix/src/matrix/credentials.ts index 832d0b4c4d44..f799a9bf08a9 100644 --- a/extensions/matrix/src/matrix/credentials.ts +++ b/extensions/matrix/src/matrix/credentials.ts @@ -1,8 +1,12 @@ // Matrix plugin module implements credentials behavior. -import { writeJsonFileAtomically } from "openclaw/plugin-sdk/json-store"; -import { createAsyncLock, type AsyncLock } from "./async-lock.js"; -import { loadMatrixCredentials, resolveMatrixCredentialsPath } from "./credentials-read.js"; -import type { MatrixStoredCredentials } from "./credentials-read.js"; +import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; +import { + isMatrixCredentialRevocation, + matrixCredentialsStoreKey, + normalizeMatrixStoredCredentials, + openMatrixCredentialsStore, +} from "./credentials-read.js"; +import type { MatrixStoredCredentialRecord, MatrixStoredCredentials } from "./credentials-read.js"; export { clearMatrixCredentials, @@ -13,29 +17,13 @@ export { } from "./credentials-read.js"; export type { MatrixStoredCredentials } from "./credentials-read.js"; -const credentialWriteLocks = new Map(); - -function withCredentialWriteLock(credPath: string, fn: () => Promise): Promise { - let withLock = credentialWriteLocks.get(credPath); - if (!withLock) { - withLock = createAsyncLock(); - credentialWriteLocks.set(credPath, withLock); +function requireCredentialStoreUpdate( + store: ReturnType, +): NonNullable["update"]> { + if (!store.update) { + throw new Error("Matrix credentials require atomic plugin-state updates"); } - return withLock(fn); -} - -async function writeMatrixCredentialsUnlocked(params: { - credPath: string; - credentials: Omit; - existing: MatrixStoredCredentials | null; -}): Promise { - const now = new Date().toISOString(); - const toSave: MatrixStoredCredentials = { - ...params.credentials, - createdAt: params.existing?.createdAt ?? now, - lastUsedAt: now, - }; - await writeJsonFileAtomically(params.credPath, toSave); + return store.update; } export async function saveMatrixCredentials( @@ -43,13 +31,20 @@ export async function saveMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): Promise { - const credPath = resolveMatrixCredentialsPath(env, accountId); - await withCredentialWriteLock(credPath, async () => { - await writeMatrixCredentialsUnlocked({ - credPath, - credentials, - existing: loadMatrixCredentials(env, accountId), - }); + const normalizedAccountId = normalizeAccountId(accountId); + const store = openMatrixCredentialsStore(env); + const now = new Date().toISOString(); + requireCredentialStoreUpdate(store)(matrixCredentialsStoreKey(normalizedAccountId), (current) => { + const existing = normalizeMatrixStoredCredentials(current, normalizedAccountId); + return { + accountId: normalizedAccountId, + homeserver: credentials.homeserver, + userId: credentials.userId, + accessToken: credentials.accessToken, + ...(typeof credentials.deviceId === "string" ? { deviceId: credentials.deviceId } : {}), + createdAt: existing?.createdAt ?? now, + lastUsedAt: now, + } satisfies MatrixStoredCredentialRecord; }); } @@ -58,39 +53,51 @@ export async function saveBackfilledMatrixDeviceId( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): Promise<"saved" | "skipped"> { - const credPath = resolveMatrixCredentialsPath(env, accountId); - return await withCredentialWriteLock(credPath, async () => { - const existing = loadMatrixCredentials(env, accountId); + const normalizedAccountId = normalizeAccountId(accountId); + const store = openMatrixCredentialsStore(env); + const now = new Date().toISOString(); + let result: "saved" | "skipped" = "saved"; + requireCredentialStoreUpdate(store)(matrixCredentialsStoreKey(normalizedAccountId), (current) => { + // A delayed login backfill must not resurrect credentials after logout. + if (isMatrixCredentialRevocation(current, normalizedAccountId)) { + result = "skipped"; + return current; + } + const existing = normalizeMatrixStoredCredentials(current, normalizedAccountId); if ( existing && (existing.homeserver !== credentials.homeserver || existing.userId !== credentials.userId || existing.accessToken !== credentials.accessToken) ) { - return "skipped"; + result = "skipped"; + return existing; } - - await writeMatrixCredentialsUnlocked({ - credPath, - credentials, - existing, - }); - return "saved"; + return { + accountId: normalizedAccountId, + homeserver: credentials.homeserver, + userId: credentials.userId, + accessToken: credentials.accessToken, + ...(typeof credentials.deviceId === "string" ? { deviceId: credentials.deviceId } : {}), + createdAt: existing?.createdAt ?? now, + lastUsedAt: now, + } satisfies MatrixStoredCredentialRecord; }); + return result; } export async function touchMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): Promise { - const credPath = resolveMatrixCredentialsPath(env, accountId); - await withCredentialWriteLock(credPath, async () => { - const existing = loadMatrixCredentials(env, accountId); - if (!existing) { - return; + const normalizedAccountId = normalizeAccountId(accountId); + const store = openMatrixCredentialsStore(env); + requireCredentialStoreUpdate(store)(matrixCredentialsStoreKey(normalizedAccountId), (current) => { + // A delayed activity touch must preserve an explicit logout tombstone. + if (isMatrixCredentialRevocation(current, normalizedAccountId)) { + return current; } - - existing.lastUsedAt = new Date().toISOString(); - await writeJsonFileAtomically(credPath, existing); + const existing = normalizeMatrixStoredCredentials(current, normalizedAccountId); + return existing ? { ...existing, lastUsedAt: new Date().toISOString() } : undefined; }); } diff --git a/extensions/msteams/doctor-contract-api.test.ts b/extensions/msteams/doctor-contract-api.test.ts index f5a9a7312745..c122f231e6f3 100644 --- a/extensions/msteams/doctor-contract-api.test.ts +++ b/extensions/msteams/doctor-contract-api.test.ts @@ -23,6 +23,12 @@ import { type MSTeamsLegacyConversationStoreData, } from "./src/conversation-store-state.js"; import type { StoredConversationReference } from "./src/conversation-store.js"; +import { + MSTEAMS_DELEGATED_TOKEN_KEY, + MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES, + MSTEAMS_DELEGATED_TOKEN_NAMESPACE, +} from "./src/delegated-state.js"; +import type { MSTeamsDelegatedTokens } from "./src/oauth.shared.js"; import { buildMSTeamsPollStateKey, buildMSTeamsPollVoteBucketKey, @@ -255,6 +261,47 @@ describe("msteams doctor state migration", () => { await expect(fs.access(`${filePath}.migrated`)).resolves.toBeUndefined(); }); + it("imports delegated OAuth tokens into plugin state before archiving the file", async () => { + const filePath = path.join(stateDir, "msteams-delegated.json"); + const token: MSTeamsDelegatedTokens = { + accessToken: "delegated-access", + refreshToken: "delegated-refresh", + expiresAt: 1_800_000_000_000, + scopes: ["User.Read", "offline_access"], + userPrincipalName: "user@example.com", + }; + await fs.writeFile(filePath, JSON.stringify(token)); + const migration = migrationById("msteams-delegated-token-json-to-plugin-state"); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: [ + `- Microsoft Teams delegated OAuth token -> plugin state (${MSTEAMS_DELEGATED_TOKEN_NAMESPACE})`, + ], + }); + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Microsoft Teams delegated OAuth token -> plugin state", + expect.stringContaining("Archived Microsoft Teams delegated OAuth token legacy source"), + ]); + const store = context.openPluginStateKeyedStore({ + namespace: MSTEAMS_DELEGATED_TOKEN_NAMESPACE, + maxEntries: MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(store.lookup(MSTEAMS_DELEGATED_TOKEN_KEY)).resolves.toEqual(token); + await expect(fs.access(`${filePath}.migrated`)).resolves.toBeUndefined(); + }); + it("does not register a doctor migration for pending-upload cache files", () => { expect(stateMigrations.map((migration) => migration.id)).not.toContain( "msteams-pending-uploads-json-to-plugin-state", diff --git a/extensions/msteams/doctor-contract-api.ts b/extensions/msteams/doctor-contract-api.ts index d339e5b994c5..5eac8eb17d13 100644 --- a/extensions/msteams/doctor-contract-api.ts +++ b/extensions/msteams/doctor-contract-api.ts @@ -27,6 +27,14 @@ import { type MSTeamsLegacyConversationStoreData, } from "./src/conversation-store-state.js"; import type { StoredConversationReference } from "./src/conversation-store.js"; +import { + MSTEAMS_DELEGATED_TOKEN_KEY, + MSTEAMS_DELEGATED_TOKEN_LEGACY_FILENAME, + MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES, + MSTEAMS_DELEGATED_TOKEN_NAMESPACE, + normalizeMSTeamsDelegatedTokens, +} from "./src/delegated-state.js"; +import type { MSTeamsDelegatedTokens } from "./src/oauth.shared.js"; import { buildMSTeamsPollStateKey, buildMSTeamsPollVoteBucketKey, @@ -492,6 +500,95 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ return { changes, warnings }; }, }, + { + id: "msteams-delegated-token-json-to-plugin-state", + label: "Microsoft Teams delegated OAuth token", + async detectLegacyState(params) { + const filePath = resolveStateFilePath( + params.stateDir, + MSTEAMS_DELEGATED_TOKEN_LEGACY_FILENAME, + ); + try { + const stat = await fs.stat(filePath); + return stat.isFile() + ? { + preview: [ + `- ${MSTEAMS_PLUGIN_ID} delegated OAuth token -> plugin state (${MSTEAMS_DELEGATED_TOKEN_NAMESPACE})`, + ], + } + : null; + } catch { + return null; + } + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const filePath = resolveStateFilePath( + params.stateDir, + MSTEAMS_DELEGATED_TOKEN_LEGACY_FILENAME, + ); + let token: MSTeamsDelegatedTokens | null; + try { + token = normalizeMSTeamsDelegatedTokens( + JSON.parse(await fs.readFile(filePath, "utf8")) as unknown, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { changes, warnings }; + } + warnings.push( + `Failed reading ${MSTEAMS_PLUGIN_ID} delegated OAuth token legacy source; left it in place`, + ); + return { changes, warnings }; + } + if (!token) { + warnings.push( + `Invalid ${MSTEAMS_PLUGIN_ID} delegated OAuth token legacy source; left it in place`, + ); + return { changes, warnings }; + } + const store = params.context.openPluginStateKeyedStore({ + namespace: MSTEAMS_DELEGATED_TOKEN_NAMESPACE, + maxEntries: MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const existing = await store.lookup(MSTEAMS_DELEGATED_TOKEN_KEY); + if (existing && JSON.stringify(existing) !== JSON.stringify(token)) { + warnings.push( + `Kept existing ${MSTEAMS_PLUGIN_ID} delegated OAuth token in plugin state; left differing legacy source in place`, + ); + return { changes, warnings }; + } + if (!existing) { + try { + await store.registerIfAbsent(MSTEAMS_DELEGATED_TOKEN_KEY, token); + } catch (error) { + warnings.push( + `Failed importing ${MSTEAMS_PLUGIN_ID} delegated OAuth token: ${String(error)}; left legacy source in place`, + ); + return { changes, warnings }; + } + } + const persisted = normalizeMSTeamsDelegatedTokens( + await store.lookup(MSTEAMS_DELEGATED_TOKEN_KEY), + ); + if (!persisted || JSON.stringify(persisted) !== JSON.stringify(token)) { + warnings.push( + `Failed verifying ${MSTEAMS_PLUGIN_ID} delegated OAuth token in plugin state; left legacy source in place`, + ); + return { changes, warnings }; + } + changes.push(`Migrated ${MSTEAMS_PLUGIN_ID} delegated OAuth token -> plugin state`); + await archiveLegacyStateSource({ + filePath, + label: `${MSTEAMS_PLUGIN_ID} delegated OAuth token`, + changes, + warnings, + }); + return { changes, warnings }; + }, + }, { id: "msteams-feedback-learnings-json-to-plugin-state", label: "Microsoft Teams feedback learnings", diff --git a/extensions/msteams/src/conversation-store-state.ts b/extensions/msteams/src/conversation-store-state.ts index 63a7bddeab1f..c6a5a093d074 100644 --- a/extensions/msteams/src/conversation-store-state.ts +++ b/extensions/msteams/src/conversation-store-state.ts @@ -29,7 +29,7 @@ export const MSTEAMS_CONVERSATIONS_NAMESPACE = "conversations"; const MSTEAMS_MAX_CONVERSATIONS = 1000; export const MSTEAMS_SQLITE_MAX_CONVERSATION_ROWS = MSTEAMS_MAX_CONVERSATIONS + 1000; const MSTEAMS_CONVERSATION_TTL_MS = 365 * 24 * 60 * 60 * 1000; -const CONVERSATION_LOCK_FILENAME = "msteams-conversations.sqlite.lock"; +const CONVERSATION_MUTATION_KEY = "conversations"; type MSTeamsConversationStoreStateOptions = { env?: NodeJS.ProcessEnv; @@ -198,7 +198,7 @@ export function createMSTeamsConversationStoreState( reference: StoredConversationReference, ): Promise => { const normalizedId = normalizeStoredConversationId(conversationId); - await withMSTeamsSqliteMutationLock(params, CONVERSATION_LOCK_FILENAME, async () => { + await withMSTeamsSqliteMutationLock(params, CONVERSATION_MUTATION_KEY, async () => { const existing = await lookupStored(normalizedId); await register( normalizedId, @@ -213,7 +213,7 @@ export function createMSTeamsConversationStoreState( const remove = async (conversationId: string): Promise => { const normalizedId = normalizeStoredConversationId(conversationId); - return await withMSTeamsSqliteMutationLock(params, CONVERSATION_LOCK_FILENAME, async () => { + return await withMSTeamsSqliteMutationLock(params, CONVERSATION_MUTATION_KEY, async () => { return await conversationStore.delete(buildMSTeamsConversationStateKey(normalizedId)); }); }; diff --git a/extensions/msteams/src/delegated-state.ts b/extensions/msteams/src/delegated-state.ts new file mode 100644 index 000000000000..f03185b0108b --- /dev/null +++ b/extensions/msteams/src/delegated-state.ts @@ -0,0 +1,65 @@ +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import type { MSTeamsDelegatedTokens } from "./oauth.shared.js"; +import { getMSTeamsRuntime } from "./runtime.js"; + +export const MSTEAMS_DELEGATED_TOKEN_LEGACY_FILENAME = "msteams-delegated.json"; +export const MSTEAMS_DELEGATED_TOKEN_NAMESPACE = "delegated-token"; +export const MSTEAMS_DELEGATED_TOKEN_KEY = "current"; +export const MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES = 1; + +function openDelegatedTokenStore( + env?: NodeJS.ProcessEnv, +): PluginStateSyncKeyedStore { + return getMSTeamsRuntime().state.openSyncKeyedStore({ + namespace: MSTEAMS_DELEGATED_TOKEN_NAMESPACE, + maxEntries: MSTEAMS_DELEGATED_TOKEN_MAX_ENTRIES, + overflowPolicy: "reject-new", + ...(env ? { env } : {}), + }); +} + +export function normalizeMSTeamsDelegatedTokens(value: unknown): MSTeamsDelegatedTokens | null { + if (!value || typeof value !== "object") { + return null; + } + const token = value as Partial; + if ( + typeof token.accessToken !== "string" || + !token.accessToken || + typeof token.refreshToken !== "string" || + !token.refreshToken || + typeof token.expiresAt !== "number" || + !Number.isFinite(token.expiresAt) || + !Array.isArray(token.scopes) || + !token.scopes.every((scope) => typeof scope === "string" && scope.length > 0) + ) { + return null; + } + return { + accessToken: token.accessToken, + refreshToken: token.refreshToken, + expiresAt: token.expiresAt, + scopes: [...token.scopes], + ...(typeof token.userPrincipalName === "string" + ? { userPrincipalName: token.userPrincipalName } + : {}), + }; +} + +export function loadMSTeamsDelegatedTokens( + env?: NodeJS.ProcessEnv, +): MSTeamsDelegatedTokens | undefined { + const stored = openDelegatedTokenStore(env).lookup(MSTEAMS_DELEGATED_TOKEN_KEY); + return normalizeMSTeamsDelegatedTokens(stored) ?? undefined; +} + +export function saveMSTeamsDelegatedTokens( + tokens: MSTeamsDelegatedTokens, + env?: NodeJS.ProcessEnv, +): void { + const normalized = normalizeMSTeamsDelegatedTokens(tokens); + if (!normalized) { + throw new Error("Invalid Microsoft Teams delegated token payload"); + } + openDelegatedTokenStore(env).register(MSTEAMS_DELEGATED_TOKEN_KEY, normalized); +} diff --git a/extensions/msteams/src/pending-uploads-fs.ts b/extensions/msteams/src/pending-uploads-fs.ts index b82e91724b46..a1011439d6ea 100644 --- a/extensions/msteams/src/pending-uploads-fs.ts +++ b/extensions/msteams/src/pending-uploads-fs.ts @@ -20,7 +20,7 @@ const PENDING_UPLOAD_META_MAX_ENTRIES = MAX_PENDING_UPLOADS + 100; const PENDING_UPLOAD_META_NAMESPACE = "pending-uploads"; const PENDING_UPLOAD_CHUNKS_NAMESPACE = "pending-upload-chunks"; -const PENDING_UPLOAD_LOCK_FILENAME = "msteams-pending-uploads.sqlite.lock"; +const PENDING_UPLOAD_MUTATION_KEY = "pending-uploads"; type PendingUploadFsRecord = { id: string; @@ -175,7 +175,7 @@ async function withPendingUploadLock( options: PendingUploadsFsOptions | undefined, run: () => Promise, ): Promise { - return await withMSTeamsSqliteMutationLock(options, PENDING_UPLOAD_LOCK_FILENAME, run); + return await withMSTeamsSqliteMutationLock(options, PENDING_UPLOAD_MUTATION_KEY, run); } async function readUploadRows( diff --git a/extensions/msteams/src/polls.ts b/extensions/msteams/src/polls.ts index b6cf0d7106a4..55d7177cba4d 100644 --- a/extensions/msteams/src/polls.ts +++ b/extensions/msteams/src/polls.ts @@ -74,7 +74,7 @@ const MSTEAMS_POLL_VOTE_BUCKET_COUNT = 32; export const MSTEAMS_MAX_POLL_VOTE_BUCKET_ROWS = (MSTEAMS_MAX_POLLS + 1) * MSTEAMS_POLL_VOTE_BUCKET_COUNT; const MSTEAMS_POLL_TTL_MS = 30 * 24 * 60 * 60 * 1000; -const POLL_LOCK_FILENAME = "msteams-polls.sqlite.lock"; +const POLL_MUTATION_KEY = "polls"; function normalizeChoiceValue(value: unknown): string | null { if (typeof value === "string") { @@ -432,7 +432,7 @@ export function createMSTeamsPollStoreState( }; const createPoll = async (poll: MSTeamsPoll) => { - await withMSTeamsSqliteMutationLock(params, POLL_LOCK_FILENAME, async () => { + await withMSTeamsSqliteMutationLock(params, POLL_MUTATION_KEY, async () => { const { metadata, votes } = splitMSTeamsPoll(poll); await pollStore.register(buildMSTeamsPollStateKey(poll.id), toPluginJsonValue(metadata)); await deletePollVotes(poll.id); @@ -453,7 +453,7 @@ export function createMSTeamsPollStoreState( }; const recordVote = async (vote: { pollId: string; voterId: string; selections: string[] }) => { - return await withMSTeamsSqliteMutationLock(params, POLL_LOCK_FILENAME, async () => { + return await withMSTeamsSqliteMutationLock(params, POLL_MUTATION_KEY, async () => { const pollKey = buildMSTeamsPollStateKey(vote.pollId); const poll = await pollStore.lookup(pollKey); if (!poll) { diff --git a/extensions/msteams/src/sqlite-state.test.ts b/extensions/msteams/src/sqlite-state.test.ts new file mode 100644 index 000000000000..d7ac1a1bb2ac --- /dev/null +++ b/extensions/msteams/src/sqlite-state.test.ts @@ -0,0 +1,47 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { setMSTeamsRuntime } from "./runtime.js"; +import { withMSTeamsSqliteMutationLock } from "./sqlite-state.js"; +import { msteamsRuntimeStub } from "./test-support/runtime.js"; + +describe("MSTeams SQLite mutation lock", () => { + let stateDir = ""; + + beforeEach(() => { + setMSTeamsRuntime(msteamsRuntimeStub); + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-msteams-lock-")); + }); + + afterEach(() => { + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + it("serializes concurrent mutations for the same state file", async () => { + let releaseFirst: (() => void) | undefined; + const firstEntered = vi.fn(); + const secondEntered = vi.fn(); + const first = withMSTeamsSqliteMutationLock({ stateDir }, "polls", async () => { + firstEntered(); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + return "first"; + }); + await vi.waitFor(() => expect(firstEntered).toHaveBeenCalledOnce()); + const second = withMSTeamsSqliteMutationLock({ stateDir }, "polls", async () => { + secondEntered(); + return "second"; + }); + await new Promise((resolve) => { + setTimeout(resolve, 25); + }); + expect(secondEntered).not.toHaveBeenCalled(); + + releaseFirst?.(); + await expect(first).resolves.toBe("first"); + await expect(second).resolves.toBe("second"); + expect(secondEntered).toHaveBeenCalledOnce(); + }); +}); diff --git a/extensions/msteams/src/sqlite-state.ts b/extensions/msteams/src/sqlite-state.ts index 38f0327857a0..7422b11019db 100644 --- a/extensions/msteams/src/sqlite-state.ts +++ b/extensions/msteams/src/sqlite-state.ts @@ -1,8 +1,8 @@ // Msteams plugin module implements sqlite state behavior. import path from "node:path"; +import { withFileLock } from "openclaw/plugin-sdk/file-lock"; import { KeyedAsyncQueue } from "openclaw/plugin-sdk/keyed-async-queue"; import { getMSTeamsRuntime } from "./runtime.js"; -import { withFileLock } from "./store-fs.js"; type MSTeamsSqliteStateOptions = { env?: NodeJS.ProcessEnv; @@ -55,6 +55,16 @@ function resolveMSTeamsSqliteStateDir(options: MSTeamsSqliteStateOptions | undef } const sqliteMutationLocks = new KeyedAsyncQueue(); +const MSTEAMS_MUTATION_LOCK_OPTIONS = { + retries: { + retries: 10, + factor: 2, + minTimeout: 100, + maxTimeout: 10_000, + randomize: true, + }, + stale: 30_000, +} as const; async function withProcessMutationLock(lockPath: string, fn: () => Promise): Promise { return await sqliteMutationLocks.enqueue(lockPath, fn); @@ -62,11 +72,11 @@ async function withProcessMutationLock(lockPath: string, fn: () => Promise export async function withMSTeamsSqliteMutationLock( options: MSTeamsSqliteStateOptions | undefined, - lockFilename: string, + mutationKey: string, fn: () => Promise, ): Promise { - const lockPath = path.join(resolveMSTeamsSqliteStateDir(options), lockFilename); - return await withProcessMutationLock(lockPath, async () => { - return await withFileLock(lockPath, { version: 1 }, fn); + const scopedMutationKey = path.join(resolveMSTeamsSqliteStateDir(options), mutationKey); + return await withProcessMutationLock(scopedMutationKey, async () => { + return await withFileLock(scopedMutationKey, MSTEAMS_MUTATION_LOCK_OPTIONS, fn); }); } diff --git a/extensions/msteams/src/sso-token-store.ts b/extensions/msteams/src/sso-token-store.ts index a436691ab732..b0e1ed99eedb 100644 --- a/extensions/msteams/src/sso-token-store.ts +++ b/extensions/msteams/src/sso-token-store.ts @@ -49,7 +49,7 @@ type MSTeamsSsoStoreData = SsoStoreData; export const MSTEAMS_SSO_TOKENS_LEGACY_FILENAME = "msteams-sso-tokens.json"; export const MSTEAMS_SSO_TOKENS_NAMESPACE = "sso-tokens"; -const SSO_TOKEN_LOCK_FILENAME = "msteams-sso-tokens.sqlite.lock"; +const SSO_TOKEN_MUTATION_KEY = "sso-tokens"; export const MSTEAMS_MAX_SSO_TOKENS = 5000; const STORE_KEY_VERSION_PREFIX = "v2:"; @@ -120,7 +120,7 @@ export function createMSTeamsSsoTokenStoreFs(params?: { }, async save(token) { - await withMSTeamsSqliteMutationLock(params, SSO_TOKEN_LOCK_FILENAME, async () => { + await withMSTeamsSqliteMutationLock(params, SSO_TOKEN_MUTATION_KEY, async () => { await tokenStore.register( makeMSTeamsSsoTokenStoreKey(token.connectionName, token.userId), toPluginJsonValue({ ...token }), @@ -130,7 +130,7 @@ export function createMSTeamsSsoTokenStoreFs(params?: { async remove({ connectionName, userId }) { let removed = false; - await withMSTeamsSqliteMutationLock(params, SSO_TOKEN_LOCK_FILENAME, async () => { + await withMSTeamsSqliteMutationLock(params, SSO_TOKEN_MUTATION_KEY, async () => { removed = await tokenStore.delete(makeMSTeamsSsoTokenStoreKey(connectionName, userId)); }); return removed; diff --git a/extensions/msteams/src/storage.ts b/extensions/msteams/src/storage.ts deleted file mode 100644 index 09b79434b654..000000000000 --- a/extensions/msteams/src/storage.ts +++ /dev/null @@ -1,26 +0,0 @@ -// Msteams plugin module implements storage behavior. -import path from "node:path"; -import { getMSTeamsRuntime } from "./runtime.js"; - -type MSTeamsStorePathOptions = { - env?: NodeJS.ProcessEnv; - homedir?: () => string; - stateDir?: string; - storePath?: string; - filename: string; -}; - -export function resolveMSTeamsStorePath(params: MSTeamsStorePathOptions): string { - if (params.storePath) { - return params.storePath; - } - if (params.stateDir) { - return path.join(params.stateDir, params.filename); - } - - const env = params.env ?? process.env; - const stateDir = params.homedir - ? getMSTeamsRuntime().state.resolveStateDir(env, params.homedir) - : getMSTeamsRuntime().state.resolveStateDir(env); - return path.join(stateDir, params.filename); -} diff --git a/extensions/msteams/src/store-fs.ts b/extensions/msteams/src/store-fs.ts deleted file mode 100644 index a81c86088eec..000000000000 --- a/extensions/msteams/src/store-fs.ts +++ /dev/null @@ -1,36 +0,0 @@ -// Msteams plugin module implements store fs behavior. -import { withFileLock as withPathLock } from "openclaw/plugin-sdk/file-lock"; -import { writeJsonFileAtomically } from "openclaw/plugin-sdk/json-store"; -import { pathExists } from "openclaw/plugin-sdk/security-runtime"; - -const STORE_LOCK_OPTIONS = { - retries: { - retries: 10, - factor: 2, - minTimeout: 100, - maxTimeout: 10_000, - randomize: true, - }, - stale: 30_000, -} as const; - -async function writeJsonFile(filePath: string, value: unknown): Promise { - await writeJsonFileAtomically(filePath, value); -} - -async function ensureJsonFile(filePath: string, fallback: unknown) { - if (!(await pathExists(filePath))) { - await writeJsonFile(filePath, fallback); - } -} - -export async function withFileLock( - filePath: string, - fallback: unknown, - fn: () => Promise, -): Promise { - await ensureJsonFile(filePath, fallback); - return await withPathLock(filePath, STORE_LOCK_OPTIONS, async () => { - return await fn(); - }); -} diff --git a/extensions/msteams/src/test-support/runtime.ts b/extensions/msteams/src/test-support/runtime.ts index 6607634c57c2..52bbb658a3a4 100644 --- a/extensions/msteams/src/test-support/runtime.ts +++ b/extensions/msteams/src/test-support/runtime.ts @@ -2,13 +2,18 @@ import os from "node:os"; import path from "node:path"; import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; -import { createPluginStateKeyedStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { + createPluginStateKeyedStoreForTests, + createPluginStateSyncKeyedStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; import type { PluginRuntime } from "../../runtime-api.js"; export const msteamsRuntimeStub = { state: { openKeyedStore: (options: OpenKeyedStoreOptions) => createPluginStateKeyedStoreForTests("msteams", options), + openSyncKeyedStore: (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("msteams", options), resolveStateDir: (env: NodeJS.ProcessEnv = process.env, homedir?: () => string) => { const override = env.OPENCLAW_STATE_DIR?.trim() || env.OPENCLAW_STATE_DIR?.trim(); if (override) { diff --git a/extensions/msteams/src/token.test.ts b/extensions/msteams/src/token.test.ts index cff5f2455753..6b9537f36c40 100644 --- a/extensions/msteams/src/token.test.ts +++ b/extensions/msteams/src/token.test.ts @@ -1,14 +1,19 @@ // Msteams tests cover token plugin behavior. -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { MSTeamsConfig } from "../runtime-api.js"; +import { setMSTeamsRuntime } from "./runtime.js"; +import { msteamsRuntimeStub } from "./test-support/runtime.js"; import { readAccessToken } from "./token-response.js"; import { hasConfiguredMSTeamsCredentials, + loadDelegatedTokens, resolveDelegatedAccessToken, resolveMSTeamsCredentials, + saveDelegatedTokens, } from "./token.js"; const oauthTokenMocks = vi.hoisted(() => ({ @@ -19,16 +24,6 @@ vi.mock("./oauth.token.js", () => ({ refreshMSTeamsDelegatedTokens: oauthTokenMocks.refreshMSTeamsDelegatedTokens, })); -vi.mock("./storage.js", () => ({ - resolveMSTeamsStorePath: ({ filename }: { filename: string }) => { - const stateDir = process.env.OPENCLAW_STATE_DIR; - if (!stateDir) { - throw new Error("OPENCLAW_STATE_DIR is required for token tests"); - } - return `${stateDir}/${filename}`; - }, -})); - vi.mock("./secret-input.js", () => ({ normalizeSecretInputString: (v: unknown) => typeof v === "string" && v.trim() ? v.trim() : undefined, @@ -300,6 +295,8 @@ describe("resolveDelegatedAccessToken", () => { let stateDir: string | undefined; beforeEach(() => { + resetPluginStateStoreForTests(); + setMSTeamsRuntime(msteamsRuntimeStub); saveAndClearEnv(); stateDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-msteams-token-")); process.env.OPENCLAW_STATE_DIR = stateDir; @@ -308,6 +305,7 @@ describe("resolveDelegatedAccessToken", () => { afterEach(() => { restoreEnv(); + resetPluginStateStoreForTests(); if (stateDir) { rmSync(stateDir, { recursive: true, force: true }); stateDir = undefined; @@ -318,18 +316,25 @@ describe("resolveDelegatedAccessToken", () => { if (!stateDir) { throw new Error("missing stateDir"); } - writeFileSync( - path.join(stateDir, "msteams-delegated.json"), - `${JSON.stringify({ - accessToken: "stale-access", - refreshToken: "refresh-token", - expiresAt, - scopes: ["User.Read"], - })}\n`, - "utf8", - ); + saveDelegatedTokens({ + accessToken: "stale-access", + refreshToken: "refresh-token", + expiresAt, + scopes: ["User.Read"], + }); } + it("roundtrips delegated tokens through plugin-state SQLite without a sidecar", () => { + writeDelegatedTokens(Date.now() + 60_000); + + expect(loadDelegatedTokens()).toMatchObject({ + accessToken: "stale-access", + refreshToken: "refresh-token", + }); + expect(existsSync(path.join(stateDir!, "state", "openclaw.sqlite"))).toBe(true); + expect(existsSync(path.join(stateDir!, "msteams-delegated.json"))).toBe(false); + }); + it("reuses a valid delegated access token before expiry", async () => { writeDelegatedTokens(Date.now() + 60_000); diff --git a/extensions/msteams/src/token.ts b/extensions/msteams/src/token.ts index 19543a06b9b0..03d977fa8a36 100644 --- a/extensions/msteams/src/token.ts +++ b/extensions/msteams/src/token.ts @@ -1,9 +1,7 @@ // Msteams plugin module implements token behavior. -import { readFileSync } from "node:fs"; -import { basename, dirname } from "node:path"; import { isFutureDateTimestampMs } from "openclaw/plugin-sdk/number-runtime"; -import { privateFileStoreSync } from "openclaw/plugin-sdk/security-runtime"; import type { MSTeamsConfig } from "../runtime-api.js"; +import { loadMSTeamsDelegatedTokens, saveMSTeamsDelegatedTokens } from "./delegated-state.js"; import type { MSTeamsDelegatedTokens } from "./oauth.shared.js"; import { refreshMSTeamsDelegatedTokens } from "./oauth.token.js"; import { @@ -11,7 +9,6 @@ import { normalizeResolvedSecretInputString, normalizeSecretInputString, } from "./secret-input.js"; -import { resolveMSTeamsStorePath } from "./storage.js"; // ── Credential types ─────────────────────────────────────────────────────── @@ -144,24 +141,12 @@ export function resolveMSTeamsCredentials(cfg?: MSTeamsConfig): MSTeamsCredentia // Delegated token storage / resolution // --------------------------------------------------------------------------- -const DELEGATED_TOKEN_FILENAME = "msteams-delegated.json"; - -function resolveDelegatedTokenPath(): string { - return resolveMSTeamsStorePath({ filename: DELEGATED_TOKEN_FILENAME }); -} - export function loadDelegatedTokens(): MSTeamsDelegatedTokens | undefined { - try { - const content = readFileSync(resolveDelegatedTokenPath(), "utf8"); - return JSON.parse(content) as MSTeamsDelegatedTokens; - } catch { - return undefined; - } + return loadMSTeamsDelegatedTokens(); } export function saveDelegatedTokens(tokens: MSTeamsDelegatedTokens): void { - const tokenPath = resolveDelegatedTokenPath(); - privateFileStoreSync(dirname(tokenPath)).writeJson(basename(tokenPath), tokens); + saveMSTeamsDelegatedTokens(tokens); } export async function resolveDelegatedAccessToken(params: { diff --git a/extensions/reef/doctor-contract-api.test.ts b/extensions/reef/doctor-contract-api.test.ts index bd803e2b443a..01446951f78a 100644 --- a/extensions/reef/doctor-contract-api.test.ts +++ b/extensions/reef/doctor-contract-api.test.ts @@ -4,8 +4,10 @@ import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { createPluginStateKeyedStoreForTests, + createPluginStateSyncKeyedStoreForTests, resetPluginStateStoreForTests, } from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; import type { OpenKeyedStoreOptions, PluginDoctorStateMigrationContext, @@ -16,13 +18,57 @@ import { normalizeCompatibilityConfig, stateMigrations, } from "./doctor-contract-api.js"; -import { generateIdentity } from "./protocol/index.js"; +import { + base64url, + generateIdentity, + MemoryAuditStore, + type ReviewRequest, +} from "./protocol/index.js"; import { ReefChannelConfigSchema } from "./src/config-schema.js"; +import { + generateAndStoreKeys, + loadKeys, + openStores, + REEF_AUDIT_MIGRATION_KEY, + REEF_AUDIT_MIGRATION_MAX_ENTRIES, + REEF_AUDIT_MIGRATION_NAMESPACE, + REEF_AUDIT_HEAD_MAX_ENTRIES, + REEF_AUDIT_HEAD_NAMESPACE, + REEF_AUDIT_HEAD_KEY, + REEF_AUDIT_NAMESPACE, + REEF_AUDIT_STORE_MAX_ENTRIES, + REEF_KEYS_KEY, + REEF_KEYS_MAX_ENTRIES, + REEF_KEYS_MIGRATION_KEY, + REEF_KEYS_MIGRATION_MAX_ENTRIES, + REEF_KEYS_MIGRATION_NAMESPACE, + REEF_KEYS_NAMESPACE, + REEF_DELIVERED_MAX_ENTRIES, + REEF_DELIVERED_NAMESPACE, + REEF_DELIVERED_TTL_MS, + REEF_REPLAY_MAX_ENTRIES, + REEF_REPLAY_NAMESPACE, + REEF_REPLAY_TTL_MS, + REEF_REGISTRATION_IDENTITY_KEY, + REEF_REGISTRATION_MAX_ENTRIES, + REEF_REGISTRATION_NAMESPACE, + REEF_REVIEWS_MAX_ENTRIES, + REEF_REVIEWS_NAMESPACE, + reefAuditEntryKey, + reefReplayStoreKey, + type ReefAuditHeadRecord, + type ReefAuditStateRecord, + type ReefIdentityBinding, + type ReefIdentityMigrationRecord, + type ReefReplayRecord, + type ReefReviewRecord, +} from "./src/state.js"; import { REEF_TRUST_STORE_MAX_ENTRIES, REEF_TRUST_STORE_NAMESPACE, resolveReefTrustStoreKey, } from "./src/trust-store.js"; +import type { ReefKeys } from "./src/types.js"; function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigrationContext { return { @@ -35,6 +81,33 @@ function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigration }; } +function migrationById(id: string) { + const migration = stateMigrations.find((entry) => entry.id === id); + if (!migration) { + throw new Error(`missing migration ${id}`); + } + return migration; +} + +function createRuntime(env: NodeJS.ProcessEnv) { + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("reef", { + ...options, + env: options.env ?? env, + }); + return runtime; +} + +function reefKeys(): ReefKeys { + return { + ...generateIdentity(), + auditKey: base64url(new Uint8Array(32).fill(1)), + replayKey: base64url(new Uint8Array(32).fill(2)), + keyEpoch: 1, + }; +} + function legacyConfig(): OpenClawConfig { const identity = generateIdentity(); return { @@ -67,10 +140,12 @@ describe("Reef doctor contract", () => { beforeEach(() => { resetPluginStateStoreForTests(); stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-doctor-")); + vi.spyOn(os, "homedir").mockReturnValue(stateDir); env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; }); afterEach(() => { + vi.restoreAllMocks(); resetPluginStateStoreForTests(); fs.rmSync(stateDir, { recursive: true, force: true }); }); @@ -94,9 +169,589 @@ describe("Reef doctor contract", () => { }); }); + it("imports identity keys into SQLite before archiving keys.json", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const filePath = path.join(legacyDir, "keys.json"); + const keys = reefKeys(); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(filePath, JSON.stringify(keys)); + const migration = migrationById("reef-keys-json-to-plugin-state"); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: ["- Reef identity keys -> plugin state (identity)"], + }); + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Reef identity keys -> plugin state", + expect.stringContaining("Archived Reef identity keys legacy source"), + ]); + const store = context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_NAMESPACE, + maxEntries: REEF_KEYS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(store.lookup(REEF_KEYS_KEY)).resolves.toEqual(keys); + expect(fs.existsSync(`${filePath}.migrated`)).toBe(true); + }); + + it("does not import the default home's Reef identity into an isolated state", async () => { + const homeDir = path.join(stateDir, "home"); + const isolatedStateDir = path.join(stateDir, "isolated"); + const homeKeysPath = path.join(homeDir, ".openclaw", "data", "reef", "keys.json"); + fs.mkdirSync(path.dirname(homeKeysPath), { recursive: true }); + fs.mkdirSync(isolatedStateDir, { recursive: true }); + fs.writeFileSync(homeKeysPath, JSON.stringify(reefKeys())); + vi.mocked(os.homedir).mockReturnValue(homeDir); + const isolatedEnv = { ...env, OPENCLAW_STATE_DIR: isolatedStateDir }; + const migration = migrationById("reef-keys-json-to-plugin-state"); + const params = { + config: {}, + env: isolatedEnv, + stateDir: isolatedStateDir, + oauthDir: path.join(isolatedStateDir, "oauth"), + context: createDoctorContext(isolatedEnv), + }; + + await expect(migration.detectLegacyState(params)).resolves.toBeNull(); + await expect(migration.migrateLegacyState(params)).resolves.toEqual({ + changes: [], + warnings: [], + }); + expect(fs.existsSync(homeKeysPath)).toBe(true); + }); + + it("imports an explicitly configured default-home Reef identity into isolated state", async () => { + const homeDir = path.join(stateDir, "explicit-home"); + const isolatedStateDir = path.join(stateDir, "explicit-isolated"); + const legacyDir = path.join(homeDir, ".openclaw", "data", "reef"); + const homeKeysPath = path.join(legacyDir, "keys.json"); + const keys = reefKeys(); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.mkdirSync(isolatedStateDir, { recursive: true }); + fs.writeFileSync(homeKeysPath, JSON.stringify(keys)); + vi.mocked(os.homedir).mockReturnValue(homeDir); + const isolatedEnv = { ...env, OPENCLAW_STATE_DIR: isolatedStateDir }; + const context = createDoctorContext(isolatedEnv); + const migration = migrationById("reef-keys-json-to-plugin-state"); + const params = { + config: { channels: { reef: { stateDir: legacyDir } } }, + env: isolatedEnv, + stateDir: isolatedStateDir, + oauthDir: path.join(isolatedStateDir, "oauth"), + context, + }; + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: ["- Reef identity keys -> plugin state (identity)"], + }); + await expect(migration.migrateLegacyState(params)).resolves.toMatchObject({ warnings: [] }); + await expect( + context + .openPluginStateKeyedStore({ + namespace: REEF_KEYS_NAMESPACE, + maxEntries: REEF_KEYS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }) + .lookup(REEF_KEYS_KEY), + ).resolves.toEqual(keys); + expect(fs.existsSync(homeKeysPath)).toBe(false); + expect(fs.existsSync(`${homeKeysPath}.migrated`)).toBe(true); + }); + + it("blocks identity regeneration after a failed keys.json import", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const filePath = path.join(legacyDir, "keys.json"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(filePath, "{broken"); + const migration = migrationById("reef-keys-json-to-plugin-state"); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context: createDoctorContext(env), + }; + + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([ + expect.stringContaining("Failed importing Reef identity keys"), + ]); + fs.rmSync(filePath); + const missingSourceResult = await migration.migrateLegacyState(params); + expect(missingSourceResult.warnings).toEqual([ + expect.stringContaining("migration is incomplete and keys.json is missing"), + ]); + await expect(generateAndStoreKeys(createRuntime(env))).rejects.toThrow( + "migration is incomplete", + ); + }); + + it("keeps legacy identity keys blocked until their handle binding is canonical", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const keys = reefKeys(); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), JSON.stringify(keys)); + fs.writeFileSync( + path.join(legacyDir, "identity.json"), + JSON.stringify({ handle: "molty", relayUrl: "https://reefwire.ai" }), + ); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + const keysResult = await migrationById("reef-keys-json-to-plugin-state").migrateLegacyState( + params, + ); + + expect(keysResult.warnings).toEqual([]); + const migrationStore = context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)).resolves.toEqual({ + pending: true, + identityBindingRequired: true, + }); + await expect(loadKeys(createRuntime(env))).rejects.toThrow( + "durable state migration is incomplete", + ); + + const registrationResult = await migrationById( + "reef-registration-json-to-plugin-state", + ).migrateLegacyState(params); + + expect(registrationResult.warnings).toEqual([]); + expect(registrationResult.changes).toContain( + "Verified Reef identity keys and binding; cleared migration marker", + ); + await expect(migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)).resolves.toBeUndefined(); + await expect(loadKeys(createRuntime(env))).rejects.toThrow( + "durable state migration is incomplete", + ); + await migrationById("reef-runtime-files-to-plugin-state").migrateLegacyState(params); + await expect(loadKeys(createRuntime(env))).resolves.toEqual(keys); + }); + + it("binds wizard-created legacy keys when unrelated Reef config is invalid", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const keys = reefKeys(); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), JSON.stringify(keys)); + const context = createDoctorContext(env); + const params = { + config: { + channels: { + reef: { + handle: "molty", + relayUrl: "https://reefwire.ai/", + email: "not-an-email", + }, + }, + }, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + await expect( + migrationById("reef-registration-json-to-plugin-state").detectLegacyState(params), + ).resolves.toEqual({ + preview: ["- Reef configured identity binding -> plugin state"], + }); + + await migrationById("reef-keys-json-to-plugin-state").migrateLegacyState(params); + const migrationStore = context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)).resolves.toEqual({ + pending: true, + identityBindingRequired: true, + }); + + const registrationResult = await migrationById( + "reef-registration-json-to-plugin-state", + ).migrateLegacyState(params); + + expect(registrationResult.warnings).toEqual([]); + expect(registrationResult.changes).toContain( + "Migrated Reef identity binding from config -> plugin state", + ); + const registrationStore = context.openPluginStateKeyedStore({ + namespace: REEF_REGISTRATION_NAMESPACE, + maxEntries: REEF_REGISTRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(registrationStore.lookup(REEF_REGISTRATION_IDENTITY_KEY)).resolves.toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + await expect(migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)).resolves.toBeUndefined(); + }); + + it("keeps identity migration blocked when config conflicts with the imported binding", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), JSON.stringify(reefKeys())); + fs.writeFileSync( + path.join(legacyDir, "identity.json"), + JSON.stringify({ handle: "canonical", relayUrl: "https://reefwire.ai" }), + ); + const context = createDoctorContext(env); + const params = { + config: { + channels: { + reef: { handle: "conflict", relayUrl: "https://reefwire.ai" }, + }, + }, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + await migrationById("reef-keys-json-to-plugin-state").migrateLegacyState(params); + const result = await migrationById("reef-registration-json-to-plugin-state").migrateLegacyState( + params, + ); + + expect(result.warnings).toEqual([ + expect.stringContaining("configured handle or relay differs"), + expect.stringContaining("identity migration is incomplete"), + ]); + const migrationStore = context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)).resolves.toEqual({ + pending: true, + identityBindingRequired: true, + }); + }); + + it("imports and verifies the append-only audit chain", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const filePath = path.join(legacyDir, "audit.jsonl"); + const audit = new MemoryAuditStore(new Uint8Array(32).fill(1)); + await audit.appendEvent("one", { id: 1 }, 10); + await audit.appendEvent("two", { id: 2 }, 11); + const entries = await audit.entries(); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(filePath, `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + const migration = migrationById("reef-audit-jsonl-to-plugin-state"); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated 2 Reef audit entries -> plugin state", + expect.stringContaining("Archived Reef audit trail legacy source"), + ]); + const store = context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_NAMESPACE, + maxEntries: REEF_AUDIT_STORE_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const headStore = context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_HEAD_NAMESPACE, + maxEntries: REEF_AUDIT_HEAD_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(headStore.lookup(REEF_AUDIT_HEAD_KEY)).resolves.toEqual({ + kind: "head", + hash: entries[1]!.entryHash, + seq: 2, + oldestHash: entries[0]!.entryHash, + }); + await expect(store.lookup(reefAuditEntryKey(entries[0]!.entryHash))).resolves.toEqual({ + kind: "entry", + entry: entries[0], + nextHash: entries[1]!.entryHash, + }); + expect(fs.existsSync(`${filePath}.migrated`)).toBe(true); + }); + + it("finishes an interrupted migration of an empty audit trail", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const filePath = path.join(legacyDir, "audit.jsonl"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(filePath, ""); + const migration = migrationById("reef-audit-jsonl-to-plugin-state"); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + const imported = await migration.migrateLegacyState(params); + expect(imported.warnings).toEqual([]); + expect(fs.existsSync(filePath)).toBe(false); + + const migrationStore = context.openPluginStateKeyedStore<{ + pending: true; + expectedEntries: number; + }>({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await migrationStore.register(REEF_AUDIT_MIGRATION_KEY, { + pending: true, + expectedEntries: 0, + }); + + const recovered = await migration.migrateLegacyState(params); + + expect(recovered.warnings).toEqual([]); + expect(recovered.changes).toContain( + "Verified Reef audit trail; cleared completed migration marker", + ); + await expect(migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY)).resolves.toBeUndefined(); + }); + + it("blocks runtime audit writes until a failed legacy import is repaired", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const filePath = path.join(legacyDir, "audit.jsonl"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(filePath, "{broken\n"); + const migration = migrationById("reef-audit-jsonl-to-plugin-state"); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + const failed = await migration.migrateLegacyState(params); + + expect(failed.warnings).toEqual([expect.stringContaining("Failed importing Reef audit trail")]); + const migrationStore = context.openPluginStateKeyedStore<{ pending: true }>({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY)).resolves.toEqual({ + pending: true, + }); + expect(() => openStores(createRuntime(env), reefKeys())).toThrow( + "Reef durable state migration is incomplete", + ); + + const audit = new MemoryAuditStore(new Uint8Array(32).fill(1)); + await audit.appendEvent("repaired", { id: 1 }, 10); + const entries = await audit.entries(); + fs.writeFileSync(filePath, `${entries.map((entry) => JSON.stringify(entry)).join("\n")}\n`); + + const repaired = await migration.migrateLegacyState(params); + + expect(repaired.warnings).toEqual([]); + await expect(migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY)).resolves.toBeUndefined(); + await migrationById("reef-runtime-files-to-plugin-state").migrateLegacyState(params); + expect(() => openStores(createRuntime(env), reefKeys())).not.toThrow(); + }); + + it("imports registration and durable runtime state before archiving files", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync( + path.join(legacyDir, "identity.json"), + JSON.stringify({ handle: "molty", relayUrl: "https://reefwire.ai" }), + ); + fs.writeFileSync( + path.join(legacyDir, "setup-session.json"), + JSON.stringify({ + session: "setup-secret", + relayUrl: "https://reefwire.ai", + email: "molty@example.com", + }), + ); + const replayId = "01JZ0000000000000000000000"; + const secondReplayId = "01JZ0000000000000000000001"; + fs.writeFileSync( + path.join(legacyDir, "replay.jsonl"), + `${JSON.stringify({ op: "claim", peer: "alice", id: replayId, envelopeHash: "a".repeat(64) })}\n${JSON.stringify({ op: "consume", peer: "alice", id: replayId })}\n${JSON.stringify({ op: "claim", peer: "bob", id: secondReplayId, envelopeHash: "d".repeat(64) })}\n`, + ); + const review: ReviewRequest = { + id: replayId, + from: "alice#1", + to: "bob#1", + direction: "outbound", + bodyHash: "b".repeat(64), + approvalDigest: "c".repeat(64), + verdict: { + decision: "review", + category: "ambiguous", + reason: "Owner review.", + model: "test-model", + policyVersion: "v1", + }, + }; + fs.writeFileSync( + path.join(legacyDir, "reviews.json"), + JSON.stringify({ [review.approvalDigest]: { review, approved: true } }), + ); + fs.writeFileSync(path.join(legacyDir, "delivered.json"), JSON.stringify([replayId])); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + const partiallyImportedReplay = context.openPluginStateKeyedStore({ + namespace: REEF_REPLAY_NAMESPACE, + maxEntries: REEF_REPLAY_MAX_ENTRIES, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_REPLAY_TTL_MS, + }); + await partiallyImportedReplay.register(reefReplayStoreKey("alice", replayId), { + peer: "alice", + id: replayId, + envelopeHash: "a".repeat(64), + state: "consumed", + }); + + const registration = await migrationById( + "reef-registration-json-to-plugin-state", + ).migrateLegacyState(params); + const runtimeState = await migrationById( + "reef-runtime-files-to-plugin-state", + ).migrateLegacyState(params); + await expect(generateAndStoreKeys(createRuntime(env))).rejects.toThrow("has no canonical keys"); + + expect(registration.warnings).toEqual([]); + expect(registration.changes).toHaveLength(4); + expect(runtimeState.warnings).toEqual([]); + expect(runtimeState.changes).toHaveLength(7); + for (const filename of [ + "identity.json", + "setup-session.json", + "replay.jsonl", + "reviews.json", + "delivered.json", + ]) { + expect(fs.existsSync(path.join(legacyDir, filename))).toBe(false); + expect(fs.existsSync(path.join(legacyDir, `${filename}.migrated`))).toBe(true); + } + const replayStore = context.openPluginStateKeyedStore({ + namespace: REEF_REPLAY_NAMESPACE, + maxEntries: REEF_REPLAY_MAX_ENTRIES, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_REPLAY_TTL_MS, + }); + await expect(replayStore.lookup(reefReplayStoreKey("alice", replayId))).resolves.toMatchObject({ + state: "consumed", + envelopeHash: "a".repeat(64), + }); + await expect( + replayStore.lookup(reefReplayStoreKey("bob", secondReplayId)), + ).resolves.toMatchObject({ + state: "available", + envelopeHash: "d".repeat(64), + }); + const reviewStore = context.openPluginStateKeyedStore({ + namespace: REEF_REVIEWS_NAMESPACE, + maxEntries: REEF_REVIEWS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(reviewStore.lookup(review.approvalDigest)).resolves.toEqual({ + review, + approved: true, + }); + const deliveredStore = context.openPluginStateKeyedStore<{ id: string }>({ + namespace: REEF_DELIVERED_NAMESPACE, + maxEntries: REEF_DELIVERED_MAX_ENTRIES, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_DELIVERED_TTL_MS, + }); + await expect(deliveredStore.lookup(replayId)).resolves.toEqual({ id: replayId }); + }); + + it("leaves oversized replay and delivered sources blocked and unarchived", async () => { + const legacyDir = path.join(stateDir, ".openclaw", "data", "reef"); + const replayPath = path.join(legacyDir, "replay.jsonl"); + const deliveredPath = path.join(legacyDir, "delivered.json"); + fs.mkdirSync(legacyDir, { recursive: true }); + const replayIds = Array.from( + { length: REEF_REPLAY_MAX_ENTRIES + 1 }, + (_, index) => `replay-${index}`, + ); + fs.writeFileSync( + replayPath, + `${replayIds + .map((id) => JSON.stringify({ op: "claim", peer: "alice", id, envelopeHash: id })) + .join("\n")}\n`, + ); + fs.writeFileSync( + deliveredPath, + JSON.stringify( + Array.from({ length: REEF_DELIVERED_MAX_ENTRIES + 1 }, (_, index) => `delivered-${index}`), + ), + ); + const migration = migrationById("reef-runtime-files-to-plugin-state"); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context: createDoctorContext(env), + }; + + const result = await migration.migrateLegacyState(params); + + expect(result.changes).toEqual([]); + expect(result.warnings).toEqual([ + expect.stringContaining( + `${REEF_REPLAY_MAX_ENTRIES + 1} replay bindings exceed plugin-state capacity`, + ), + expect.stringContaining( + `${REEF_DELIVERED_MAX_ENTRIES + 1} delivered markers exceed plugin-state capacity`, + ), + expect.stringContaining("Reef durable state migration is incomplete"), + ]); + for (const filePath of [replayPath, deliveredPath]) { + expect(fs.existsSync(filePath)).toBe(true); + expect(fs.existsSync(`${filePath}.migrated`)).toBe(false); + } + }); + it("imports config-backed trust into scoped plugin state without overwriting canonical rows", async () => { const cfg = legacyConfig(); - const migration = stateMigrations[0]!; + const migration = migrationById("reef-config-trust-to-plugin-state"); const context = createDoctorContext(env); const params = { config: cfg, env, stateDir, oauthDir: path.join(stateDir, "oauth"), context }; @@ -146,7 +801,7 @@ describe("Reef doctor contract", () => { ...(reef.friends as Record), broken: { autonomy: "extended" }, }; - const migration = stateMigrations[0]!; + const migration = migrationById("reef-config-trust-to-plugin-state"); const context = createDoctorContext(env); const params = { config: cfg, env, stateDir, oauthDir: path.join(stateDir, "oauth"), context }; @@ -182,7 +837,7 @@ describe("Reef doctor contract", () => { } as PluginDoctorStateMigrationContext; await expect( - stateMigrations[0]!.migrateLegacyState({ + migrationById("reef-config-trust-to-plugin-state").migrateLegacyState({ config: cfg, env, stateDir, diff --git a/extensions/reef/doctor-contract-api.ts b/extensions/reef/doctor-contract-api.ts index c10f02f4d05e..2b69438f61cd 100644 --- a/extensions/reef/doctor-contract-api.ts +++ b/extensions/reef/doctor-contract-api.ts @@ -1,19 +1,61 @@ +import fs from "node:fs/promises"; +import path from "node:path"; import type { ChannelDoctorLegacyConfigRule } from "openclaw/plugin-sdk/channel-contract"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; -import type { PluginDoctorStateMigration } from "openclaw/plugin-sdk/runtime-doctor"; +import { + archiveLegacyStateSource, + type PluginDoctorStateMigration, +} from "openclaw/plugin-sdk/runtime-doctor"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import { z } from "zod"; -import { ReefChannelConfigSchema, normalizeReefTarget } from "./src/config-schema.js"; +import { + parseReefRelayUrl, + ReefChannelConfigSchema, + normalizeReefTarget, +} from "./src/config-schema.js"; +import { reefAuditStateMigration, reefRuntimeStateMigration } from "./src/doctor-durable-state.js"; +import { + legacyReefFileExists, + REEF_DURABLE_LEGACY_FILENAMES, + resolveLegacyReefStateDir, +} from "./src/doctor-state-paths.js"; import { ReefPeerTrustSchema, type ReefPeerTrust } from "./src/friend-types.js"; +import { + REEF_DURABLE_MIGRATION_KEY, + REEF_DURABLE_MIGRATION_MAX_ENTRIES, + REEF_DURABLE_MIGRATION_NAMESPACE, + REEF_KEYS_KEY, + REEF_KEYS_MAX_ENTRIES, + REEF_KEYS_MIGRATION_KEY, + REEF_KEYS_MIGRATION_MAX_ENTRIES, + REEF_KEYS_MIGRATION_NAMESPACE, + REEF_KEYS_NAMESPACE, + REEF_REGISTRATION_IDENTITY_KEY, + REEF_REGISTRATION_MAX_ENTRIES, + REEF_REGISTRATION_NAMESPACE, + REEF_REGISTRATION_SESSION_KEY, + parseReefIdentityBinding, + parseReefKeys, + parseReefSetupSession, + type ReefIdentityMigrationRecord, + type ReefDurableMigrationRecord, + type ReefIdentityBinding, + type ReefSetupSession, +} from "./src/state.js"; import { REEF_TRUST_STORE_MAX_ENTRIES, REEF_TRUST_STORE_NAMESPACE, resolveReefTrustStoreKey, } from "./src/trust-store.js"; +import type { ReefKeys } from "./src/types.js"; const RETIRED_REEF_CONFIG_KEYS = ["friends", "dmPolicy", "allowFrom"] as const; const REEF_CONFIG_IMPORT_NAMESPACE = "peer-state-config-imports"; const LegacyReefFriendSchema = ReefPeerTrustSchema.omit({ approvedAt: true }); +const ReefIdentityConfigSchema = ReefChannelConfigSchema.pick({ + handle: true, + relayUrl: true, +}); type ReefPeerStateSnapshot = { revision: number; @@ -25,6 +67,61 @@ type ReefConfigImportMarker = { importedAt: number; }; +type ReefLegacyRegistrationSource = + | { + filename: "identity.json"; + key: typeof REEF_REGISTRATION_IDENTITY_KEY; + parse: typeof parseReefIdentityBinding; + label: string; + } + | { + filename: "setup-session.json"; + key: typeof REEF_REGISTRATION_SESSION_KEY; + parse: typeof parseReefSetupSession; + label: string; + }; + +const REEF_LEGACY_REGISTRATION_SOURCES: ReefLegacyRegistrationSource[] = [ + { + filename: "identity.json", + key: REEF_REGISTRATION_IDENTITY_KEY, + parse: parseReefIdentityBinding, + label: "Reef identity binding", + }, + { + filename: "setup-session.json", + key: REEF_REGISTRATION_SESSION_KEY, + parse: parseReefSetupSession, + label: "Reef setup session", + }, +]; + +type ConfiguredReefIdentityBinding = + | { status: "absent" } + | { status: "invalid" } + | { status: "valid"; binding: ReefIdentityBinding }; + +function configuredReefIdentityBinding(cfg: OpenClawConfig): ConfiguredReefIdentityBinding { + const reef = cfg.channels?.reef; + if (!isRecord(reef) || !Object.hasOwn(reef, "handle") || reef.handle === undefined) { + return { status: "absent" }; + } + const parsed = ReefIdentityConfigSchema.safeParse({ + handle: reef.handle, + relayUrl: reef.relayUrl, + }); + if (!parsed.success || !parsed.data.handle) { + return { status: "invalid" }; + } + return { + status: "valid", + binding: { + handle: parsed.data.handle, + relayUrl: parseReefRelayUrl(parsed.data.relayUrl), + }, + }; +} + function hasRetiredReefPolicyConfig(value: unknown): boolean { return isRecord(value) && ["dmPolicy", "allowFrom"].some((key) => Object.hasOwn(value, key)); } @@ -90,6 +187,343 @@ export function normalizeCompatibilityConfig({ cfg }: { cfg: OpenClawConfig }): } export const stateMigrations: PluginDoctorStateMigration[] = [ + { + id: "reef-keys-json-to-plugin-state", + label: "Reef identity keys", + async detectLegacyState(params) { + const stateDir = resolveLegacyReefStateDir(params); + const filePath = path.join(stateDir, "keys.json"); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const sourceExists = await legacyReefFileExists(filePath); + const pending = await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY); + const durableSourceExists = ( + await Promise.all( + REEF_DURABLE_LEGACY_FILENAMES.map((filename) => + legacyReefFileExists(path.join(stateDir, filename)), + ), + ) + ).some(Boolean); + const durablePending = await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY); + return sourceExists || pending || durableSourceExists || durablePending + ? { + preview: [ + sourceExists + ? "- Reef identity keys -> plugin state (identity)" + : pending + ? "- Verify Reef identity-key migration marker" + : "- Prepare Reef durable state migration barrier", + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const stateDir = resolveLegacyReefStateDir(params); + const filePath = path.join(stateDir, "keys.json"); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const store = params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_NAMESPACE, + maxEntries: REEF_KEYS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durableSourceExists = ( + await Promise.all( + REEF_DURABLE_LEGACY_FILENAMES.map((filename) => + legacyReefFileExists(path.join(stateDir, filename)), + ), + ) + ).some(Boolean); + const durablePending = await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY); + if (durableSourceExists || durablePending) { + await durableMigrationStore.register(REEF_DURABLE_MIGRATION_KEY, { pending: true }); + } + if (!(await legacyReefFileExists(filePath))) { + const pending = await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY); + if (!pending) { + return { changes, warnings }; + } + try { + parseReefKeys(await store.lookup(REEF_KEYS_KEY)); + if (!pending?.identityBindingRequired) { + await migrationStore.delete(REEF_KEYS_MIGRATION_KEY); + changes.push("Verified Reef identity keys; cleared completed migration marker"); + } + } catch { + warnings.push( + "Reef identity key migration is incomplete and keys.json is missing; left migration blocker in place", + ); + } + return { changes, warnings }; + } + const existingMarker = await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY); + const configuredBinding = configuredReefIdentityBinding(params.config); + const identityBindingRequired = + existingMarker?.identityBindingRequired || + (await legacyReefFileExists( + path.join(resolveLegacyReefStateDir(params), "identity.json"), + )) || + configuredBinding.status !== "absent"; + await migrationStore.register(REEF_KEYS_MIGRATION_KEY, { + pending: true, + identityBindingRequired, + }); + let keys: ReefKeys; + try { + keys = parseReefKeys(JSON.parse(await fs.readFile(filePath, "utf8")) as unknown); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { changes, warnings }; + } + warnings.push( + `Failed importing Reef identity keys: ${String(error)}; left source in place`, + ); + return { changes, warnings }; + } + const existing = await store.lookup(REEF_KEYS_KEY); + if (existing && JSON.stringify(existing) !== JSON.stringify(keys)) { + warnings.push("Kept existing Reef identity keys; left differing legacy source in place"); + return { changes, warnings }; + } + if (!existing) { + try { + await store.registerIfAbsent(REEF_KEYS_KEY, keys); + } catch (error) { + warnings.push( + `Failed importing Reef identity keys: ${String(error)}; left source in place`, + ); + return { changes, warnings }; + } + } + const persisted = await store.lookup(REEF_KEYS_KEY); + try { + if (JSON.stringify(parseReefKeys(persisted)) !== JSON.stringify(keys)) { + throw new Error("persisted value differs"); + } + } catch (error) { + warnings.push( + `Failed verifying Reef identity keys after import: ${String(error)}; left source in place`, + ); + return { changes, warnings }; + } + changes.push("Migrated Reef identity keys -> plugin state"); + const warningCount = warnings.length; + await archiveLegacyStateSource({ + filePath, + label: "Reef identity keys", + changes, + warnings, + }); + if (warnings.length === warningCount && !identityBindingRequired) { + await migrationStore.delete(REEF_KEYS_MIGRATION_KEY); + } + return { changes, warnings }; + }, + }, + { + id: "reef-registration-json-to-plugin-state", + label: "Reef registration state", + async detectLegacyState(params) { + const stateDir = resolveLegacyReefStateDir(params); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const files = ( + await Promise.all( + REEF_LEGACY_REGISTRATION_SOURCES.map(async (source) => ({ + source, + exists: await legacyReefFileExists(path.join(stateDir, source.filename)), + })), + ) + ).filter((entry) => entry.exists); + const pending = await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY); + const configuredBinding = configuredReefIdentityBinding(params.config); + const configuredBindingNeedsImport = + configuredBinding.status !== "absent" && + (await legacyReefFileExists(path.join(stateDir, "keys.json"))); + return files.length > 0 || pending?.identityBindingRequired || configuredBindingNeedsImport + ? { + preview: [ + files.length > 0 + ? `- Reef registration state -> plugin state (${files.map((entry) => entry.source.filename).join(", ")})` + : configuredBindingNeedsImport + ? "- Reef configured identity binding -> plugin state" + : "- Verify Reef identity binding migration marker", + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const stateDir = resolveLegacyReefStateDir(params); + const store = params.context.openPluginStateKeyedStore< + ReefIdentityBinding | ReefSetupSession + >({ + namespace: REEF_REGISTRATION_NAMESPACE, + maxEntries: REEF_REGISTRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const hasRegistrationSource = ( + await Promise.all( + REEF_LEGACY_REGISTRATION_SOURCES.map((source) => + legacyReefFileExists(path.join(stateDir, source.filename)), + ), + ) + ).some(Boolean); + if ( + hasRegistrationSource || + (await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY)) || + (await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY)) + ) { + await durableMigrationStore.register(REEF_DURABLE_MIGRATION_KEY, { pending: true }); + } + for (const source of REEF_LEGACY_REGISTRATION_SOURCES) { + const filePath = path.join(stateDir, source.filename); + if (!(await legacyReefFileExists(filePath))) { + continue; + } + let legacy: ReefIdentityBinding | ReefSetupSession | undefined; + try { + legacy = source.parse(JSON.parse(await fs.readFile(filePath, "utf8")) as unknown); + } catch { + // The structural validation below owns the fail-closed warning. + } + if (!legacy) { + warnings.push(`Failed importing ${source.label}: invalid JSON; left source in place`); + continue; + } + const existing = await store.lookup(source.key); + const normalizedExisting = source.parse(existing); + if (normalizedExisting && JSON.stringify(normalizedExisting) !== JSON.stringify(legacy)) { + warnings.push(`Kept existing ${source.label}; left differing legacy source in place`); + continue; + } + if (!normalizedExisting) { + try { + await store.registerIfAbsent(source.key, legacy); + } catch (error) { + warnings.push( + `Failed importing ${source.label}: ${String(error)}; left source in place`, + ); + continue; + } + } + const persisted = source.parse(await store.lookup(source.key)); + if (!persisted || JSON.stringify(persisted) !== JSON.stringify(legacy)) { + warnings.push(`Failed verifying ${source.label}; left source in place`); + continue; + } + changes.push(`Migrated ${source.label} -> plugin state`); + await archiveLegacyStateSource({ + filePath, + label: source.label, + changes, + warnings, + }); + } + const configuredBindingResult = configuredReefIdentityBinding(params.config); + const configuredBinding = + configuredBindingResult.status === "valid" ? configuredBindingResult.binding : undefined; + if (configuredBinding) { + const existing = parseReefIdentityBinding( + await store.lookup(REEF_REGISTRATION_IDENTITY_KEY), + ); + if (existing && JSON.stringify(existing) !== JSON.stringify(configuredBinding)) { + warnings.push("Kept existing Reef identity binding; configured handle or relay differs"); + } else if (!existing) { + try { + await store.registerIfAbsent(REEF_REGISTRATION_IDENTITY_KEY, configuredBinding); + const persisted = parseReefIdentityBinding( + await store.lookup(REEF_REGISTRATION_IDENTITY_KEY), + ); + if (JSON.stringify(persisted) !== JSON.stringify(configuredBinding)) { + throw new Error("persisted value differs"); + } + changes.push("Migrated Reef identity binding from config -> plugin state"); + } catch (error) { + warnings.push(`Failed importing Reef identity binding from config: ${String(error)}`); + } + } + } + const pending = await migrationStore.lookup(REEF_KEYS_MIGRATION_KEY); + if (pending?.identityBindingRequired) { + const keysPath = path.join(stateDir, "keys.json"); + const identityPath = path.join(stateDir, "identity.json"); + try { + parseReefKeys( + await params.context + .openPluginStateKeyedStore({ + namespace: REEF_KEYS_NAMESPACE, + maxEntries: REEF_KEYS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }) + .lookup(REEF_KEYS_KEY), + ); + const binding = parseReefIdentityBinding( + await store.lookup(REEF_REGISTRATION_IDENTITY_KEY), + ); + if (!binding) { + throw new Error("canonical identity binding is missing"); + } + if (configuredBindingResult.status === "invalid") { + throw new Error("configured handle or relay is invalid"); + } + if (configuredBinding && JSON.stringify(binding) !== JSON.stringify(configuredBinding)) { + throw new Error("configured handle or relay differs from canonical identity binding"); + } + if ( + (await legacyReefFileExists(keysPath)) || + (await legacyReefFileExists(identityPath)) + ) { + throw new Error("legacy identity sources remain"); + } + await migrationStore.delete(REEF_KEYS_MIGRATION_KEY); + changes.push("Verified Reef identity keys and binding; cleared migration marker"); + } catch (error) { + warnings.push( + `Reef identity migration is incomplete: ${String(error)}; left migration blocker in place`, + ); + } + } + return { changes, warnings }; + }, + }, + reefAuditStateMigration, + reefRuntimeStateMigration, { id: "reef-config-trust-to-plugin-state", label: "Reef peer trust", diff --git a/extensions/reef/protocol/audit.ts b/extensions/reef/protocol/audit.ts index 14dba2ea25b5..5f7ea75a8011 100644 --- a/extensions/reef/protocol/audit.ts +++ b/extensions/reef/protocol/audit.ts @@ -96,11 +96,22 @@ export function verifyChain( if (expected?.length !== undefined && entries.length !== expected.length) { return false; } - let previous = ""; + return verifyChainSegment(entries, { + previousHash: "", + previousSeq: 0, + ...(expected?.head === undefined ? {} : { head: expected.head }), + }); +} + +export function verifyChainSegment( + entries: readonly AuditEntry[], + expected: { previousHash: string; previousSeq: number; head?: string }, +): boolean { + let previous = expected.previousHash; for (let index = 0; index < entries.length; index++) { const entry = entries[index]!; if ( - entry.event.seq !== index + 1 || + entry.event.seq !== expected.previousSeq + index + 1 || entry.prevHash !== previous || entry.entryHash !== hashEntry(previous, entry.event) ) { @@ -108,7 +119,7 @@ export function verifyChain( } previous = entry.entryHash; } - return expected?.head === undefined || previous === expected.head; + return expected.head === undefined || previous === expected.head; } export function signCheckpoint( diff --git a/extensions/reef/protocol/envelope.ts b/extensions/reef/protocol/envelope.ts index d6660ef815a2..979e012b37a9 100644 --- a/extensions/reef/protocol/envelope.ts +++ b/extensions/reef/protocol/envelope.ts @@ -19,6 +19,8 @@ export interface CompletedReplay { export interface ReplayStore { claim(peer: string, id: string, envelopeHash: string): Promise; + /** Renews an in-flight claim while slow guard or review work is active. */ + refresh?(peer: string, id: string): Promise; complete(peer: string, id: string, receipt: SignedReceipt, body?: MessageBody): Promise; consume(peer: string, id: string): Promise; release(peer: string, id: string): Promise; @@ -137,6 +139,7 @@ export type ClaimedOpenResult = const MAX_PLAINTEXT = 32 * 1024; const MAX_CIPHERTEXT_BASE64 = 44_752; const MAX_ENVELOPE_BYTES = 48 * 1024; +export const REEF_ENVELOPE_MAX_AGE_SECONDS = 30 * 24 * 60 * 60; const HKDF_INFO = utf8("reef-v1"); const ULID_PATTERN = /^[0-7][0-9A-HJKMNP-TV-Z]{25}$/; @@ -237,7 +240,7 @@ export async function openClaimed(options: OpenOptions): Promise now + maxFutureSkew || envelope.ts < now - maxAge) { throw new ExpiredError(); diff --git a/extensions/reef/protocol/node.ts b/extensions/reef/protocol/node.ts index 3b2b82a10722..5012e51fd793 100644 --- a/extensions/reef/protocol/node.ts +++ b/extensions/reef/protocol/node.ts @@ -136,6 +136,8 @@ export class FileReplayStore implements ReplayStore { }); } + async refresh(_peer: string, _id: string): Promise {} + async complete( peer: string, id: string, diff --git a/extensions/reef/protocol/pipeline.ts b/extensions/reef/protocol/pipeline.ts index 31d8a5b3a437..a92eed91947c 100644 --- a/extensions/reef/protocol/pipeline.ts +++ b/extensions/reef/protocol/pipeline.ts @@ -157,6 +157,8 @@ export type InboundResult = | { disposition: "accepted"; body: MessageBody; verdict: Verdict; receipt: SignedReceipt } | { disposition: "duplicate"; body?: MessageBody; receipt: SignedReceipt }; +const REPLAY_CLAIM_HEARTBEAT_MS = 60_000; + // Caller MUST ack the relay with receipt. For accepted or duplicate-accepted results, it MUST // idempotently deliver every present body to channel ingress, keyed by envelope id. export async function composeInbound(options: ComposeInboundOptions): Promise { @@ -171,6 +173,15 @@ export async function composeInbound(options: ComposeInboundOptions): Promise { + await options.replayStore.refresh?.(peer, options.envelope.id); + }; + const heartbeat = options.replayStore.refresh + ? setInterval(() => { + void refreshClaim().catch(() => undefined); + }, REPLAY_CLAIM_HEARTBEAT_MS) + : undefined; + heartbeat?.unref?.(); try { const proposalHash = bodyHash(opened.body); const approvalDigest = computeApprovalDigest( @@ -183,6 +194,7 @@ export async function composeInbound(options: ComposeInboundOptions): Promise {} + async complete( peer: string, id: string, diff --git a/extensions/reef/src/audit-state.ts b/extensions/reef/src/audit-state.ts new file mode 100644 index 000000000000..784373a1021f --- /dev/null +++ b/extensions/reef/src/audit-state.ts @@ -0,0 +1,412 @@ +import { randomUUID } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; +import { randomBytes } from "@noble/hashes/utils.js"; +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createAuditEntry, + verifyChainSegment, + type AuditEntry, + type AuditStore, +} from "../protocol/index.js"; + +export const REEF_AUDIT_NAMESPACE = "audit"; +export const REEF_AUDIT_HEAD_NAMESPACE = "audit-head"; +export const REEF_AUDIT_HEAD_KEY = "head"; +export const REEF_AUDIT_MAX_ENTRIES = 30_000; +export const REEF_AUDIT_STORE_MAX_ENTRIES = REEF_AUDIT_MAX_ENTRIES + 1; +export const REEF_AUDIT_HEAD_MAX_ENTRIES = 1; +export const REEF_AUDIT_MIGRATION_NAMESPACE = "audit-migration"; +export const REEF_AUDIT_MIGRATION_KEY = "audit-jsonl"; +export const REEF_AUDIT_MIGRATION_MAX_ENTRIES = 1; + +type ReefAuditPendingAppend = { + owner: string; + expiresAt: number; + entryKey?: string; +}; + +export type ReefAuditHeadRecord = { + kind: "head"; + hash: string; + seq: number; + oldestHash: string; + pending?: ReefAuditPendingAppend; + garbageEntryKey?: string; +}; + +export type ReefAuditStateRecord = { kind: "entry"; entry: AuditEntry; nextHash?: string }; + +const REEF_AUDIT_APPEND_LEASE_MS = 30_000; +const REEF_AUDIT_APPEND_RETRY_MS = 25; +const REEF_AUDIT_APPEND_ATTEMPTS = 120; + +export function reefAuditEntryKey(entryHash: string): string { + return `entry:${entryHash}`; +} + +export function parseReefAuditHead(value: ReefAuditHeadRecord | undefined): ReefAuditHeadRecord { + if (value === undefined) { + return { kind: "head", hash: "", seq: 0, oldestHash: "" }; + } + if ( + value.kind !== "head" || + typeof value.hash !== "string" || + !Number.isSafeInteger(value.seq) || + value.seq < 0 || + (value.seq === 0) !== (value.hash === "") || + typeof value.oldestHash !== "string" || + (value.seq === 0) !== (value.oldestHash === "") || + (value.garbageEntryKey !== undefined && + (typeof value.garbageEntryKey !== "string" || value.garbageEntryKey.length === 0)) || + (value.pending !== undefined && + (typeof value.pending.owner !== "string" || + value.pending.owner.length === 0 || + !Number.isSafeInteger(value.pending.expiresAt) || + value.pending.expiresAt <= 0 || + (value.pending.entryKey !== undefined && + (typeof value.pending.entryKey !== "string" || value.pending.entryKey.length === 0)))) + ) { + throw new Error("invalid Reef audit head"); + } + return value; +} + +function parseAuditEntryRecord(value: ReefAuditStateRecord | undefined): AuditEntry { + if (!value || value.kind !== "entry") { + throw new Error("missing Reef audit entry"); + } + return value.entry; +} + +function parseAuditStateRecord(value: ReefAuditStateRecord | undefined): ReefAuditStateRecord { + parseAuditEntryRecord(value); + if ( + value?.nextHash !== undefined && + (typeof value.nextHash !== "string" || value.nextHash.length === 0) + ) { + throw new Error("invalid Reef audit next pointer"); + } + return value!; +} + +class ReefSqliteAuditStore implements AuditStore { + readonly #auditKey: Uint8Array; + readonly #rng: (length: number) => Uint8Array; + readonly #maxEntries: number; + readonly #store: PluginStateSyncKeyedStore; + readonly #headStore: PluginStateSyncKeyedStore; + + constructor( + runtime: PluginRuntime, + auditKey: Uint8Array, + rng: (length: number) => Uint8Array = randomBytes, + maxEntries = REEF_AUDIT_MAX_ENTRIES, + ) { + if (auditKey.length !== 32) { + throw new Error("audit key must be 32 bytes"); + } + this.#auditKey = auditKey.slice(); + this.#rng = rng; + this.#maxEntries = maxEntries; + const migration = runtime.state.openSyncKeyedStore<{ pending: true }>({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + if (migration.lookup(REEF_AUDIT_MIGRATION_KEY)) { + throw new Error( + "Reef audit migration is incomplete; repair audit.jsonl and rerun openclaw doctor --fix", + ); + } + this.#store = runtime.state.openSyncKeyedStore({ + namespace: REEF_AUDIT_NAMESPACE, + maxEntries: maxEntries + 1, + overflowPolicy: "reject-new", + }); + this.#headStore = runtime.state.openSyncKeyedStore({ + namespace: REEF_AUDIT_HEAD_NAMESPACE, + maxEntries: REEF_AUDIT_HEAD_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + } + + async appendEvent( + type: string, + payload: unknown, + ts = Math.floor(Date.now() / 1000), + ): Promise { + const update = this.#headStore.update; + const updateEntry = this.#store.update; + if (!update || !updateEntry) { + throw new Error("Reef audit state requires atomic plugin-state updates"); + } + const owner = randomUUID(); + for (let attempt = 0; attempt < REEF_AUDIT_APPEND_ATTEMPTS; attempt++) { + let acquired = false; + let staleEntryKey: string | undefined; + let head: ReefAuditHeadRecord = { kind: "head", hash: "", seq: 0, oldestHash: "" }; + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if (latest.pending && latest.pending.expiresAt > Date.now()) { + return latest; + } + acquired = true; + staleEntryKey = latest.pending?.entryKey; + head = { + kind: "head", + hash: latest.hash, + seq: latest.seq, + oldestHash: latest.oldestHash, + ...(latest.garbageEntryKey ? { garbageEntryKey: latest.garbageEntryKey } : {}), + }; + return { + ...head, + pending: { + owner, + expiresAt: Date.now() + REEF_AUDIT_APPEND_LEASE_MS, + ...(staleEntryKey ? { entryKey: staleEntryKey } : {}), + }, + }; + }); + if (!acquired) { + await sleep(REEF_AUDIT_APPEND_RETRY_MS); + continue; + } + + let entryKey: string | undefined; + let entryHash: string | undefined; + let inserted = false; + let staleCleanupComplete = !staleEntryKey; + try { + if (staleEntryKey) { + if (!staleEntryKey.startsWith("entry:") || staleEntryKey.length === "entry:".length) { + throw new Error("invalid Reef audit staged entry key"); + } + const staleEntryHash = staleEntryKey.slice("entry:".length); + if (head.hash) { + updateEntry(reefAuditEntryKey(head.hash), (current) => { + const previous = parseAuditStateRecord(current); + if (previous.nextHash !== staleEntryHash) { + return previous; + } + const { nextHash: _nextHash, ...unlinked } = previous; + return unlinked; + }); + } + this.#store.delete(staleEntryKey); + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if (latest.pending?.owner !== owner || latest.pending.entryKey !== staleEntryKey) { + return latest; + } + return { + ...latest, + pending: { + owner, + expiresAt: latest.pending.expiresAt, + }, + }; + }); + staleCleanupComplete = true; + staleEntryKey = undefined; + } + if (head.garbageEntryKey) { + this.#store.delete(head.garbageEntryKey); + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if (latest.pending?.owner !== owner) { + return latest; + } + const { garbageEntryKey: _garbageEntryKey, ...cleaned } = latest; + return cleaned; + }); + const { garbageEntryKey: _garbageEntryKey, ...cleanedHead } = head; + head = cleanedHead; + } + const entry = createAuditEntry(type, payload, ts, this.#auditKey, head, this.#rng); + entryHash = entry.entryHash; + entryKey = reefAuditEntryKey(entry.entryHash); + let staged = false; + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if ( + latest.hash !== head.hash || + latest.seq !== head.seq || + latest.pending?.owner !== owner + ) { + return latest; + } + staged = true; + return { ...latest, pending: { ...latest.pending, entryKey } }; + }); + if (!staged) { + throw new Error("Reef audit append lease was lost before staging"); + } + inserted = this.#store.registerIfAbsent(entryKey, { kind: "entry", entry }); + if (!inserted) { + throw new Error("Reef audit entry already exists before head advancement"); + } + if (head.hash) { + updateEntry(reefAuditEntryKey(head.hash), (current) => { + const previous = parseAuditStateRecord(current); + if (previous.entry.entryHash !== head.hash) { + throw new Error("Reef audit head entry differs before linking append"); + } + const latestHead = parseReefAuditHead(this.#headStore.lookup(REEF_AUDIT_HEAD_KEY)); + if (latestHead.pending?.owner !== owner) { + throw new Error("Reef audit append lease was lost before linking"); + } + const replacesStaleLink = + previous.nextHash !== undefined && + staleEntryKey === reefAuditEntryKey(previous.nextHash); + if (previous.nextHash === entry.entryHash) { + return previous; + } + if (previous.nextHash !== undefined && !replacesStaleLink) { + throw new Error("Reef audit head already links a committed successor"); + } + return { ...previous, nextHash: entry.entryHash }; + }); + } + let oldestHash = head.seq === 0 ? entry.entryHash : head.oldestHash; + let garbageEntryKey: string | undefined; + if (head.seq >= this.#maxEntries) { + const oldest = parseAuditStateRecord( + this.#store.lookup(reefAuditEntryKey(head.oldestHash)), + ); + if (!oldest.nextHash) { + throw new Error("Reef audit retention pointer is missing"); + } + oldestHash = oldest.nextHash; + garbageEntryKey = reefAuditEntryKey(head.oldestHash); + } + let advanced = false; + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if ( + latest.hash !== head.hash || + latest.seq !== head.seq || + latest.pending?.owner !== owner || + latest.pending.entryKey !== entryKey + ) { + return latest; + } + advanced = true; + return { + kind: "head", + hash: entry.entryHash, + seq: entry.event.seq, + oldestHash, + ...(garbageEntryKey ? { garbageEntryKey } : {}), + }; + }); + if (!advanced) { + throw new Error("Reef audit append lease was lost before commit"); + } + if (garbageEntryKey) { + try { + this.#store.delete(garbageEntryKey); + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if (latest.hash !== entry.entryHash || latest.garbageEntryKey !== garbageEntryKey) { + return latest; + } + const { garbageEntryKey: _garbageEntryKey, ...cleaned } = latest; + return cleaned; + }); + } catch { + // The committed head names the orphan. The next lease holder + // removes it before consuming the single overflow slot. + } + } + return structuredClone(entry); + } catch (error) { + const latestHead = parseReefAuditHead(this.#headStore.lookup(REEF_AUDIT_HEAD_KEY)); + const entryOwnedElsewhere = + entryKey !== undefined && + ((latestHead.hash === entryHash && latestHead.seq === head.seq + 1) || + (latestHead.pending?.owner !== owner && latestHead.pending?.entryKey === entryKey)); + if (inserted && entryKey && !entryOwnedElsewhere) { + this.#store.delete(entryKey); + } + if (entryKey && head.hash && !entryOwnedElsewhere) { + updateEntry(reefAuditEntryKey(head.hash), (current) => { + const previous = parseAuditStateRecord(current); + if (previous.nextHash !== entryHash) { + return previous; + } + const { nextHash: _nextHash, ...unlinked } = previous; + return unlinked; + }); + } + update(REEF_AUDIT_HEAD_KEY, (current) => { + const latest = parseReefAuditHead(current); + if (latest.pending?.owner !== owner) { + return latest; + } + if (!staleCleanupComplete && staleEntryKey) { + return { + ...latest, + pending: { + owner, + expiresAt: Math.max(1, Date.now() - 1), + entryKey: staleEntryKey, + }, + }; + } + const { pending: _pending, ...committed } = latest; + return committed; + }); + throw error; + } + } + throw new Error("Reef audit append contention exceeded retry budget"); + } + + async entries(): Promise { + const head = parseReefAuditHead(this.#headStore.lookup(REEF_AUDIT_HEAD_KEY)); + if (head.seq === 0) { + return []; + } + const reversed: AuditEntry[] = []; + let hash = head.hash; + for (let seq = head.seq; seq > 0 && reversed.length < this.#maxEntries; seq--) { + const record = this.#store.lookup(reefAuditEntryKey(hash)); + if (!record) { + break; + } + const entry = parseAuditEntryRecord(record); + if (entry.entryHash !== hash || entry.event.seq !== seq) { + throw new Error("invalid Reef audit chain state"); + } + reversed.push(entry); + hash = entry.prevHash; + } + const expectedEntries = Math.min(head.seq, this.#maxEntries); + if (reversed.length !== expectedEntries) { + throw new Error("Reef audit chain is shorter than its committed retention window"); + } + const entries = reversed.toReversed(); + const first = entries[0]; + if ( + !first || + !verifyChainSegment(entries, { + previousHash: first.prevHash, + previousSeq: first.event.seq - 1, + head: head.hash, + }) + ) { + throw new Error("invalid Reef audit chain state"); + } + return structuredClone(entries); + } +} + +export function openReefAuditStore( + runtime: PluginRuntime, + auditKey: Uint8Array, + maxEntries?: number, +): AuditStore { + return new ReefSqliteAuditStore(runtime, auditKey, randomBytes, maxEntries); +} diff --git a/extensions/reef/src/channel.ts b/extensions/reef/src/channel.ts index 9c8fc527ddac..cdf0f267bc30 100644 --- a/extensions/reef/src/channel.ts +++ b/extensions/reef/src/channel.ts @@ -14,6 +14,7 @@ import { ReefChannelConfigSchema, autonomyBudget, normalizeReefTarget, + parseReefRelayUrl, resolveReefConfig, type ReefCoreConfig, } from "./config-schema.js"; @@ -29,7 +30,7 @@ import { import { isRephrasedReefResend } from "./rejection-resend.js"; import { getActiveReef, getOptionalReefRuntime, getReefRuntime, setActiveReef } from "./runtime.js"; import { reefSetupAdapter, reefSetupWizard } from "./setup.js"; -import { loadKeys, openStores, resolveStateDir, ReviewApprovalStore } from "./state.js"; +import { assertReefIdentityBinding, loadKeys, openStores } from "./state.js"; import { ReefInboxConnection, ReefTransportClient, @@ -191,15 +192,18 @@ export const reefPlugin: ChannelPlugin = { throw new Error("Reef requires handle, email, and guard config"); } const runtime = getReefRuntime(); - const stateDir = resolveStateDir(ctx.account.config.stateDir); - const keys = await loadKeys(stateDir); + const keys = await loadKeys(runtime); + assertReefIdentityBinding(runtime, { + handle: ctx.account.config.handle!, + relayUrl: parseReefRelayUrl(ctx.account.config.relayUrl), + }); const transport = new ReefTransportClient( ctx.account.config.relayUrl, ctx.account.config.handle!, keys, ); - const stores = openStores(stateDir, keys); - const reviews = new ReviewApprovalStore(stateDir); + const stores = openStores(runtime, keys); + const reviews = stores.reviews; const pairing = createChannelPairingController({ core: runtime, channel: "reef", @@ -270,12 +274,12 @@ export const reefPlugin: ChannelPlugin = { config: ctx.account.config, trust, keys, - stateDir, transport, guard: createConfiguredGuard(ctx.account.config), audit: stores.audit, replay: stores.replay, reviews, + delivered: stores.delivered, onIngress, onOwnerNotice: async (text) => ownerNotice({ diff --git a/extensions/reef/src/cli.ts b/extensions/reef/src/cli.ts index d652b9b7176d..80bdcdc55c61 100644 --- a/extensions/reef/src/cli.ts +++ b/extensions/reef/src/cli.ts @@ -1,8 +1,6 @@ // Reef plugin module implements headless CLI behavior. Every command is // non-interactive so agents can register a claw and manage friendships when // asked to by their owner; --json emits machine-readable results. -import { readFile, rm } from "node:fs/promises"; -import { join } from "node:path"; import type { Command } from "commander"; import { createChannelPairingController } from "openclaw/plugin-sdk/channel-pairing"; import { mutateConfigFile } from "openclaw/plugin-sdk/config-mutation"; @@ -15,9 +13,25 @@ import { } from "./config-schema.js"; import { ReefAutonomySchema } from "./friend-types.js"; import { ReefFriendManager } from "./friends.js"; +import { assertLegacyReefKeysMigrated, REEF_LEGACY_KEYS_PENDING_CODE } from "./legacy-key-guard.js"; import { getReefRuntime } from "./runtime.js"; -import { generateAndStoreKeys, loadKeys, resolveStateDir, writePrivateJson } from "./state.js"; -import { ReefTransportClient } from "./transport.js"; +import { + assertReefIdentityBinding, + clearReefSetupSession, + finalizeReefIdentityBinding, + generateAndStoreKeys, + loadKeys, + loadReefIdentityBinding, + loadReefSetupSession, + releaseReefIdentityReservation, + reserveReefIdentityBinding, + saveReefSetupSession, +} from "./state.js"; +import { + isDefinitiveReefRegistrationFailure, + isReefOwnershipRejection, + ReefTransportClient, +} from "./transport.js"; import { openReefTrustStore } from "./trust-store.js"; import type { ReefKeys } from "./types.js"; @@ -73,15 +87,20 @@ function reefCliAction { +async function loadOrCreateKeys( + createMissing: boolean, + legacyStateDir?: string, +): Promise { + const runtime = getReefRuntime(); try { - return await loadKeys(stateDir); + return await loadKeys(runtime); } catch (error) { // Only a missing key file may mint a new identity. Replacing keys on // corruption or I/O failures would orphan the relay handle and every // pinned friendship bound to the old public keys. if (createMissing && (error as NodeJS.ErrnoException).code === "ENOENT") { - return await generateAndStoreKeys(stateDir); + await assertLegacyReefKeysMigrated(legacyStateDir); + return await generateAndStoreKeys(runtime); } throw error; } @@ -106,10 +125,11 @@ async function loadConfiguredManager(output: ReefCliOutput): Promise<{ if (!config?.handle) { return await fail(output, "Reef is not configured. Run `openclaw reef register` first."); } - const stateDir = resolveStateDir(config.stateDir); - const keys = await loadOrCreateKeys(stateDir, false); - const transport = new ReefTransportClient(config.relayUrl, config.handle, keys); + const keys = await loadOrCreateKeys(false); const runtime = getReefRuntime(); + const relayUrl = parseReefRelayUrl(config.relayUrl); + assertReefIdentityBinding(runtime, { handle: config.handle, relayUrl }); + const transport = new ReefTransportClient(relayUrl, config.handle, keys); const pairing = createChannelPairingController({ core: runtime, channel: "reef", @@ -151,6 +171,22 @@ async function writeReefRegistration(candidate: ReefChannelConfig): Promise { + await mutateConfigFile({ + afterWrite: { mode: "auto" }, + mutate(draft: OpenClawConfig) { + const existing = draft.channels?.reef; + draft.channels = { + ...draft.channels, + reef: { + ...(existing && typeof existing === "object" && !Array.isArray(existing) ? existing : {}), + stateDir, + }, + }; + }, + }); +} + async function runRegister(output: ReefCliOutput, options: RegisterOptions): Promise { if (!options.email.includes("@")) { return await fail(output, "A valid --email is required."); @@ -161,36 +197,49 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro return await fail(output, "--guard-provider must be one of: anthropic, openai."); } const relayUrl = parseReefRelayUrl(options.relay); - const stateDir = resolveStateDir(options.stateDir); - const requestedHandle = options.handle?.toLowerCase(); - // One state dir is one identity. Reusing existing keys for a different handle - // or relay would link supposedly separate identities under a single - // fingerprint. The binding is persisted beside the keys so the check holds - // even if the channel config was deleted or points elsewhere. - const identityPath = join(stateDir, "identity.json"); - const identity = await readFile(identityPath, "utf8").then( - (raw) => JSON.parse(raw) as { handle?: string; relayUrl?: string }, - () => undefined, - ); - if (identity?.handle && (identity.handle !== requestedHandle || identity.relayUrl !== relayUrl)) { + const legacyStateDir = options.stateDir ?? currentReefConfig()?.stateDir; + const explicitHandle = options.handle?.toLowerCase(); + // One plugin-state identity may bind to one handle and relay. This check + // survives config deletion and prevents linking peers under reused keys. + const runtime = getReefRuntime(); + const identity = loadReefIdentityBinding(runtime); + if ( + identity?.handle && + (identity.relayUrl !== relayUrl || + (explicitHandle !== undefined && identity.handle !== explicitHandle)) + ) { return await fail( output, - `This state dir already holds the identity @${identity.handle} on ${identity.relayUrl}. Re-register the same handle and relay, or pass a fresh --state-dir for a new identity.`, + `This OpenClaw state already holds the Reef identity @${identity.handle} on ${identity.relayUrl}. Re-register the same handle and relay.`, ); } - const keys = await loadOrCreateKeys(stateDir, true); + const requestedHandle = explicitHandle ?? identity?.handle; + let keys: ReefKeys; + try { + keys = await loadOrCreateKeys(true, legacyStateDir); + } catch (error) { + if ( + options.stateDir && + (error as NodeJS.ErrnoException).code === REEF_LEGACY_KEYS_PENDING_CODE + ) { + try { + await writeReefMigrationStateDir(options.stateDir); + } catch (writeError) { + throw new Error("Failed to save the Reef legacy state directory for Doctor", { + cause: writeError, + }); + } + } + throw error; + } - const bootstrap = new ReefTransportClient(relayUrl, options.handle ?? "pending", keys); - const sessionPath = join(stateDir, "setup-session.json"); - // A previously exchanged session is reused from the key store so retries + const bootstrap = new ReefTransportClient(relayUrl, requestedHandle ?? "pending", keys); + // A previously exchanged session is reused from plugin state so retries // never need the single-use token again and the credential never appears in // command output or automation logs. It is scoped to the relay and email it // was minted for, and explicit --session/--token always take precedence, so // stale state can never reach another account or origin. - const stored = await readFile(sessionPath, "utf8").then( - (raw) => JSON.parse(raw) as { session?: string; relayUrl?: string; email?: string }, - () => undefined, - ); + const stored = loadReefSetupSession(runtime); const token = options.token?.trim(); const storedSession = !options.session?.trim() && stored?.relayUrl === relayUrl && stored?.email === options.email @@ -222,7 +271,7 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro // Validate everything that completion needs BEFORE consuming the single-use // token or mutating relay state, so a bad flag cannot burn the credential or // claim a handle that a retry then finds taken. - const handle = options.handle?.toLowerCase(); + const handle = requestedHandle; if (!handle || !HANDLE_PATTERN.test(handle)) { return await fail(output, "A valid --handle is required (lowercase letters, digits, - or _)."); } @@ -243,19 +292,36 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro handle, email: options.email, requestPolicy: options.policy, - stateDir, + ...(legacyStateDir ? { stateDir: legacyStateDir } : {}), guard, }; ReefChannelConfigSchema.parse(provisional); + // Reserve keys to this handle before consuming auth or mutating the relay. + // Retries are idempotent; mismatched concurrent registrations fail closed. + const reservation = reserveReefIdentityBinding(runtime, { handle, relayUrl }); let resolvedSession = session; if (!resolvedSession) { - resolvedSession = (await bootstrap.authComplete(token ?? "")).session; - await writePrivateJson(sessionPath, { - session: resolvedSession, - relayUrl, - email: options.email, - }); + try { + resolvedSession = (await bootstrap.authComplete(token ?? "")).session; + } catch (error) { + if (isDefinitiveReefRegistrationFailure(error)) { + releaseReefIdentityReservation(runtime, reservation); + } else { + finalizeReefIdentityBinding(runtime, reservation); + } + throw error; + } + try { + saveReefSetupSession(runtime, { + session: resolvedSession, + relayUrl, + email: options.email, + }); + } catch (error) { + releaseReefIdentityReservation(runtime, reservation); + throw error; + } } const transport = new ReefTransportClient(relayUrl, handle, keys); let effectivePolicy = options.policy; @@ -267,15 +333,33 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro // A device-signed read only succeeds when OUR key owns the handle; treat // that as the claim already being done instead of stranding the retry. const unavailable = error instanceof Error && error.message.includes("handle_unavailable"); - const owned = - unavailable && - (await transport.listFriends().then( - () => true, - () => false, - )); + let owned = false; + if (unavailable) { + try { + await transport.listFriends(); + owned = true; + } catch (verificationError) { + if (isReefOwnershipRejection(verificationError)) { + releaseReefIdentityReservation(runtime, reservation); + } else { + // A failed probe proves non-ownership only for the relay's explicit + // unknown-handle result. Keep all other outcomes bound to these keys. + finalizeReefIdentityBinding(runtime, reservation); + } + throw verificationError; + } + } if (!owned) { + if (isDefinitiveReefRegistrationFailure(error)) { + releaseReefIdentityReservation(runtime, reservation); + } else { + finalizeReefIdentityBinding(runtime, reservation); + } throw error; } + // Signed access proves these keys already own the handle. Persist that + // invariant before the account-list request, which can fail ambiguously. + finalizeReefIdentityBinding(runtime, reservation); const { handles } = await transport.listOwnHandles(resolvedSession); const existingHandle = handles.find((entry) => entry.handle === handle); if (!existingHandle) { @@ -289,6 +373,7 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro } effectivePolicy = existingHandle.request_policy; } + finalizeReefIdentityBinding(runtime, reservation); const candidate = ReefChannelConfigSchema.parse({ ...provisional, @@ -303,11 +388,10 @@ async function runRegister(output: ReefCliOutput, options: RegisterOptions): Pro `Handle @${handle} is claimed, but writing the local config failed: ${error instanceof Error ? error.message : String(error)}. Fix the local issue and rerun the exact same command — the retry reuses the stored session and recognizes the existing claim.`, ); } - await writePrivateJson(identityPath, { handle, relayUrl }); - await rm(sessionPath, { force: true }); + clearReefSetupSession(runtime); const printed = fingerprint(keys.signing.publicKey, keys.encryption.publicKey); - emit(output, { status: "registered", handle, relayUrl, stateDir, fingerprint: printed }, [ + emit(output, { status: "registered", handle, relayUrl, fingerprint: printed }, [ `Registered @${handle} on ${relayUrl}.`, `Safety fingerprint (share out of band): ${printed}`, "Restart the gateway to connect: openclaw gateway restart", @@ -328,7 +412,7 @@ export function registerReefCli({ program }: { program: Command }): void { .option("--token ", "Magic-link token to exchange for a session") .option("--relay ", "Relay origin URL", "https://reefwire.ai") .option("--policy ", "Inbound friend-request policy", "code-only") - .option("--state-dir ", "Local key/state directory") + .option("--state-dir ", "Legacy Reef file directory for Doctor import") .option("--guard-provider ", "Guard provider (anthropic|openai)", "openai") .option("--guard-model ", "Immutable guard model id (default depends on provider)") .option("--guard-env ", "Env var holding the guard API key (default depends on provider)") diff --git a/extensions/reef/src/doctor-durable-state.ts b/extensions/reef/src/doctor-durable-state.ts new file mode 100644 index 000000000000..67bbbccafef3 --- /dev/null +++ b/extensions/reef/src/doctor-durable-state.ts @@ -0,0 +1,669 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { PluginStateKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + archiveLegacyStateSource, + type PluginDoctorStateMigration, +} from "openclaw/plugin-sdk/runtime-doctor"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + verifyChain, + verifyChainSegment, + type AuditEntry, + type ReviewRequest, + type SignedReceipt, +} from "../protocol/index.js"; +import { + legacyReefFileExists, + REEF_DURABLE_LEGACY_FILENAMES, + resolveLegacyReefStateDir, +} from "./doctor-state-paths.js"; +import { + REEF_AUDIT_HEAD_KEY, + REEF_AUDIT_HEAD_MAX_ENTRIES, + REEF_AUDIT_HEAD_NAMESPACE, + REEF_AUDIT_MAX_ENTRIES, + REEF_AUDIT_MIGRATION_KEY, + REEF_AUDIT_MIGRATION_MAX_ENTRIES, + REEF_AUDIT_MIGRATION_NAMESPACE, + REEF_AUDIT_NAMESPACE, + REEF_AUDIT_STORE_MAX_ENTRIES, + REEF_DELIVERED_MAX_ENTRIES, + REEF_DELIVERED_NAMESPACE, + REEF_DELIVERED_TTL_MS, + REEF_DURABLE_MIGRATION_KEY, + REEF_DURABLE_MIGRATION_MAX_ENTRIES, + REEF_DURABLE_MIGRATION_NAMESPACE, + REEF_REPLAY_MAX_ENTRIES, + REEF_REPLAY_NAMESPACE, + REEF_REPLAY_TTL_MS, + REEF_REVIEWS_MAX_ENTRIES, + REEF_REVIEWS_NAMESPACE, + parseReefAuditHead, + reefAuditEntryKey, + reefReplayStoreKey, + type ReefAuditHeadRecord, + type ReefAuditStateRecord, + type ReefReplayRecord, + type ReefReviewRecord, + type ReefDurableMigrationRecord, + type ReefIdentityMigrationRecord, + REEF_KEYS_MIGRATION_KEY, + REEF_KEYS_MIGRATION_MAX_ENTRIES, + REEF_KEYS_MIGRATION_NAMESPACE, +} from "./state.js"; + +const REEF_RUNTIME_LEGACY_FILENAMES = ["replay.jsonl", "reviews.json", "delivered.json"]; + +type ReefAuditMigrationRecord = { pending: true; expectedEntries?: number }; + +async function readLegacyReefAudit(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf8"); + const entries = raw + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as AuditEntry); + if (!verifyChain(entries)) { + throw new Error("invalid Reef audit chain"); + } + return entries; +} + +async function readStoredReefAudit( + store: PluginStateKeyedStore, + headStore: PluginStateKeyedStore, +): Promise { + const headValue = await headStore.lookup(REEF_AUDIT_HEAD_KEY); + if (!headValue) { + return []; + } + const head = parseReefAuditHead(headValue); + const reversed: AuditEntry[] = []; + let hash = head.hash; + for (let seq = head.seq; seq > 0 && reversed.length < REEF_AUDIT_MAX_ENTRIES; seq--) { + const record = await store.lookup(reefAuditEntryKey(hash)); + if (!record) { + break; + } + if (record.entry.entryHash !== hash || record.entry.event.seq !== seq) { + throw new Error("invalid Reef audit chain state"); + } + reversed.push(record.entry); + hash = record.entry.prevHash; + } + const expectedEntries = Math.min(head.seq, REEF_AUDIT_MAX_ENTRIES); + if (reversed.length !== expectedEntries) { + throw new Error("Reef audit chain is shorter than its committed retention window"); + } + const entries = reversed.toReversed(); + const first = entries[0]; + if ( + !first || + !verifyChainSegment(entries, { + previousHash: first.prevHash, + previousSeq: first.event.seq - 1, + head: head.hash, + }) + ) { + throw new Error("invalid Reef audit chain state"); + } + return entries; +} + +type LegacyReefReplayLogRecord = + | { op: "claim"; peer: string; id: string; envelopeHash: string } + | { op: "complete"; peer: string; id: string; receipt: SignedReceipt; body?: { enc: string } } + | { op: "consume" | "release"; peer: string; id: string }; + +function requireLegacyReplayString(record: Record, field: string): string { + const value = record[field]; + if (typeof value !== "string" || value.length === 0) { + throw new Error(`invalid Reef replay ${field}`); + } + return value; +} + +function parseLegacyReefReplayLine(value: unknown): LegacyReefReplayLogRecord { + if (!isRecord(value)) { + throw new Error("invalid Reef replay record"); + } + const peer = requireLegacyReplayString(value, "peer"); + const id = requireLegacyReplayString(value, "id"); + if (value.op === "claim") { + return { + op: "claim", + peer, + id, + envelopeHash: requireLegacyReplayString(value, "envelopeHash"), + }; + } + if (value.op === "consume" || value.op === "release") { + return { op: value.op, peer, id }; + } + if (value.op !== "complete" || !isRecord(value.receipt)) { + throw new Error("invalid Reef replay operation"); + } + const receipt = value.receipt as unknown as SignedReceipt; + if (receipt.id !== id || !["accepted", "rejected"].includes(receipt.status)) { + throw new Error("invalid Reef replay receipt"); + } + const body = value.body; + if ( + (receipt.status === "accepted" && (!isRecord(body) || typeof body.enc !== "string")) || + (receipt.status === "rejected" && body !== undefined) + ) { + throw new Error("invalid Reef replay completion"); + } + return { + op: "complete", + peer, + id, + receipt, + ...(isRecord(body) && typeof body.enc === "string" ? { body: { enc: body.enc } } : {}), + }; +} + +async function readLegacyReefReplay(filePath: string): Promise { + const raw = await fs.readFile(filePath, "utf8"); + const lines = raw.split("\n").filter((line) => line.length > 0); + const records = new Map(); + for (const [index, line] of lines.entries()) { + let log: LegacyReefReplayLogRecord; + try { + log = parseLegacyReefReplayLine(JSON.parse(line) as unknown); + } catch (error) { + // The old append-only store tolerated only a torn final write. + if (index === lines.length - 1 && !raw.endsWith("\n")) { + break; + } + throw error; + } + const key = reefReplayStoreKey(log.peer, log.id); + const existing = records.get(key); + let next: ReefReplayRecord; + if (log.op === "claim") { + if (existing && existing.envelopeHash !== log.envelopeHash) { + throw new Error("conflicting Reef replay binding"); + } + next = { + peer: log.peer, + id: log.id, + envelopeHash: log.envelopeHash, + state: "available", + }; + } else { + if (!existing) { + throw new Error(`Reef replay ${log.op} lacks claim`); + } + if (log.op === "complete") { + next = { + ...existing, + state: "completed", + receipt: log.receipt, + ...(log.body ? { body: log.body } : {}), + }; + } else if (log.op === "consume") { + next = { + peer: existing.peer, + id: existing.id, + envelopeHash: existing.envelopeHash, + state: "consumed", + }; + } else { + next = { ...existing, state: "available" }; + } + } + records.delete(key); + records.set(key, next); + } + return [...records.values()]; +} + +async function readLegacyReefReviews(filePath: string): Promise> { + const value = JSON.parse(await fs.readFile(filePath, "utf8")) as unknown; + if (!isRecord(value)) { + throw new Error("invalid Reef reviews file"); + } + const records = new Map(); + for (const [digest, raw] of Object.entries(value)) { + if (!isRecord(raw) || !isRecord(raw.review)) { + throw new Error(`invalid Reef review ${digest}`); + } + const review = raw.review as unknown as ReviewRequest; + if ( + review.approvalDigest !== digest || + (raw.approved !== undefined && typeof raw.approved !== "boolean") + ) { + throw new Error(`invalid Reef review ${digest}`); + } + records.set(digest, { + review, + ...(typeof raw.approved === "boolean" ? { approved: raw.approved } : {}), + }); + } + return records; +} + +async function readLegacyReefDelivered(filePath: string): Promise { + const value = JSON.parse(await fs.readFile(filePath, "utf8")) as unknown; + if (!Array.isArray(value) || value.some((id) => typeof id !== "string" || id.length === 0)) { + throw new Error("invalid Reef delivered file"); + } + return [...new Set(value)]; +} + +export const reefAuditStateMigration: PluginDoctorStateMigration = { + id: "reef-audit-jsonl-to-plugin-state", + label: "Reef audit trail", + async detectLegacyState(params) { + const filePath = path.join(resolveLegacyReefStateDir(params), "audit.jsonl"); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const sourceExists = await legacyReefFileExists(filePath); + const pending = await migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY); + return sourceExists || pending + ? { + preview: [ + sourceExists + ? "- Reef audit trail -> plugin state (audit)" + : "- Verify Reef audit migration marker", + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const filePath = path.join(resolveLegacyReefStateDir(params), "audit.jsonl"); + const migrationStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const store = params.context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_NAMESPACE, + maxEntries: REEF_AUDIT_STORE_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const headStore = params.context.openPluginStateKeyedStore({ + namespace: REEF_AUDIT_HEAD_NAMESPACE, + maxEntries: REEF_AUDIT_HEAD_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + if ( + (await legacyReefFileExists(filePath)) || + (await migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY)) || + (await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY)) + ) { + await durableMigrationStore.register(REEF_DURABLE_MIGRATION_KEY, { pending: true }); + } + if (!(await legacyReefFileExists(filePath))) { + const pending = await migrationStore.lookup(REEF_AUDIT_MIGRATION_KEY); + if (!pending) { + return { changes, warnings }; + } + try { + const canonical = await readStoredReefAudit(store, headStore); + if ( + pending.expectedEntries === undefined + ? canonical.length === 0 + : canonical.length !== pending.expectedEntries + ) { + throw new Error("canonical audit trail does not match the verified import"); + } + await migrationStore.delete(REEF_AUDIT_MIGRATION_KEY); + changes.push("Verified Reef audit trail; cleared completed migration marker"); + } catch (error) { + warnings.push( + `Reef audit migration is incomplete and audit.jsonl is missing: ${String(error)}; left migration blocker in place`, + ); + } + return { changes, warnings }; + } + await migrationStore.register(REEF_AUDIT_MIGRATION_KEY, { pending: true }); + let legacy: AuditEntry[]; + try { + legacy = await readLegacyReefAudit(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { changes, warnings }; + } + warnings.push(`Failed importing Reef audit trail: ${String(error)}; left source in place`); + return { changes, warnings }; + } + let canonical: AuditEntry[]; + try { + canonical = await readStoredReefAudit(store, headStore); + } catch (error) { + warnings.push( + `Failed reading canonical Reef audit trail: ${String(error)}; left legacy source in place`, + ); + return { changes, warnings }; + } + if ( + canonical.length > 0 && + JSON.stringify(canonical) !== JSON.stringify(legacy.slice(-canonical.length)) + ) { + warnings.push("Kept existing Reef audit trail; left differing legacy source in place"); + return { changes, warnings }; + } + const retained = legacy.slice(-REEF_AUDIT_MAX_ENTRIES); + if (canonical.length === 0 && retained.length > 0) { + try { + for (const [index, entry] of retained.entries()) { + const key = reefAuditEntryKey(entry.entryHash); + const nextHash = retained[index + 1]?.entryHash; + const record: ReefAuditStateRecord = { + kind: "entry", + entry, + ...(nextHash ? { nextHash } : {}), + }; + const existing = await store.lookup(key); + if (existing && JSON.stringify(existing) !== JSON.stringify(record)) { + throw new Error(`conflicting audit entry ${entry.entryHash}`); + } + await store.registerIfAbsent(key, record); + } + const last = retained.at(-1)!; + const first = retained[0]!; + if ( + !(await headStore.registerIfAbsent(REEF_AUDIT_HEAD_KEY, { + kind: "head", + hash: last.entryHash, + seq: last.event.seq, + oldestHash: first.entryHash, + })) + ) { + throw new Error("audit head appeared during import"); + } + } catch (error) { + warnings.push(`Failed importing Reef audit trail: ${String(error)}; left source in place`); + return { changes, warnings }; + } + } + const persisted = await readStoredReefAudit(store, headStore); + if (JSON.stringify(persisted) !== JSON.stringify(retained)) { + warnings.push("Failed verifying Reef audit trail after import; left source in place"); + return { changes, warnings }; + } + changes.push( + `Migrated ${legacy.length} Reef audit ${legacy.length === 1 ? "entry" : "entries"} -> plugin state`, + ); + // Persist the verified cardinality before archiving. A rerun can then + // distinguish an interrupted empty import from a missing legacy source. + await migrationStore.register(REEF_AUDIT_MIGRATION_KEY, { + pending: true, + expectedEntries: persisted.length, + }); + const warningCount = warnings.length; + await archiveLegacyStateSource({ + filePath, + label: "Reef audit trail", + changes, + warnings, + }); + if (persisted.length < legacy.length && warnings.length === warningCount) { + changes.push( + `Retained the newest ${persisted.length} Reef audit entries in SQLite; preserved the complete ${legacy.length}-entry chain in the archived legacy source`, + ); + } + if (warnings.length === warningCount) { + await migrationStore.delete(REEF_AUDIT_MIGRATION_KEY); + } + return { changes, warnings }; + }, +}; + +export const reefRuntimeStateMigration: PluginDoctorStateMigration = { + id: "reef-runtime-files-to-plugin-state", + label: "Reef durable runtime state", + async detectLegacyState(params) { + const stateDir = resolveLegacyReefStateDir(params); + const files = ( + await Promise.all( + REEF_RUNTIME_LEGACY_FILENAMES.map(async (filename) => ({ + filename, + exists: await legacyReefFileExists(path.join(stateDir, filename)), + })), + ) + ).filter((entry) => entry.exists); + const durableSourceExists = ( + await Promise.all( + REEF_DURABLE_LEGACY_FILENAMES.map((filename) => + legacyReefFileExists(path.join(stateDir, filename)), + ), + ) + ).some(Boolean); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durablePending = await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY); + return files.length > 0 || durableSourceExists || durablePending + ? { + preview: [ + files.length > 0 + ? `- Reef runtime state -> plugin state (${files.map((entry) => entry.filename).join(", ")})` + : durableSourceExists + ? "- Finalize Reef durable state migration barrier" + : "- Verify Reef durable state migration barrier", + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const stateDir = resolveLegacyReefStateDir(params); + const durableMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const durablePending = await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY); + const runtimeSourceExists = ( + await Promise.all( + REEF_RUNTIME_LEGACY_FILENAMES.map((filename) => + legacyReefFileExists(path.join(stateDir, filename)), + ), + ) + ).some(Boolean); + if (runtimeSourceExists || durablePending) { + await durableMigrationStore.register(REEF_DURABLE_MIGRATION_KEY, { pending: true }); + } + const replayPath = path.join(stateDir, "replay.jsonl"); + if (await legacyReefFileExists(replayPath)) { + try { + const legacy = await readLegacyReefReplay(replayPath); + const store = params.context.openPluginStateKeyedStore({ + namespace: REEF_REPLAY_NAMESPACE, + maxEntries: REEF_REPLAY_MAX_ENTRIES, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_REPLAY_TTL_MS, + }); + const canonicalEntries = await store.entries(); + const canonical = new Map(canonicalEntries.map((entry) => [entry.key, entry.value])); + for (const record of legacy) { + const key = reefReplayStoreKey(record.peer, record.id); + const existing = canonical.get(key); + if (existing && JSON.stringify(existing) !== JSON.stringify(record)) { + throw new Error(`canonical replay state ${key} differs`); + } + } + const missing = legacy.filter( + (record) => !canonical.has(reefReplayStoreKey(record.peer, record.id)), + ); + if (canonical.size + missing.length > REEF_REPLAY_MAX_ENTRIES) { + throw new Error( + `${canonical.size + missing.length} replay bindings exceed plugin-state capacity`, + ); + } + for (const record of missing) { + await store.registerIfAbsent(reefReplayStoreKey(record.peer, record.id), record); + } + for (const entry of canonicalEntries) { + if (JSON.stringify(await store.lookup(entry.key)) !== JSON.stringify(entry.value)) { + throw new Error(`canonical replay state ${entry.key} changed during import`); + } + } + for (const record of missing) { + if ( + JSON.stringify(await store.lookup(reefReplayStoreKey(record.peer, record.id))) !== + JSON.stringify(record) + ) { + throw new Error("persisted replay state differs"); + } + } + changes.push(`Migrated ${legacy.length} Reef replay bindings -> plugin state`); + await archiveLegacyStateSource({ + filePath: replayPath, + label: "Reef replay state", + changes, + warnings, + }); + } catch (error) { + warnings.push(`Failed importing Reef replay state: ${String(error)}; left source in place`); + } + } + + const reviewsPath = path.join(stateDir, "reviews.json"); + if (await legacyReefFileExists(reviewsPath)) { + try { + const legacy = await readLegacyReefReviews(reviewsPath); + const pending = [...legacy].filter(([, record]) => record.approved === undefined); + if (pending.length > REEF_REVIEWS_MAX_ENTRIES) { + throw new Error(`${pending.length} pending reviews exceed plugin-state capacity`); + } + const completed = [...legacy].filter(([, record]) => record.approved !== undefined); + const completedCapacity = REEF_REVIEWS_MAX_ENTRIES - pending.length; + const retainedCompleted = completedCapacity > 0 ? completed.slice(-completedCapacity) : []; + const retainedKeys = new Set([...pending, ...retainedCompleted].map(([digest]) => digest)); + const retained = new Map([...legacy].filter(([digest]) => retainedKeys.has(digest))); + const store = params.context.openPluginStateKeyedStore({ + namespace: REEF_REVIEWS_NAMESPACE, + maxEntries: REEF_REVIEWS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + for (const [digest, record] of retained) { + const existing = await store.lookup(digest); + if (existing && JSON.stringify(existing) !== JSON.stringify(record)) { + throw new Error(`canonical review ${digest} differs`); + } + if (!existing) { + await store.registerIfAbsent(digest, record); + } + } + for (const [digest, record] of retained) { + if (JSON.stringify(await store.lookup(digest)) !== JSON.stringify(record)) { + throw new Error(`persisted review ${digest} differs`); + } + } + changes.push(`Migrated ${retained.size} of ${legacy.size} Reef reviews -> plugin state`); + await archiveLegacyStateSource({ + filePath: reviewsPath, + label: "Reef reviews", + changes, + warnings, + }); + } catch (error) { + warnings.push(`Failed importing Reef reviews: ${String(error)}; left source in place`); + } + } + + const deliveredPath = path.join(stateDir, "delivered.json"); + if (await legacyReefFileExists(deliveredPath)) { + try { + const legacy = await readLegacyReefDelivered(deliveredPath); + const store = params.context.openPluginStateKeyedStore<{ id: string }>({ + namespace: REEF_DELIVERED_NAMESPACE, + maxEntries: REEF_DELIVERED_MAX_ENTRIES, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_DELIVERED_TTL_MS, + }); + const canonicalEntries = await store.entries(); + const canonical = new Map(canonicalEntries.map((entry) => [entry.key, entry.value])); + for (const id of legacy) { + const existing = canonical.get(id); + if (existing && existing.id !== id) { + throw new Error(`canonical delivered marker ${id} differs`); + } + } + const missing = legacy.filter((id) => !canonical.has(id)); + if (canonical.size + missing.length > REEF_DELIVERED_MAX_ENTRIES) { + throw new Error( + `${canonical.size + missing.length} delivered markers exceed plugin-state capacity`, + ); + } + for (const id of missing) { + await store.registerIfAbsent(id, { id }); + } + for (const entry of canonicalEntries) { + if (JSON.stringify(await store.lookup(entry.key)) !== JSON.stringify(entry.value)) { + throw new Error(`canonical delivered marker ${entry.key} changed during import`); + } + } + for (const id of missing) { + if ((await store.lookup(id))?.id !== id) { + throw new Error(`persisted delivered marker ${id} differs`); + } + } + changes.push(`Migrated ${legacy.length} Reef delivered markers -> plugin state`); + await archiveLegacyStateSource({ + filePath: deliveredPath, + label: "Reef delivered markers", + changes, + warnings, + }); + } catch (error) { + warnings.push( + `Failed importing Reef delivered markers: ${String(error)}; left source in place`, + ); + } + } + const remainingSources = ( + await Promise.all( + REEF_DURABLE_LEGACY_FILENAMES.map(async (filename) => ({ + filename, + exists: await legacyReefFileExists(path.join(stateDir, filename)), + })), + ) + ).filter((entry) => entry.exists); + const identityMigrationStore = + params.context.openPluginStateKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + const auditMigrationStore = params.context.openPluginStateKeyedStore<{ pending: true }>({ + namespace: REEF_AUDIT_MIGRATION_NAMESPACE, + maxEntries: REEF_AUDIT_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + if ( + remainingSources.length === 0 && + !(await identityMigrationStore.lookup(REEF_KEYS_MIGRATION_KEY)) && + !(await auditMigrationStore.lookup(REEF_AUDIT_MIGRATION_KEY)) + ) { + if (await durableMigrationStore.delete(REEF_DURABLE_MIGRATION_KEY)) { + changes.push("Verified all Reef durable state; cleared migration barrier"); + } + } else if (await durableMigrationStore.lookup(REEF_DURABLE_MIGRATION_KEY)) { + warnings.push( + `Reef durable state migration is incomplete; left migration blocker in place${remainingSources.length > 0 ? ` (${remainingSources.map((entry) => entry.filename).join(", ")})` : ""}`, + ); + } + return { changes, warnings }; + }, +}; diff --git a/extensions/reef/src/doctor-state-paths.ts b/extensions/reef/src/doctor-state-paths.ts new file mode 100644 index 000000000000..ff1fa6d70a36 --- /dev/null +++ b/extensions/reef/src/doctor-state-paths.ts @@ -0,0 +1,50 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { resolveUserPath } from "openclaw/plugin-sdk/account-resolution"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; + +export const REEF_DURABLE_LEGACY_FILENAMES = [ + "keys.json", + "identity.json", + "setup-session.json", + "audit.jsonl", + "replay.jsonl", + "reviews.json", + "delivered.json", +] as const; + +export function resolveLegacyReefStateDir(params: { + config: OpenClawConfig; + env: NodeJS.ProcessEnv; + stateDir: string; + homeDir?: string; +}): string { + const reef = params.config.channels?.reef; + const configured = isRecord(reef) && typeof reef.stateDir === "string" ? reef.stateDir : null; + const defaultDir = resolveDefaultLegacyReefStateDir(params.homeDir); + const configuredDir = configured ? resolveUserPath(configured, params.env) : null; + if (configuredDir) { + return configuredDir; + } + const relativeToActiveState = path.relative(path.resolve(params.stateDir), defaultDir); + return relativeToActiveState === "" || + (!relativeToActiveState.startsWith(`..${path.sep}`) && + relativeToActiveState !== ".." && + !path.isAbsolute(relativeToActiveState)) + ? defaultDir + : path.join(params.stateDir, "data", "reef"); +} + +function resolveDefaultLegacyReefStateDir(homeDir = os.homedir()): string { + return path.join(homeDir, ".openclaw", "data", "reef"); +} + +export async function legacyReefFileExists(filePath: string): Promise { + try { + return (await fs.stat(filePath)).isFile(); + } catch { + return false; + } +} diff --git a/extensions/reef/src/flow-receipts.test.ts b/extensions/reef/src/flow-receipts.test.ts index 40f70f26a4cf..8b0c979bb770 100644 --- a/extensions/reef/src/flow-receipts.test.ts +++ b/extensions/reef/src/flow-receipts.test.ts @@ -1,5 +1,4 @@ -import { randomUUID } from "node:crypto"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { canonicalBytes, composeOutbound, @@ -14,15 +13,16 @@ import { ReefMessageFlow } from "./flow.js"; import { allow, config, + flowStores, guard, peerTrust, reefKeys, + resetFlowStoresForTests, transport, trust, } from "./flow.test-helpers.js"; import { reefPeerIdentity } from "./friend-types.js"; import { processReefInboxEntriesInOrder, ReefReceiptNotifier } from "./owner-notice.js"; -import { ReviewApprovalStore } from "./state.js"; import type { ReefTransportClient } from "./transport.js"; import { REEF_OUTBOUND_DELIVERY_MAX_ENTRIES, @@ -30,6 +30,9 @@ import { } from "./trust-store.js"; import type { InboxEntry } from "./types.js"; +beforeEach(resetFlowStoresForTests); +afterEach(resetFlowStoresForTests); + describe("ReefMessageFlow delivery receipts", () => { it("quarantines an unmatched forged receipt without scanning audit history", async () => { const alice = generateIdentity(); @@ -40,12 +43,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trust({ alice: peerTrust(alice) }).store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -89,12 +92,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -167,12 +170,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trust({ alice: peerTrust(alice) }).store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -224,12 +227,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trust({ alice: peerTrust(alice) }).store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -280,12 +283,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -347,12 +350,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -419,12 +422,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: relay as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -545,12 +548,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -609,12 +612,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -679,12 +682,12 @@ describe("ReefMessageFlow delivery receipts", () => { config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: transport() as unknown as ReefTransportClient, guard: guard(allow), audit, replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); diff --git a/extensions/reef/src/flow-send.test.ts b/extensions/reef/src/flow-send.test.ts index 9e672d5f944a..c0b29337ed87 100644 --- a/extensions/reef/src/flow-send.test.ts +++ b/extensions/reef/src/flow-send.test.ts @@ -1,21 +1,24 @@ -import { randomUUID } from "node:crypto"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { generateIdentity, MemoryAuditStore, MemoryReplayStore } from "../protocol/index.js"; import { ReefMessageFlow } from "./flow.js"; import { allow, config, + flowStores, guard, peerTrust, reefKeys, + resetFlowStoresForTests, transport, trust, } from "./flow.test-helpers.js"; import { reefPeerIdentity } from "./friend-types.js"; import { reefMessageTextHash } from "./rejection-resend.js"; -import { ReviewApprovalStore } from "./state.js"; import type { ReefTransportClient } from "./transport.js"; +beforeEach(resetFlowStoresForTests); +afterEach(resetFlowStoresForTests); + describe("ReefMessageFlow send recovery", () => { it("persists automatic resends as non-resendable deliveries", async () => { const alice = reefKeys(); @@ -29,12 +32,12 @@ describe("ReefMessageFlow send recovery", () => { config: cfg, trust: trusted.store, keys: alice, - stateDir: `/tmp/reef-flow-${randomUUID()}`, + transport: relay as unknown as ReefTransportClient, guard: guard(allow), audit: new MemoryAuditStore(new Uint8Array(32).fill(7)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...flowStores(), onIngress: async () => {}, onOwnerNotice: async () => {}, }); diff --git a/extensions/reef/src/flow.test-helpers.ts b/extensions/reef/src/flow.test-helpers.ts index 8745c3ac6341..7a768914a82f 100644 --- a/extensions/reef/src/flow.test-helpers.ts +++ b/extensions/reef/src/flow.test-helpers.ts @@ -1,3 +1,12 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { vi } from "vitest"; import { base64url, @@ -10,11 +19,36 @@ import { } from "../protocol/index.js"; import { ReefChannelConfigSchema } from "./config-schema.js"; import { sameReefPeerIdentity, type ReefPeerIdentity, type ReefPeerTrust } from "./friend-types.js"; +import { ReefDeliveredStore, ReviewApprovalStore } from "./state.js"; import type { ReefTransportClient } from "./transport.js"; import type { ReefTrustStore } from "./trust-store.js"; import type { ReefKeys, ReefRejectionNoticeState } from "./types.js"; const model = "mock-2026-07-12"; +const stateDirs: string[] = []; + +export function resetFlowStoresForTests(): void { + resetPluginStateStoreForTests(); + for (const stateDir of stateDirs.splice(0)) { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +} + +export function flowStores() { + const stateDir = fs.mkdtempSync(path.join(resolvePreferredOpenClawTmpDir(), "reef-flow-")); + stateDirs.push(stateDir); + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("reef", { + ...options, + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + return { + reviews: new ReviewApprovalStore(runtime), + delivered: new ReefDeliveredStore(runtime), + }; +} + export const allow: Verdict = { decision: "allow", category: "safe", diff --git a/extensions/reef/src/flow.test.ts b/extensions/reef/src/flow.test.ts index d4b6f1ecbe80..e6274391f144 100644 --- a/extensions/reef/src/flow.test.ts +++ b/extensions/reef/src/flow.test.ts @@ -1,6 +1,4 @@ -import { randomUUID } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { canonicalBytes, generateIdentity, @@ -16,22 +14,26 @@ import { allow, config, envelope, + flowStores, guard, peerTrust, reefKeys, + resetFlowStoresForTests, transport, trust, } from "./flow.test-helpers.js"; -import { ReviewApprovalStore } from "./state.js"; import type { ReefTransportClient } from "./transport.js"; import type { InboxEntry } from "./types.js"; +beforeEach(resetFlowStoresForTests); +afterEach(resetFlowStoresForTests); + describe("ReefMessageFlow inbound", () => { it("delivers and persists before ack, then acks duplicate redelivery without delivering twice", async () => { const alice = generateIdentity(); const bob = reefKeys(); const id = "01JZ0000000000000000000104"; - const stateDir = `/tmp/reef-flow-${randomUUID()}`; + const stores = flowStores(); const order: string[] = []; const onIngress = vi.fn(async () => { order.push("ingress"); @@ -39,10 +41,7 @@ describe("ReefMessageFlow inbound", () => { const relay = transport(); const trusted = trust({ alice: peerTrust(alice) }); relay.acknowledge.mockImplementation(async () => { - const delivered = JSON.parse( - await readFile(`${stateDir}/delivered.json`, "utf8"), - ) as string[]; - expect(delivered).toContain(id); + await expect(stores.delivered.has(id)).resolves.toBe(true); order.push("ack"); return { result: "deleted" }; }); @@ -50,12 +49,11 @@ describe("ReefMessageFlow inbound", () => { config: config(), trust: trusted.store, keys: bob, - stateDir, transport: relay as unknown as ReefTransportClient, guard: guard(allow), audit: new MemoryAuditStore(new Uint8Array(32).fill(10)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress, onOwnerNotice: async () => {}, }); @@ -70,7 +68,7 @@ describe("ReefMessageFlow inbound", () => { await flow.processEntries([entry]); expect(order).toEqual(["ingress", "ack"]); - expect(JSON.parse(await readFile(`${stateDir}/delivered.json`, "utf8"))).toContain(id); + await expect(stores.delivered.has(id)).resolves.toBe(true); await flow.processEntries([{ ...entry, seq: 2 }]); expect(order).toEqual(["ingress", "ack", "ack"]); @@ -84,16 +82,16 @@ describe("ReefMessageFlow inbound", () => { const relay = transport(); const trusted = trust({ alice: peerTrust(alice) }); const ingress = new Map(); + const stores = flowStores(); const flow = new ReefMessageFlow({ config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: guard(allow), audit: new MemoryAuditStore(new Uint8Array(32).fill(4)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress: async (message) => { ingress.set(message.id, message); }, @@ -129,16 +127,16 @@ describe("ReefMessageFlow inbound", () => { const onIngress = vi.fn(); const trusted = trust({ alice: peerTrust(alice) }); const deny: Verdict = { ...allow, decision: "deny", category: "injection", reason: "Denied." }; + const stores = flowStores(); const flow = new ReefMessageFlow({ config: config(), trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: guard(deny), audit: new MemoryAuditStore(new Uint8Array(32).fill(5)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress, onOwnerNotice: async () => {}, }); @@ -169,16 +167,16 @@ describe("ReefMessageFlow inbound", () => { const classifier = guard(allow); const cfg = config(); const trusted = trust({ alice: peerTrust(alice) }); + const stores = flowStores(); const flow = new ReefMessageFlow({ config: cfg, trust: trusted.store, keys: bob, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: classifier, audit: new MemoryAuditStore(new Uint8Array(32).fill(6)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -223,16 +221,16 @@ describe("ReefMessageFlow outbound", () => { cfg.handle = "alice"; const trusted = trust({ bob: peerTrust(bob) }); const relay = transport(); + const stores = flowStores(); const flow = new ReefMessageFlow({ config: cfg, trust: trusted.store, keys: alice, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: guard(allow), audit: new MemoryAuditStore(new Uint8Array(32).fill(7)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -259,7 +257,8 @@ describe("ReefMessageFlow outbound", () => { cfg.handle = "alice"; const trusted = trust({ bob: peerTrust(bob) }); const relay = transport(); - const reviews = new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`); + const stores = flowStores(); + const { reviews } = stores; const review: Verdict = { ...allow, decision: "review", @@ -270,12 +269,11 @@ describe("ReefMessageFlow outbound", () => { config: cfg, trust: trusted.store, keys: alice, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: guard(review), audit: new MemoryAuditStore(new Uint8Array(32).fill(8)), replay: new MemoryReplayStore(), - reviews, + ...stores, onIngress: async () => {}, onOwnerNotice: async () => {}, }); @@ -323,16 +321,16 @@ describe("ReefMessageFlow outbound", () => { category: "confidential", reason: "Denied.", }; + const stores = flowStores(); const flow = new ReefMessageFlow({ config: cfg, trust: trusted.store, keys: alice, - stateDir: `/tmp/reef-flow-${randomUUID()}`, transport: relay as unknown as ReefTransportClient, guard: guard(deny), audit: new MemoryAuditStore(new Uint8Array(32).fill(9)), replay: new MemoryReplayStore(), - reviews: new ReviewApprovalStore(`/tmp/reef-reviews-${randomUUID()}`), + ...stores, onIngress: async () => {}, onOwnerNotice: async () => {}, }); diff --git a/extensions/reef/src/flow.ts b/extensions/reef/src/flow.ts index e16dfb1e079f..bc5004b24deb 100644 --- a/extensions/reef/src/flow.ts +++ b/extensions/reef/src/flow.ts @@ -1,5 +1,3 @@ -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; import { appendAudit, appendInboxRead, @@ -28,7 +26,7 @@ import { type ReefPeerIdentity, } from "./friend-types.js"; import { reefMessageTextHash } from "./rejection-resend.js"; -import { ReviewApprovalStore, writePrivateJson } from "./state.js"; +import { ReefDeliveredStore, ReviewApprovalStore } from "./state.js"; import { ReefTransportClient } from "./transport.js"; import { REEF_OUTBOUND_DELIVERY_MAX_ENTRIES, @@ -97,8 +95,6 @@ function buildLegacyDeliveryIndex( } export class ReefMessageFlow { - private readonly delivered = new Set(); - private deliveredLoaded = false; private legacyDeliveryIndex?: Promise>; private readonly ulid = createMonotonicUlidFactory(); @@ -107,12 +103,12 @@ export class ReefMessageFlow { config: ReefChannelConfig; trust: ReefTrustStore; keys: ReefKeys; - stateDir: string; transport: ReefTransportClient; guard: GuardAdapter; audit: AuditStore; replay: ReplayStore; reviews: ReviewApprovalStore; + delivered: ReefDeliveredStore; onIngress: (message: ReefIngressMessage) => Promise; onOwnerNotice: (text: string) => Promise; }, @@ -367,8 +363,7 @@ export class ReefMessageFlow { await this.options.transport.acknowledge(relayPeer, envelope.id, result.receipt); return; } - await this.loadDelivered(); - if (this.delivered.has(envelope.id)) { + if (await this.options.delivered.has(envelope.id)) { await this.options.transport.acknowledge(relayPeer, envelope.id, result.receipt); return; } @@ -388,30 +383,10 @@ export class ReefMessageFlow { autonomy: friend.autonomy, }); } - this.delivered.add(envelope.id); - await writePrivateJson(join(this.options.stateDir, "delivered.json"), [...this.delivered]); + await this.options.delivered.add(envelope.id); await this.options.transport.acknowledge(relayPeer, envelope.id, result.receipt); } - private async loadDelivered(): Promise { - if (this.deliveredLoaded) { - return; - } - try { - const ids = JSON.parse( - await readFile(join(this.options.stateDir, "delivered.json"), "utf8"), - ) as string[]; - for (const id of ids) { - this.delivered.add(id); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - this.deliveredLoaded = true; - } - private requireHandle(): string { if (!this.options.config.handle) { throw new Error("Reef handle is not configured"); diff --git a/extensions/reef/src/legacy-key-guard.test.ts b/extensions/reef/src/legacy-key-guard.test.ts new file mode 100644 index 000000000000..ea7756a14ab8 --- /dev/null +++ b/extensions/reef/src/legacy-key-guard.test.ts @@ -0,0 +1,83 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { assertLegacyReefKeysMigrated } from "./legacy-key-guard.js"; + +describe("Reef legacy key guard", () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("blocks identity generation while a legacy keys file awaits Doctor", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-keys-")); + tempDirs.push(stateRoot); + const legacyDir = path.join(stateRoot, ".openclaw", "data", "reef"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), "{}"); + + await expect(assertLegacyReefKeysMigrated(undefined, {}, stateRoot)).rejects.toThrow( + "Legacy Reef identity keys must be imported", + ); + }); + + it("uses the configured legacy directory when one is present", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-keys-")); + const legacyDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-custom-")); + tempDirs.push(stateRoot, legacyDir); + fs.writeFileSync(path.join(legacyDir, "keys.json"), "{}"); + + await expect(assertLegacyReefKeysMigrated(legacyDir)).rejects.toThrow( + "Legacy Reef identity keys must be imported", + ); + }); + + it("blocks when the legacy keys path exists but is not a regular file", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-keys-")); + tempDirs.push(stateRoot); + fs.mkdirSync(path.join(stateRoot, ".openclaw", "data", "reef", "keys.json"), { + recursive: true, + }); + + await expect(assertLegacyReefKeysMigrated(undefined, {}, stateRoot)).rejects.toThrow( + "Legacy Reef identity keys must be imported", + ); + }); + + it("allows a new identity when no legacy key file exists", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-keys-")); + tempDirs.push(stateRoot); + + await expect(assertLegacyReefKeysMigrated(undefined, {}, stateRoot)).resolves.toBeUndefined(); + }); + + it("ignores default-home keys for an isolated active state", async () => { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-home-")); + const isolatedStateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-isolated-")); + tempDirs.push(homeDir, isolatedStateDir); + const legacyDir = path.join(homeDir, ".openclaw", "data", "reef"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), "{}"); + + await expect( + assertLegacyReefKeysMigrated(undefined, { OPENCLAW_STATE_DIR: isolatedStateDir }, homeDir), + ).resolves.toBeUndefined(); + }); + + it("honors explicitly configured default-home keys for an isolated active state", async () => { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-home-")); + const isolatedStateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-isolated-")); + tempDirs.push(homeDir, isolatedStateDir); + const legacyDir = path.join(homeDir, ".openclaw", "data", "reef"); + fs.mkdirSync(legacyDir, { recursive: true }); + fs.writeFileSync(path.join(legacyDir, "keys.json"), "{}"); + + await expect( + assertLegacyReefKeysMigrated(legacyDir, { OPENCLAW_STATE_DIR: isolatedStateDir }, homeDir), + ).rejects.toThrow("Legacy Reef identity keys must be imported"); + }); +}); diff --git a/extensions/reef/src/legacy-key-guard.ts b/extensions/reef/src/legacy-key-guard.ts new file mode 100644 index 000000000000..98bbc088db38 --- /dev/null +++ b/extensions/reef/src/legacy-key-guard.ts @@ -0,0 +1,34 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; +import { resolveLegacyReefStateDir } from "./doctor-state-paths.js"; + +export const REEF_LEGACY_KEYS_PENDING_CODE = "REEF_LEGACY_KEYS_PENDING"; + +export async function assertLegacyReefKeysMigrated( + configuredStateDir?: string, + env: NodeJS.ProcessEnv = process.env, + homeDir = os.homedir(), +): Promise { + const legacyStateDir = resolveLegacyReefStateDir({ + config: configuredStateDir ? { channels: { reef: { stateDir: configuredStateDir } } } : {}, + env, + stateDir: resolveStateDir(env, () => homeDir), + homeDir, + }); + const filePath = `${legacyStateDir}/keys.json`; + try { + await fs.stat(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + throw Object.assign( + new Error( + "Legacy Reef identity keys must be imported before registration. Run `openclaw doctor --fix`, then retry.", + ), + { code: REEF_LEGACY_KEYS_PENDING_CODE }, + ); +} diff --git a/extensions/reef/src/registration-state.ts b/extensions/reef/src/registration-state.ts new file mode 100644 index 000000000000..afc2556111c9 --- /dev/null +++ b/extensions/reef/src/registration-state.ts @@ -0,0 +1,235 @@ +import { randomUUID } from "node:crypto"; +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; + +export const REEF_REGISTRATION_NAMESPACE = "registration"; +export const REEF_REGISTRATION_IDENTITY_KEY = "identity"; +export const REEF_REGISTRATION_SESSION_KEY = "setup-session"; +export const REEF_REGISTRATION_MAX_ENTRIES = 2; + +export type ReefIdentityBinding = { handle: string; relayUrl: string }; +type ReefIdentityPendingRecord = ReefIdentityBinding & { + kind: "pending"; + owner: string; + expiresAt: number; +}; +type ReefIdentityReservation = { + binding: ReefIdentityBinding; + owner?: string; +}; +export type ReefSetupSession = { session: string; relayUrl: string; email: string }; + +const REEF_IDENTITY_RESERVATION_MS = 10 * 60_000; + +function openRegistrationStore( + runtime: PluginRuntime, +): PluginStateSyncKeyedStore { + return runtime.state.openSyncKeyedStore< + ReefIdentityBinding | ReefIdentityPendingRecord | ReefSetupSession + >({ + namespace: REEF_REGISTRATION_NAMESPACE, + maxEntries: REEF_REGISTRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); +} + +export function parseReefIdentityBinding(value: unknown): ReefIdentityBinding | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const parsed = value as Partial; + if (parsed.kind === "pending") { + return undefined; + } + return typeof parsed.handle === "string" && + parsed.handle.length > 0 && + typeof parsed.relayUrl === "string" && + parsed.relayUrl.length > 0 + ? { handle: parsed.handle, relayUrl: parsed.relayUrl } + : undefined; +} + +function parseReefIdentityPendingRecord(value: unknown): ReefIdentityPendingRecord | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const parsed = value as Partial; + return parsed.kind === "pending" && + typeof parsed.handle === "string" && + parsed.handle.length > 0 && + typeof parsed.relayUrl === "string" && + parsed.relayUrl.length > 0 && + typeof parsed.owner === "string" && + parsed.owner.length > 0 && + Number.isSafeInteger(parsed.expiresAt) && + (parsed.expiresAt ?? 0) > 0 + ? { + kind: "pending", + handle: parsed.handle, + relayUrl: parsed.relayUrl, + owner: parsed.owner, + expiresAt: parsed.expiresAt!, + } + : undefined; +} + +function reefIdentityConflict(binding: ReefIdentityBinding): Error { + return new Error( + `This OpenClaw state already holds the Reef identity @${binding.handle} on ${binding.relayUrl}. Re-register the same handle and relay.`, + ); +} + +export function parseReefSetupSession(value: unknown): ReefSetupSession | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const parsed = value as Partial; + return typeof parsed.session === "string" && + parsed.session.length > 0 && + typeof parsed.relayUrl === "string" && + parsed.relayUrl.length > 0 && + typeof parsed.email === "string" && + parsed.email.length > 0 + ? { session: parsed.session, relayUrl: parsed.relayUrl, email: parsed.email } + : undefined; +} + +export function loadReefIdentityBinding(runtime: PluginRuntime): ReefIdentityBinding | undefined { + return parseReefIdentityBinding( + openRegistrationStore(runtime).lookup(REEF_REGISTRATION_IDENTITY_KEY), + ); +} + +export function assertReefIdentityBinding( + runtime: PluginRuntime, + binding: ReefIdentityBinding, +): void { + const existing = loadReefIdentityBinding(runtime); + if (!existing) { + throw new Error( + "Reef identity binding is missing; run openclaw doctor --fix or register this claw", + ); + } + if (existing.handle !== binding.handle || existing.relayUrl !== binding.relayUrl) { + throw reefIdentityConflict(existing); + } +} + +export function reserveReefIdentityBinding( + runtime: PluginRuntime, + binding: ReefIdentityBinding, +): ReefIdentityReservation { + const parsed = parseReefIdentityBinding(binding); + if (!parsed) { + throw new Error("invalid Reef identity binding"); + } + const store = openRegistrationStore(runtime); + const update = store.update; + if (!update) { + throw new Error("Reef identity reservation requires atomic plugin-state updates"); + } + let reservation: ReefIdentityReservation | undefined; + let conflict: ReefIdentityBinding | undefined; + update(REEF_REGISTRATION_IDENTITY_KEY, (current) => { + const existing = parseReefIdentityBinding(current); + if (existing) { + if (existing.handle !== parsed.handle || existing.relayUrl !== parsed.relayUrl) { + conflict = existing; + } else { + reservation = { binding: parsed }; + } + return existing; + } + const pending = parseReefIdentityPendingRecord(current); + if (pending) { + const sameBinding = pending.handle === parsed.handle && pending.relayUrl === parsed.relayUrl; + // Never transfer a live reservation. After expiry, only the same target + // may retry because the original relay request may already have committed. + if (pending.expiresAt > Date.now() || !sameBinding) { + conflict = pending; + return pending; + } + } + const owner = randomUUID(); + reservation = { binding: parsed, owner }; + return { + kind: "pending", + ...parsed, + owner, + expiresAt: Date.now() + REEF_IDENTITY_RESERVATION_MS, + }; + }); + if (conflict) { + throw reefIdentityConflict(conflict); + } + return reservation!; +} + +export function finalizeReefIdentityBinding( + runtime: PluginRuntime, + reservation: ReefIdentityReservation, +): void { + if (!reservation.owner) { + return; + } + const store = openRegistrationStore(runtime); + const update = store.update; + if (!update) { + throw new Error("Reef identity reservation requires atomic plugin-state updates"); + } + let finalized = false; + update(REEF_REGISTRATION_IDENTITY_KEY, (current) => { + const existing = parseReefIdentityBinding(current); + if ( + existing?.handle === reservation.binding.handle && + existing.relayUrl === reservation.binding.relayUrl + ) { + finalized = true; + return existing; + } + const pending = parseReefIdentityPendingRecord(current); + if (pending?.owner !== reservation.owner) { + return current; + } + finalized = true; + return reservation.binding; + }); + if (!finalized) { + throw new Error("Reef identity reservation was replaced before registration completed"); + } +} + +export function releaseReefIdentityReservation( + runtime: PluginRuntime, + reservation: ReefIdentityReservation, +): void { + if (!reservation.owner) { + return; + } + const deleteIf = openRegistrationStore(runtime).deleteIf; + if (!deleteIf) { + throw new Error("Reef identity reservation requires atomic plugin-state updates"); + } + deleteIf( + REEF_REGISTRATION_IDENTITY_KEY, + (current) => parseReefIdentityPendingRecord(current)?.owner === reservation.owner, + ); +} + +export function loadReefSetupSession(runtime: PluginRuntime): ReefSetupSession | undefined { + return parseReefSetupSession( + openRegistrationStore(runtime).lookup(REEF_REGISTRATION_SESSION_KEY), + ); +} + +export function saveReefSetupSession(runtime: PluginRuntime, session: ReefSetupSession): void { + const parsed = parseReefSetupSession(session); + if (!parsed) { + throw new Error("invalid Reef setup session"); + } + openRegistrationStore(runtime).register(REEF_REGISTRATION_SESSION_KEY, parsed); +} + +export function clearReefSetupSession(runtime: PluginRuntime): void { + openRegistrationStore(runtime).delete(REEF_REGISTRATION_SESSION_KEY); +} diff --git a/extensions/reef/src/setup.test.ts b/extensions/reef/src/setup.test.ts new file mode 100644 index 000000000000..b87e01884f66 --- /dev/null +++ b/extensions/reef/src/setup.test.ts @@ -0,0 +1,173 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { setReefRuntime } from "./runtime.js"; +import { reefSetupWizard } from "./setup.js"; +import { + finalizeReefIdentityBinding, + generateAndStoreKeys, + loadReefIdentityBinding, + reserveReefIdentityBinding, +} from "./state.js"; +import { ReefRelayError, ReefTransportClient } from "./transport.js"; + +describe("Reef setup wizard identity binding", () => { + let stateDir = ""; + + beforeEach(() => { + resetPluginStateStoreForTests(); + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-setup-")); + }); + + afterEach(() => { + vi.restoreAllMocks(); + resetPluginStateStoreForTests(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + function installRuntime() { + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("reef", { + ...options, + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + runtime.state.resolveStateDir = () => stateDir; + setReefRuntime(runtime); + return runtime; + } + + function bindIdentity(runtime: ReturnType, handle: string): void { + finalizeReefIdentityBinding( + runtime, + reserveReefIdentityBinding(runtime, { handle, relayUrl: "https://reefwire.ai" }), + ); + } + + it("rejects a different handle before reusing the stored identity keys", async () => { + const runtime = installRuntime(); + bindIdentity(runtime, "existing"); + const textAnswers = [ + "https://reefwire.ai", + "owner@example.com", + "setup-session", + "replacement", + ]; + const prompter = { + note: vi.fn(async () => undefined), + text: vi.fn(async () => textAnswers.shift() ?? ""), + select: vi.fn(async () => "code-only"), + }; + + await expect( + reefSetupWizard.configureInteractive({ cfg: {}, prompter: prompter as never }), + ).rejects.toThrow("already holds the Reef identity @existing"); + }); + + it("persists the identity binding immediately after claiming the handle", async () => { + const runtime = installRuntime(); + await generateAndStoreKeys(runtime); + vi.spyOn(ReefTransportClient.prototype, "createHandle").mockResolvedValue({ + handle: "molty", + key_epoch: 1, + }); + const textAnswers = [ + "https://reefwire.ai", + "owner@example.com", + "setup-session", + "molty", + "gpt-5.6-terra", + "REEF_GUARD_OPENAI_KEY", + "reef-v1", + ]; + const selectAnswers = ["code-only", "openai"]; + const prompter = { + note: vi.fn(async () => undefined), + text: vi.fn(async () => textAnswers.shift() ?? ""), + select: vi.fn(async () => selectAnswers.shift()), + }; + + await reefSetupWizard.configureInteractive({ cfg: {}, prompter: prompter as never }); + + expect(loadReefIdentityBinding(runtime)).toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + }); + + it("releases a reservation after a definitively rejected handle claim", async () => { + const runtime = installRuntime(); + await generateAndStoreKeys(runtime); + vi.spyOn(ReefTransportClient.prototype, "createHandle").mockRejectedValue( + new ReefRelayError(409, "handle_unavailable"), + ); + vi.spyOn(ReefTransportClient.prototype, "listFriends").mockRejectedValue( + new ReefRelayError(401, "unknown_handle"), + ); + const textAnswers = ["https://reefwire.ai", "owner@example.com", "setup-session", "molty"]; + const prompter = { + note: vi.fn(async () => undefined), + text: vi.fn(async () => textAnswers.shift() ?? ""), + select: vi.fn(async () => "code-only"), + }; + + await expect( + reefSetupWizard.configureInteractive({ cfg: {}, prompter: prompter as never }), + ).rejects.toThrow("handle_unavailable"); + expect(loadReefIdentityBinding(runtime)).toBeUndefined(); + }); + + it("keeps a binding after an ambiguous handle-claim failure", async () => { + const runtime = installRuntime(); + await generateAndStoreKeys(runtime); + vi.spyOn(ReefTransportClient.prototype, "createHandle").mockRejectedValue( + new TypeError("connection reset"), + ); + const textAnswers = ["https://reefwire.ai", "owner@example.com", "setup-session", "molty"]; + const prompter = { + note: vi.fn(async () => undefined), + text: vi.fn(async () => textAnswers.shift() ?? ""), + select: vi.fn(async () => "code-only"), + }; + + await expect( + reefSetupWizard.configureInteractive({ cfg: {}, prompter: prompter as never }), + ).rejects.toThrow("connection reset"); + expect(loadReefIdentityBinding(runtime)).toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + }); + + it("keeps a binding when an ownership probe fails without proving non-ownership", async () => { + const runtime = installRuntime(); + await generateAndStoreKeys(runtime); + vi.spyOn(ReefTransportClient.prototype, "createHandle").mockRejectedValue( + new ReefRelayError(409, "handle_unavailable"), + ); + vi.spyOn(ReefTransportClient.prototype, "listFriends").mockRejectedValue( + new ReefRelayError(401, "invalid_signature"), + ); + const textAnswers = ["https://reefwire.ai", "owner@example.com", "setup-session", "molty"]; + const prompter = { + note: vi.fn(async () => undefined), + text: vi.fn(async () => textAnswers.shift() ?? ""), + select: vi.fn(async () => "code-only"), + }; + + await expect( + reefSetupWizard.configureInteractive({ cfg: {}, prompter: prompter as never }), + ).rejects.toThrow("invalid_signature"); + expect(loadReefIdentityBinding(runtime)).toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + }); +}); diff --git a/extensions/reef/src/setup.ts b/extensions/reef/src/setup.ts index 2d1d578788b8..eaf76fdde659 100644 --- a/extensions/reef/src/setup.ts +++ b/extensions/reef/src/setup.ts @@ -5,8 +5,21 @@ import { ReefChannelConfigSchema, type ReefChannelConfig, } from "./config-schema.js"; -import { generateAndStoreKeys, resolveStateDir } from "./state.js"; -import { ReefTransportClient } from "./transport.js"; +import { assertLegacyReefKeysMigrated } from "./legacy-key-guard.js"; +import { getReefRuntime } from "./runtime.js"; +import { + finalizeReefIdentityBinding, + generateAndStoreKeys, + loadKeys, + loadReefIdentityBinding, + releaseReefIdentityReservation, + reserveReefIdentityBinding, +} from "./state.js"; +import { + isDefinitiveReefRegistrationFailure, + isReefOwnershipRejection, + ReefTransportClient, +} from "./transport.js"; type Prompt = { note(message: string, title?: string): Promise; @@ -104,23 +117,83 @@ export const reefSetupWizard = { }, ], }); - const stateDir = resolveStateDir( - await prompter.text({ - message: "Local Reef state directory", - initialValue: resolveStateDir(), - }), - ); - const keys = await generateAndStoreKeys(stateDir); + const runtime = getReefRuntime(); + const identity = loadReefIdentityBinding(runtime); + if (identity && (identity.handle !== handle || identity.relayUrl !== relayUrl)) { + throw new Error( + `This OpenClaw state already holds the Reef identity @${identity.handle} on ${identity.relayUrl}. Re-register the same handle and relay.`, + ); + } + const configuredStateDir = (cfg.channels?.reef as { stateDir?: unknown } | undefined)?.stateDir; + const keys = await loadKeys(runtime).catch(async (error: unknown) => { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + await assertLegacyReefKeysMigrated( + typeof configuredStateDir === "string" ? configuredStateDir : undefined, + ); + return await generateAndStoreKeys(runtime); + }); const client = new ReefTransportClient(relayUrl, handle, keys); + let token: string | undefined; if (!setupSession) { const started = await client.authStart(email); if (started.magicLink) { await prompter.note(started.magicLink, "Development magic link"); } - const token = await prompter.text({ message: "Magic-link token", sensitive: true }); - setupSession = (await client.authComplete(token)).session; + token = await prompter.text({ message: "Magic-link token", sensitive: true }); + } + // Reserve the keys immediately before consuming auth or claiming a handle. + // Definitive relay rejection releases it; ambiguous transport failure keeps + // the binding because the relay may have committed the request. + const reservation = reserveReefIdentityBinding(runtime, { handle, relayUrl }); + let effectiveRequestPolicy = requestPolicy; + try { + if (!setupSession) { + setupSession = (await client.authComplete(token ?? "")).session; + } + try { + await client.createHandle(setupSession, requestPolicy); + } catch (error) { + const unavailable = error instanceof Error && error.message.includes("handle_unavailable"); + if (!unavailable) { + throw error; + } + try { + await client.listFriends(); + } catch (verificationError) { + if (isReefOwnershipRejection(verificationError)) { + releaseReefIdentityReservation(runtime, reservation); + throw error; + } + finalizeReefIdentityBinding(runtime, reservation); + throw verificationError; + } + // Signed access proves these keys already own the handle. Finalize + // before checking account ownership so an account mismatch cannot + // redirect the same keys to a different handle. + finalizeReefIdentityBinding(runtime, reservation); + const { handles } = await client.listOwnHandles(setupSession); + const existing = handles.find((entry) => entry.handle === handle); + if (!existing) { + throw new Error( + `Handle @${handle} is owned by this claw's keys, but the setup session belongs to a different relay account`, + { cause: error }, + ); + } + effectiveRequestPolicy = ReefChannelConfigSchema.shape.requestPolicy.parse( + existing.request_policy, + ); + } + finalizeReefIdentityBinding(runtime, reservation); + } catch (error) { + if (isDefinitiveReefRegistrationFailure(error)) { + releaseReefIdentityReservation(runtime, reservation); + } else { + finalizeReefIdentityBinding(runtime, reservation); + } + throw error; } - await client.createHandle(setupSession, requestPolicy); const provider = await prompter.select({ message: "Guard provider", options: [ @@ -141,8 +214,7 @@ export const reefSetupWizard = { relayUrl, handle, email, - requestPolicy, - stateDir, + requestPolicy: effectiveRequestPolicy, guard: { provider, pinnedModel, apiKeyEnv, policyVersion, timeoutMs: 30_000 }, }); await prompter.note( diff --git a/extensions/reef/src/state.test.ts b/extensions/reef/src/state.test.ts new file mode 100644 index 000000000000..5afa297995f2 --- /dev/null +++ b/extensions/reef/src/state.test.ts @@ -0,0 +1,612 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { + OpenKeyedStoreOptions, + PluginStateSyncKeyedStore, +} from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + base64url, + generateIdentity, + signReceipt, + verifyChain, + verifyChainSegment, + type ReviewRequest, +} from "../protocol/index.js"; +import { + assertReefIdentityBinding, + clearReefSetupSession, + generateAndStoreKeys, + loadKeys, + loadReefIdentityBinding, + loadReefSetupSession, + openStores, + finalizeReefIdentityBinding, + REEF_DELIVERED_NAMESPACE, + REEF_REPLAY_TTL_MS, + REEF_REVIEWS_NAMESPACE, + releaseReefIdentityReservation, + reserveReefIdentityBinding, + ReviewApprovalStore, + reefReplayStoreKey, + saveReefSetupSession, +} from "./state.js"; + +const auditKey = base64url(Uint8Array.from({ length: 32 }, (_, index) => index + 1)); +const replayKey = base64url(Uint8Array.from({ length: 32 }, (_, index) => 255 - index)); +const receiptId = "01JZ0000000000000000000000"; + +function createRuntime(stateDir: string) { + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("reef", { + ...options, + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + return runtime; +} + +function bindIdentity(runtime: ReturnType, handle: string): void { + finalizeReefIdentityBinding( + runtime, + reserveReefIdentityBinding(runtime, { handle, relayUrl: "https://reefwire.ai" }), + ); +} + +describe("Reef SQLite state", () => { + let stateDir = ""; + + beforeEach(() => { + resetPluginStateStoreForTests(); + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-reef-state-")); + }); + + afterEach(() => { + vi.useRealTimers(); + resetPluginStateStoreForTests(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + it("does not let an expired audit writer replace a committed successor link", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-16T00:00:00.000Z")); + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + await openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit.appendEvent( + "initial", + { id: 1 }, + 10, + ); + const competing = openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit; + const runtime = createRuntime(stateDir); + const openSyncKeyedStore = runtime.state.openSyncKeyedStore; + let triggerCompetingWriter = true; + let competingAppend: Promise | undefined; + runtime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = openSyncKeyedStore(options); + if (options.namespace !== "audit") { + return store; + } + return { + ...store, + registerIfAbsent(key, value, opts) { + const inserted = store.registerIfAbsent(key, value, opts); + if (triggerCompetingWriter && inserted) { + triggerCompetingWriter = false; + vi.advanceTimersByTime(31_000); + competingAppend = competing.appendEvent("winner", { id: 2 }, 12); + } + return inserted; + }, + }; + }; + + const expired = openStores(runtime, keys, { auditMaxEntries: 2 }).audit.appendEvent( + "expired", + { id: 3 }, + 11, + ); + await expect(expired).rejects.toThrow(); + await expect(competingAppend).resolves.toBeDefined(); + const retained = await competing.entries(); + expect(retained.map((entry) => entry.event.type)).toEqual(["initial", "winner"]); + }); + + it("retains expired audit cleanup state when takeover cleanup fails", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-16T00:00:00.000Z")); + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + await openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit.appendEvent( + "initial", + { id: 1 }, + 10, + ); + + const takeoverRuntime = createRuntime(stateDir); + const takeoverOpenStore = takeoverRuntime.state.openSyncKeyedStore; + let failCleanup = true; + takeoverRuntime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = takeoverOpenStore(options); + if (options.namespace !== "audit") { + return store; + } + return { + ...store, + delete(key) { + const deleted = store.delete(key); + if (failCleanup) { + failCleanup = false; + throw new Error("simulated cleanup interruption"); + } + return deleted; + }, + }; + }; + const takeover = openStores(takeoverRuntime, keys, { auditMaxEntries: 2 }).audit; + const stalledRuntime = createRuntime(stateDir); + const stalledOpenStore = stalledRuntime.state.openSyncKeyedStore; + let takeoverAppend: Promise | undefined; + stalledRuntime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = stalledOpenStore(options); + if (options.namespace !== "audit") { + return store; + } + return { + ...store, + registerIfAbsent(key, value, opts) { + const inserted = store.registerIfAbsent(key, value, opts); + if (inserted && !takeoverAppend) { + vi.advanceTimersByTime(31_000); + takeoverAppend = takeover.appendEvent("interrupted-takeover", { id: 2 }, 12); + } + return inserted; + }, + }; + }; + + await expect( + openStores(stalledRuntime, keys, { auditMaxEntries: 2 }).audit.appendEvent( + "stalled", + { id: 3 }, + 11, + ), + ).rejects.toThrow(); + await expect(takeoverAppend).rejects.toThrow("simulated cleanup interruption"); + await expect( + openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit.appendEvent( + "recovered", + { id: 4 }, + 13, + ), + ).resolves.toBeDefined(); + const retained = await openStores(createRuntime(stateDir), keys, { + auditMaxEntries: 2, + }).audit.entries(); + expect(retained.map((entry) => entry.event.type)).toEqual(["initial", "recovered"]); + }); + + it("persists keys and registration state without creating Reef files", async () => { + const runtime = createRuntime(stateDir); + const keys = await generateAndStoreKeys(runtime); + bindIdentity(runtime, "molty"); + saveReefSetupSession(runtime, { + session: "setup-secret", + relayUrl: "https://reefwire.ai", + email: "molty@example.com", + }); + + expect(await loadKeys(createRuntime(stateDir))).toEqual(keys); + expect(loadReefIdentityBinding(createRuntime(stateDir))).toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + expect(loadReefSetupSession(createRuntime(stateDir))?.session).toBe("setup-secret"); + clearReefSetupSession(runtime); + expect(loadReefSetupSession(runtime)).toBeUndefined(); + expect(fs.existsSync(path.join(stateDir, "state", "openclaw.sqlite"))).toBe(true); + expect(fs.existsSync(path.join(stateDir, "data", "reef"))).toBe(false); + }); + + it("atomically rejects redirecting stored identity keys to another handle", () => { + const runtime = createRuntime(stateDir); + bindIdentity(runtime, "molty"); + + expect(() => + reserveReefIdentityBinding(runtime, { + handle: "other", + relayUrl: "https://reefwire.ai", + }), + ).toThrow("already holds the Reef identity @molty"); + expect(loadReefIdentityBinding(runtime)).toEqual({ + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + expect(() => + assertReefIdentityBinding(runtime, { + handle: "other", + relayUrl: "https://reefwire.ai", + }), + ).toThrow("already holds the Reef identity @molty"); + }); + + it("conditionally releases or finalizes an identity reservation", () => { + const runtime = createRuntime(stateDir); + const released = reserveReefIdentityBinding(runtime, { + handle: "first", + relayUrl: "https://reefwire.ai", + }); + releaseReefIdentityReservation(runtime, released); + expect(loadReefIdentityBinding(runtime)).toBeUndefined(); + + const finalized = reserveReefIdentityBinding(runtime, { + handle: "second", + relayUrl: "https://reefwire.ai", + }); + finalizeReefIdentityBinding(runtime, finalized); + releaseReefIdentityReservation(runtime, finalized); + expect(loadReefIdentityBinding(runtime)).toEqual({ + handle: "second", + relayUrl: "https://reefwire.ai", + }); + }); + + it("does not transfer a live reservation to a concurrent retry", () => { + const runtime = createRuntime(stateDir); + const reservation = reserveReefIdentityBinding(runtime, { + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + + expect(() => + reserveReefIdentityBinding(runtime, { + handle: "molty", + relayUrl: "https://reefwire.ai", + }), + ).toThrow("already holds the Reef identity @molty"); + finalizeReefIdentityBinding(runtime, reservation); + expect(loadReefIdentityBinding(runtime)?.handle).toBe("molty"); + }); + + it("allows only the same binding to take over an expired reservation", () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-16T00:00:00.000Z")); + const runtime = createRuntime(stateDir); + reserveReefIdentityBinding(runtime, { + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + vi.advanceTimersByTime(10 * 60_000 + 1); + + expect(() => + reserveReefIdentityBinding(runtime, { + handle: "other", + relayUrl: "https://reefwire.ai", + }), + ).toThrow("already holds the Reef identity @molty"); + const retry = reserveReefIdentityBinding(runtime, { + handle: "molty", + relayUrl: "https://reefwire.ai", + }); + finalizeReefIdentityBinding(runtime, retry); + expect(loadReefIdentityBinding(runtime)?.handle).toBe("molty"); + }); + + it("appends and reopens a verified audit chain", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const first = openStores(createRuntime(stateDir), keys); + await Promise.all( + Array.from({ length: 20 }, (_, index) => + first.audit.appendEvent("test", { id: index }, 10 + index), + ), + ); + + const reopened = await openStores(createRuntime(stateDir), keys).audit.entries(); + expect(reopened).toHaveLength(20); + expect(verifyChain(reopened)).toBe(true); + }); + + it("retains a verifiable audit suffix after bounded eviction", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const store = openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit; + await store.appendEvent("one", { id: 1 }, 10); + await store.appendEvent("two", { id: 2 }, 11); + await store.appendEvent("three", { id: 3 }, 12); + + const retained = await store.entries(); + expect(retained.map((entry) => entry.event.seq)).toEqual([2, 3]); + expect( + verifyChainSegment(retained, { + previousHash: retained[0]!.prevHash, + previousSeq: 1, + head: retained[1]!.entryHash, + }), + ).toBe(true); + }); + + it("does not evict committed audit history when head advancement fails", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const initial = openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit; + await initial.appendEvent("one", { id: 1 }, 10); + await initial.appendEvent("two", { id: 2 }, 11); + + const runtime = createRuntime(stateDir); + const openSyncKeyedStore = runtime.state.openSyncKeyedStore; + let failAdvance = true; + runtime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = openSyncKeyedStore(options); + if (options.namespace !== "audit-head" || !store.update) { + return store; + } + const update = store.update; + return { + ...store, + update(key, updateValue, opts) { + return update( + key, + (current) => { + const next = updateValue(current); + const head = next as { seq?: number; pending?: unknown } | undefined; + if (failAdvance && head?.seq === 3 && head.pending === undefined) { + failAdvance = false; + throw new Error("simulated head write failure"); + } + return next; + }, + opts, + ); + }, + }; + }; + + const failing = openStores(runtime, keys, { auditMaxEntries: 2 }).audit; + await expect(failing.appendEvent("three", { id: 3 }, 12)).rejects.toThrow(); + const unchanged = await openStores(createRuntime(stateDir), keys, { + auditMaxEntries: 2, + }).audit.entries(); + expect(unchanged.map((entry) => entry.event.type)).toEqual(["one", "two"]); + + await openStores(createRuntime(stateDir), keys, { auditMaxEntries: 2 }).audit.appendEvent( + "three", + { id: 3 }, + 12, + ); + const recovered = await openStores(createRuntime(stateDir), keys, { + auditMaxEntries: 2, + }).audit.entries(); + expect(recovered.map((entry) => entry.event.type)).toEqual(["two", "three"]); + }); + + it("roundtrips encrypted replay completions and durable dedupe state", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const stores = openStores(createRuntime(stateDir), keys); + const receipt = signReceipt( + { + id: receiptId, + bodyHash: "a".repeat(64), + auditHead: "b".repeat(64), + status: "accepted", + }, + identity.signing.secretKey, + ); + const body = { text: "RECOVERABLE SECRET BODY" }; + + await expect(stores.replay.claim("alice", receiptId, "c".repeat(64))).resolves.toBe("new"); + await stores.replay.complete("alice", receiptId, receipt, body); + const reopened = openStores(createRuntime(stateDir), keys).replay; + await expect(reopened.claim("alice", receiptId, "c".repeat(64))).resolves.toBe("duplicate"); + await expect(reopened.completed("alice", receiptId)).resolves.toEqual({ receipt, body }); + await expect(reopened.claim("alice", receiptId, "d".repeat(64))).resolves.toBe("mismatch"); + + const raw = createPluginStateSyncKeyedStoreForTests("reef", { + namespace: "replay", + maxEntries: 3_000, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_REPLAY_TTL_MS, + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + expect(JSON.stringify(raw.entries())).not.toContain(body.text); + }); + + it("does not steal a live replay claim owned by another process", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const runtime = createRuntime(stateDir); + const raw = createPluginStateSyncKeyedStoreForTests<{ + peer: string; + id: string; + envelopeHash: string; + state: "in_flight"; + claimOwner: string; + claimExpiresAt: number; + }>("reef", { + namespace: "replay", + maxEntries: 3_000, + overflowPolicy: "reject-new", + defaultTtlMs: REEF_REPLAY_TTL_MS, + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + const key = reefReplayStoreKey("alice", receiptId); + raw.register(key, { + peer: "alice", + id: receiptId, + envelopeHash: "c".repeat(64), + state: "in_flight", + claimOwner: "other-process", + claimExpiresAt: Date.now() + 5 * 60_000, + }); + + const replay = openStores(runtime, keys).replay; + await expect(replay.claim("alice", receiptId, "c".repeat(64))).resolves.toBe("in_flight"); + expect(raw.lookup(key)?.claimOwner).toBe("other-process"); + + raw.register(key, { + ...raw.lookup(key)!, + claimExpiresAt: Date.now() - 1, + }); + await expect(replay.claim("alice", receiptId, "c".repeat(64))).resolves.toBe("new"); + const firstOwner = raw.lookup(key)?.claimOwner; + expect(firstOwner).not.toBe("other-process"); + const firstExpiry = raw.lookup(key)?.claimExpiresAt ?? 0; + await replay.refresh?.("alice", receiptId); + expect(raw.lookup(key)?.claimExpiresAt).toBeGreaterThanOrEqual(firstExpiry); + + raw.register(key, { + ...raw.lookup(key)!, + claimExpiresAt: Date.now() - 1, + }); + await expect(replay.claim("alice", receiptId, "c".repeat(64))).resolves.toBe("new"); + expect(raw.lookup(key)?.claimOwner).not.toBe(firstOwner); + }); + + it("persists review decisions and delivered ids", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const stores = openStores(createRuntime(stateDir), keys); + const review: ReviewRequest = { + id: receiptId, + from: "alice#1", + to: "bob#1", + direction: "outbound", + bodyHash: "a".repeat(64), + approvalDigest: "b".repeat(64), + verdict: { + decision: "review", + category: "ambiguous", + reason: "Owner review.", + model: "test-model", + policyVersion: "v1", + }, + }; + + await expect(stores.reviews.request(review)).resolves.toBeUndefined(); + await expect(stores.reviews.decide(review.approvalDigest, true)).resolves.toBe(true); + await expect( + openStores(createRuntime(stateDir), keys).reviews.request(review), + ).resolves.toEqual({ + approved: true, + approvalDigest: review.approvalDigest, + }); + await stores.delivered.add(receiptId); + await expect(openStores(createRuntime(stateDir), keys).delivered.has(receiptId)).resolves.toBe( + true, + ); + }); + + it("fails closed instead of evicting live replay and delivered state", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const stores = openStores(createRuntime(stateDir), keys, { + replayMaxEntries: 1, + deliveredMaxEntries: 1, + }); + + await expect(stores.replay.claim("alice", "first", "a".repeat(64))).resolves.toBe("new"); + await stores.replay.consume("alice", "first"); + await expect(stores.replay.claim("alice", "second", "b".repeat(64))).rejects.toThrow(); + await expect(stores.replay.claim("alice", "first", "a".repeat(64))).resolves.toBe("duplicate"); + + await stores.delivered.add("first"); + await expect(stores.delivered.add("second")).rejects.toThrow(); + await expect(stores.delivered.has("first")).resolves.toBe(true); + }); + + it("fails when a pending review claim does not persist", async () => { + const runtime = createRuntime(stateDir); + const openSyncKeyedStore = runtime.state.openSyncKeyedStore; + runtime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = openSyncKeyedStore(options); + return options.namespace === REEF_REVIEWS_NAMESPACE + ? { ...store, registerIfAbsent: () => false } + : store; + }; + const review: ReviewRequest = { + id: receiptId, + from: "alice#1", + to: "bob#1", + direction: "outbound", + bodyHash: "a".repeat(64), + approvalDigest: "b".repeat(64), + verdict: { + decision: "review", + category: "ambiguous", + reason: "Owner review.", + model: "test-model", + policyVersion: "v1", + }, + }; + + await expect(new ReviewApprovalStore(runtime).request(review)).rejects.toThrow( + "Failed persisting Reef pending review", + ); + }); + + it("fails when a delivered marker claim does not persist", async () => { + const identity = generateIdentity(); + const keys = { ...identity, auditKey, replayKey, keyEpoch: 1 }; + const runtime = createRuntime(stateDir); + const openSyncKeyedStore = runtime.state.openSyncKeyedStore; + runtime.state.openSyncKeyedStore = ( + options: OpenKeyedStoreOptions, + ): PluginStateSyncKeyedStore => { + const store = openSyncKeyedStore(options); + return options.namespace === REEF_DELIVERED_NAMESPACE + ? { ...store, registerIfAbsent: () => false } + : store; + }; + + await expect(openStores(runtime, keys).delivered.add(receiptId)).rejects.toThrow( + "Failed persisting Reef delivered marker", + ); + }); + + it("evicts completed review decisions before rejecting new pending work", async () => { + const runtime = createRuntime(stateDir); + const store = new ReviewApprovalStore(runtime, 2); + const review = (id: string, digest: string): ReviewRequest => ({ + id, + from: "alice#1", + to: "bob#1", + direction: "outbound", + bodyHash: "a".repeat(64), + approvalDigest: digest, + verdict: { + decision: "review", + category: "ambiguous", + reason: "Owner review.", + model: "test-model", + policyVersion: "v1", + }, + }); + const first = review("first", "1".repeat(64)); + const second = review("second", "2".repeat(64)); + const third = review("third", "3".repeat(64)); + + await store.request(first); + await store.decide(first.approvalDigest, false); + await store.request(second); + await store.request(third); + + await expect(store.list()).resolves.toEqual([second, third]); + }); +}); diff --git a/extensions/reef/src/state.ts b/extensions/reef/src/state.ts index 1cc15ab865b6..da3901f42636 100644 --- a/extensions/reef/src/state.ts +++ b/extensions/reef/src/state.ts @@ -1,21 +1,131 @@ -import { chmod, mkdir, open, readFile, rename } from "node:fs/promises"; -import { homedir } from "node:os"; -import { dirname, join } from "node:path"; +import { createHash, randomUUID } from "node:crypto"; +import { gcm } from "@noble/ciphers/aes.js"; +import { concatBytes, randomBytes } from "@noble/hashes/utils.js"; +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; import { + base64, base64url, + canonicalBytes, + decodeUtf8, + fromBase64, fromBase64url, generateIdentity, + REEF_ENVELOPE_MAX_AGE_SECONDS, + validateMessageBody, + type CompletedReplay, + type MessageBody, + type ReplayClaim, + type ReplayStore, type ReviewApproval, type ReviewRequest, + type SignedReceipt, } from "../protocol/index.js"; -import { JsonlAuditStore, FileReplayStore } from "../protocol/node.js"; +import { openReefAuditStore } from "./audit-state.js"; +import { loadReefIdentityBinding } from "./registration-state.js"; import type { ReefKeys } from "./types.js"; -export function resolveStateDir(configured?: string): string { - return configured ?? join(homedir(), ".openclaw", "data", "reef"); +export * from "./audit-state.js"; +export * from "./registration-state.js"; + +export const REEF_KEYS_NAMESPACE = "identity"; +export const REEF_KEYS_KEY = "keys"; +export const REEF_KEYS_MAX_ENTRIES = 1; +export const REEF_KEYS_MIGRATION_NAMESPACE = "identity-migration"; +export const REEF_KEYS_MIGRATION_KEY = "keys-json"; +export const REEF_KEYS_MIGRATION_MAX_ENTRIES = 1; +export const REEF_DURABLE_MIGRATION_NAMESPACE = "durable-migration"; +export const REEF_DURABLE_MIGRATION_KEY = "legacy-files"; +export const REEF_DURABLE_MIGRATION_MAX_ENTRIES = 1; +export const REEF_REPLAY_NAMESPACE = "replay"; +export const REEF_REPLAY_MAX_ENTRIES = 3_000; +export const REEF_REPLAY_TTL_MS = (REEF_ENVELOPE_MAX_AGE_SECONDS + 24 * 60 * 60) * 1_000; +export const REEF_REVIEWS_NAMESPACE = "reviews"; +export const REEF_REVIEWS_MAX_ENTRIES = 2_000; +export const REEF_DELIVERED_NAMESPACE = "delivered"; +export const REEF_DELIVERED_MAX_ENTRIES = 5_000; +export const REEF_DELIVERED_TTL_MS = REEF_REPLAY_TTL_MS; + +export type ReefReplayRecord = { + peer: string; + id: string; + envelopeHash: string; + state: "available" | "in_flight" | "completed" | "consumed"; + claimOwner?: string; + claimExpiresAt?: number; + receipt?: SignedReceipt; + body?: { enc: string }; +}; + +const REEF_REPLAY_CLAIM_LEASE_MS = 5 * 60_000; + +export type ReefReviewRecord = { review: ReviewRequest; approved?: boolean }; + +export type ReefIdentityMigrationRecord = { + pending: true; + identityBindingRequired: boolean; +}; +export type ReefDurableMigrationRecord = { pending: true }; + +export function parseReefKeys(value: unknown): ReefKeys { + if (!value || typeof value !== "object") { + throw new Error("invalid Reef keys"); + } + const keys = value as ReefKeys; + if ( + fromBase64url(keys.signing?.publicKey ?? "").length !== 32 || + fromBase64url(keys.signing?.secretKey ?? "").length !== 32 || + fromBase64url(keys.encryption?.publicKey ?? "").length !== 32 || + fromBase64url(keys.encryption?.secretKey ?? "").length !== 32 || + fromBase64url(keys.auditKey ?? "").length !== 32 || + fromBase64url(keys.replayKey ?? "").length !== 32 || + !Number.isSafeInteger(keys.keyEpoch) || + keys.keyEpoch < 1 + ) { + throw new Error("invalid Reef keys"); + } + return structuredClone(keys); } -export async function generateAndStoreKeys(stateDir: string): Promise { +function openKeysStore(runtime: PluginRuntime): PluginStateSyncKeyedStore { + return runtime.state.openSyncKeyedStore({ + namespace: REEF_KEYS_NAMESPACE, + maxEntries: REEF_KEYS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); +} + +function assertReefIdentityMigrationComplete(runtime: PluginRuntime): void { + const durableMigration = runtime.state.openSyncKeyedStore({ + namespace: REEF_DURABLE_MIGRATION_NAMESPACE, + maxEntries: REEF_DURABLE_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + if (durableMigration.lookup(REEF_DURABLE_MIGRATION_KEY)) { + throw new Error( + "Reef durable state migration is incomplete; repair the legacy state files and rerun openclaw doctor --fix", + ); + } + const migration = runtime.state.openSyncKeyedStore({ + namespace: REEF_KEYS_MIGRATION_NAMESPACE, + maxEntries: REEF_KEYS_MIGRATION_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + if (migration.lookup(REEF_KEYS_MIGRATION_KEY)) { + throw new Error( + "Reef identity migration is incomplete; repair the legacy identity files and rerun openclaw doctor --fix", + ); + } +} + +export async function generateAndStoreKeys(runtime: PluginRuntime): Promise { + assertReefIdentityMigrationComplete(runtime); + const binding = loadReefIdentityBinding(runtime); + if (binding) { + throw new Error( + `Reef identity @${binding.handle} on ${binding.relayUrl} has no canonical keys; restore the original keys before registration`, + ); + } const identity = generateIdentity(); const random = (length: number) => crypto.getRandomValues(new Uint8Array(length)); const keys: ReefKeys = { @@ -24,91 +134,396 @@ export async function generateAndStoreKeys(stateDir: string): Promise replayKey: base64url(random(32)), keyEpoch: 1, }; - await writePrivateJson(join(stateDir, "keys.json"), keys); + if (!openKeysStore(runtime).registerIfAbsent(REEF_KEYS_KEY, keys)) { + throw new Error("Reef keys already exist in plugin state"); + } return keys; } -export async function loadKeys(stateDir: string): Promise { - const value = JSON.parse(await readFile(join(stateDir, "keys.json"), "utf8")) as ReefKeys; - if ( - fromBase64url(value.signing.secretKey).length !== 32 || - fromBase64url(value.encryption.secretKey).length !== 32 || - fromBase64url(value.auditKey).length !== 32 || - fromBase64url(value.replayKey).length !== 32 || - !Number.isSafeInteger(value.keyEpoch) || - value.keyEpoch < 1 - ) { - throw new Error("invalid Reef key file"); +export async function loadKeys(runtime: PluginRuntime): Promise { + assertReefIdentityMigrationComplete(runtime); + const value = openKeysStore(runtime).lookup(REEF_KEYS_KEY); + if (!value) { + const error = new Error("Reef keys are missing from plugin state") as Error & { + code?: string; + }; + error.code = "ENOENT"; + throw error; + } + return parseReefKeys(value); +} + +export function reefReplayStoreKey(peer: string, id: string): string { + return `binding:${createHash("sha256") + .update(JSON.stringify([peer, id])) + .digest("hex")}`; +} + +function parseReplayRecord(value: ReefReplayRecord | undefined): ReefReplayRecord | undefined { + if (!value) { + return undefined; + } + if ( + typeof value.peer !== "string" || + typeof value.id !== "string" || + typeof value.envelopeHash !== "string" || + !["available", "in_flight", "completed", "consumed"].includes(value.state) || + (value.state === "in_flight" && + (typeof value.claimOwner !== "string" || + value.claimOwner.length === 0 || + !Number.isSafeInteger(value.claimExpiresAt) || + (value.claimExpiresAt ?? 0) <= 0)) + ) { + throw new Error("invalid Reef replay state"); } - await chmod(join(stateDir, "keys.json"), 0o600); return value; } -export function openStores(stateDir: string, keys: ReefKeys) { - return { - audit: new JsonlAuditStore(join(stateDir, "audit.jsonl"), fromBase64url(keys.auditKey)), - replay: new FileReplayStore(join(stateDir, "replay.jsonl"), fromBase64url(keys.replayKey)), - }; +function encryptReplayBody( + body: MessageBody, + key: Uint8Array, + rng: (length: number) => Uint8Array, +): { enc: string } { + validateMessageBody(body); + const nonce = rng(12); + if (nonce.length !== 12) { + throw new Error("replay body rng returned invalid nonce"); + } + return { enc: base64(concatBytes(nonce, gcm(key, nonce).encrypt(canonicalBytes(body)))) }; +} + +function decryptReplayBody(body: { enc: string }, key: Uint8Array): MessageBody { + const packed = fromBase64(body.enc); + if (packed.length < 28) { + throw new Error("invalid encrypted replay body"); + } + const value = JSON.parse( + decodeUtf8(gcm(key, packed.slice(0, 12)).decrypt(packed.slice(12))), + ) as unknown; + validateMessageBody(value); + return value; +} + +function validateReplayCompletion(receipt: SignedReceipt, body: MessageBody | undefined): void { + if ((receipt.status === "accepted") !== (body !== undefined)) { + throw new Error("accepted replay completion requires body; rejected completion forbids body"); + } +} + +class ReefSqliteReplayStore implements ReplayStore { + readonly #bodyKey: Uint8Array; + readonly #rng: (length: number) => Uint8Array; + readonly #store: PluginStateSyncKeyedStore; + readonly #claimOwners = new Map(); + + constructor( + runtime: PluginRuntime, + bodyKey: Uint8Array, + rng: (length: number) => Uint8Array = randomBytes, + maxEntries = REEF_REPLAY_MAX_ENTRIES, + ) { + if (bodyKey.length !== 32) { + throw new Error("replay body key must be 32 bytes"); + } + this.#bodyKey = bodyKey.slice(); + this.#rng = rng; + this.#store = runtime.state.openSyncKeyedStore({ + namespace: REEF_REPLAY_NAMESPACE, + maxEntries, + overflowPolicy: "reject-new", + // Once this expires, the protocol rejects the original envelope by age. + // The margin covers clock skew and delayed local processing. + defaultTtlMs: REEF_REPLAY_TTL_MS, + }); + } + + #update( + peer: string, + id: string, + updateValue: (current: ReefReplayRecord | undefined) => ReefReplayRecord | undefined, + ): boolean { + const update = this.#store.update; + if (!update) { + throw new Error("Reef replay state requires atomic plugin-state updates"); + } + return update(reefReplayStoreKey(peer, id), (current) => + updateValue(parseReplayRecord(current)), + ); + } + + async claim(peer: string, id: string, envelopeHash: string): Promise { + const key = reefReplayStoreKey(peer, id); + let result: ReplayClaim = "new"; + const owner = randomUUID(); + const claimExpiresAt = Date.now() + REEF_REPLAY_CLAIM_LEASE_MS; + this.#update(peer, id, (existing) => { + if (!existing) { + return { + peer, + id, + envelopeHash, + state: "in_flight", + claimOwner: owner, + claimExpiresAt, + }; + } + if (existing.peer !== peer || existing.id !== id || existing.envelopeHash !== envelopeHash) { + result = "mismatch"; + return existing; + } + if (existing.state === "completed" || existing.state === "consumed") { + result = "duplicate"; + return existing; + } + if (existing.state === "in_flight" && (existing.claimExpiresAt ?? 0) > Date.now()) { + result = "in_flight"; + return existing; + } + return { + ...existing, + state: "in_flight", + claimOwner: owner, + claimExpiresAt, + }; + }); + if (result === "new") { + this.#claimOwners.set(key, owner); + } + return result; + } + + async refresh(peer: string, id: string): Promise { + const key = reefReplayStoreKey(peer, id); + const owner = this.#claimOwners.get(key); + let refreshed = false; + if (owner) { + this.#update(peer, id, (existing) => { + if (existing?.state !== "in_flight" || existing.claimOwner !== owner) { + return existing; + } + refreshed = true; + return { ...existing, claimExpiresAt: Date.now() + REEF_REPLAY_CLAIM_LEASE_MS }; + }); + } + if (!refreshed) { + this.#claimOwners.delete(key); + throw new Error("replay claim is not in flight"); + } + } + + async complete( + peer: string, + id: string, + receipt: SignedReceipt, + body?: MessageBody, + ): Promise { + if (receipt.id !== id) { + throw new Error("receipt id does not match replay claim"); + } + validateReplayCompletion(receipt, body); + const key = reefReplayStoreKey(peer, id); + const owner = this.#claimOwners.get(key); + let completed = false; + this.#update(peer, id, (existing) => { + if (existing?.state !== "in_flight" || existing.claimOwner !== owner) { + return existing; + } + completed = true; + const { claimOwner: _claimOwner, claimExpiresAt: _claimExpiresAt, ...rest } = existing; + return { + ...rest, + state: "completed", + receipt: structuredClone(receipt), + ...(body ? { body: encryptReplayBody(body, this.#bodyKey, this.#rng) } : {}), + }; + }); + if (!completed) { + throw new Error("replay claim is not in flight"); + } + this.#claimOwners.delete(key); + } + + async consume(peer: string, id: string): Promise { + const key = reefReplayStoreKey(peer, id); + const owner = this.#claimOwners.get(key); + let consumed = false; + this.#update(peer, id, (existing) => { + if (existing?.state !== "in_flight" || existing.claimOwner !== owner) { + return existing; + } + consumed = true; + const { + receipt: _receipt, + body: _body, + claimOwner: _claimOwner, + claimExpiresAt: _claimExpiresAt, + ...rest + } = existing; + return { ...rest, state: "consumed" }; + }); + if (!consumed) { + throw new Error("replay claim is not in flight"); + } + this.#claimOwners.delete(key); + } + + async release(peer: string, id: string): Promise { + const key = reefReplayStoreKey(peer, id); + const owner = this.#claimOwners.get(key); + this.#update(peer, id, (existing) => + existing?.state === "in_flight" && existing.claimOwner === owner + ? { + peer: existing.peer, + id: existing.id, + envelopeHash: existing.envelopeHash, + state: "available", + } + : existing, + ); + this.#claimOwners.delete(key); + } + + async completed(peer: string, id: string): Promise { + const existing = parseReplayRecord(this.#store.lookup(reefReplayStoreKey(peer, id))); + if ( + existing?.peer !== peer || + existing.id !== id || + existing.state !== "completed" || + !existing.receipt + ) { + return undefined; + } + return existing.body + ? { + receipt: structuredClone(existing.receipt), + body: decryptReplayBody(existing.body, this.#bodyKey), + } + : { receipt: structuredClone(existing.receipt) }; + } } export class ReviewApprovalStore { - readonly path: string; - constructor(stateDir: string) { - this.path = join(stateDir, "reviews.json"); + readonly #store: PluginStateSyncKeyedStore; + readonly #maxEntries: number; + + constructor(runtime: PluginRuntime, maxEntries = REEF_REVIEWS_MAX_ENTRIES) { + this.#maxEntries = maxEntries; + this.#store = runtime.state.openSyncKeyedStore({ + namespace: REEF_REVIEWS_NAMESPACE, + maxEntries, + overflowPolicy: "reject-new", + }); + } + + #makeRoomForPendingReview(): void { + const deleteIf = this.#store.deleteIf; + if (!deleteIf) { + throw new Error("Reef review retention requires atomic plugin-state deleteIf"); + } + while (true) { + const entries = this.#store.entries(); + if (entries.length < this.#maxEntries) { + return; + } + const completed = entries + .filter((entry) => entry.value.approved !== undefined) + .toSorted((left, right) => left.createdAt - right.createdAt)[0]; + if (!completed) { + throw new Error("Reef pending review capacity is exhausted"); + } + deleteIf(completed.key, (current) => current.approved !== undefined); + } } async request(review: ReviewRequest): Promise { - const records = await this.read(); - const current = records[review.approvalDigest]; + const current = this.#store.lookup(review.approvalDigest); if (current?.approved !== undefined) { return { approved: current.approved, approvalDigest: review.approvalDigest }; } - records[review.approvalDigest] = { review }; - await writePrivateJson(this.path, records); - return undefined; + if (!current) { + this.#makeRoomForPendingReview(); + } + this.#store.registerIfAbsent(review.approvalDigest, { review: structuredClone(review) }); + const persisted = this.#store.lookup(review.approvalDigest); + if (!persisted) { + throw new Error("Failed persisting Reef pending review"); + } + return persisted?.approved === undefined + ? undefined + : { approved: persisted.approved, approvalDigest: review.approvalDigest }; } async decide(digest: string, approved: boolean): Promise { - const records = await this.read(); - if (!records[digest]) { - return false; + const update = this.#store.update; + if (!update) { + throw new Error("Reef review state requires atomic plugin-state updates"); } - records[digest] = { ...records[digest], approved }; - await writePrivateJson(this.path, records); - return true; + let found = false; + update(digest, (current) => { + if (!current) { + return undefined; + } + found = true; + return { ...current, approved }; + }); + return found; } async list(): Promise { - return Object.values(await this.read()) - .filter((entry) => entry.approved === undefined) - .map((entry) => entry.review); + return this.#store + .entries() + .filter((entry) => entry.value.approved === undefined) + .map((entry) => structuredClone(entry.value.review)); + } +} + +export class ReefDeliveredStore { + readonly #store: PluginStateSyncKeyedStore<{ id: string }>; + + constructor(runtime: PluginRuntime, maxEntries = REEF_DELIVERED_MAX_ENTRIES) { + this.#store = runtime.state.openSyncKeyedStore<{ id: string }>({ + namespace: REEF_DELIVERED_NAMESPACE, + maxEntries, + overflowPolicy: "reject-new", + // Relay redelivery is bounded by the same envelope-age contract as replay. + // Keep markers longer than that window and fail closed at live capacity. + defaultTtlMs: REEF_DELIVERED_TTL_MS, + }); } - private async read(): Promise> { - try { - return JSON.parse(await readFile(this.path, "utf8")) as Record< - string, - { review: ReviewRequest; approved?: boolean } - >; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return {}; - } - throw error; + async has(id: string): Promise { + return this.#store.lookup(id)?.id === id; + } + + async add(id: string): Promise { + if (this.#store.lookup(id)?.id === id) { + return; + } + if (!this.#store.registerIfAbsent(id, { id }) && this.#store.lookup(id)?.id !== id) { + throw new Error("Failed persisting Reef delivered marker"); } } } -export async function writePrivateJson(path: string, value: unknown): Promise { - await mkdir(dirname(path), { recursive: true, mode: 0o700 }); - const temporary = `${path}.${process.pid}.tmp`; - const file = await open(temporary, "w", 0o600); - try { - await file.writeFile(`${JSON.stringify(value, null, 2)}\n`, "utf8"); - await file.sync(); - } finally { - await file.close(); - } - await rename(temporary, path); - await chmod(path, 0o600); +export function openStores( + runtime: PluginRuntime, + keys: ReefKeys, + options: { + auditMaxEntries?: number; + replayMaxEntries?: number; + deliveredMaxEntries?: number; + } = {}, +) { + assertReefIdentityMigrationComplete(runtime); + return { + audit: openReefAuditStore(runtime, fromBase64url(keys.auditKey), options.auditMaxEntries), + replay: new ReefSqliteReplayStore( + runtime, + fromBase64url(keys.replayKey), + randomBytes, + options.replayMaxEntries, + ), + reviews: new ReviewApprovalStore(runtime), + delivered: new ReefDeliveredStore(runtime, options.deliveredMaxEntries), + }; } diff --git a/extensions/reef/src/transport.test.ts b/extensions/reef/src/transport.test.ts index 4e87b4e2576f..bc4f2db67f85 100644 --- a/extensions/reef/src/transport.test.ts +++ b/extensions/reef/src/transport.test.ts @@ -450,3 +450,38 @@ describe("ReefInboxConnection response frame bounds", () => { expect(result.states).toEqual(["connected", "disconnected"]); }); }); + +describe("createReefWebSocket handshake deadline", () => { + it("errors when the relay accepts TCP but never completes the upgrade", async () => { + const peers = new Set(); + const server = http.createServer(); + server.on("connection", (socket) => { + peers.add(socket); + socket.once("close", () => peers.delete(socket)); + }); + server.on("upgrade", () => { + // Leave the HTTP upgrade pending until the client deadline aborts it. + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", () => resolve()); + }); + + try { + const { port } = server.address() as AddressInfo; + const socket = createReefWebSocket(`ws://127.0.0.1:${port}`, { + handshakeTimeoutMs: 50, + }) as WebSocket; + const [error] = await once(socket, "error"); + + expect(error).toMatchObject({ message: "Opening handshake has timed out" }); + } finally { + for (const peer of peers) { + peer.destroy(); + } + await new Promise((resolve) => { + server.close(() => resolve()); + }); + } + }); +}); diff --git a/extensions/reef/src/transport.ts b/extensions/reef/src/transport.ts index 682e0bb034c2..a649a4e60ef9 100644 --- a/extensions/reef/src/transport.ts +++ b/extensions/reef/src/transport.ts @@ -28,6 +28,20 @@ export class ReefRelayError extends Error { } } +export function isDefinitiveReefRegistrationFailure(error: unknown): boolean { + return ( + error instanceof ReefRelayError && + error.status >= 400 && + error.status < 500 && + error.status !== 408 && + error.status !== 429 + ); +} + +export function isReefOwnershipRejection(error: unknown): boolean { + return error instanceof ReefRelayError && error.message === "unknown_handle"; +} + export class ReefTransportClient { // Ed25519 is deterministic: identical (method, path, ts, body) requests produce // identical signatures, which collide with the relay's replay key. Keep ts diff --git a/extensions/zalouser/doctor-contract-api.test.ts b/extensions/zalouser/doctor-contract-api.test.ts new file mode 100644 index 000000000000..3a3a44e0acfc --- /dev/null +++ b/extensions/zalouser/doctor-contract-api.test.ts @@ -0,0 +1,147 @@ +// Zalouser tests cover Doctor-owned credential migration. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + createPluginStateSyncKeyedStoreForTests, + createPluginStateKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; +import type { + OpenKeyedStoreOptions, + PluginDoctorStateMigrationContext, +} from "openclaw/plugin-sdk/runtime-doctor"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { stateMigrations } from "./doctor-contract-api.js"; +import { setZalouserRuntime } from "./src/runtime.js"; +import { + clearStoredZaloCredentials, + resolveLegacyZalouserCredentialsPath, + zalouserCredentialStoreKey, + ZALOUSER_CREDENTIALS_MAX_ENTRIES, + ZALOUSER_CREDENTIALS_NAMESPACE, + type StoredZaloCredentials, +} from "./src/session-state.js"; + +function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigrationContext { + return { + openPluginStateKeyedStore(options: OpenKeyedStoreOptions) { + return createPluginStateKeyedStoreForTests("zalouser", { + ...options, + env: options.env ?? env, + }); + }, + }; +} + +describe("zalouser doctor state migration", () => { + let stateDir = ""; + let env: NodeJS.ProcessEnv; + + beforeEach(async () => { + resetPluginStateStoreForTests(); + stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-doctor-")); + env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + }); + + afterEach(async () => { + resetPluginStateStoreForTests(); + await fs.rm(stateDir, { recursive: true, force: true }); + }); + + it("imports a profile credential blob into SQLite before archiving it", async () => { + const profile = "work"; + const filePath = resolveLegacyZalouserCredentialsPath(profile, env); + const legacy = { + imei: "imei-1", + cookie: [{ key: "zpsid", value: "secret", domain: "chat.zalo.me" }], + userAgent: "user-agent", + language: "vi", + lastUsedAt: "2026-07-02T12:00:00.000Z", + }; + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, JSON.stringify(legacy)); + const createdAt = (await fs.stat(filePath)).mtime.toISOString(); + const migration = stateMigrations.find( + (entry) => entry.id === "zalouser-credentials-json-to-plugin-state", + ); + if (!migration) { + throw new Error("missing Zalouser credential migration"); + } + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: [ + `- Zalo Personal credentials: 1 file -> plugin state (${ZALOUSER_CREDENTIALS_NAMESPACE})`, + ], + }); + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Zalo Personal credentials for profile work", + expect.stringContaining("Archived Zalo Personal credentials legacy source"), + ]); + const store = context.openPluginStateKeyedStore({ + namespace: ZALOUSER_CREDENTIALS_NAMESPACE, + maxEntries: ZALOUSER_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + await expect(store.lookup(zalouserCredentialStoreKey(profile))).resolves.toEqual({ + profile, + ...legacy, + createdAt, + }); + await expect(fs.access(`${filePath}.migrated`)).resolves.toBeUndefined(); + }); + + it("archives legacy credentials without restoring an explicitly cleared profile", async () => { + const profile = "work"; + const filePath = resolveLegacyZalouserCredentialsPath(profile, env); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile( + filePath, + JSON.stringify({ + imei: "legacy-imei", + cookie: [{ key: "zpsid", value: "legacy", domain: "chat.zalo.me" }], + userAgent: "legacy-agent", + }), + ); + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("zalouser", { + ...options, + env: options.env ?? env, + }); + setZalouserRuntime(runtime); + clearStoredZaloCredentials(profile, env); + const context = createDoctorContext(env); + const params = { + config: {}, + env, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + context, + }; + const migration = stateMigrations.find( + (entry) => entry.id === "zalouser-credentials-json-to-plugin-state", + )!; + + const result = await migration.migrateLegacyState(params); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Archived revoked Zalo Personal credential legacy source for profile work", + expect.stringContaining("Archived Zalo Personal credentials legacy source"), + ]); + await expect(fs.access(`${filePath}.migrated`)).resolves.toBeUndefined(); + }); +}); diff --git a/extensions/zalouser/doctor-contract-api.ts b/extensions/zalouser/doctor-contract-api.ts index 35980450a7f0..2a3bd12c51c3 100644 --- a/extensions/zalouser/doctor-contract-api.ts +++ b/extensions/zalouser/doctor-contract-api.ts @@ -1,2 +1,157 @@ // Zalouser API module exposes the plugin public contract. +import type { Dirent } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { + archiveLegacyStateSource, + type PluginDoctorStateMigration, +} from "openclaw/plugin-sdk/runtime-doctor"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + isZaloCredentialRevocation, + normalizeStoredZaloCredentials, + normalizeZalouserCredentialProfile, + resolveLegacyZalouserCredentialsDir, + resolveLegacyZalouserCredentialsPath, + zalouserCredentialStoreKey, + ZALOUSER_CREDENTIALS_MAX_ENTRIES, + ZALOUSER_CREDENTIALS_NAMESPACE, + type ZaloCredentialStateRecord, + type StoredZaloCredentials, +} from "./src/session-state.js"; + export { normalizeCompatibilityConfig, legacyConfigRules } from "./src/doctor-contract.js"; + +type LegacyZalouserCredentialSource = { + filePath: string; + profile: string; +}; + +async function collectLegacyZalouserCredentialSources( + env: NodeJS.ProcessEnv, +): Promise { + const credentialsDir = resolveLegacyZalouserCredentialsDir(env); + let entries: Dirent[]; + try { + entries = await fs.readdir(credentialsDir, { withFileTypes: true }); + } catch { + return []; + } + return entries + .filter( + (entry) => + entry.isFile() && + (entry.name === "credentials.json" || + (entry.name.startsWith("credentials-") && entry.name.endsWith(".json"))), + ) + .flatMap((entry) => { + let profile = "default"; + if (entry.name !== "credentials.json") { + try { + profile = decodeURIComponent(entry.name.slice("credentials-".length, -".json".length)); + } catch { + return []; + } + } + const normalizedProfile = normalizeZalouserCredentialProfile(profile); + const filePath = path.join(credentialsDir, entry.name); + return resolveLegacyZalouserCredentialsPath(normalizedProfile, env) === filePath + ? [{ filePath, profile: normalizedProfile }] + : []; + }) + .toSorted((left, right) => left.profile.localeCompare(right.profile)); +} + +export const stateMigrations: PluginDoctorStateMigration[] = [ + { + id: "zalouser-credentials-json-to-plugin-state", + label: "Zalo Personal credentials", + async detectLegacyState(params) { + const sources = await collectLegacyZalouserCredentialSources(params.env); + return sources.length > 0 + ? { + preview: [ + `- Zalo Personal credentials: ${sources.length} ${sources.length === 1 ? "file" : "files"} -> plugin state (${ZALOUSER_CREDENTIALS_NAMESPACE})`, + ], + } + : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + const store = params.context.openPluginStateKeyedStore({ + namespace: ZALOUSER_CREDENTIALS_NAMESPACE, + maxEntries: ZALOUSER_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + }); + for (const source of await collectLegacyZalouserCredentialSources(params.env)) { + let credentials: StoredZaloCredentials | null = null; + try { + const raw = JSON.parse(await fs.readFile(source.filePath, "utf8")) as unknown; + const createdAt = + isRecord(raw) && typeof raw.createdAt === "string" && raw.createdAt + ? raw.createdAt + : (await fs.stat(source.filePath)).mtime.toISOString(); + credentials = normalizeStoredZaloCredentials( + isRecord(raw) ? { ...raw, createdAt } : raw, + source.profile, + ); + } catch { + // Report the same fail-closed result as a structurally invalid file. + } + if (!credentials) { + warnings.push( + `Left invalid Zalo Personal credential legacy source in place for profile ${source.profile}`, + ); + continue; + } + const key = zalouserCredentialStoreKey(source.profile); + const stored = await store.lookup(key); + if (isZaloCredentialRevocation(stored, source.profile)) { + changes.push( + `Archived revoked Zalo Personal credential legacy source for profile ${source.profile}`, + ); + await archiveLegacyStateSource({ + filePath: source.filePath, + label: "Zalo Personal credentials", + changes, + warnings, + }); + continue; + } + const existing = normalizeStoredZaloCredentials(stored, source.profile); + if (existing && JSON.stringify(existing) !== JSON.stringify(credentials)) { + warnings.push( + `Kept existing Zalo Personal credentials for profile ${source.profile}; left differing legacy source in place`, + ); + continue; + } + if (!existing) { + try { + await store.registerIfAbsent(key, credentials); + } catch (error) { + warnings.push( + `Failed importing Zalo Personal credentials for profile ${source.profile}: ${String(error)}; left legacy source in place`, + ); + continue; + } + } + const persisted = normalizeStoredZaloCredentials(await store.lookup(key), source.profile); + if (!persisted || JSON.stringify(persisted) !== JSON.stringify(credentials)) { + warnings.push( + `Failed verifying Zalo Personal credentials for profile ${source.profile}; left legacy source in place`, + ); + continue; + } + changes.push(`Migrated Zalo Personal credentials for profile ${source.profile}`); + await archiveLegacyStateSource({ + filePath: source.filePath, + label: "Zalo Personal credentials", + changes, + warnings, + }); + } + return { changes, warnings }; + }, + }, +]; diff --git a/extensions/zalouser/src/session-state.ts b/extensions/zalouser/src/session-state.ts new file mode 100644 index 000000000000..f9f358f538c3 --- /dev/null +++ b/extensions/zalouser/src/session-state.ts @@ -0,0 +1,182 @@ +import { createHash } from "node:crypto"; +import os from "node:os"; +import path from "node:path"; +import type { PluginStateSyncKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; +import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { getZalouserRuntime } from "./runtime.js"; +import type { Credentials } from "./zca-client.js"; + +export type StoredZaloCredentials = { + profile: string; + imei: string; + cookie: Credentials["cookie"]; + userAgent: string; + language?: string; + createdAt: string; + lastUsedAt?: string; +}; + +type ZaloCredentialRevocationRecord = { + kind: "revoked"; + profile: string; + revokedAt: string; +}; + +export type ZaloCredentialStateRecord = StoredZaloCredentials | ZaloCredentialRevocationRecord; + +export const ZALOUSER_CREDENTIALS_NAMESPACE = "credentials"; +export const ZALOUSER_CREDENTIALS_MAX_ENTRIES = 256; + +export function normalizeZalouserCredentialProfile(profile?: string | null): string { + return normalizeLowercaseStringOrEmpty(profile) || "default"; +} + +export function zalouserCredentialStoreKey(profile?: string | null): string { + return `profile:${createHash("sha256") + .update(normalizeZalouserCredentialProfile(profile)) + .digest("hex")}`; +} + +export function resolveLegacyZalouserCredentialsDir(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveStateDir(env, os.homedir), "credentials", "zalouser"); +} + +export function resolveLegacyZalouserCredentialsPath( + profile: string, + env: NodeJS.ProcessEnv = process.env, +): string { + const normalized = normalizeZalouserCredentialProfile(profile); + const filename = + normalized === "default" + ? "credentials.json" + : `credentials-${encodeURIComponent(normalized)}.json`; + return path.join(resolveLegacyZalouserCredentialsDir(env), filename); +} + +export function normalizeStoredZaloCredentials( + value: unknown, + profile?: string | null, +): StoredZaloCredentials | null { + if (!value || typeof value !== "object") { + return null; + } + const parsed = value as Partial; + if ( + typeof parsed.imei !== "string" || + !parsed.imei || + !parsed.cookie || + typeof parsed.userAgent !== "string" || + !parsed.userAgent || + typeof parsed.createdAt !== "string" || + !parsed.createdAt + ) { + return null; + } + return { + profile: normalizeZalouserCredentialProfile(profile ?? parsed.profile), + imei: parsed.imei, + cookie: parsed.cookie, + userAgent: parsed.userAgent, + ...(typeof parsed.language === "string" ? { language: parsed.language } : {}), + createdAt: parsed.createdAt, + ...(typeof parsed.lastUsedAt === "string" ? { lastUsedAt: parsed.lastUsedAt } : {}), + }; +} + +export function isZaloCredentialRevocation( + value: unknown, + profile?: string | null, +): value is ZaloCredentialRevocationRecord { + if (!value || typeof value !== "object") { + return false; + } + const parsed = value as Partial; + return ( + parsed.kind === "revoked" && + typeof parsed.revokedAt === "string" && + parsed.revokedAt.length > 0 && + normalizeZalouserCredentialProfile(parsed.profile) === + normalizeZalouserCredentialProfile(profile ?? parsed.profile) + ); +} + +function openZalouserCredentialsStore( + env: NodeJS.ProcessEnv = process.env, +): PluginStateSyncKeyedStore { + return getZalouserRuntime().state.openSyncKeyedStore({ + namespace: ZALOUSER_CREDENTIALS_NAMESPACE, + maxEntries: ZALOUSER_CREDENTIALS_MAX_ENTRIES, + overflowPolicy: "reject-new", + env, + }); +} + +export function loadStoredZaloCredentials( + profile: string, + env: NodeJS.ProcessEnv = process.env, +): StoredZaloCredentials | null { + const normalizedProfile = normalizeZalouserCredentialProfile(profile); + const stored = openZalouserCredentialsStore(env).lookup( + zalouserCredentialStoreKey(normalizedProfile), + ); + const parsed = normalizeStoredZaloCredentials(stored, normalizedProfile); + return parsed?.profile === normalizedProfile ? parsed : null; +} + +export function saveStoredZaloCredentials( + profile: string, + credentials: Omit, + env: NodeJS.ProcessEnv = process.env, +): void { + const normalizedProfile = normalizeZalouserCredentialProfile(profile); + openZalouserCredentialsStore(env).register(zalouserCredentialStoreKey(normalizedProfile), { + profile: normalizedProfile, + ...credentials, + }); +} + +export function refreshStoredZaloCredentials( + profile: string, + credentials: Omit, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const normalizedProfile = normalizeZalouserCredentialProfile(profile); + const store = openZalouserCredentialsStore(env); + const update = store.update; + if (!update) { + throw new Error("Zalo credential refresh requires atomic plugin-state updates"); + } + let saved = true; + update(zalouserCredentialStoreKey(normalizedProfile), (current) => { + // Background refreshes can finish after logout. Preserve the revocation; + // only an explicit QR login may replace it with a new authenticated session. + if (isZaloCredentialRevocation(current, normalizedProfile)) { + saved = false; + return current; + } + return { profile: normalizedProfile, ...credentials }; + }); + return saved; +} + +export function clearStoredZaloCredentials( + profile: string, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const normalizedProfile = normalizeZalouserCredentialProfile(profile); + const store = openZalouserCredentialsStore(env); + const hadCredentials = + normalizeStoredZaloCredentials( + store.lookup(zalouserCredentialStoreKey(normalizedProfile)), + normalizedProfile, + ) !== null; + // Keep a durable revocation marker so doctor cannot resurrect explicitly + // cleared credentials from an older profile file. + store.register(zalouserCredentialStoreKey(normalizedProfile), { + kind: "revoked", + profile: normalizedProfile, + revokedAt: new Date().toISOString(), + }); + return hadCredentials; +} diff --git a/extensions/zalouser/src/zalo-js.credentials.test.ts b/extensions/zalouser/src/zalo-js.credentials.test.ts index f10eca9ec232..0734adcd170f 100644 --- a/extensions/zalouser/src/zalo-js.credentials.test.ts +++ b/extensions/zalouser/src/zalo-js.credentials.test.ts @@ -1,21 +1,17 @@ // Zalouser tests cover zalo js.credentials plugin behavior. -import { - lstat, - mkdir, - mkdtemp, - readFile, - rm, - stat, - symlink, - utimes, - writeFile, -} from "node:fs/promises"; +import { access, mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; +import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; import { withEnvAsync } from "openclaw/plugin-sdk/test-env"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { API, Credentials, LoginQRCallbackEvent } from "./zca-client.js"; +import type { API, LoginQRCallbackEvent } from "./zca-client.js"; import { LoginQRCallbackEventType } from "./zca-constants.js"; const createZaloMock = vi.hoisted(() => vi.fn()); @@ -26,6 +22,15 @@ vi.mock("./zca-client.js", () => ({ TextStyle: { Indent: 9 }, })); +import { setZalouserRuntime } from "./runtime.js"; +import { + clearStoredZaloCredentials, + loadStoredZaloCredentials, + refreshStoredZaloCredentials, + resolveLegacyZalouserCredentialsPath, + saveStoredZaloCredentials, + type StoredZaloCredentials, +} from "./session-state.js"; import { checkZaloAuthenticated, listZaloFriends, @@ -35,31 +40,23 @@ import { waitForZaloQrLogin, } from "./zalo-js.js"; -type StoredCredentialFile = { - imei: string; - cookie: Credentials["cookie"]; - userAgent: string; - language?: string; - createdAt?: string; - lastUsedAt?: string; -}; - -function credentialPath(stateDir: string, profile: string): string { - const trimmed = profile.trim().toLowerCase(); - const filename = - !trimmed || trimmed === "default" - ? "credentials.json" - : `credentials-${encodeURIComponent(trimmed)}.json`; - return path.join(stateDir, "credentials", "zalouser", filename); -} - async function readStoredCredentials( stateDir: string, profile: string, -): Promise { - return JSON.parse( - await readFile(credentialPath(stateDir, profile), "utf8"), - ) as StoredCredentialFile; +): Promise { + const stored = loadStoredZaloCredentials(profile, { OPENCLAW_STATE_DIR: stateDir }); + if (!stored) { + throw new Error("Expected stored Zalo credentials"); + } + return stored; +} + +function seedStoredCredentials( + stateDir: string, + profile: string, + credentials: Omit, +): void { + saveStoredZaloCredentials(profile, credentials, { OPENCLAW_STATE_DIR: stateDir }); } function createMockApi(params: { @@ -99,9 +96,41 @@ function createMockApi(params: { describe("zalouser credential persistence", () => { beforeEach(() => { + resetPluginStateStoreForTests(); + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("zalouser", options); + setZalouserRuntime(runtime); createZaloMock.mockReset(); }); + it("does not let a delayed credential refresh undo explicit logout", async () => { + const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const profile = "revoked-refresh"; + const stored = { + imei: "device", + cookie: [{ key: "zpsid", value: "old", domain: "chat.zalo.me" }], + userAgent: "agent", + createdAt: "2026-04-01T00:00:00.000Z", + }; + try { + saveStoredZaloCredentials(profile, stored, env); + clearStoredZaloCredentials(profile, env); + + expect( + refreshStoredZaloCredentials( + profile, + { ...stored, cookie: [{ key: "zpsid", value: "late", domain: "chat.zalo.me" }] }, + env, + ), + ).toBe(false); + expect(loadStoredZaloCredentials(profile, env)).toBeNull(); + } finally { + await rm(stateDir, { recursive: true, force: true }); + } + }); + it("persists the final API cookie jar after QR login", async () => { const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); const profile = "qr-refresh"; @@ -162,7 +191,6 @@ describe("zalouser credential persistence", () => { it("revalidates setup ownership immediately before QR credentials are written", async () => { const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); const profile = "qr-stale-owner"; - const filePath = credentialPath(stateDir, profile); const guardError = new Error("verified inference changed"); const beforeCredentialPersistence = vi.fn(async () => { throw guardError; @@ -202,7 +230,7 @@ describe("zalouser credential persistence", () => { expect(`${started.message} ${waited.message}`).toContain(guardError.message); expect(beforeCredentialPersistence).toHaveBeenCalledTimes(1); - await expect(readFile(filePath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + expect(loadStoredZaloCredentials(profile)).toBeNull(); }); } finally { await rm(stateDir, { recursive: true, force: true }); @@ -210,14 +238,20 @@ describe("zalouser credential persistence", () => { }); it("caps oversized QR start timeout before computing the polling deadline", async () => { - createZaloMock.mockResolvedValueOnce({ - loginQR: async () => new Promise(() => {}), + let loginStarted = false; + let postStartClockReads = 0; + createZaloMock.mockImplementationOnce(async () => { + loginStarted = true; + return { + loginQR: async () => new Promise(() => {}), + }; + }); + const nowSpy = vi.spyOn(Date, "now").mockImplementation(() => { + if (!loginStarted) { + return 0; + } + return postStartClockReads++ === 0 ? 0 : MAX_TIMER_TIMEOUT_MS + 1; }); - const nowSpy = vi.spyOn(Date, "now"); - nowSpy - .mockReturnValueOnce(0) - .mockReturnValueOnce(0) - .mockReturnValueOnce(MAX_TIMER_TIMEOUT_MS + 1); try { const result = await startZaloQrLogin({ profile: "qr-timeout-cap", @@ -227,7 +261,7 @@ describe("zalouser credential persistence", () => { expect(result.message).toBe( "Still preparing QR. Call wait to continue checking login status.", ); - expect(nowSpy).toHaveBeenCalledTimes(3); + expect(postStartClockReads).toBeGreaterThanOrEqual(2); } finally { nowSpy.mockRestore(); } @@ -238,21 +272,12 @@ describe("zalouser credential persistence", () => { const profile = "restore-refresh"; const storedCookie = [{ key: "zpsid", value: "stored", domain: "chat.zalo.me" }]; const refreshedCookie = [{ key: "zpsid", value: "refreshed", domain: "chat.zalo.me" }]; - const filePath = credentialPath(stateDir, profile); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile( - filePath, - JSON.stringify( - { - imei: "stored-imei", - cookie: storedCookie, - userAgent: "stored-user-agent", - createdAt: "2026-04-01T00:00:00.000Z", - }, - null, - 2, - ), - ); + seedStoredCredentials(stateDir, profile, { + imei: "stored-imei", + cookie: storedCookie, + userAgent: "stored-user-agent", + createdAt: "2026-04-01T00:00:00.000Z", + }); const api = createMockApi({ imei: "stored-imei", @@ -289,19 +314,13 @@ describe("zalouser credential persistence", () => { const storedCookie = [{ key: "zpsid", value: "stored", domain: "chat.zalo.me" }]; const loginCookie = [{ key: "zpsid", value: "login", domain: "chat.zalo.me" }]; const refreshedCookie = [{ key: "zpsid", value: "refreshed", domain: "chat.zalo.me" }]; - const filePath = credentialPath(stateDir, profile); - await mkdir(path.dirname(filePath), { recursive: true }); - const storedRaw = JSON.stringify( - { - imei: "stored-imei", - cookie: storedCookie, - userAgent: "stored-user-agent", - createdAt: "2026-04-01T00:00:00.000Z", - }, - null, - 2, - ); - await writeFile(filePath, storedRaw); + seedStoredCredentials(stateDir, profile, { + imei: "stored-imei", + cookie: storedCookie, + userAgent: "stored-user-agent", + createdAt: "2026-04-01T00:00:00.000Z", + }); + const storedBefore = await readStoredCredentials(stateDir, profile); let currentCookie = loginCookie; const api = createMockApi({ @@ -322,7 +341,7 @@ describe("zalouser credential persistence", () => { listZaloFriends(profile, { credentialPersistence: "read-only" }), ).resolves.toStrictEqual([]); - expect(await readFile(filePath, "utf8")).toBe(storedRaw); + expect(await readStoredCredentials(stateDir, profile)).toEqual(storedBefore); }); } finally { await rm(stateDir, { recursive: true, force: true }); @@ -337,21 +356,12 @@ describe("zalouser credential persistence", () => { const refreshedCookie: unknown[] = [ { key: "zpsid", value: "api-refreshed", domain: "chat.zalo.me" }, ]; - const filePath = credentialPath(stateDir, profile); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile( - filePath, - JSON.stringify( - { - imei: "stored-imei", - cookie: storedCookie, - userAgent: "stored-user-agent", - createdAt: "2026-04-01T00:00:00.000Z", - }, - null, - 2, - ), - ); + seedStoredCredentials(stateDir, profile, { + imei: "stored-imei", + cookie: storedCookie, + userAgent: "stored-user-agent", + createdAt: "2026-04-01T00:00:00.000Z", + }); let currentCookie = loginCookie; const api = createMockApi({ @@ -402,21 +412,12 @@ describe("zalouser credential persistence", () => { { key: "zpw", value: "same-secondary", domain: "chat.zalo.me" }, ]; const cookieB = [...cookieA].toReversed(); - const filePath = credentialPath(stateDir, profile); - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile( - filePath, - JSON.stringify( - { - imei: "stored-imei", - cookie: cookieA, - userAgent: "stored-user-agent", - createdAt: "2026-04-01T00:00:00.000Z", - }, - null, - 2, - ), - ); + seedStoredCredentials(stateDir, profile, { + imei: "stored-imei", + cookie: cookieA, + userAgent: "stored-user-agent", + createdAt: "2026-04-01T00:00:00.000Z", + }); let currentCookie = cookieA; const api = createMockApi({ @@ -431,16 +432,12 @@ describe("zalouser credential persistence", () => { try { await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { await expect(listZaloFriends(profile)).resolves.toStrictEqual([]); - const firstRaw = await readFile(filePath, "utf8"); - const stableMtime = new Date("2026-04-01T00:00:10.000Z"); - await utimes(filePath, stableMtime, stableMtime); - const firstMtimeMs = (await stat(filePath)).mtimeMs; + const firstStored = await readStoredCredentials(stateDir, profile); currentCookie = cookieB; await expect(listZaloFriends(profile)).resolves.toStrictEqual([]); - expect(await readFile(filePath, "utf8")).toBe(firstRaw); - expect((await stat(filePath)).mtimeMs).toBe(firstMtimeMs); + expect(await readStoredCredentials(stateDir, profile)).toEqual(firstStored); }); } finally { await rm(stateDir, { recursive: true, force: true }); @@ -486,102 +483,25 @@ describe("zalouser credential persistence", () => { } }); - it.skipIf(process.platform === "win32")( - "writes credentials with private permissions", - async () => { - const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); - const profile = "private-mode"; - const api = createMockApi({ - imei: "api-imei", - userAgent: "api-user-agent", - cookies: [{ key: "zpsid", value: "private", domain: "chat.zalo.me" }], - }); + it("writes plugin-state SQLite without recreating the retired credential blob", async () => { + const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); + const profile = "sqlite-only"; + seedStoredCredentials(stateDir, profile, { + imei: "api-imei", + userAgent: "api-user-agent", + cookie: [{ key: "zpsid", value: "sqlite", domain: "chat.zalo.me" }], + createdAt: "2026-04-01T00:00:00.000Z", + }); - createZaloMock.mockResolvedValueOnce({ - loginQR: async (_options: unknown, callback?: (event: LoginQRCallbackEvent) => unknown) => { - callback?.({ - type: LoginQRCallbackEventType.QRCodeGenerated, - data: { - code: "qr-code", - image: "data:image/png;base64,abc123", - }, - actions: { - saveToFile: vi.fn(async () => undefined), - retry: vi.fn(), - abort: vi.fn(), - }, - }); - return api; - }, - }); - - try { - await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { - await startZaloQrLogin({ profile, timeoutMs: 1000 }); - const loginResult = await waitForZaloQrLogin({ profile, timeoutMs: 1000 }); - expect(loginResult.connected).toBe(true); - - const filePath = credentialPath(stateDir, profile); - const dirMode = (await stat(path.dirname(filePath))).mode & 0o777; - const fileMode = (await stat(filePath)).mode & 0o777; - expect(dirMode).toBe(0o700); - expect(fileMode).toBe(0o600); - }); - } finally { - await rm(stateDir, { recursive: true, force: true }); - } - }, - ); - - it.skipIf(process.platform === "win32")( - "refuses to write credentials through a symlinked file", - async () => { - const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-credentials-")); - const profile = "symlink-target"; - const filePath = credentialPath(stateDir, profile); - const targetPath = path.join(stateDir, "outside.json"); - const api = createMockApi({ - imei: "api-imei", - userAgent: "api-user-agent", - cookies: [{ key: "zpsid", value: "symlink", domain: "chat.zalo.me" }], - }); - - await mkdir(path.dirname(filePath), { recursive: true }); - await writeFile(targetPath, "sentinel", "utf8"); - await symlink(targetPath, filePath); - - createZaloMock.mockResolvedValueOnce({ - loginQR: async (_options: unknown, callback?: (event: LoginQRCallbackEvent) => unknown) => { - callback?.({ - type: LoginQRCallbackEventType.QRCodeGenerated, - data: { - code: "qr-code", - image: "data:image/png;base64,abc123", - }, - actions: { - saveToFile: vi.fn(async () => undefined), - retry: vi.fn(), - abort: vi.fn(), - }, - }); - return api; - }, - }); - - try { - await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { - const started = await startZaloQrLogin({ profile, timeoutMs: 1000 }); - const waited = await waitForZaloQrLogin({ profile, timeoutMs: 1000 }); - expect(`${started.message} ${waited.message}`).toMatch( - /Refusing to write Zalo credentials to symlinked path|private store target must be a regular file/, - ); - }); - - expect(await readFile(targetPath, "utf8")).toBe("sentinel"); - expect((await lstat(filePath)).isSymbolicLink()).toBe(true); - } finally { - await rm(stateDir, { recursive: true, force: true }); - } - }, - ); + try { + await expect( + access(resolveLegacyZalouserCredentialsPath(profile, { OPENCLAW_STATE_DIR: stateDir })), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + access(path.join(stateDir, "state", "openclaw.sqlite")), + ).resolves.toBeUndefined(); + } finally { + await rm(stateDir, { recursive: true, force: true }); + } + }); }); diff --git a/extensions/zalouser/src/zalo-js.ts b/extensions/zalouser/src/zalo-js.ts index 50f2b048478d..973c2ed74f9f 100644 --- a/extensions/zalouser/src/zalo-js.ts +++ b/extensions/zalouser/src/zalo-js.ts @@ -1,6 +1,4 @@ import { randomUUID } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; // Zalouser plugin module implements zalo js behavior. import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; @@ -15,13 +13,7 @@ import { resolveTimerTimeoutMs, } from "openclaw/plugin-sdk/number-runtime"; import { loadOutboundMediaFromUrl } from "openclaw/plugin-sdk/outbound-media"; -import { - privateFileStoreSync, - readRegularFileSync, - statRegularFileSync, - withTimeout, -} from "openclaw/plugin-sdk/security-runtime"; -import { resolveStateDir as resolvePluginStateDir } from "openclaw/plugin-sdk/state-paths"; +import { withTimeout } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, @@ -30,6 +22,13 @@ import { import { sleep, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { normalizeZaloReactionIcon } from "./reaction.js"; import { createZalouserSendReceipt } from "./send-receipt.js"; +import { + clearStoredZaloCredentials, + loadStoredZaloCredentials, + refreshStoredZaloCredentials, + saveStoredZaloCredentials, + type StoredZaloCredentials, +} from "./session-state.js"; import type { ZaloAuthStatus, ZaloEventMessage, @@ -72,6 +71,7 @@ const credentialSignaturesByProfile = new Map(); type CredentialPersistenceMode = "persist" | "read-only"; type CredentialPersistenceOptions = { credentialPersistence?: CredentialPersistenceMode }; +type ZaloCredentialPayload = Omit; type ActiveZaloQrLogin = { id: string; @@ -98,59 +98,6 @@ const groupContextCache = new Map>; -type StoredZaloCredentials = { - imei: string; - cookie: Credentials["cookie"]; - userAgent: string; - language?: string; - createdAt: string; - lastUsedAt?: string; -}; - -function resolveStateDir(env: NodeJS.ProcessEnv = process.env): string { - return resolvePluginStateDir(env, os.homedir); -} - -function resolveCredentialsDir(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveStateDir(env), "credentials", "zalouser"); -} - -function credentialsFilename(profile: string): string { - const trimmed = normalizeLowercaseStringOrEmpty(profile); - if (!trimmed || trimmed === "default") { - return "credentials.json"; - } - return `credentials-${encodeURIComponent(trimmed)}.json`; -} - -function resolveCredentialsPath(profile: string, env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveCredentialsDir(env), credentialsFilename(profile)); -} - -function isNodeErrorCode(error: unknown, code: string): boolean { - return ( - typeof error === "object" && - error !== null && - "code" in error && - (error as { code?: unknown }).code === code - ); -} - -function isReadableCredentialFile(filePath: string): boolean { - try { - return !statRegularFileSync(filePath).missing; - } catch (error) { - if (isNodeErrorCode(error, "ENOENT")) { - return false; - } - throw error; - } -} - -function writeCredentialFileAtomic(filePath: string, payload: string): void { - privateFileStoreSync(resolveCredentialsDir()).writeText(path.basename(filePath), payload); -} - function normalizeProfile(profile?: string | null): string { const trimmed = profile?.trim(); return trimmed && trimmed.length > 0 ? trimmed : "default"; @@ -558,30 +505,11 @@ function mapGroup(groupId: string, group: GroupInfo & Record): } function readCredentials(profile: string): StoredZaloCredentials | null { - const filePath = resolveCredentialsPath(profile); try { - if (!isReadableCredentialFile(filePath)) { + const credentials = loadStoredZaloCredentials(profile); + if (!credentials) { return null; } - const raw = readRegularFileSync({ filePath }).buffer.toString("utf-8"); - const parsed = JSON.parse(raw) as Partial; - if ( - typeof parsed.imei !== "string" || - !parsed.imei || - !parsed.cookie || - typeof parsed.userAgent !== "string" || - !parsed.userAgent - ) { - return null; - } - const credentials = { - imei: parsed.imei, - cookie: parsed.cookie as Credentials["cookie"], - userAgent: parsed.userAgent, - language: typeof parsed.language === "string" ? parsed.language : undefined, - createdAt: typeof parsed.createdAt === "string" ? parsed.createdAt : new Date().toISOString(), - lastUsedAt: typeof parsed.lastUsedAt === "string" ? parsed.lastUsedAt : undefined, - }; credentialSignaturesByProfile.set(profile, credentialSignature(credentials)); return credentials; } catch { @@ -589,9 +517,7 @@ function readCredentials(profile: string): StoredZaloCredentials | null { } } -function credentialSignature( - credentials: Omit, -): string { +function credentialSignature(credentials: ZaloCredentialPayload): string { return JSON.stringify({ imei: credentials.imei, cookie: canonicalCredentialCookie(credentials.cookie), @@ -647,23 +573,32 @@ function canonicalCredentialCookie(cookie: Credentials["cookie"]): unknown { function writeCredentials( profile: string, - credentials: Omit, -): void { + credentials: ZaloCredentialPayload, + allowRevokedReplace: boolean, +): boolean { const existing = readCredentials(profile); const now = new Date().toISOString(); const next: StoredZaloCredentials = { + profile, ...credentials, createdAt: existing?.createdAt ?? now, lastUsedAt: now, }; - writeCredentialFileAtomic(resolveCredentialsPath(profile), JSON.stringify(next, null, 2)); + const { profile: _profile, ...stored } = next; + const saved = allowRevokedReplace + ? (saveStoredZaloCredentials(profile, stored), true) + : refreshStoredZaloCredentials(profile, stored); + if (!saved) { + return false; + } credentialSignaturesByProfile.set(profile, credentialSignature(next)); + return true; } function snapshotApiCredentials( api: API, - fallback?: Partial>, -): Omit { + fallback?: Partial, +): ZaloCredentialPayload { const ctx = api.getContext(); const cookieJson = api.getCookie().toJSON(); const refreshedCookies = @@ -687,9 +622,10 @@ function snapshotApiCredentials( function writeApiCredentials( profile: string, api: API, - fallback?: Partial>, + fallback?: Partial, + allowRevokedReplace = false, ): void { - writeCredentials(profile, snapshotApiCredentials(api, fallback)); + writeCredentials(profile, snapshotApiCredentials(api, fallback), allowRevokedReplace); } function writeApiCredentialsIfChanged(profile: string, api: API): boolean { @@ -698,8 +634,7 @@ function writeApiCredentialsIfChanged(profile: string, api: API): boolean { if (credentialSignaturesByProfile.get(profile) === signature) { return false; } - writeCredentials(profile, credentials); - return true; + return writeCredentials(profile, credentials, false); } function persistApiCredentialsIfChanged(profile: string, api: API): void { @@ -712,10 +647,8 @@ function persistApiCredentialsIfChanged(profile: string, api: API): void { } function clearCredentials(profile: string): boolean { - const filePath = resolveCredentialsPath(profile); try { - if (fs.existsSync(filePath)) { - fs.unlinkSync(filePath); + if (clearStoredZaloCredentials(profile)) { credentialSignaturesByProfile.delete(profile); return true; } @@ -1572,8 +1505,7 @@ export async function startZaloQrLogin(params: { }; login.waitPromise = (async () => { - let capturedCredentials: Omit | null = - null; + let capturedCredentials: ZaloCredentialPayload | null = null; try { const zalo = await createZalo({ logging: false, selfListen: false }); const api = await zalo.loginQR(undefined, (event: LoginQRCallbackEvent) => { @@ -1647,7 +1579,7 @@ export async function startZaloQrLogin(params: { if (!owned || owned.id !== login.id) { return; } - writeApiCredentials(profile, api, capturedCredentials ?? undefined); + writeApiCredentials(profile, api, capturedCredentials ?? undefined, true); invalidateApi(profile); apiByProfile.set(profile, api); current.connected = true; diff --git a/extensions/zalouser/src/zalo-quote-metadata.test.ts b/extensions/zalouser/src/zalo-quote-metadata.test.ts index cbfc8d09e4f4..bac8c98605c7 100644 --- a/extensions/zalouser/src/zalo-quote-metadata.test.ts +++ b/extensions/zalouser/src/zalo-quote-metadata.test.ts @@ -1,7 +1,13 @@ // Zalouser tests cover inbound normalization and outbound bounds through public plugin paths. -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginStateSyncKeyedStoreForTests, + resetPluginStateStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; import { withEnvAsync } from "openclaw/plugin-sdk/test-env"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { API, Message } from "./zca-client.js"; @@ -14,6 +20,8 @@ vi.mock("./zca-client.js", () => ({ TextStyle: { Indent: 9 }, })); +import { setZalouserRuntime } from "./runtime.js"; +import { saveStoredZaloCredentials } from "./session-state.js"; import { resolveZaloGroupContext, sendZaloTextMessage, startZaloListener } from "./zalo-js.js"; type ListenerOn = ReturnType; @@ -45,20 +53,15 @@ async function withStoredSession(params: { run: () => Promise; }): Promise { const stateDir = await mkdtemp(path.join(os.tmpdir(), "openclaw-zalouser-message-")); - const credentialFile = path.join( - stateDir, - "credentials", - "zalouser", - `credentials-${encodeURIComponent(params.profile)}.json`, - ); - await mkdir(path.dirname(credentialFile), { recursive: true }); - await writeFile( - credentialFile, - JSON.stringify({ + saveStoredZaloCredentials( + params.profile, + { imei: "test-imei", cookie: [{ key: "zpsid", value: "test" }], userAgent: "test-agent", - }), + createdAt: new Date().toISOString(), + }, + { OPENCLAW_STATE_DIR: stateDir }, ); createZaloMock.mockResolvedValueOnce({ login: vi.fn(async () => params.api) }); try { @@ -91,6 +94,11 @@ function createInboundMessage(data: Record): Message { } beforeEach(() => { + resetPluginStateStoreForTests(); + const runtime = createPluginRuntimeMock(); + runtime.state.openSyncKeyedStore = (options: OpenKeyedStoreOptions) => + createPluginStateSyncKeyedStoreForTests("zalouser", options); + setZalouserRuntime(runtime); createZaloMock.mockReset(); });