diff --git a/docs/cli/backup.md b/docs/cli/backup.md index 9d090d5bec1b..e276e9d4437b 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -1,8 +1,9 @@ --- -summary: "CLI reference for `openclaw backup` (archives and SQLite snapshots)" +summary: "CLI reference for `openclaw backup` (archives, SQLite snapshots, and Git history)" read_when: - You want a first-class backup archive for local OpenClaw state - You need a compact, verified snapshot of one OpenClaw SQLite database + - You want scheduled, versioned database backups in an operator-owned Git repository - You want to preview which paths would be included before reset or uninstall - You want to restore from a `.tar.gz` archive previously created by `openclaw backup` title: "Backup" @@ -26,6 +27,13 @@ openclaw backup sqlite list --repository ~/Backups/openclaw-sqlite openclaw backup sqlite verify ~/Backups/openclaw-sqlite/ openclaw backup sqlite verify ~/Backups/openclaw-sqlite/ --scratch ~/Private/openclaw-scratch openclaw backup sqlite restore ~/Backups/openclaw-sqlite/ --target ./restored/openclaw.sqlite +openclaw backup git init --repository ~/Backups/openclaw-git --remote +openclaw backup git create --repository ~/Backups/openclaw-git --all --push +openclaw backup git log --repository ~/Backups/openclaw-git +openclaw backup git verify --repository ~/Backups/openclaw-git --global +openclaw backup git restore --repository ~/Backups/openclaw-git --agent main --target ./restored/agent.sqlite +openclaw backup enable --repository ~/Backups/openclaw-git --every 24h --push +openclaw backup disable ``` Archive `create` and `verify`, plus SQLite `create`, `list`, `verify`, and @@ -80,6 +88,115 @@ Restore repeats verification and writes only to a fresh target. It refuses an ex Snapshot repositories are local directories. Scheduling, upload, retention, incremental WAL bundles, failover, and restore-on-boot behavior are intentionally outside this command. +## Versioned Git backups + +`openclaw backup git` stores deterministic, per-table JSONL dumps in a plain Git repository owned by the operator. One repository can hold the shared database and every per-agent database: + +```text +global/manifest.json +global/schema.sql +global/tables/.jsonl +agents//manifest.json +agents//schema.sql +agents//tables/
.jsonl +``` + +Initialize the repository, then create a snapshot of all registered databases: + +```bash +openclaw backup git init --repository ~/Backups/openclaw-git --remote +openclaw backup git create --repository ~/Backups/openclaw-git --all --push +``` + +The repository root must be owned by the current user and must not be group- or +world-writable. OpenClaw checks this when initializing or adopting a repository +and before every create. On POSIX systems, repair unsafe permissions with +`chmod 700 ` after confirming its ownership. + +The repository must be dedicated to OpenClaw backups. An existing `global/` or +`agents//` scope is backup-owned only when it is empty or contains a +valid schema-version-1 `manifest.json`. OpenClaw refuses to replace any other +scope. With `--all`, it validates every existing entry under `agents/` before +removing stale backup-owned agent scopes, so an unowned entry aborts the cleanup +before anything is deleted. + +You can also select `--global`, repeat `--agent `, or combine the shared database with selected agents. Snapshot creation uses the same online backup, sanitizer, `VACUUM`, owner validation, and integrity checks as `backup sqlite create`; it never reads live SQLite files directly. Rows and schema entries have deterministic ordering, and integers and blobs use lossless encodings. The command creates one commit named `openclaw backup `. If the database content is unchanged, it prints `no changes` and creates no commit. + +Git staging is restricted to the backup-owned `global` and `agents` paths; +unrelated files elsewhere in an adopted repository are never staged. + +`--push` pushes the current branch to `origin`. A push failure after a successful local commit is a warning and does not discard or mark the local backup as failed. + + + Git history is durable. Without `--exclude-secrets`, snapshots include + credential material and any pushed remote must be private. + +`src/state/secret-state-tables.ts` is the source of truth for redaction. At this revision, `--exclude-secrets` omits these shared-state tables: + +- `audit_identity_keys` +- `auth_profile_state` +- `auth_profile_stores` +- `apns_registrations` +- `channel_ingress_events` +- `channel_pairing_requests` +- `clawhub_promotion_claims` +- `device_auth_tokens` +- `device_bootstrap_tokens` +- `device_identities` +- `device_pairing_join_codes` +- `device_pairing_paired` +- `gateway_origin_device_tokens` +- `mcp_oauth_pending_authorizations` +- `mcp_oauth_stores` +- `native_hook_relay_bridges` +- `node_host_config` +- `secret_store_entries` +- `web_push_subscriptions` +- `web_push_vapid_keys` +- `worker_environment_credentials` + +It omits these per-agent tables: + +- `auth_profile_state` +- `auth_profile_store` +- `session_suggestions` + +Restore reports the omitted tables so a redacted snapshot cannot be mistaken +for a complete credential backup. + + +Inspect or verify history without changing the live databases: + +```bash +openclaw backup git log --repository ~/Backups/openclaw-git --limit 20 +openclaw backup git verify --repository ~/Backups/openclaw-git --ref --global +openclaw backup git verify --repository ~/Backups/openclaw-git --ref --agent main +``` + +Verification restores the selected snapshot into private scratch space, checks each table's row count and SHA-256, runs `PRAGMA integrity_check` and `PRAGMA foreign_key_check`, and removes the scratch copy. Restore writes only to a fresh target and refuses existing `-wal`, `-shm`, and `-journal` sidecars: + +```bash +openclaw backup git restore --repository ~/Backups/openclaw-git --ref --global --target ./restored/openclaw.sqlite +``` + +Restore rebuilds content-backed FTS5 indexes after loading their content tables. It deliberately omits the derived `session_transcript_index_state` projection so Gateway startup reconciliation rebuilds transcript search. `vec0` virtual tables are not materialized because the extension is unavailable in the restore process; memory indexing recreates them and schedules a full reindex. + +## Schedule backups + +Provision one Gateway-owned automation with a fixed name: + +```bash +openclaw backup enable --repository ~/Backups/openclaw-git --every 24h --push +``` + +The default scope is every database. Use `--global-only` or `--agent ` to narrow it, and add `--exclude-secrets` for a redacted history. Pushed schedules (`--push`) redact credential-bearing tables by default because an unattended recurring push retains them durably in remote history; pass `--include-secrets` for explicit full-fidelity remote backups (restores from redacted history need device re-pairing and provider re-authentication). `--push` also requires the repository to already have an `origin` remote. Re-running `backup enable` updates the existing automation instead of creating a duplicate. `openclaw backup disable` removes it; disabling an already-missing job is a successful no-op. Backup scheduling currently requires a local Gateway because the command job runs on the Gateway host; for a remote Gateway, create the cron job manually with `openclaw cron add`. + +## Recorded runs and freshness + +Every real archive, SQLite snapshot, and Git create attempt records a compact outcome in the existing shared state database. Dry runs are not recorded. The log retains the newest 200 attempts, so frequent schedules remain bounded. + +`openclaw status` shows one `Backups` overview row, and `openclaw status --json` includes the latest attempt and latest successful run. `openclaw doctor` prints an informational hint when no successful backup is recorded or the newest successful backup is more than 14 days old. Recording is best-effort: a record-write failure prints a warning but never changes a successful backup into a failed command. + ## What gets backed up `openclaw backup create` plans sources from your local OpenClaw install: diff --git a/docs/install/backups.md b/docs/install/backups.md index 2778053261e9..530f72790e89 100644 --- a/docs/install/backups.md +++ b/docs/install/backups.md @@ -35,7 +35,8 @@ committed state safely. - One-off, everything, portable: `openclaw backup create` archive. - One database, compact and verified: `openclaw backup sqlite create`. -- Regular protection: schedule either command and sync the output offsite. +- Versioned and incremental by content: `openclaw backup git create`. +- Regular protection: provision the Gateway-owned backup automation. - Continuous, incremental, seconds of data loss: replicate the databases with Litestream. @@ -75,8 +76,42 @@ below cover them. ## Schedule backups -Use your platform scheduler. A nightly cron example that snapshots the -control-plane database and the `main` agent database: +The recommended schedule is one Gateway-owned automation. This example backs +up every registered database daily and pushes the current branch to `origin`. +Pushing requires the repository to have an `origin` remote first, so +initialize it once before enabling a pushed schedule: + +```bash +openclaw backup git init --repository ~/Backups/openclaw-git --remote git@github.com:you/openclaw-backups.git +openclaw backup enable --repository ~/Backups/openclaw-git --every 24h --push +``` + +`backup enable --push` refuses to schedule when no `origin` remote is +configured, so a fresh install cannot silently create a schedule whose pushes +always fail. + +Pushed schedules redact credential-bearing tables by default: an unattended +recurring push would otherwise retain credentials durably in remote Git +history. Pass `--include-secrets` to schedule full-fidelity remote backups +when you accept that tradeoff and the remote is private; restores from +redacted history require re-pairing devices and re-authenticating providers +afterward. Local (non-push) schedules keep full fidelity so restores are +complete. + +Use `--global-only` or `--agent ` to narrow the scope. Add +`--exclude-secrets` for a redacted Git history. Re-running the command updates +the fixed scheduled job instead of creating another one. Disable it with: + +```bash +openclaw backup disable +``` + +The Gateway must be reachable while enabling or disabling the schedule. There +is no local fallback scheduler. + +As an alternative, use your platform scheduler directly. A nightly cron +example that snapshots the control-plane database and the `main` agent +database: ```bash 0 3 * * * openclaw backup sqlite create --global --repository "$HOME/Backups/openclaw-sqlite" --json >> "$HOME/Backups/openclaw-backup.log" 2>&1 @@ -88,6 +123,11 @@ On macOS, a `launchd` job works the same way; on servers provisioned from the emits one machine-readable result per run, so the log doubles as a backup audit trail. Prune old snapshot directories on your own retention schedule. +Every non-dry-run archive, local SQLite snapshot, and Git backup attempt is +also recorded in the shared state database. `openclaw status` shows the newest +attempt, and `openclaw doctor` suggests a one-off or scheduled backup when no +successful run is recorded or the newest success is more than 14 days old. + ## Copy backups offsite Archives and snapshot repositories are plain files, so any sync tool works. @@ -97,10 +137,54 @@ An `rclone` example targeting an S3-compatible bucket: rclone sync ~/Backups/openclaw-sqlite remote:openclaw-backups/sqlite ``` -Because every archive and snapshot is a full copy, offsite syncs re-upload +Because every archive and local snapshot is a full copy, offsite syncs re-upload each new backup in full. Deduplicating backup tools such as `restic` reduce storage at the destination but still read full snapshots as input. When -upload size per backup matters, use continuous replication instead. +upload size per backup matters, use Git-backed snapshots or continuous +replication. + +## Versioned backups to a Git repository + +Git-backed backups dump each selected database into deterministic `schema.sql`, +`manifest.json`, and per-table JSONL files, then create one commit for the +whole run. Unchanged database content produces no commit, so Git stores and +pushes only content changes by construction. OpenClaw stages only the +backup-owned `global` and `agents` paths, not unrelated files elsewhere in the +repository. + +```bash +openclaw backup git init --repository ~/Backups/openclaw-git --remote +openclaw backup git create --repository ~/Backups/openclaw-git --all --push +openclaw backup git log --repository ~/Backups/openclaw-git +``` + +Use a repository dedicated to OpenClaw backups. Existing `global/` and +`agents//` scopes must be empty or contain a valid schema-version-1 +OpenClaw backup manifest. OpenClaw refuses to replace any other scope, and an +`--all` run validates every existing agent scope before deleting stale +backup-owned entries. + +The repository root must be owned by the current user and must not be group- or +world-writable. This is checked during init and every create. On POSIX systems, +confirm ownership and run `chmod 700 ` to repair unsafe permissions. + +The repository is ordinary Git and can use any remote, including GitHub. Keep +the remote private: the default dump includes auth profiles, tokens, and other +credential-bearing state. `--exclude-secrets` omits the documented secret +tables when a redacted history is more useful than a credential-complete +backup; see [Backup CLI](/cli/backup#versioned-git-backups) for the exact list. + +Verify or restore one database at any commit without overwriting a live file: + +```bash +openclaw backup git verify --repository ~/Backups/openclaw-git --ref --global +openclaw backup git restore --repository ~/Backups/openclaw-git --ref --agent main --target ./restored-agent.sqlite +``` + +Git restore converges derived search state: it rebuilds content-backed FTS5 +indexes, leaves transcript projection state for Gateway startup reconciliation, +and leaves vector tables for memory indexing to recreate. It then verifies +table hashes, SQLite integrity, and foreign keys. ## Continuous replication with Litestream @@ -236,10 +320,12 @@ credentials directory, and workspace directories. See ### Restore a database For a snapshot, `openclaw backup sqlite restore --target -` writes a re-verified database to a fresh target. For -Litestream, `litestream restore` writes a fresh database file. Move either -result into place while the Gateway is stopped, then start the Gateway and -check `openclaw health` and `openclaw doctor`. +` writes a re-verified database to a fresh target. For Git +history, `openclaw backup git restore --repository --ref +(--global | --agent ) --target ` materializes and +verifies a fresh database. For Litestream, `litestream restore` writes a fresh +database file. Move the result into place while the Gateway is stopped, then +start the Gateway and check `openclaw health` and `openclaw doctor`. After restoring onto a different OpenClaw version, preflight the database first with `openclaw database preflight`; see diff --git a/scripts/check-kysely-guardrails.mts b/scripts/check-kysely-guardrails.mts index 7aeed326b7d8..442096aa80a6 100644 --- a/scripts/check-kysely-guardrails.mts +++ b/scripts/check-kysely-guardrails.mts @@ -72,6 +72,7 @@ const rawSqliteAllowPathGroups = { "backup snapshot maintenance": [ "src/commands/backup-verify.ts", "src/infra/backup-create.ts", + "src/snapshot/git-backup-codec.ts", "src/snapshot/local-repository.ts", ], "agent auth profile read-only bootstrap": ["src/agents/auth-profiles/sqlite.ts"], diff --git a/src/agents/worktrees/git-lock.ts b/src/agents/worktrees/git-lock.ts index ee552d92b9f0..1c3837a10b18 100644 --- a/src/agents/worktrees/git-lock.ts +++ b/src/agents/worktrees/git-lock.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { isPidDefinitelyDead } from "../../shared/pid-alive.js"; -import { commandError, listGitWorktrees, runGit } from "./git.js"; +import { commandError, runGit } from "./git.js"; +import { listGitWorktrees } from "./git.js"; import type { ManagedWorktreeRecord } from "./types.js"; const OPENCLAW_LOCK_PATTERN = /^openclaw pid=(\d+)$/; diff --git a/src/agents/worktrees/git.ts b/src/agents/worktrees/git.ts index 9b0bb6d1ef29..db2c6d61dd88 100644 --- a/src/agents/worktrees/git.ts +++ b/src/agents/worktrees/git.ts @@ -1,9 +1,13 @@ import { existsSync } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; -import { runCommandBuffered, runCommandWithTimeout } from "../../process/exec.js"; - -const GIT_TIMEOUT_MS = 120_000; +import { + createGitCommandError, + executeGitCommand, + requireGitCommand, + requireGitCommandBuffer, + requireGitCommandRaw, +} from "../../infra/git-exec.js"; export type GitResult = { stdout: string; @@ -16,21 +20,18 @@ type WorktreeListEntry = { lockedReason?: string; }; +// Preserve the worktree-facing dependency contract while generic Git execution +// remains owned by infra/git-exec. export async function runGit( cwd: string, args: string[], options: { env?: NodeJS.ProcessEnv; input?: string | Uint8Array } = {}, ): Promise { - return await runCommandWithTimeout(["git", "-C", cwd, ...args], { - timeoutMs: GIT_TIMEOUT_MS, - env: options.env, - input: options.input, - }); + return await executeGitCommand(cwd, args, options); } export function commandError(command: string, result: GitResult): Error { - const detail = (result.stderr || result.stdout).trim().split("\n").slice(-12).join("\n"); - return new Error(`${command} failed${detail ? `:\n${detail}` : ""}`); + return createGitCommandError(command, result); } export async function requireGit( @@ -38,19 +39,11 @@ export async function requireGit( args: string[], options: { env?: NodeJS.ProcessEnv; input?: string | Uint8Array } = {}, ): Promise { - const result = await runGit(cwd, args, options); - if (result.code !== 0) { - throw commandError(`git ${args.join(" ")}`, result); - } - return result.stdout.trim(); + return await requireGitCommand(cwd, args, options); } export async function requireGitRaw(cwd: string, args: string[]): Promise { - const result = await runGit(cwd, args); - if (result.code !== 0) { - throw commandError(`git ${args.join(" ")}`, result); - } - return result.stdout; + return await requireGitCommandRaw(cwd, args); } export async function requireGitBuffer( @@ -58,21 +51,7 @@ export async function requireGitBuffer( args: string[], options: { env?: NodeJS.ProcessEnv; input?: Uint8Array } = {}, ): Promise { - const result = await runCommandBuffered(["git", "-C", cwd, ...args], { - timeoutMs: GIT_TIMEOUT_MS, - env: options.env, - input: options.input, - }); - if (result.code !== 0) { - const detail = (result.stderr.length > 0 ? result.stderr : result.stdout) - .toString("utf8") - .trim() - .split("\n") - .slice(-12) - .join("\n"); - throw new Error(`git ${args.join(" ")} failed${detail ? `:\n${detail}` : ""}`); - } - return result.stdout; + return await requireGitCommandBuffer(cwd, args, options); } function parseWorktreeList(output: string): WorktreeListEntry[] { diff --git a/src/agents/worktrees/provisioned-files.ts b/src/agents/worktrees/provisioned-files.ts index 324fd035c277..ddec81c87ae4 100644 --- a/src/agents/worktrees/provisioned-files.ts +++ b/src/agents/worktrees/provisioned-files.ts @@ -1,7 +1,8 @@ import { constants as fsConstants } from "node:fs"; import fs, { type FileHandle } from "node:fs/promises"; import path from "node:path"; -import { requireGitRaw, worktreePathExists } from "./git.js"; +import { requireGitBuffer, requireGitRaw } from "./git.js"; +import { worktreePathExists } from "./git.js"; import { clearRegistryWorktreeProvisionedChunks, getRegistryWorktreeProvisionedChunk, @@ -257,7 +258,8 @@ export async function snapshotProvisionedFiles( (await requireGitRaw(worktreePath, ["ls-files", "--cached", "-z"])).split("\0").filter(Boolean), ); const trackedAtHead = new Set( - (await requireGitRaw(worktreePath, ["ls-tree", "-r", "--name-only", "-z", "HEAD"])) + (await requireGitBuffer(worktreePath, ["ls-tree", "-r", "--name-only", "-z", "HEAD"])) + .toString("utf8") .split("\0") .filter(Boolean), ); diff --git a/src/agents/worktrees/service.ts b/src/agents/worktrees/service.ts index b7e87068c50c..86d941c91976 100644 --- a/src/agents/worktrees/service.ts +++ b/src/agents/worktrees/service.ts @@ -7,6 +7,11 @@ import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { resolveStateDir } from "../../config/paths.js"; import { isMissingPathError } from "../../infra/errors.js"; import { formatErrorMessage } from "../../infra/errors.js"; +import { + executeGitCommand as runGit, + requireGitCommand as requireGit, + requireGitCommandBuffer as requireGitBuffer, +} from "../../infra/git-exec.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { runCommandWithTimeout } from "../../process/exec.js"; import { withOpenClawStateLease } from "../../state/openclaw-state-lease.js"; @@ -19,9 +24,6 @@ import { listGitWorktrees, worktreePathExists, removeEmptyParents, - requireGit, - requireGitBuffer, - runGit, type GitResult, } from "./git.js"; import { worktreeNameAllocationFamily } from "./name.js"; diff --git a/src/cli/gateway-rpc.runtime.test.ts b/src/cli/gateway-rpc.runtime.test.ts index 47f0ab601449..b610ca8150d6 100644 --- a/src/cli/gateway-rpc.runtime.test.ts +++ b/src/cli/gateway-rpc.runtime.test.ts @@ -5,15 +5,18 @@ import { addGatewayClientOptions } from "./gateway-rpc.js"; import type { GatewayRpcOpts } from "./gateway-rpc.types.js"; const callGatewayMock = vi.fn(async () => ({ ok: true })); +const isImplicitLocalGatewayTargetMock = vi.fn(async () => true); vi.mock("../gateway/call.js", () => ({ callGateway: callGatewayMock, + isImplicitLocalGatewayTarget: isImplicitLocalGatewayTargetMock, })); vi.mock("./progress.js", () => ({ withProgress: async (_options: unknown, action: () => Promise) => await action(), })); -const { callGatewayFromCliRuntime } = await import("./gateway-rpc.runtime.js"); +const { callGatewayFromCliRuntime, isImplicitLocalGatewayTargetFromCliRuntime } = + await import("./gateway-rpc.runtime.js"); describe("addGatewayClientOptions", () => { it.each([ @@ -170,3 +173,21 @@ describe("callGatewayFromCliRuntime", () => { ); }); }); + +describe("isImplicitLocalGatewayTargetFromCliRuntime", () => { + it("forwards CLI target options to the canonical Gateway classifier", async () => { + isImplicitLocalGatewayTargetMock.mockResolvedValueOnce(false); + + await expect( + isImplicitLocalGatewayTargetFromCliRuntime({ + url: "ws://127.0.0.1:18789", + token: "token", + }), + ).resolves.toBe(false); + expect(isImplicitLocalGatewayTargetMock).toHaveBeenCalledWith({ + config: undefined, + url: "ws://127.0.0.1:18789", + localPortOverride: undefined, + }); + }); +}); diff --git a/src/cli/gateway-rpc.runtime.ts b/src/cli/gateway-rpc.runtime.ts index e058b93bd1e6..b8c7e02651e5 100644 --- a/src/cli/gateway-rpc.runtime.ts +++ b/src/cli/gateway-rpc.runtime.ts @@ -4,7 +4,7 @@ import { GATEWAY_CLIENT_NAMES, } from "../../packages/gateway-protocol/src/client-info.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { callGateway } from "../gateway/call.js"; +import { callGateway, isImplicitLocalGatewayTarget } from "../gateway/call.js"; import type { GatewayRpcOpts } from "./gateway-rpc.types.js"; import { parseTimeoutMsWithFallback } from "./parse-timeout.js"; import { withProgress } from "./progress.js"; @@ -35,6 +35,16 @@ type GatewayCliTransportRpcOpts = Omit & { const DEFAULT_GATEWAY_RPC_TIMEOUT_MS = 30_000; +export async function isImplicitLocalGatewayTargetFromCliRuntime( + opts: GatewayCliTransportRpcOpts, +): Promise { + return await isImplicitLocalGatewayTarget({ + config: opts.config, + url: opts.url, + localPortOverride: opts.localPortOverride, + }); +} + export async function callGatewayFromCliRuntime( method: string, opts: GatewayCliTransportRpcOpts, diff --git a/src/cli/gateway-rpc.ts b/src/cli/gateway-rpc.ts index a5753da352ad..253577331736 100644 --- a/src/cli/gateway-rpc.ts +++ b/src/cli/gateway-rpc.ts @@ -47,6 +47,12 @@ export async function callGatewayFromCli( return await callGatewayFromCliWithTransport(method, opts, params, extra); } +/** Resolve whether CLI Gateway options select the implicit local Gateway. */ +export async function isImplicitLocalGatewayTargetFromCli(opts: GatewayRpcOpts): Promise { + const runtime = await loadGatewayRpcRuntime(); + return await runtime.isImplicitLocalGatewayTargetFromCliRuntime(opts); +} + /** Internal CLI facade for callers that need transport or auth policy overrides. */ export async function callGatewayFromCliWithTransport( method: string, diff --git a/src/cli/program/register.backup.ts b/src/cli/program/register.backup.ts index 31d64427bffb..68e467ddf0d3 100644 --- a/src/cli/program/register.backup.ts +++ b/src/cli/program/register.backup.ts @@ -2,6 +2,14 @@ import type { Command } from "commander"; import { formatDocsLink } from "../../../packages/terminal-core/src/links.js"; import { theme } from "../../../packages/terminal-core/src/theme.js"; +import { + backupGitCreateCommand, + backupGitInitCommand, + backupGitLogCommand, + backupGitRestoreCommand, + backupGitVerifyCommand, +} from "../../commands/backup-git.js"; +import { backupDisableCommand, backupEnableCommand } from "../../commands/backup-schedule.js"; import { backupSqliteCreateCommand, backupSqliteListCommand, @@ -12,6 +20,7 @@ import { backupVerifyCommand } from "../../commands/backup-verify.js"; import { backupCreateCommand } from "../../commands/backup.js"; import { defaultRuntime } from "../../runtime.js"; import { runCommandWithRuntime } from "../cli-utils.js"; +import { addGatewayClientOptions } from "../gateway-rpc.js"; import { formatHelpExamples } from "../help-format.js"; /** Register backup create/verify subcommands. */ @@ -99,6 +108,134 @@ export function registerBackupCommand(program: Command) { }); registerBackupSqliteCommands(backup); + registerBackupGitCommands(backup); + registerBackupScheduleCommands(backup); +} + +function collectAgent(value: string, previous: string[]): string[] { + return [...previous, value]; +} + +function registerBackupScheduleCommands(backup: Command): void { + addGatewayClientOptions( + backup + .command("enable") + .description("Provision a Gateway automation for scheduled Git backups") + .requiredOption("--repository ", "Git backup repository directory") + .option("--every ", "Backup interval", "24h") + .option("--push", "Push the current branch to origin after each backup", false) + .option("--exclude-secrets", "Omit credential-bearing database tables", false) + .option( + "--include-secrets", + "Keep credential-bearing tables in pushed scheduled backups", + false, + ) + .option("--global-only", "Back up only the shared state database", false) + .option("--agent ", "Back up only one agent database") + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupEnableCommand(defaultRuntime, opts); + }); + }), + ); + + addGatewayClientOptions( + backup + .command("disable") + .description("Remove the scheduled Git backup automation") + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupDisableCommand(defaultRuntime, opts); + }); + }), + ); +} + +function registerBackupGitCommands(backup: Command): void { + const git = backup + .command("git") + .description("Create and restore deterministic versioned SQLite dumps in Git") + .action(() => { + git.outputHelp(); + process.exitCode = 1; + }); + + git + .command("init") + .description("Initialize or adopt an operator-owned Git backup repository") + .requiredOption("--repository ", "Git backup repository directory") + .option("--remote ", "Add the remote as origin") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupGitInitCommand(defaultRuntime, opts); + }); + }); + + git + .command("create") + .description("Dump selected OpenClaw databases and commit one Git revision") + .requiredOption("--repository ", "Git backup repository directory") + .option("--all", "Back up the shared database and every registered agent database", false) + .option("--global", "Back up the shared OpenClaw state database", false) + .option("--agent ", "Back up an agent database (repeatable)", collectAgent, []) + .option("--push", "Push the current branch to origin", false) + .option("--exclude-secrets", "Omit credential-bearing database tables", false) + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupGitCreateCommand(defaultRuntime, { + repository: opts.repository as string, + all: Boolean(opts.all), + global: Boolean(opts.global), + agents: opts.agent as string[], + push: Boolean(opts.push), + excludeSecrets: Boolean(opts.excludeSecrets), + json: Boolean(opts.json), + }); + }); + }); + + git + .command("log") + .description("Show Git backup commits") + .requiredOption("--repository ", "Git backup repository directory") + .option("--limit ", "Maximum commits to show", (value) => Number.parseInt(value, 10), 20) + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupGitLogCommand(defaultRuntime, opts); + }); + }); + + git + .command("verify") + .description("Restore and verify one database snapshot from a Git ref") + .requiredOption("--repository ", "Git backup repository directory") + .option("--ref ", "Commit or ref to verify", "HEAD") + .option("--global", "Verify the shared state database", false) + .option("--agent ", "Verify one agent database") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupGitVerifyCommand(defaultRuntime, opts); + }); + }); + + git + .command("restore") + .description("Restore one database snapshot from a Git ref to a fresh SQLite file") + .requiredOption("--repository ", "Git backup repository directory") + .requiredOption("--target ", "Fresh target path; existing files and sidecars are refused") + .option("--ref ", "Commit or ref to restore", "HEAD") + .option("--global", "Restore the shared state database", false) + .option("--agent ", "Restore one agent database") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupGitRestoreCommand(defaultRuntime, opts); + }); + }); } function registerBackupSqliteCommands(backup: Command): void { diff --git a/src/cli/program/root-command-descriptions.test.ts b/src/cli/program/root-command-descriptions.test.ts index 76fa11da6092..2a6a485878e7 100644 --- a/src/cli/program/root-command-descriptions.test.ts +++ b/src/cli/program/root-command-descriptions.test.ts @@ -28,6 +28,7 @@ const JSON_NOT_APPLICABLE = { reason: "command group only; reporting subcommands declare JSON output individually", commands: [ "backup", + "backup git", "backup sqlite", "database", "database ownership", @@ -143,6 +144,8 @@ const JSON_NOT_APPLICABLE = { commands: [ "reset", "uninstall", + "backup enable", + "backup disable", "config set", "mcp add", "mcp set", diff --git a/src/commands/backup-git.ts b/src/commands/backup-git.ts new file mode 100644 index 000000000000..3554783db982 --- /dev/null +++ b/src/commands/backup-git.ts @@ -0,0 +1,261 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { normalizeAgentId } from "../routing/session-key.js"; +import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; +import type { GitBackupIdentity } from "../snapshot/git-backup-codec.js"; +import { + createGitBackup, + initializeGitBackupRepository, + readGitBackupLog, + restoreGitBackupRef, + verifyGitBackupRef, +} from "../snapshot/git-backup.js"; +import { recordBackupRunOutcome } from "../state/backup-run-records.js"; +import { listOpenClawRegisteredAgentDatabases } from "../state/openclaw-agent-db.js"; +import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { resolveUserPath, shortenHomePath } from "../utils.js"; + +type BackupGitCreateOptions = { + repository?: string; + all?: boolean; + global?: boolean; + agents?: string[]; + push?: boolean; + excludeSecrets?: boolean; + json?: boolean; +}; + +type BackupGitScopeOptions = { + global?: boolean; + agent?: string; +}; + +export const GIT_BACKUP_PUSH_CREDENTIAL_WARNING = + "Warning: pushed backup history contains credential material; keep the Git remote private."; + +function resolveRequiredPath(value: string | undefined, label: string): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`Missing required ${label} value.`); + } + return path.resolve(resolveUserPath(trimmed)); +} + +async function resolveCreateDatabases(runtime: RuntimeEnv, options: BackupGitCreateOptions) { + const agents = [...new Set((options.agents ?? []).map((agent) => normalizeAgentId(agent)))]; + const explicit = options.global === true || agents.length > 0; + if (options.all && explicit) { + throw new Error("Use --all by itself, or select --global and --agent scopes explicitly."); + } + if (!options.all && !explicit) { + throw new Error("Choose at least one Git backup scope: --all, --global, or --agent ."); + } + const databases: Array<{ + path: string; + identity: GitBackupIdentity; + }> = []; + if (options.all || options.global) { + databases.push({ + path: await fs.realpath(resolveOpenClawStateSqlitePath()), + identity: { role: "global" }, + }); + } + // Registry rows can carry stale or foreign absolute paths (deleted agents, + // retired temp state dirs), so --all resolves each distinct agent id to its + // canonical database under the current state dir and skips absent files + // instead of aborting the whole scheduled run on one dead registration. + const allAgentIds = options.all + ? [...new Set(listOpenClawRegisteredAgentDatabases().map((entry) => entry.agentId))].toSorted() + : agents; + for (const agentId of allAgentIds) { + const canonicalPath = resolveOpenClawAgentSqlitePath({ agentId }); + let resolvedPath: string; + try { + resolvedPath = await fs.realpath(canonicalPath); + } catch (error) { + if (options.all && (error as NodeJS.ErrnoException).code === "ENOENT") { + runtime.error(`Warning: skipping agent ${agentId}: no database at ${canonicalPath}`); + continue; + } + throw error; + } + databases.push({ path: resolvedPath, identity: { role: "agent", agentId } }); + } + if (databases.length === 0) { + throw new Error("No Git backup databases were found for the selected scope."); + } + return databases; +} + +function resolveOneIdentity(options: BackupGitScopeOptions): GitBackupIdentity { + const agent = options.agent?.trim(); + if (options.global === true && agent) { + throw new Error("Choose exactly one Git backup scope: --global or --agent ."); + } + if (options.global !== true && !agent) { + throw new Error("Choose a Git backup scope: --global or --agent ."); + } + return options.global === true + ? { role: "global" } + : { role: "agent", agentId: normalizeAgentId(agent) }; +} + +function recordGitOutcomeBestEffort( + runtime: RuntimeEnv, + params: { + repositoryPath: string; + status: "ok" | "failed"; + target?: string; + error?: string; + pushFailed?: true; + }, +): void { + try { + recordBackupRunOutcome({ + kind: "git", + archivePath: params.repositoryPath, + status: params.status, + target: params.target, + error: params.error, + pushFailed: params.pushFailed, + }); + } catch (error) { + runtime.error( + `Warning: the Git backup outcome could not be recorded: ${formatErrorMessage(error)}`, + ); + } +} + +export async function backupGitInitCommand( + runtime: RuntimeEnv, + options: { repository?: string; remote?: string; json?: boolean }, +): Promise<{ repositoryPath: string }> { + const result = await initializeGitBackupRepository({ + repositoryPath: resolveRequiredPath(options.repository, "--repository"), + stateDir: resolveStateDir(), + remote: options.remote, + }); + if (options.json) { + writeRuntimeJson(runtime, result); + } else { + runtime.log(`Git backup repository ready: ${shortenHomePath(result.repositoryPath)}`); + } + return result; +} + +export async function backupGitCreateCommand(runtime: RuntimeEnv, options: BackupGitCreateOptions) { + const repositoryPath = resolveRequiredPath(options.repository, "--repository"); + if (options.push && !options.excludeSecrets) { + runtime.error(GIT_BACKUP_PUSH_CREDENTIAL_WARNING); + } + try { + const result = await createGitBackup({ + repositoryPath, + stateDir: resolveStateDir(), + databases: await resolveCreateDatabases(runtime, options), + all: options.all, + excludeSecrets: options.excludeSecrets, + push: options.push, + }); + // A completed local backup remains successful even when requested remote replication fails; + // pushFailed records that durable degradation without discarding the recoverable local commit. + recordGitOutcomeBestEffort(runtime, { + repositoryPath, + status: "ok", + target: result.commit, + error: result.pushWarning, + ...(result.pushWarning ? { pushFailed: true } : {}), + }); + if (options.json) { + writeRuntimeJson(runtime, result); + } else if (result.noChanges) { + runtime.log(`Git backup: no changes (${shortenHomePath(repositoryPath)})`); + } else { + runtime.log(`Git backup committed: ${result.commit}`); + } + if (result.pushWarning) { + runtime.error(`Warning: Git backup committed, but push failed: ${result.pushWarning}`); + } + return result; + } catch (error) { + recordGitOutcomeBestEffort(runtime, { + repositoryPath, + status: "failed", + error: formatErrorMessage(error), + }); + throw error; + } +} + +export async function backupGitLogCommand( + runtime: RuntimeEnv, + options: { repository?: string; limit?: number; json?: boolean }, +) { + const repositoryPath = resolveRequiredPath(options.repository, "--repository"); + const limit = options.limit ?? 20; + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error("--limit must be a positive integer."); + } + const entries = await readGitBackupLog({ repositoryPath, limit }); + if (options.json) { + writeRuntimeJson(runtime, { repositoryPath, entries }); + } else if (entries.length === 0) { + runtime.log(`No Git backup commits in ${shortenHomePath(repositoryPath)}.`); + } else { + runtime.log( + entries.map((entry) => `${entry.commit}\t${entry.date}\t${entry.message}`).join("\n"), + ); + } + return entries; +} + +export async function backupGitVerifyCommand( + runtime: RuntimeEnv, + options: BackupGitScopeOptions & { repository?: string; ref?: string; json?: boolean }, +) { + const result = await verifyGitBackupRef({ + repositoryPath: resolveRequiredPath(options.repository, "--repository"), + identity: resolveOneIdentity(options), + ref: options.ref, + }); + if (options.json) { + writeRuntimeJson(runtime, result); + } else { + for (const table of result.tables) { + runtime.log(`${table.ok ? "ok" : "failed"}\t${table.table}\t${table.rows}\t${table.sha256}`); + } + runtime.log(`Git backup verified: ${result.commit}`); + } + return result; +} + +export async function backupGitRestoreCommand( + runtime: RuntimeEnv, + options: BackupGitScopeOptions & { + repository?: string; + ref?: string; + target?: string; + json?: boolean; + }, +) { + const result = await restoreGitBackupRef({ + repositoryPath: resolveRequiredPath(options.repository, "--repository"), + identity: resolveOneIdentity(options), + ref: options.ref, + targetPath: resolveRequiredPath(options.target, "--target"), + }); + if (options.json) { + writeRuntimeJson(runtime, result); + } else { + runtime.log(`Git backup restored: ${shortenHomePath(result.targetPath)} (${result.commit})`); + if (result.excludedTables.length > 0) { + runtime.error( + `Warning: this redacted backup omits tables: ${result.excludedTables.join(", ")}`, + ); + } + } + return result; +} diff --git a/src/commands/backup-health.ts b/src/commands/backup-health.ts new file mode 100644 index 000000000000..e33a5bd97fa4 --- /dev/null +++ b/src/commands/backup-health.ts @@ -0,0 +1,79 @@ +import { note } from "../../packages/terminal-core/src/note.js"; +import { formatCliCommand } from "../cli/command-format.js"; +import { + readLatestBackupRun, + readLatestSuccessfulBackupRun, + type BackupRunRecord, +} from "../state/backup-run-records.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; + +// Backups older than two weeks no longer provide a useful routine recovery point. +const BACKUP_STALE_AFTER_MS = 14 * 24 * 60 * 60 * 1_000; + +type BackupFreshness = { + latest?: BackupRunRecord; + latestOk?: BackupRunRecord; +}; + +/** Read backup freshness without creating or repairing an absent state database. */ +export function readBackupFreshness(env: NodeJS.ProcessEnv): BackupFreshness { + return ( + withExistingOpenClawStateDatabaseReadOnly( + ({ db }) => ({ + latest: readLatestBackupRun(db), + latestOk: readLatestSuccessfulBackupRun(db), + }), + { env }, + ) ?? {} + ); +} + +/** Format the compact status overview value for the latest backup attempt. */ +export function buildBackupStatusValue(params: { + freshness: BackupFreshness; + now?: number; + formatTimeAgo: (ageMs: number) => string; +}): string { + const latest = params.freshness.latest; + if (!latest) { + return "none recorded"; + } + const age = params.formatTimeAgo(Math.max(0, (params.now ?? Date.now()) - latest.createdAt)); + return latest.status === "ok" + ? `last ok ${age} (${latest.kind}${latest.pushFailed ? ", push failing" : ""})` + : `last attempt failed ${age} (${latest.kind})`; +} + +/** Build the informational Doctor hint for missing or stale successful backups. */ +function buildBackupDoctorHint(params: { + freshness: BackupFreshness; + now?: number; +}): string | null { + const latestOk = params.freshness.latestOk; + if (latestOk?.pushFailed) { + return [ + "The newest local Git backup succeeded, but its requested push failed.", + `Check the configured Git remote for ${latestOk.archivePath}, then retry the backup.`, + ].join("\n"); + } + const stale = + !latestOk || (params.now ?? Date.now()) - latestOk.createdAt > BACKUP_STALE_AFTER_MS; + if (!stale) { + return null; + } + return [ + latestOk + ? "The newest successful backup is more than 14 days old." + : "No successful backup is recorded.", + `Create one now with ${formatCliCommand("openclaw backup create")}.`, + `Schedule versioned backups with ${formatCliCommand("openclaw backup enable --repository ")}.`, + ].join("\n"); +} + +/** Emit the non-repairing backup freshness hint when it applies. */ +export function noteBackupDoctorHint(env: NodeJS.ProcessEnv): void { + const hint = buildBackupDoctorHint({ freshness: readBackupFreshness(env) }); + if (hint) { + note(hint, "Backups"); + } +} diff --git a/src/commands/backup-schedule.test.ts b/src/commands/backup-schedule.test.ts new file mode 100644 index 000000000000..dd48dcfb438a --- /dev/null +++ b/src/commands/backup-schedule.test.ts @@ -0,0 +1,216 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createTestRuntime } from "./test-runtime-config-helpers.js"; + +const gatewayRpc = vi.hoisted(() => ({ + call: vi.fn(), + isImplicitLocalTarget: vi.fn(async () => true), +})); + +vi.mock("../cli/gateway-rpc.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + callGatewayFromCli: gatewayRpc.call, + isImplicitLocalGatewayTargetFromCli: gatewayRpc.isImplicitLocalTarget, + }; +}); + +import { GIT_BACKUP_PUSH_CREDENTIAL_WARNING } from "./backup-git.js"; +import { backupDisableCommand, backupEnableCommand } from "./backup-schedule.js"; + +const BACKUP_CRON_JOB_NAME = "openclaw-backup-scheduled"; + +const roots: string[] = []; + +// enable --push preflights an origin remote, so push fixtures need a real repo. +async function pushReadyRepository(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-backup-schedule-test-")); + roots.push(root); + execFileSync("git", ["-C", root, "init"], { stdio: "ignore" }); + execFileSync("git", ["-C", root, "remote", "add", "origin", "git@example.invalid:backups.git"], { + stdio: "ignore", + }); + return root; +} + +describe("scheduled backups", () => { + beforeEach(() => { + gatewayRpc.call.mockReset(); + gatewayRpc.isImplicitLocalTarget.mockReset().mockResolvedValue(true); + }); + + afterEach(async () => { + await Promise.all( + roots.splice(0).map(async (root) => await fs.rm(root, { recursive: true, force: true })), + ); + }); + + it("adds one isolated command job with the selected Git backup argv", async () => { + gatewayRpc.call.mockImplementation(async (method: string) => { + if (method === "cron.add") { + return { created: true, job: { id: "backup-job" } }; + } + throw new Error(`unexpected method ${method}`); + }); + const runtime = createTestRuntime(); + const repository = await pushReadyRepository(); + await expect( + backupEnableCommand(runtime, { + repository, + every: "6h", + push: true, + excludeSecrets: true, + }), + ).resolves.toEqual({ id: "backup-job", updated: false }); + expect(gatewayRpc.call).toHaveBeenCalledWith( + "cron.add", + expect.anything(), + expect.objectContaining({ + declarationKey: BACKUP_CRON_JOB_NAME, + name: BACKUP_CRON_JOB_NAME, + schedule: { kind: "every", everyMs: 21_600_000 }, + sessionTarget: "isolated", + payload: { + kind: "command", + argv: [ + "openclaw", + "backup", + "git", + "create", + "--repository", + repository, + "--all", + "--push", + "--exclude-secrets", + ], + }, + }), + ); + expect(gatewayRpc.call).toHaveBeenCalledOnce(); + expect(runtime.error).not.toHaveBeenCalled(); + }); + + it("atomically converges an existing declaration and removes it idempotently", async () => { + gatewayRpc.call.mockResolvedValueOnce({ + created: false, + updated: true, + job: { id: "existing" }, + }); + const runtime = createTestRuntime(); + await expect( + backupEnableCommand(runtime, { + repository: "/tmp/openclaw-backups", + globalOnly: true, + }), + ).resolves.toEqual({ id: "existing", updated: true }); + expect(gatewayRpc.call).toHaveBeenCalledOnce(); + expect(gatewayRpc.call).toHaveBeenCalledWith( + "cron.add", + expect.anything(), + expect.objectContaining({ + declarationKey: BACKUP_CRON_JOB_NAME, + payload: expect.objectContaining({ argv: expect.arrayContaining(["--global"]) }), + }), + ); + + gatewayRpc.call.mockReset(); + gatewayRpc.call.mockImplementation(async (method: string) => { + if (method === "cron.list") { + return { + jobs: [ + { id: "decoy", name: BACKUP_CRON_JOB_NAME }, + { + id: "existing", + name: "operator display name", + declarationKey: BACKUP_CRON_JOB_NAME, + }, + ], + }; + } + return { ok: true }; + }); + await expect(backupDisableCommand(runtime, {})).resolves.toEqual({ removed: true }); + expect(gatewayRpc.call).toHaveBeenCalledWith("cron.remove", {}, { id: "existing" }); + expect(gatewayRpc.call).not.toHaveBeenCalledWith("cron.remove", {}, { id: "decoy" }); + + gatewayRpc.call.mockReset(); + gatewayRpc.call.mockResolvedValueOnce({ + jobs: [{ id: "decoy", name: BACKUP_CRON_JOB_NAME }], + }); + await expect(backupDisableCommand(runtime, {})).resolves.toEqual({ removed: false }); + }); + + it("redacts pushed schedules by default and warns only on explicit full fidelity", async () => { + const runtime = createTestRuntime(); + gatewayRpc.call.mockResolvedValue({ created: true, job: { id: "backup-job" } }); + + // Default pushed schedule: redacted, no credential warning. + await backupEnableCommand(runtime, { + repository: await pushReadyRepository(), + push: true, + }); + expect(gatewayRpc.call).toHaveBeenLastCalledWith( + "cron.add", + expect.anything(), + expect.objectContaining({ + payload: expect.objectContaining({ argv: expect.arrayContaining(["--exclude-secrets"]) }), + }), + ); + expect(runtime.error).not.toHaveBeenCalled(); + + // Explicit --include-secrets keeps full fidelity and warns. + await backupEnableCommand(runtime, { + repository: await pushReadyRepository(), + push: true, + includeSecrets: true, + }); + const lastSpec = gatewayRpc.call.mock.calls.at(-1)?.[2] as { + payload: { argv: string[] }; + }; + expect(lastSpec.payload.argv).not.toContain("--exclude-secrets"); + expect(runtime.error).toHaveBeenCalledWith(GIT_BACKUP_PUSH_CREDENTIAL_WARNING); + + await expect( + backupEnableCommand(runtime, { + repository: await pushReadyRepository(), + push: true, + includeSecrets: true, + excludeSecrets: true, + }), + ).rejects.toThrow(/not both/); + }); + + it("refuses a pushed schedule when the repository has no origin remote", async () => { + const runtime = createTestRuntime(); + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-backup-schedule-test-")); + roots.push(root); + execFileSync("git", ["-C", root, "init"], { stdio: "ignore" }); + await expect(backupEnableCommand(runtime, { repository: root, push: true })).rejects.toThrow( + /--push requires an origin remote/, + ); + expect(gatewayRpc.call).not.toHaveBeenCalled(); + }); + + it("rejects scheduling through a non-local Gateway before touching local paths", async () => { + gatewayRpc.isImplicitLocalTarget.mockResolvedValue(false); + const runtime = createTestRuntime(); + const expected = + "backup enable manages backups on the Gateway host and currently requires a local Gateway. Create the cron job manually with openclaw cron add for remote Gateways."; + + await expect( + backupEnableCommand(runtime, { + repository: "/path/that/does/not/exist", + push: true, + url: "ws://127.0.0.1:18789", + }), + ).rejects.toThrow(expected); + await expect( + backupDisableCommand(runtime, { url: "wss://gateway.example.invalid" }), + ).rejects.toThrow(expected); + expect(gatewayRpc.call).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/backup-schedule.ts b/src/commands/backup-schedule.ts new file mode 100644 index 000000000000..25c8f6e52c9a --- /dev/null +++ b/src/commands/backup-schedule.ts @@ -0,0 +1,164 @@ +import path from "node:path"; +import { + callGatewayFromCli, + isImplicitLocalGatewayTargetFromCli, + type GatewayRpcOpts, +} from "../cli/gateway-rpc.js"; +import { parseDurationMs } from "../cli/parse-duration.js"; +import type { CronJob } from "../cron/types.js"; +import { executeGitCommand } from "../infra/git-exec.js"; +import { normalizeAgentId } from "../routing/session-key.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { resolveUserPath, shortenHomePath } from "../utils.js"; +import { GIT_BACKUP_PUSH_CREDENTIAL_WARNING } from "./backup-git.js"; + +const BACKUP_CRON_JOB_NAME = "openclaw-backup-scheduled"; +const LOCAL_GATEWAY_REQUIRED_ERROR = + "backup enable manages backups on the Gateway host and currently requires a local Gateway. Create the cron job manually with openclaw cron add for remote Gateways."; + +type BackupScheduleOptions = GatewayRpcOpts & { + repository?: string; + every?: string; + push?: boolean; + excludeSecrets?: boolean; + includeSecrets?: boolean; + globalOnly?: boolean; + agent?: string; +}; + +/** + * Unattended pushed schedules make credential retention durable in remote + * history, so they redact by default; --include-secrets is the explicit + * full-fidelity override. Local (non-push) schedules keep full fidelity for + * complete restores. + */ +function resolveScheduledRedaction(options: BackupScheduleOptions): boolean { + if (options.excludeSecrets && options.includeSecrets) { + throw new Error("Use either --exclude-secrets or --include-secrets, not both."); + } + if (!options.push) { + return options.excludeSecrets === true; + } + return options.includeSecrets !== true; +} + +function resolveRepository(value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error("Missing required --repository value."); + } + return path.resolve(resolveUserPath(trimmed)); +} + +function buildScheduledArgv( + options: BackupScheduleOptions, + repositoryPath: string, + redactSecrets: boolean, +): string[] { + const agent = options.agent?.trim(); + if (options.globalOnly && agent) { + throw new Error("Use either --global-only or --agent , not both."); + } + return [ + "openclaw", + "backup", + "git", + "create", + "--repository", + repositoryPath, + ...(options.globalOnly + ? ["--global"] + : agent + ? ["--agent", normalizeAgentId(agent)] + : ["--all"]), + ...(options.push ? ["--push"] : []), + ...(redactSecrets ? ["--exclude-secrets"] : []), + ]; +} + +async function findScheduledBackup(options: GatewayRpcOpts): Promise { + const response = (await callGatewayFromCli("cron.list", options, { + includeDisabled: true, + query: BACKUP_CRON_JOB_NAME, + limit: 200, + offset: 0, + })) as { jobs?: CronJob[] }; + return response.jobs?.find((job) => job.declarationKey === BACKUP_CRON_JOB_NAME); +} + +async function assertLocalGatewayScheduleTarget(options: GatewayRpcOpts): Promise { + // V1 tradeoff: the CLI validates host-local repository paths, while cron runs + // on the Gateway host. Reject remote targets until Gateway-owned setup exists. + if (!(await isImplicitLocalGatewayTargetFromCli(options))) { + throw new Error(LOCAL_GATEWAY_REQUIRED_ERROR); + } +} + +export async function backupEnableCommand( + runtime: RuntimeEnv, + options: BackupScheduleOptions, +): Promise<{ id: string; updated: boolean }> { + await assertLocalGatewayScheduleTarget(options); + const repositoryPath = resolveRepository(options.repository); + const every = options.every?.trim() || "24h"; + const everyMs = parseDurationMs(every, { defaultUnit: "ms" }); + if (!Number.isSafeInteger(everyMs) || everyMs <= 0) { + throw new Error("--every must be a positive duration such as 6h or 24h."); + } + const redactSecrets = resolveScheduledRedaction(options); + const spec = { + declarationKey: BACKUP_CRON_JOB_NAME, + name: BACKUP_CRON_JOB_NAME, + enabled: true, + schedule: { kind: "every" as const, everyMs }, + sessionTarget: "isolated" as const, + wakeMode: "now" as const, + payload: { + kind: "command" as const, + argv: buildScheduledArgv(options, repositoryPath, redactSecrets), + }, + delivery: { mode: "none" as const }, + }; + if (options.push) { + // The unattended job cannot configure a remote; without this preflight the + // first scheduled run records a degraded push-failed backup instead. + const origin = await executeGitCommand(repositoryPath, ["remote", "get-url", "origin"]); + if (origin.code !== 0) { + throw new Error( + `--push requires an origin remote. Run: openclaw backup git init --repository ${shortenHomePath(repositoryPath)} --remote `, + ); + } + if (!redactSecrets) { + runtime.error(GIT_BACKUP_PUSH_CREDENTIAL_WARNING); + } + } + const result = (await callGatewayFromCli("cron.add", options, spec)) as { + created?: boolean; + updated?: boolean; + job?: { id?: string }; + }; + const id = result.job?.id; + if (!id) { + throw new Error("cron.add returned no scheduled backup job id."); + } + const updated = result.created === false; + runtime.log( + `Scheduled Git backups ${updated ? "updated" : "enabled"}: every ${every} to ${shortenHomePath(repositoryPath)}`, + ); + return { id, updated }; +} + +export async function backupDisableCommand( + runtime: RuntimeEnv, + options: GatewayRpcOpts, +): Promise<{ removed: boolean }> { + await assertLocalGatewayScheduleTarget(options); + const existing = await findScheduledBackup(options); + if (!existing) { + runtime.log("Scheduled Git backups are already disabled."); + return { removed: false }; + } + await callGatewayFromCli("cron.remove", options, { id: existing.id }); + runtime.log("Scheduled Git backups disabled."); + return { removed: true }; +} diff --git a/src/commands/backup-sqlite.ts b/src/commands/backup-sqlite.ts index 3a9014bdb538..ce10339637f0 100644 --- a/src/commands/backup-sqlite.ts +++ b/src/commands/backup-sqlite.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { formatErrorMessage } from "../infra/errors.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { createLocalSqliteSnapshotProvider } from "../snapshot/local-repository.js"; @@ -9,6 +10,7 @@ import type { SnapshotRef, SnapshotSummary, } from "../snapshot/snapshot-provider.js"; +import { recordBackupRunOutcome } from "../state/backup-run-records.js"; import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { resolveUserPath, shortenHomePath } from "../utils.js"; @@ -73,15 +75,41 @@ export async function backupSqliteCreateCommand( options: BackupSqliteCreateOptions, ): Promise { const repositoryPath = resolveRequiredPath(options.repository, "--repository"); - const database = await resolveSnapshotDatabase(options); - const result = await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database); - const report: BackupSqliteCreateResult = { - ok: true, - snapshotPath: result.ref.path, - manifest: result.manifest, - }; - writeCreateResult(runtime, options, report); - return report; + try { + const database = await resolveSnapshotDatabase(options); + const result = await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database); + const report: BackupSqliteCreateResult = { + ok: true, + snapshotPath: result.ref.path, + manifest: result.manifest, + }; + recordSqliteOutcomeBestEffort(runtime, { + archivePath: report.snapshotPath, + status: "ok", + }); + writeCreateResult(runtime, options, report); + return report; + } catch (error) { + recordSqliteOutcomeBestEffort(runtime, { + archivePath: repositoryPath, + status: "failed", + error: formatErrorMessage(error), + }); + throw error; + } +} + +function recordSqliteOutcomeBestEffort( + runtime: RuntimeEnv, + params: { archivePath: string; status: "ok" | "failed"; error?: string }, +): void { + try { + recordBackupRunOutcome({ kind: "sqlite-snapshot", ...params }); + } catch (error) { + runtime.error( + `Warning: backup completed, but its run record could not be written: ${formatErrorMessage(error)}`, + ); + } } export async function backupSqliteListCommand( diff --git a/src/commands/backup.ts b/src/commands/backup.ts index 2c8db2ad2f1e..6095e310a81f 100644 --- a/src/commands/backup.ts +++ b/src/commands/backup.ts @@ -5,8 +5,10 @@ import { type BackupCreateOptions, type BackupCreateResult, } from "../infra/backup-create.js"; +import { formatErrorMessage } from "../infra/errors.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; +import { recordBackupRunOutcome } from "../state/backup-run-records.js"; type BackupVerifyRuntime = typeof import("./backup-verify.js"); @@ -23,25 +25,57 @@ export async function backupCreateCommand( runtime: RuntimeEnv, opts: BackupCreateOptions = {}, ): Promise { - const result = await createBackupArchive({ - ...opts, - log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)), - }); - if (opts.verify && !opts.dryRun) { - const { backupVerifyCommand } = await loadBackupVerifyRuntime(); - await backupVerifyCommand( - { - ...runtime, - log: () => {}, - }, - { archive: result.archivePath, json: false }, - ); - result.verified = true; + let archivePath = opts.output ?? process.cwd(); + try { + const result = await createBackupArchive({ + ...opts, + log: opts.log ?? (opts.json ? undefined : (message: string) => runtime.log(message)), + }); + archivePath = result.archivePath; + if (opts.verify && !opts.dryRun) { + const { backupVerifyCommand } = await loadBackupVerifyRuntime(); + await backupVerifyCommand( + { + ...runtime, + log: () => {}, + }, + { archive: result.archivePath, json: false }, + ); + result.verified = true; + } + if (!opts.dryRun) { + recordBackupOutcomeBestEffort(runtime, { + archivePath, + status: "ok", + }); + } + if (opts.json) { + writeRuntimeJson(runtime, result); + } else { + runtime.log(formatBackupCreateSummary(result).join("\n")); + } + return result; + } catch (error) { + if (!opts.dryRun) { + recordBackupOutcomeBestEffort(runtime, { + archivePath, + status: "failed", + error: formatErrorMessage(error), + }); + } + throw error; + } +} + +function recordBackupOutcomeBestEffort( + runtime: RuntimeEnv, + params: { archivePath: string; status: "ok" | "failed"; error?: string }, +): void { + try { + recordBackupRunOutcome({ kind: "archive", ...params }); + } catch (error) { + runtime.error( + `Warning: backup completed, but its run record could not be written: ${formatErrorMessage(error)}`, + ); } - if (opts.json) { - writeRuntimeJson(runtime, result); - } else { - runtime.log(formatBackupCreateSummary(result).join("\n")); - } - return result; } diff --git a/src/commands/status-all.ts b/src/commands/status-all.ts index b602af740af6..3171e668ebd3 100644 --- a/src/commands/status-all.ts +++ b/src/commands/status-all.ts @@ -16,6 +16,7 @@ export async function statusAllCommand( ): Promise { await withProgress({ label: "Scanning status --all…", total: 11 }, async (progress) => { const overview = await collectStatusScanOverview({ + env: process.env, commandName: "status --all", opts: { timeoutMs: opts?.timeoutMs, diff --git a/src/commands/status-json-payload.test.ts b/src/commands/status-json-payload.test.ts index 0da02bb86d60..4be19e191143 100644 --- a/src/commands/status-json-payload.test.ts +++ b/src/commands/status-json-payload.test.ts @@ -137,7 +137,6 @@ describe("status-json-payload", () => { }, }); }); - it("omits optional sections when they are absent", () => { expect( buildStatusJsonPayload({ diff --git a/src/commands/status-json-runtime.test.ts b/src/commands/status-json-runtime.test.ts index d9f60d218908..7e5212ba1bca 100644 --- a/src/commands/status-json-runtime.test.ts +++ b/src/commands/status-json-runtime.test.ts @@ -4,9 +4,22 @@ import { resolveStatusJsonOutput } from "./status-json-runtime.ts"; const mocks = vi.hoisted(() => ({ buildStatusJsonPayload: vi.fn((input) => ({ built: true, input })), + readBackupFreshness: vi.fn(() => ({ + latest: { + id: "backup-1", + createdAt: 123, + archivePath: "/backups/git", + status: "ok" as const, + kind: "git" as const, + }, + })), resolveStatusRuntimeSnapshot: vi.fn(), })); +vi.mock("./backup-health.js", () => ({ + readBackupFreshness: mocks.readBackupFreshness, +})); + vi.mock("./status-json-payload.ts", () => ({ buildStatusJsonPayload: mocks.buildStatusJsonPayload, })); @@ -17,6 +30,7 @@ vi.mock("./status-runtime-shared.ts", () => ({ function createScan() { return { + env: { OPENCLAW_STATE_DIR: "/tmp/status-json-runtime-state" }, cfg: { update: { channel: "stable" }, gateway: {} }, sourceConfig: { gateway: {} }, summary: { ok: true }, @@ -72,8 +86,9 @@ describe("status-json-runtime", () => { }); it("builds the full json output for status --json", async () => { + const scan = createScan(); const result = await resolveStatusJsonOutput({ - scan: createScan(), + scan, opts: { deep: true, usage: true, timeoutMs: 1234 }, includeSecurityAudit: true, includePluginCompatibility: true, @@ -90,6 +105,7 @@ describe("status-json-runtime", () => { suppressHealthErrors: undefined, }); expect(mocks.buildStatusJsonPayload).toHaveBeenCalledOnce(); + expect(mocks.readBackupFreshness).toHaveBeenCalledWith(scan.env); const payloadInput = requireStatusPayloadInput(); expect(payloadInput.surface.gatewayConnection).toStrictEqual({ url: "ws://127.0.0.1:18789", @@ -113,6 +129,7 @@ describe("status-json-runtime", () => { expect(result).toEqual({ built: true, input: payloadInput, + backups: mocks.readBackupFreshness(), }); }); @@ -126,8 +143,9 @@ describe("status-json-runtime", () => { nodeService: { label: "node" }, }); + const { env: _env, ...scanWithoutEnv } = createScan(); await resolveStatusJsonOutput({ - scan: createScan(), + scan: scanWithoutEnv, opts: { deep: false, usage: false, timeoutMs: 500 }, includeSecurityAudit: false, includePluginCompatibility: false, @@ -144,6 +162,7 @@ describe("status-json-runtime", () => { suppressHealthErrors: undefined, }); expect(mocks.buildStatusJsonPayload).toHaveBeenCalledOnce(); + expect(mocks.readBackupFreshness).toHaveBeenCalledWith({}); const payloadInput = requireStatusPayloadInput(); expect(payloadInput.surface.gatewayProbeAuth).toStrictEqual({ token: "tok" }); expect(payloadInput.securityAudit).toBeUndefined(); diff --git a/src/commands/status-json-runtime.ts b/src/commands/status-json-runtime.ts index 163ddd821721..08f32c8e188f 100644 --- a/src/commands/status-json-runtime.ts +++ b/src/commands/status-json-runtime.ts @@ -3,11 +3,13 @@ import type { OpenClawConfig } from "../config/types.js"; import type { UpdateCheckResult } from "../infra/update-check.js"; +import { readBackupFreshness } from "./backup-health.js"; import { buildStatusJsonPayload } from "./status-json-payload.ts"; import { buildStatusOverviewSurfaceFromScan } from "./status-overview-surface.ts"; import { resolveStatusRuntimeSnapshot } from "./status-runtime-shared.ts"; type StatusJsonScanLike = { + env?: NodeJS.ProcessEnv; cfg: OpenClawConfig; sourceConfig: OpenClawConfig; summary: Record; @@ -76,7 +78,7 @@ export async function resolveStatusJsonOutput(params: { suppressHealthErrors: params.suppressHealthErrors, }); - return buildStatusJsonPayload({ + const payload = buildStatusJsonPayload({ summary: scan.summary, surface: buildStatusOverviewSurfaceFromScan({ // The scan shape is intentionally narrower than the surface helper's full scan type. @@ -95,4 +97,9 @@ export async function resolveStatusJsonOutput(params: { lastHeartbeat, pluginCompatibility: params.includePluginCompatibility ? scan.pluginCompatibility : undefined, }); + const backups = readBackupFreshness(scan.env ?? {}); + if (backups.latest || backups.latestOk) { + Object.assign(payload, { backups }); + } + return payload; } diff --git a/src/commands/status-overview-rows.ts b/src/commands/status-overview-rows.ts index d6d957cda754..d34911577a0f 100644 --- a/src/commands/status-overview-rows.ts +++ b/src/commands/status-overview-rows.ts @@ -6,6 +6,7 @@ import type { HeartbeatEventPayload } from "../infra/heartbeat-events.js"; import type { PluginCompatibilityNotice } from "../plugins/status.js"; import type { StatusSummary } from "../status/types.js"; import { VERSION } from "../version.js"; +import { buildBackupStatusValue, readBackupFreshness } from "./backup-health.js"; import type { HealthSummary } from "./health.js"; import { buildStatusOverviewRowsFromSurface, @@ -33,6 +34,7 @@ import type { MemoryPluginStatus, MemoryStatusSnapshot } from "./status.scan.sha /** Builds the default `openclaw status` overview rows from scan, health, memory, and session inputs. */ export function buildStatusCommandOverviewRows( params: { + env: NodeJS.ProcessEnv; opts: { deep?: boolean; }; @@ -149,6 +151,13 @@ export function buildStatusCommandOverviewRows( { Item: "Probes", Value: probesValue }, { Item: "Events", Value: eventsValue }, { Item: "Tasks", Value: tasksValue }, + { + Item: "Backups", + Value: buildBackupStatusValue({ + freshness: readBackupFreshness(params.env), + formatTimeAgo: params.formatTimeAgo, + }), + }, { Item: "Heartbeat", Value: heartbeatValue }, ...(lastHeartbeatValue ? [{ Item: "Last heartbeat", Value: lastHeartbeatValue }] : []), { diff --git a/src/commands/status.command-report-data.ts b/src/commands/status.command-report-data.ts index b142fcd58856..124992f3737a 100644 --- a/src/commands/status.command-report-data.ts +++ b/src/commands/status.command-report-data.ts @@ -35,6 +35,7 @@ import type { MemoryPluginStatus, MemoryStatusSnapshot } from "./status.scan.sha /** Builds all table rows, section lines, and footer data needed by the status report renderer. */ export async function buildStatusCommandReportData( params: { + env: NodeJS.ProcessEnv; opts: { deep?: boolean; verbose?: boolean; @@ -99,6 +100,7 @@ export async function buildStatusCommandReportData( } & StatusMemoryStateResolvers, ) { const overviewRows = buildStatusCommandOverviewRows({ + env: params.env, opts: params.opts, surface: params.surface, osLabel: params.osSummary.label, diff --git a/src/commands/status.command.ts b/src/commands/status.command.ts index ab2a6d121fff..935cd2c7e744 100644 --- a/src/commands/status.command.ts +++ b/src/commands/status.command.ts @@ -165,6 +165,7 @@ export async function statusCommand( memory, memoryPlugin, pluginCompatibility, + env, } = scan; const { @@ -325,6 +326,7 @@ export async function statusCommand( ); const lines = await buildStatusCommandReportLines( await buildStatusCommandReportData({ + env: env ?? {}, opts, surface: overviewSurface, osSummary, diff --git a/src/commands/status.scan-execute.ts b/src/commands/status.scan-execute.ts index af351de8e882..5d093925b0ba 100644 --- a/src/commands/status.scan-execute.ts +++ b/src/commands/status.scan-execute.ts @@ -39,6 +39,7 @@ export async function executeStatusScanFromOverview(params: { ]); return buildStatusScanResult({ + env: params.overview.env ?? {}, cfg: params.overview.cfg, sourceConfig: params.overview.sourceConfig, secretDiagnostics: params.overview.secretDiagnostics, diff --git a/src/commands/status.scan-overview.ts b/src/commands/status.scan-overview.ts index 539abeb7a02a..163797481190 100644 --- a/src/commands/status.scan-overview.ts +++ b/src/commands/status.scan-overview.ts @@ -69,6 +69,7 @@ async function resolveStatusChannelsStatus(params: { } export type StatusScanOverviewResult = { + env?: NodeJS.ProcessEnv; coldStart: boolean; hasConfiguredChannels: boolean; skipColdStartNetworkChecks: boolean; @@ -101,6 +102,7 @@ export type StatusScanOverviewResult = { /** Collects the common status scan data shared by text, JSON, and status-all commands. */ export async function collectStatusScanOverview(params: { + env?: NodeJS.ProcessEnv; commandName: string; opts: { timeoutMs?: number; all?: boolean }; showSecrets: boolean; @@ -137,6 +139,7 @@ export async function collectStatusScanOverview(params: { summarizingChannels?: string; }; }): Promise { + const env = params.env ?? process.env; if (params.labels?.loadingConfig) { params.progress?.setLabel(params.labels.loadingConfig); } @@ -146,6 +149,7 @@ export async function collectStatusScanOverview(params: { resolvedConfig: cfg, secretDiagnostics, } = await loadStatusScanCommandConfig({ + env, commandName: params.commandName, allowMissingConfigFastPath: params.allowMissingConfigFastPath, readConfigSnapshot: async () => @@ -161,7 +165,7 @@ export async function collectStatusScanOverview(params: { commandName: params.commandName, targetIds: (await commandSecretTargetsModuleLoader.load()).getStatusCommandSecretTargetIds( loadedConfig, - process.env, + env, { includeChannelTargets: params.includeChannelSecretTargets }, ), mode: "read_only_status", @@ -298,6 +302,7 @@ export async function collectStatusScanOverview(params: { }; return { + env, coldStart, hasConfiguredChannels, skipColdStartNetworkChecks: bootstrap.skipColdStartNetworkChecks, diff --git a/src/commands/status.scan.fast-json.ts b/src/commands/status.scan.fast-json.ts index 93f18b2bd979..deca9ffc1670 100644 --- a/src/commands/status.scan.fast-json.ts +++ b/src/commands/status.scan.fast-json.ts @@ -93,6 +93,7 @@ export async function scanStatusJsonWithPolicy( policy: StatusJsonScanPolicy, ): Promise { const overview = await collectStatusScanOverview({ + env: process.env, commandName: policy.commandName, opts, showSecrets: false, diff --git a/src/commands/status.scan.ts b/src/commands/status.scan.ts index bd315f0f9b16..54dbf6536540 100644 --- a/src/commands/status.scan.ts +++ b/src/commands/status.scan.ts @@ -54,6 +54,7 @@ export async function scanStatus( async (progress) => { const isFullScan = opts.all === true || opts.deep === true; const overview = await collectStatusScanOverview({ + env: process.env, commandName: "status", opts, showSecrets: process.env.OPENCLAW_SHOW_SECRETS?.trim() !== "0", diff --git a/src/commands/status.test-support.ts b/src/commands/status.test-support.ts index f47552bcc24b..acddb6344e7b 100644 --- a/src/commands/status.test-support.ts +++ b/src/commands/status.test-support.ts @@ -1,4 +1,6 @@ // Status test support builds reusable gateway, update, heartbeat, and service fixtures for command tests. +import os from "node:os"; +import path from "node:path"; import type { HeartbeatEventPayload } from "../infra/heartbeat-events.js"; import { isBetaTag } from "../infra/update-channels.js"; import type { Tone } from "../memory-host-sdk/status.js"; @@ -14,6 +16,8 @@ import type { MemoryPluginStatus, MemoryStatusSnapshot } from "./status.scan.sha type StatusCommandOverviewRowsParams = Parameters[0]; type StatusCommandReportDataParams = Parameters[0]; +const STATUS_TEST_STATE_DIR = path.join(os.tmpdir(), `openclaw-status-test-${process.pid}-absent`); + export const baseStatusCfg = { update: { channel: "stable" }, gateway: { bind: "loopback" }, @@ -222,6 +226,7 @@ export function createStatusCommandOverviewRowsParams( overrides: Partial = {}, ): StatusCommandOverviewRowsParams { return { + env: { OPENCLAW_STATE_DIR: STATUS_TEST_STATE_DIR }, opts: { deep: true }, surface: baseStatusOverviewSurface, osLabel: "macOS", @@ -244,6 +249,7 @@ export function createStatusCommandReportDataParams( overrides: Partial = {}, ): StatusCommandReportDataParams { return { + env: { OPENCLAW_STATE_DIR: STATUS_TEST_STATE_DIR }, opts: { deep: true, verbose: true }, surface: baseStatusOverviewSurface, osSummary: { label: "macOS" } as never, diff --git a/src/flows/doctor-health-contribution-runners.state.ts b/src/flows/doctor-health-contribution-runners.state.ts index d9e84d9d2682..c4a8b15e809a 100644 --- a/src/flows/doctor-health-contribution-runners.state.ts +++ b/src/flows/doctor-health-contribution-runners.state.ts @@ -1,3 +1,4 @@ +import { noteBackupDoctorHint } from "../commands/backup-health.js"; import { isLegacyParentWritableUpdateDoctorPass } from "../commands/doctor/shared/update-phase.js"; import { writeConfigMachineState } from "../state/config-machine-state.js"; import type { DoctorHealthFlowContext } from "./doctor-health-contribution-types.js"; @@ -100,6 +101,7 @@ export async function runStateIntegrityHealth(ctx: DoctorHealthFlowContext): Pro await noteStateIntegrity(ctx.cfg, ctx.prompter, ctx.configPath, { stateDirExistedAtStart: ctx.stateDirExistedAtStart, }); + noteBackupDoctorHint(ctx.env ?? process.env); } export async function runCodexSessionRouteHealth(ctx: DoctorHealthFlowContext): Promise { diff --git a/src/gateway/call.test.ts b/src/gateway/call.test.ts index d96a49c7a88b..4c5dcbc51d63 100644 --- a/src/gateway/call.test.ts +++ b/src/gateway/call.test.ts @@ -201,6 +201,7 @@ const { formatGatewayTransportErrorJson, GatewayCredentialsRequiredError, GatewayExplicitAuthRequiredError, + isImplicitLocalGatewayTarget, isGatewayTransportError, } = await import("./call.js"); const { GatewaySecretRefUnavailableError } = await import("./credentials.js"); @@ -326,6 +327,22 @@ describe("callGateway url resolution", () => { resetGatewayCallMocks(); }); + it("classifies only the implicit configured local Gateway as local", async () => { + setLocalLoopbackGatewayConfig(); + await expect(isImplicitLocalGatewayTarget({})).resolves.toBe(true); + + setGatewayConfig({ mode: "remote", remote: { url: "wss://gateway.example/ws" } }); + await expect(isImplicitLocalGatewayTarget({})).resolves.toBe(false); + + setLocalLoopbackGatewayConfig(); + await expect(isImplicitLocalGatewayTarget({ url: "ws://127.0.0.1:18789" })).resolves.toBe( + false, + ); + + process.env.OPENCLAW_GATEWAY_URL = "wss://gateway.example/ws"; + await expect(isImplicitLocalGatewayTarget({})).resolves.toBe(false); + }); + afterEach(() => { resetConfigRuntimeState(); envSnapshot.restore(); diff --git a/src/gateway/call.ts b/src/gateway/call.ts index df7a9bade273..5bd79bde99ee 100644 --- a/src/gateway/call.ts +++ b/src/gateway/call.ts @@ -650,6 +650,11 @@ type ResolvedGatewayCallContext = { explicitAuth: ExplicitGatewayAuth; }; +export type GatewayTargetClassificationOptions = Pick< + CallGatewayBaseOptions, + "config" | "url" | "localPortOverride" | "ignoreEnvUrlOverride" +>; + function resolveGatewayCallTimeout(timeoutValue: unknown): { timeoutMs: number | null; startupTimeoutMs: number; @@ -700,6 +705,23 @@ async function resolveGatewayCallContext( }; } +/** Whether the caller selected the configured local Gateway without a URL override. */ +export async function isImplicitLocalGatewayTarget( + opts: GatewayTargetClassificationOptions, +): Promise { + const urlOverride = resolveGatewayUrlOverride({ + gatewayUrl: opts.url, + env: process.env, + ignoreEnvUrlOverride: opts.ignoreEnvUrlOverride, + localPortOverride: opts.localPortOverride, + }); + if (urlOverride.url) { + return false; + } + const config = opts.config ?? (await loadGatewayConfig()); + return config.gateway?.mode !== "remote"; +} + function ensureRemoteModeUrlConfigured(params: { context: ResolvedGatewayCallContext; urlOverrideSource?: "cli" | "env"; diff --git a/src/infra/git-exec.ts b/src/infra/git-exec.ts new file mode 100644 index 000000000000..cb9b4d7fc01c --- /dev/null +++ b/src/infra/git-exec.ts @@ -0,0 +1,69 @@ +import { runCommandBuffered, runCommandWithTimeout } from "../process/exec.js"; + +const GIT_TIMEOUT_MS = 120_000; + +type GitCommandResult = { + stdout: string; + stderr: string; + code: number | null; +}; + +export async function executeGitCommand( + cwd: string, + args: string[], + options: { env?: NodeJS.ProcessEnv; input?: string | Uint8Array } = {}, +): Promise { + return await runCommandWithTimeout(["git", "-C", cwd, ...args], { + timeoutMs: GIT_TIMEOUT_MS, + env: options.env, + input: options.input, + }); +} + +export function createGitCommandError(command: string, result: GitCommandResult): Error { + const detail = (result.stderr || result.stdout).trim().split("\n").slice(-12).join("\n"); + return new Error(`${command} failed${detail ? `:\n${detail}` : ""}`); +} + +export async function requireGitCommand( + cwd: string, + args: string[], + options: { env?: NodeJS.ProcessEnv; input?: string | Uint8Array } = {}, +): Promise { + const result = await executeGitCommand(cwd, args, options); + if (result.code !== 0) { + throw createGitCommandError(`git ${args.join(" ")}`, result); + } + return result.stdout.trim(); +} + +export async function requireGitCommandRaw(cwd: string, args: string[]): Promise { + const result = await executeGitCommand(cwd, args); + if (result.code !== 0) { + throw createGitCommandError(`git ${args.join(" ")}`, result); + } + return result.stdout; +} + +export async function requireGitCommandBuffer( + cwd: string, + args: string[], + options: { env?: NodeJS.ProcessEnv; input?: Uint8Array; maxOutputBytes?: number } = {}, +): Promise { + const result = await runCommandBuffered(["git", "-C", cwd, ...args], { + timeoutMs: GIT_TIMEOUT_MS, + env: options.env, + input: options.input, + ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}), + }); + if (result.code !== 0) { + const detail = (result.stderr.length > 0 ? result.stderr : result.stdout) + .toString("utf8") + .trim() + .split("\n") + .slice(-12) + .join("\n"); + throw new Error(`git ${args.join(" ")} failed${detail ? `:\n${detail}` : ""}`); + } + return result.stdout; +} diff --git a/src/snapshot/git-backup-codec.ts b/src/snapshot/git-backup-codec.ts new file mode 100644 index 000000000000..1c5b3f0dad1c --- /dev/null +++ b/src/snapshot/git-backup-codec.ts @@ -0,0 +1,634 @@ +import { createHash } from "node:crypto"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import { applyPrivateModeSync } from "../infra/private-mode.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; +import { createPrivateSqliteTempDirectory } from "../infra/sqlite-private-directory.js"; +import { publishVerifiedSqliteFile } from "../infra/sqlite-snapshot.js"; +import { normalizeAgentId } from "../routing/session-key.js"; +import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.js"; +import { getOpenClawStateRuntimeSchema } from "../state/openclaw-state-schema-compatibility.js"; +import { + AGENT_SECRET_TABLE_NAMES, + STATE_SECRET_TABLE_NAMES, +} from "../state/secret-state-tables.js"; +import { hashSnapshotArtifact } from "./manifest.js"; +import { buildSnapshotValidator } from "./openclaw-snapshot-copy.js"; +import { SNAPSHOT_SQLITE_FILENAME } from "./snapshot-provider.js"; + +export const GIT_BACKUP_MANIFEST = "manifest.json"; +export const GIT_BACKUP_SCHEMA = "schema.sql"; +export const GIT_BACKUP_TABLES = "tables"; + +const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; +const SAFE_TABLE_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +// session_transcript_index_state: Gateway startup transcript reconciliation owns +// rebuilding that FTS projection when the state rows are absent. +// backup_runs: the backup outcome log is written by every backup run, so dumping +// it would make each cycle dirty the next one and defeat no-change detection. +const GIT_BACKUP_PROJECTION_TABLES = ["backup_runs", "session_transcript_index_state"] as const; + +export type GitBackupIdentity = { role: "global" } | { role: "agent"; agentId: string }; + +export type GitBackupManifest = { + schemaVersion: 1; + identity: GitBackupIdentity; + userVersion: number; + excludedTables: string[]; + tables: Record; +}; + +type GitBackupTableResult = { + table: string; + rows: number; + sha256: string; + ok: boolean; +}; + +export type GitBackupRestoreResult = { + manifest: GitBackupManifest; + targetPath: string; + tables: GitBackupTableResult[]; + excludedTables: string[]; +}; + +type SchemaEntry = { + type: "index" | "table" | "trigger"; + name: string; + tableName: string; + sql: string; +}; + +type TableColumn = { name: string; pk: number }; + +function quoteIdentifier(value: string): string { + return `"${value.replaceAll('"', '""')}"`; +} + +function requireSafeTableName(value: string): string { + if (!SAFE_TABLE_NAME.test(value)) { + throw new Error(`Git backup table name is not filesystem-safe: ${value}`); + } + return value; +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function normalizeIdentity(identity: GitBackupIdentity): GitBackupIdentity { + if (identity.role === "global") { + return identity; + } + const agentId = normalizeAgentId(identity.agentId); + if (agentId !== identity.agentId) { + throw new Error(`Git backup agent id must be canonical: ${identity.agentId}`); + } + return { role: "agent", agentId }; +} + +export function gitBackupScopePath(identity: GitBackupIdentity): string { + const normalized = normalizeIdentity(identity); + return normalized.role === "global" ? "global" : path.join("agents", normalized.agentId); +} + +function readSchemaEntries(database: DatabaseSync): SchemaEntry[] { + return database + .prepare( + `SELECT type, name, tbl_name AS tableName, sql + FROM sqlite_master + WHERE type IN ('table', 'index', 'trigger') + AND name NOT LIKE 'sqlite_%' + AND sql IS NOT NULL + ORDER BY CASE type WHEN 'table' THEN 0 WHEN 'index' THEN 1 ELSE 2 END, name`, + ) + .all() + .map((row) => row as SchemaEntry); +} + +function virtualTableNames(entries: SchemaEntry[]): string[] { + return entries + .filter((entry) => /^\s*CREATE\s+VIRTUAL\s+TABLE\b/iu.test(entry.sql)) + .map((entry) => entry.name); +} + +function isVirtualShadow(name: string, virtualTables: readonly string[]): boolean { + return virtualTables.some( + (virtualTable) => name === virtualTable || name.startsWith(`${virtualTable}_`), + ); +} + +function readTableColumns(database: DatabaseSync, table: string): TableColumn[] { + return database + .prepare(`PRAGMA table_info(${quoteIdentifier(table)})`) + .all() + .map((row) => { + const value = row as { name?: unknown; pk?: unknown }; + if (typeof value.name !== "string" || typeof value.pk !== "number") { + throw new Error(`Unable to read columns for Git backup table ${table}.`); + } + return { name: value.name, pk: value.pk }; + }); +} + +function encodeSqliteValue(value: unknown): unknown { + if (value === null || typeof value === "string") { + return value; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) { + throw new Error("Git backup cannot encode a non-finite SQLite REAL value."); + } + return value; + } + if (typeof value === "bigint") { + return value >= Number.MIN_SAFE_INTEGER && value <= Number.MAX_SAFE_INTEGER + ? Number(value) + : { $int: value.toString() }; + } + if (value instanceof Uint8Array) { + return { $hex: Buffer.from(value).toString("hex") }; + } + throw new Error(`Git backup cannot encode SQLite value type ${typeof value}.`); +} + +function serializeTable(database: DatabaseSync, table: string): { content: string; rows: number } { + const columns = readTableColumns(database, table); + if (columns.length === 0) { + throw new Error(`Git backup table has no readable columns: ${table}`); + } + const primaryKey = columns + .filter((column) => column.pk > 0) + .toSorted((left, right) => left.pk - right.pk) + .map((column) => quoteIdentifier(column.name)); + const orderBy = primaryKey.length > 0 ? primaryKey.join(", ") : "rowid"; + const statement = database.prepare( + `SELECT ${columns.map((column) => quoteIdentifier(column.name)).join(", ")} + FROM ${quoteIdentifier(table)} ORDER BY ${orderBy}`, + ); + statement.setReadBigInts(true); + const lines: string[] = []; + for (const rawRow of statement.iterate()) { + const source = rawRow as Record; + const encoded: Record = {}; + for (const column of columns) { + encoded[column.name] = encodeSqliteValue(source[column.name]); + } + lines.push(JSON.stringify(encoded)); + } + return { content: lines.length > 0 ? `${lines.join("\n")}\n` : "", rows: lines.length }; +} + +function schemaText(entries: SchemaEntry[], userVersion: number): string { + const statements = entries.map((entry) => + entry.sql.trimEnd().endsWith(";") ? entry.sql : `${entry.sql};`, + ); + return `${statements.join("\n\n")}\n-- PRAGMA user_version = ${userVersion}\n`; +} + +function redactedSecretTables(identity: GitBackupIdentity, excludeSecrets: boolean): Set { + if (!excludeSecrets) { + return new Set(); + } + return new Set(identity.role === "global" ? STATE_SECRET_TABLE_NAMES : AGENT_SECRET_TABLE_NAMES); +} + +/** Dump one verified SQLite copy into the deterministic Git repository layout. */ +export async function dumpGitBackupDatabase(params: { + snapshotPath: string; + outputPath: string; + identity: GitBackupIdentity; + excludeSecrets?: boolean; +}): Promise { + const identity = normalizeIdentity(params.identity); + const database = openNodeSqliteDatabase(params.snapshotPath, { readOnly: true }); + try { + const entries = readSchemaEntries(database); + const virtualTables = virtualTableNames(entries); + const redacted = redactedSecretTables(identity, params.excludeSecrets === true); + const existingTables = new Set( + entries.filter((entry) => entry.type === "table").map((entry) => entry.name), + ); + // manifest.excludedTables documents redaction only; operational projection + // tables are always omitted and converge on next gateway startup. + const excludedTables = [...redacted].filter((table) => existingTables.has(table)).toSorted(); + const excluded = new Set([...excludedTables, ...GIT_BACKUP_PROJECTION_TABLES]); + const includedSchema = entries.filter( + (entry) => !excluded.has(entry.name) && !excluded.has(entry.tableName), + ); + const dataTables = entries + .filter( + (entry) => + entry.type === "table" && + !isVirtualShadow(entry.name, virtualTables) && + !excluded.has(entry.name), + ) + .map((entry) => requireSafeTableName(entry.name)) + .toSorted(); + const userVersionRow = database.prepare("PRAGMA user_version").get() as { + user_version?: unknown; + }; + if (typeof userVersionRow.user_version !== "number") { + throw new Error("Unable to read SQLite user_version for Git backup."); + } + await fs.rm(params.outputPath, { recursive: true, force: true }); + const tablesPath = path.join(params.outputPath, GIT_BACKUP_TABLES); + await fs.mkdir(tablesPath, { recursive: true, mode: 0o700 }); + const tables: Record = {}; + for (const table of dataTables) { + const serialized = serializeTable(database, table); + await fs.writeFile(path.join(tablesPath, `${table}.jsonl`), serialized.content, { + encoding: "utf8", + mode: 0o600, + }); + tables[table] = { rows: serialized.rows, sha256: sha256(serialized.content) }; + } + const manifest: GitBackupManifest = { + schemaVersion: 1, + identity, + userVersion: userVersionRow.user_version, + excludedTables, + tables, + }; + await fs.writeFile( + path.join(params.outputPath, GIT_BACKUP_SCHEMA), + schemaText(includedSchema, manifest.userVersion), + { encoding: "utf8", mode: 0o600 }, + ); + await fs.writeFile( + path.join(params.outputPath, GIT_BACKUP_MANIFEST), + `${JSON.stringify(manifest, null, 2)}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + return manifest; + } finally { + database.close(); + } +} + +export function parseGitBackupManifest(value: string, source: string): GitBackupManifest { + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch (error) { + throw new Error(`Git backup manifest is invalid JSON: ${source}`, { cause: error }); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`Git backup manifest is invalid: ${source}`); + } + const manifest = parsed as Partial; + if ( + manifest.schemaVersion !== 1 || + !manifest.identity || + (manifest.identity.role !== "global" && manifest.identity.role !== "agent") || + !Number.isSafeInteger(manifest.userVersion) || + !Array.isArray(manifest.excludedTables) || + !manifest.tables || + typeof manifest.tables !== "object" + ) { + throw new Error(`Git backup manifest has unsupported fields: ${source}`); + } + const validated = manifest as GitBackupManifest; + normalizeIdentity(validated.identity); + for (const [table, entry] of Object.entries(validated.tables)) { + requireSafeTableName(table); + if ( + !Number.isSafeInteger(entry.rows) || + entry.rows < 0 || + !/^[a-f0-9]{64}$/u.test(entry.sha256) + ) { + throw new Error(`Git backup manifest has an invalid table entry: ${table}`); + } + } + return validated; +} + +function splitSchemaStatements(schema: string): string[] { + const statements: string[] = []; + let start = 0; + let quote: "'" | '"' | "`" | "]" | undefined; + let lineComment = false; + let blockComment = false; + for (let index = 0; index < schema.length; index += 1) { + const character = schema[index]!; + const next = schema[index + 1]; + if (lineComment) { + if (character === "\n") { + lineComment = false; + } + continue; + } + if (blockComment) { + if (character === "*" && next === "/") { + blockComment = false; + index += 1; + } + continue; + } + if (quote) { + if ((quote === "]" && character === "]") || (quote !== "]" && character === quote)) { + if (quote !== "]" && next === quote) { + index += 1; + } else { + quote = undefined; + } + } + continue; + } + if (character === "-" && next === "-") { + lineComment = true; + index += 1; + continue; + } + if (character === "/" && next === "*") { + blockComment = true; + index += 1; + continue; + } + if (character === "'" || character === '"' || character === "`") { + quote = character; + continue; + } + if (character === "[") { + quote = "]"; + continue; + } + if (character !== ";") { + continue; + } + const candidate = schema.slice(start, index + 1).trim(); + if (/^CREATE\s+TRIGGER\b/iu.test(candidate) && !/\bEND\s*;$/iu.test(candidate)) { + continue; + } + if (candidate && !candidate.startsWith("-- PRAGMA user_version")) { + statements.push(candidate); + } + start = index + 1; + } + return statements; +} + +function unquoteSqlIdentifier(value: string): string { + if (value.startsWith("'")) { + return value.slice(1, -1).replaceAll("''", "'"); + } + if (value.startsWith('"')) { + return value.slice(1, -1).replaceAll('""', '"'); + } + if (value.startsWith("`")) { + return value.slice(1, -1).replaceAll("``", "`"); + } + if (value.startsWith("[")) { + return value.slice(1, -1); + } + return value; +} + +function schemaObjectName(statement: string, kind: "table" | "virtual"): string | undefined { + const prefix = kind === "virtual" ? "CREATE\\s+VIRTUAL\\s+TABLE" : "CREATE\\s+TABLE"; + const match = new RegExp( + `^${prefix}\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?('(?:[^']|'')*'|"(?:[^"]|"")*"|\\[[^\\]]+\\]|\`(?:[^\`]|\`\`)*\`|[^\\s(]+)`, + "iu", + ).exec(statement); + return match?.[1] ? unquoteSqlIdentifier(match[1]) : undefined; +} + +function decodeSqliteValue(value: unknown): null | string | number | bigint | Buffer { + if (value === null || typeof value === "string" || typeof value === "number") { + return value; + } + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("Git backup row contains an invalid encoded value."); + } + const record = value as Record; + if (Object.keys(record).length === 1 && typeof record.$int === "string") { + return BigInt(record.$int); + } + if ( + Object.keys(record).length === 1 && + typeof record.$hex === "string" && + /^(?:[a-f0-9]{2})*$/u.test(record.$hex) + ) { + return Buffer.from(record.$hex, "hex"); + } + throw new Error("Git backup row contains an invalid encoded object."); +} + +async function assertFreshRestoreTarget(targetPath: string): Promise { + for (const candidate of [ + targetPath, + ...SQLITE_SIDECAR_SUFFIXES.map((suffix) => `${targetPath}${suffix}`), + ]) { + try { + await fs.lstat(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw error; + } + throw new Error(`Fresh SQLite restore path already exists: ${candidate}`); + } +} + +function assertNoSqliteSidecarsSync(targetPath: string): void { + for (const suffix of SQLITE_SIDECAR_SUFFIXES) { + const sidecarPath = `${targetPath}${suffix}`; + try { + fsSync.lstatSync(sidecarPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw error; + } + throw new Error(`Fresh SQLite restore path already exists: ${sidecarPath}`); + } +} + +function convergeRestoredSchema(database: DatabaseSync, identity: GitBackupIdentity): void { + database.exec( + identity.role === "global" + ? getOpenClawStateRuntimeSchema({ includeVersionLazyAdditiveTables: false }) + : OPENCLAW_AGENT_SCHEMA_SQL, + ); +} + +function validateRestoredOwner( + database: DatabaseSync, + databasePath: string, + identity: GitBackupIdentity, +): void { + assertSqliteIntegrity(database, databasePath); + const foreignKeys = database.prepare("PRAGMA foreign_key_check").all(); + if (foreignKeys.length > 0) { + throw new Error(`SQLite foreign_key_check failed for restored Git backup: ${databasePath}`); + } + buildSnapshotValidator(identity)(database, databasePath); +} + +function loadTable(database: DatabaseSync, table: string, content: string): number { + const columns = readTableColumns(database, table); + const statement = database.prepare( + `INSERT INTO ${quoteIdentifier(table)} (${columns.map((column) => quoteIdentifier(column.name)).join(", ")}) + VALUES (${columns.map(() => "?").join(", ")})`, + ); + let rows = 0; + for (const line of content.split("\n")) { + if (!line) { + continue; + } + const parsed = JSON.parse(line) as Record; + statement.run(...columns.map((column) => decodeSqliteValue(parsed[column.name]))); + rows += 1; + } + return rows; +} + +/** Restore one materialized Git snapshot scope into a fresh SQLite file. */ +export async function restoreGitBackupDirectory(params: { + sourcePath: string; + targetPath: string; + expectedIdentity?: GitBackupIdentity; +}): Promise { + const targetPath = path.resolve(params.targetPath); + await assertFreshRestoreTarget(targetPath); + const manifest = parseGitBackupManifest( + await fs.readFile(path.join(params.sourcePath, GIT_BACKUP_MANIFEST), "utf8"), + params.sourcePath, + ); + const restoreIdentity = normalizeIdentity(params.expectedIdentity ?? manifest.identity); + if ( + params.expectedIdentity && + JSON.stringify(normalizeIdentity(manifest.identity)) !== JSON.stringify(restoreIdentity) + ) { + throw new Error("Git backup manifest database identity does not match the requested scope."); + } + const schema = await fs.readFile(path.join(params.sourcePath, GIT_BACKUP_SCHEMA), "utf8"); + const statements = splitSchemaStatements(schema); + const virtual = statements.filter((statement) => /^CREATE\s+VIRTUAL\s+TABLE\b/iu.test(statement)); + const triggers = statements.filter((statement) => /^CREATE\s+TRIGGER\b/iu.test(statement)); + const virtualNames = virtual + .map((statement) => schemaObjectName(statement, "virtual")) + .filter((value): value is string => Boolean(value)); + const plainTables = statements.filter((statement) => { + if (!/^CREATE\s+TABLE\b/iu.test(statement)) { + return false; + } + const name = schemaObjectName(statement, "table"); + return !name || !isVirtualShadow(name, virtualNames); + }); + const indexes = statements.filter((statement) => + /^CREATE\s+(?:UNIQUE\s+)?INDEX\b/iu.test(statement), + ); + const targetDirectory = path.dirname(targetPath); + await fs.mkdir(targetDirectory, { recursive: true, mode: 0o700 }); + const stagingDirectory = await createPrivateSqliteTempDirectory( + targetDirectory, + ".git-backup-restore-", + ); + applyPrivateModeSync(stagingDirectory, 0o700); + const stagedPath = path.join(stagingDirectory, SNAPSHOT_SQLITE_FILENAME); + const stagedHandle = await fs.open(stagedPath, "wx", 0o600); + await stagedHandle.close(); + const database = openNodeSqliteDatabase(stagedPath); + try { + database.exec("PRAGMA foreign_keys = OFF; PRAGMA journal_mode = DELETE;"); + for (const statement of [...plainTables, ...indexes]) { + database.exec(statement); + } + database.exec("BEGIN IMMEDIATE;"); + try { + for (const [table, expected] of Object.entries(manifest.tables)) { + requireSafeTableName(table); + const content = await fs.readFile( + path.join(params.sourcePath, GIT_BACKUP_TABLES, `${table}.jsonl`), + "utf8", + ); + if (sha256(content) !== expected.sha256) { + throw new Error(`Git backup table hash mismatch: ${table}`); + } + const rows = loadTable(database, table, content); + if (rows !== expected.rows) { + throw new Error(`Git backup table row count mismatch: ${table}`); + } + } + database.exec("COMMIT;"); + } catch (error) { + database.exec("ROLLBACK;"); + throw error; + } + for (const statement of virtual) { + if (/\bUSING\s+vec0\b/iu.test(statement)) { + continue; + } + database.exec(statement); + } + for (const statement of triggers) { + database.exec(statement); + } + for (const statement of virtual) { + const name = schemaObjectName(statement, "virtual"); + if (name && /\bUSING\s+fts5\b/iu.test(statement) && /\bcontent\s*=/iu.test(statement)) { + database + .prepare( + `INSERT INTO ${quoteIdentifier(name)} (${quoteIdentifier(name)}) VALUES ('rebuild')`, + ) + .run(); + } + } + // Contentless transcript FTS stays empty. Omission of session_transcript_index_state + // makes Gateway startup reconciliation rebuild that projection from transcripts. + database.exec(`PRAGMA user_version = ${manifest.userVersion};`); + // Redacted and operational projection tables are absent from Git. Recreate + // their canonical empty schemas before enforcing database ownership. + convergeRestoredSchema(database, restoreIdentity); + validateRestoredOwner(database, stagedPath, restoreIdentity); + const tables = Object.entries(manifest.tables).map(([table, expected]) => { + const actual = serializeTable(database, table); + const actualSha256 = sha256(actual.content); + return { + table, + rows: actual.rows, + sha256: actualSha256, + ok: actual.rows === expected.rows && actualSha256 === expected.sha256, + }; + }); + if (tables.some((table) => !table.ok)) { + throw new Error(`Restored Git backup does not match its table manifest: ${stagedPath}`); + } + database.close(); + applyPrivateModeSync(stagedPath, 0o600); + const artifact = await hashSnapshotArtifact(stagingDirectory); + await publishVerifiedSqliteFile({ + sourceIdentity: artifact.stat, + sourcePath: stagedPath, + targetPath, + expectedContent: artifact, + requireAtomicPublication: true, + beforePublish: async () => await assertFreshRestoreTarget(targetPath), + validatePublished: async (publishedPath) => { + const published = openNodeSqliteDatabase(publishedPath, { readOnly: true }); + try { + validateRestoredOwner(published, publishedPath, restoreIdentity); + } finally { + published.close(); + } + }, + afterPublish: (guard) => { + guard.assertTargetMatchesExpectedContent(() => assertNoSqliteSidecarsSync(targetPath)); + }, + }); + return { manifest, targetPath, tables, excludedTables: manifest.excludedTables }; + } catch (error) { + if (database.isOpen) { + database.close(); + } + throw error; + } finally { + await fs.rm(stagingDirectory, { recursive: true, force: true }).catch(() => undefined); + } +} diff --git a/src/snapshot/git-backup.test.ts b/src/snapshot/git-backup.test.ts new file mode 100644 index 000000000000..c646442d9abd --- /dev/null +++ b/src/snapshot/git-backup.test.ts @@ -0,0 +1,700 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; +import { backupGitCreateCommand } from "../commands/backup-git.js"; +import { readBackupFreshness } from "../commands/backup-health.js"; +import { createTestRuntime } from "../commands/test-runtime-config-helpers.js"; +import { executeGitCommand, requireGitCommand as requireGit } from "../infra/git-exec.js"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { createPathResolutionEnv, withEnvAsync } from "../test-utils/env.js"; +import { dumpGitBackupDatabase, restoreGitBackupDirectory } from "./git-backup-codec.js"; +import { createGitBackup, initializeGitBackupRepository } from "./git-backup.js"; + +const mocks = vi.hoisted(() => ({ pushDiagnostic: undefined as string | undefined })); + +vi.mock("../infra/git-exec.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + executeGitCommand: async ( + ...args: Parameters + ): ReturnType => { + if (args[1][0] === "push" && mocks.pushDiagnostic) { + return { code: 1, stdout: "", stderr: mocks.pushDiagnostic }; + } + return await actual.executeGitCommand(...args); + }, + }; +}); + +const roots: string[] = []; + +async function tempRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-git-backup-test-")); + roots.push(root); + return root; +} + +afterEach(async () => { + mocks.pushDiagnostic = undefined; + closeOpenClawStateDatabaseForTest(); + await Promise.all( + roots.splice(0).map(async (root) => await fs.rm(root, { recursive: true, force: true })), + ); +}); + +async function createFormatFixture(databasePath: string): Promise { + const database = new DatabaseSync(databasePath, { allowExtension: true }); + try { + await loadSqliteVecExtension({ db: database }); + database.exec(` + PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION}; + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE device_auth_tokens ( + device_id TEXT NOT NULL, + role TEXT NOT NULL, + token TEXT NOT NULL, + scopes_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (device_id, role) + ) STRICT; + CREATE TABLE channel_pairing_requests ( + channel_key TEXT NOT NULL, + account_id TEXT NOT NULL, + request_id TEXT NOT NULL, + code TEXT NOT NULL, + created_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + meta_json TEXT, + PRIMARY KEY (channel_key, account_id, request_id) + ) STRICT; + CREATE TABLE device_pairing_join_codes ( + shortcode TEXT, + payload_json TEXT, + created_at_ms INTEGER, + expires_at_ms INTEGER + ) STRICT; + CREATE TABLE content ( + id INTEGER PRIMARY KEY, + body TEXT NOT NULL, + huge INTEGER NOT NULL, + bytes BLOB NOT NULL, + optional TEXT + ); + CREATE VIRTUAL TABLE content_fts USING fts5(body, content='content', content_rowid='id'); + CREATE TRIGGER content_ai AFTER INSERT ON content BEGIN + INSERT INTO content_fts(rowid, body) VALUES (new.id, new.body); + END; + CREATE VIRTUAL TABLE memory_vec USING vec0(embedding float[2]); + CREATE TABLE empty_table (id INTEGER PRIMARY KEY, value TEXT); + CREATE TABLE session_transcript_index_state (id TEXT PRIMARY KEY, cursor INTEGER); + `); + database + .prepare( + `INSERT INTO schema_meta + (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) + VALUES ('primary', 'global', ?, NULL, NULL, 1, 1)`, + ) + .run(OPENCLAW_STATE_SCHEMA_VERSION); + database + .prepare("INSERT INTO content (id, body, huge, bytes, optional) VALUES (?, ?, ?, ?, ?)") + .run(1, "hello lobster", 9_007_199_254_740_993n, Buffer.from([0, 1, 254, 255]), ""); + database + .prepare("INSERT INTO content (id, body, huge, bytes, optional) VALUES (?, ?, ?, ?, ?)") + .run(2, "second row", -9_007_199_254_740_994n, Buffer.from([42]), null); + database.prepare("INSERT INTO session_transcript_index_state VALUES (?, ?)").run("main", 99); + database + .prepare( + `INSERT INTO device_auth_tokens + (device_id, role, token, scopes_json, updated_at_ms) + VALUES (?, ?, ?, ?, ?)`, + ) + .run("device", "operator", "secret-token", "[]", 1); + database + .prepare( + `INSERT INTO channel_pairing_requests + (channel_key, account_id, request_id, code, created_at, last_seen_at, meta_json) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run("telegram", "default", "request", "pairing-code", "now", "now", null); + database + .prepare( + `INSERT INTO device_pairing_join_codes + (shortcode, payload_json, created_at_ms, expires_at_ms) + VALUES (?, ?, ?, ?)`, + ) + .run( + "join-code", + JSON.stringify({ url: "wss://gateway.example", bootstrapToken: "bootstrap-secret" }), + 1, + 2, + ); + } finally { + database.close(); + } +} + +function createAgentFixture(databasePath: string, agentId: string): void { + const database = new DatabaseSync(databasePath); + try { + database.exec(` + PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION}; + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + `); + database + .prepare( + `INSERT INTO schema_meta + (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) + VALUES ('primary', 'agent', ?, ?, NULL, 1, 1)`, + ) + .run(OPENCLAW_AGENT_SCHEMA_VERSION, agentId); + } finally { + database.close(); + } +} + +async function writeBackupManifest(scopePath: string, agentId: string): Promise { + await fs.mkdir(scopePath, { recursive: true }); + await fs.writeFile( + path.join(scopePath, "manifest.json"), + `${JSON.stringify({ + schemaVersion: 1, + identity: { role: "agent", agentId }, + userVersion: 1, + excludedTables: [], + tables: {}, + })}\n`, + ); +} + +async function listTree(root: string): Promise> { + const result: Array<[string, string]> = []; + async function visit(directory: string): Promise { + for (const entry of (await fs.readdir(directory, { withFileTypes: true })).toSorted((a, b) => + a.name.localeCompare(b.name), + )) { + const entryPath = path.join(directory, entry.name); + const relative = path.relative(root, entryPath); + if (entry.isDirectory()) { + await visit(entryPath); + } else { + result.push([relative, (await fs.readFile(entryPath)).toString("hex")]); + } + } + } + await visit(root); + return result; +} + +function createStateDatabaseFixture(root: string): { + stateDir: string; + database: { path: string; identity: { role: "global" } }; +} { + const stateDir = path.join(root, "state"); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + openOpenClawStateDatabase({ env }); + closeOpenClawStateDatabaseForTest(); + return { + stateDir, + database: { + path: resolveOpenClawStateSqlitePath(env), + identity: { role: "global" }, + }, + }; +} + +describe("Git-backed SQLite snapshots", () => { + it("rejects state and repository overlap in either canonical direction", async () => { + const root = await fs.realpath(await tempRoot()); + const stateDir = path.join(root, "state"); + await fs.mkdir(stateDir, { recursive: true }); + const stateAlias = path.join(root, "state-alias"); + await fs.symlink(stateDir, stateAlias, process.platform === "win32" ? "junction" : "dir"); + + for (const repositoryPath of [ + path.join(stateDir, "backup"), + root, + path.join(stateAlias, "backup"), + ]) { + await expect(initializeGitBackupRepository({ repositoryPath, stateDir })).rejects.toThrow( + `Git backup repository must be outside the OpenClaw state directory: ${stateDir}`, + ); + } + }); + + it("dumps byte-identical trees and skips a second unchanged create commit", async () => { + const root = await tempRoot(); + const source = path.join(root, "source.sqlite"); + const first = path.join(root, "first"); + const second = path.join(root, "second"); + await createFormatFixture(source); + + await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: first, + identity: { role: "global" }, + }); + await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: second, + identity: { role: "global" }, + }); + expect(await listTree(second)).toEqual(await listTree(first)); + + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "repository"); + await initializeGitBackupRepository({ repositoryPath, stateDir }); + await requireGit(repositoryPath, ["config", "user.name", "OpenClaw Backup Test"]); + await requireGit(repositoryPath, ["config", "user.email", "backup@example.invalid"]); + const created = await createGitBackup({ repositoryPath, stateDir, databases: [database] }); + const unchanged = await createGitBackup({ repositoryPath, stateDir, databases: [database] }); + expect(created.noChanges).toBe(false); + expect(unchanged.noChanges).toBe(true); + expect(unchanged).not.toHaveProperty("commit"); + expect(await requireGit(repositoryPath, ["rev-list", "--count", "HEAD"])).toBe("1"); + }); + + it("stages only backup-owned paths in an adopted repository", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "repository"); + await initializeGitBackupRepository({ repositoryPath, stateDir }); + await requireGit(repositoryPath, ["config", "user.name", "OpenClaw Backup Test"]); + await requireGit(repositoryPath, ["config", "user.email", "backup@example.invalid"]); + await fs.writeFile(path.join(repositoryPath, "unrelated.txt"), "operator-owned\n"); + await requireGit(repositoryPath, ["add", "unrelated.txt"]); + + const created = await createGitBackup({ repositoryPath, stateDir, databases: [database] }); + const unchanged = await createGitBackup({ repositoryPath, stateDir, databases: [database] }); + + expect(created.noChanges).toBe(false); + expect(unchanged.noChanges).toBe(true); + expect(await requireGit(repositoryPath, ["status", "--porcelain", "--", "unrelated.txt"])).toBe( + "A unrelated.txt", + ); + const committedPaths = ( + await requireGit(repositoryPath, ["show", "--pretty=format:", "--name-only", "HEAD"]) + ) + .split("\n") + .filter(Boolean); + expect(committedPaths.length).toBeGreaterThan(0); + expect( + committedPaths.every( + (entry) => + entry === "global" || + entry.startsWith("global/") || + entry === "agents" || + entry.startsWith("agents/"), + ), + ).toBe(true); + expect(committedPaths).not.toContain("unrelated.txt"); + expect( + await requireGit(repositoryPath, ["ls-tree", "-r", "--name-only", "HEAD"]), + ).not.toContain("unrelated.txt"); + expect(await requireGit(repositoryPath, ["rev-list", "--count", "HEAD"])).toBe("1"); + }); + + it("preserves an unowned global namespace in an adopted repository", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "repository"); + const operatorFile = path.join(repositoryPath, "global", "operator.txt"); + await initializeGitBackupRepository({ repositoryPath, stateDir }); + await fs.mkdir(path.dirname(operatorFile), { recursive: true }); + await fs.writeFile(operatorFile, "operator-owned\n"); + + await expect( + createGitBackup({ repositoryPath, stateDir, databases: [database] }), + ).rejects.toThrow(/repository must be dedicated to OpenClaw backups/u); + await expect(fs.readFile(operatorFile, "utf8")).resolves.toBe("operator-owned\n"); + }); + + it("removes stale backup-owned agent scopes for an all-database backup", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "repository"); + const staleAgentPath = path.join(repositoryPath, "agents", "old-agent"); + await initializeGitBackupRepository({ repositoryPath, stateDir }); + await writeBackupManifest(staleAgentPath, "old-agent"); + + await createGitBackup({ repositoryPath, stateDir, databases: [database], all: true }); + + await expect(fs.lstat(staleAgentPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("aborts all-database cleanup before deleting an unowned agent scope", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "repository"); + const ownedAgentPath = path.join(repositoryPath, "agents", "owned-agent"); + const unownedFile = path.join(repositoryPath, "agents", "operator", "operator.txt"); + await initializeGitBackupRepository({ repositoryPath, stateDir }); + await writeBackupManifest(ownedAgentPath, "owned-agent"); + await fs.mkdir(path.dirname(unownedFile), { recursive: true }); + await fs.writeFile(unownedFile, "operator-owned\n"); + + await expect( + createGitBackup({ repositoryPath, stateDir, databases: [database], all: true }), + ).rejects.toThrow(/repository must be dedicated to OpenClaw backups/u); + await expect(fs.readFile(unownedFile, "utf8")).resolves.toBe("operator-owned\n"); + await expect( + fs.readFile(path.join(ownedAgentPath, "manifest.json"), "utf8"), + ).resolves.toContain('"schemaVersion":1'); + }); + + it.skipIf(process.platform === "win32")( + "rejects group-writable adopted roots with a chmod hint", + async () => { + const root = await tempRoot(); + const stateDir = path.join(root, "state"); + const repositoryPath = path.join(root, "repository"); + await fs.mkdir(stateDir); + await fs.mkdir(repositoryPath, { mode: 0o700 }); + await fs.chmod(repositoryPath, 0o770); + + await expect(initializeGitBackupRepository({ repositoryPath, stateDir })).rejects.toThrow( + /chmod 700/u, + ); + }, + ); + + it("accepts a private adopted root", async () => { + const root = await tempRoot(); + const stateDir = path.join(root, "state"); + const repositoryPath = path.join(root, "repository"); + await fs.mkdir(stateDir); + await fs.mkdir(repositoryPath, { mode: 0o700 }); + + await expect(initializeGitBackupRepository({ repositoryPath, stateDir })).resolves.toEqual({ + repositoryPath, + }); + }); + + it("uses a commit-scoped fallback identity when Git has no configured email", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "identity-free-repository"); + const isolatedHome = path.join(root, "git-home"); + await fs.mkdir(isolatedHome, { recursive: true }); + const gitEnv = createPathResolutionEnv(isolatedHome, { + GIT_CONFIG_GLOBAL: os.devNull, + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + }); + + const result = await createGitBackup({ + repositoryPath, + stateDir, + databases: [database], + gitEnv, + }); + + expect(result.commit).toMatch(/^[a-f0-9]{40}$/u); + expect( + await requireGit(repositoryPath, ["log", "-1", "--format=%an <%ae>"], { env: gitEnv }), + ).toBe("OpenClaw "); + expect( + await requireGit(repositoryPath, ["config", "--local", "--get", "user.email"], { + env: gitEnv, + }).catch(() => undefined), + ).toBeUndefined(); + }); + + it("redacts and bounds credential-bearing push diagnostics", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "push-repository"); + const username = ["synthetic", "user"].join("-"); + const password = ["synthetic", "password"].join("-"); + const remote = `https://${username}:${password}@example.invalid/repository`; + mocks.pushDiagnostic = `fatal: unable to access '${remote}': ${"x".repeat(600)}`; + await initializeGitBackupRepository({ repositoryPath, stateDir, remote }); + await requireGit(repositoryPath, ["config", "user.name", "OpenClaw Backup Test"]); + await requireGit(repositoryPath, ["config", "user.email", "backup@example.invalid"]); + + const result = await createGitBackup({ + repositoryPath, + stateDir, + databases: [database], + push: true, + }); + + expect(result.pushWarning).toContain("https://***@example.invalid/repository"); + expect(result.pushWarning).not.toContain(username); + expect(result.pushWarning).not.toContain(password); + expect(result.pushWarning?.length).toBeLessThanOrEqual(500); + }); + + it("refuses adopted non-backup ancestry and records local push degradation", async () => { + const root = await tempRoot(); + const { stateDir } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "adopted-repository"); + const remotePath = path.join(root, "remote.git"); + await requireGit(root, ["init", "--bare", remotePath]); + await initializeGitBackupRepository({ repositoryPath, stateDir, remote: remotePath }); + await requireGit(repositoryPath, ["config", "user.name", "OpenClaw Backup Test"]); + await requireGit(repositoryPath, ["config", "user.email", "backup@example.invalid"]); + await fs.writeFile(path.join(repositoryPath, "unrelated.txt"), "operator-owned\n"); + await requireGit(repositoryPath, ["add", "unrelated.txt"]); + await requireGit(repositoryPath, ["commit", "-m", "operator history"]); + + const warning = + "repository history contains non-backup commits; use a dedicated backup repository"; + await withEnvAsync({ OPENCLAW_STATE_DIR: stateDir }, async () => { + const result = await backupGitCreateCommand(createTestRuntime(), { + repository: repositoryPath, + global: true, + push: true, + excludeSecrets: true, + }); + + expect(result).toMatchObject({ noChanges: false, pushed: false, pushWarning: warning }); + expect(result.commit).toMatch(/^[a-f0-9]{40}$/u); + expect(readBackupFreshness(process.env)).toMatchObject({ + latest: { status: "ok", kind: "git", pushFailed: true, error: warning }, + latestOk: { status: "ok", kind: "git", pushFailed: true, error: warning }, + }); + }); + expect((await executeGitCommand(remotePath, ["show-ref"])).code).not.toBe(0); + }); + + it("pushes backup-only ancestry to a new remote", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const repositoryPath = path.join(root, "backup-repository"); + const remotePath = path.join(root, "remote.git"); + await requireGit(root, ["init", "--bare", remotePath]); + await initializeGitBackupRepository({ repositoryPath, stateDir, remote: remotePath }); + await requireGit(repositoryPath, ["config", "user.name", "OpenClaw Backup Test"]); + await requireGit(repositoryPath, ["config", "user.email", "backup@example.invalid"]); + + const result = await createGitBackup({ + repositoryPath, + stateDir, + databases: [database], + push: true, + }); + + const branch = await requireGit(repositoryPath, ["branch", "--show-current"]); + expect(result).toMatchObject({ noChanges: false, pushed: true }); + expect(result).not.toHaveProperty("pushWarning"); + expect(await requireGit(remotePath, ["rev-parse", `refs/heads/${branch}`])).toBe(result.commit); + }); + + it("redacts credential-bearing origins in conflict errors", async () => { + const root = await tempRoot(); + const stateDir = path.join(root, "state"); + const repositoryPath = path.join(root, "repository"); + const username = ["synthetic", "origin-user"].join("-"); + const password = ["synthetic", "origin-password"].join("-"); + await fs.mkdir(stateDir); + await initializeGitBackupRepository({ + repositoryPath, + stateDir, + remote: `https://${username}:${password}@example.invalid/first`, + }); + + const conflict = initializeGitBackupRepository({ + repositoryPath, + stateDir, + remote: "https://example.invalid/second", + }); + await expect(conflict).rejects.toThrow( + "Git backup repository already has a different origin: https://***@example.invalid/first", + ); + await expect(conflict).rejects.not.toThrow(username); + await expect(conflict).rejects.not.toThrow(password); + }); + + it("round-trips losslessly, converges FTS, and omits derived vec and transcript state", async () => { + const root = await tempRoot(); + const source = path.join(root, "source.sqlite"); + const dump = path.join(root, "dump"); + const restoredPath = path.join(root, "restored.sqlite"); + await createFormatFixture(source); + const manifest = await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: dump, + identity: { role: "global" }, + }); + const restored = await restoreGitBackupDirectory({ + sourcePath: dump, + targetPath: restoredPath, + expectedIdentity: { role: "global" }, + }); + expect(restored.tables.every((table) => table.ok)).toBe(true); + expect(restored.manifest.tables).toEqual(manifest.tables); + expect(manifest.tables).not.toHaveProperty("session_transcript_index_state"); + if (process.platform !== "win32") { + expect((await fs.stat(restoredPath)).mode & 0o777).toBe(0o600); + } + + const database = new DatabaseSync(restoredPath, { readOnly: true }); + try { + const statement = database.prepare( + "SELECT id, huge, bytes, optional FROM content ORDER BY id", + ); + statement.setReadBigInts(true); + const rows = statement.all() as Array<{ + id: bigint; + huge: bigint; + bytes: Uint8Array; + optional: string | null; + }>; + expect( + rows.map((row) => ({ + id: row.id, + huge: row.huge, + bytes: [...row.bytes], + optional: row.optional, + })), + ).toEqual([ + { + id: 1n, + huge: 9_007_199_254_740_993n, + bytes: [0, 1, 254, 255], + optional: "", + }, + { id: 2n, huge: -9_007_199_254_740_994n, bytes: [42], optional: null }, + ]); + expect( + database.prepare("SELECT rowid FROM content_fts WHERE content_fts MATCH 'lobster'").all(), + ).toEqual([{ rowid: 1 }]); + const tables = database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table'") + .all() as Array<{ name: string }>; + expect(tables.some((table) => table.name === "memory_vec")).toBe(false); + expect(tables.some((table) => table.name === "session_transcript_index_state")).toBe(false); + } finally { + database.close(); + } + }); + + it("omits secret tables and reports the restore gap", async () => { + const root = await tempRoot(); + const source = path.join(root, "source.sqlite"); + const dump = path.join(root, "dump"); + await createFormatFixture(source); + const manifest = await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: dump, + identity: { role: "global" }, + excludeSecrets: true, + }); + expect(manifest.excludedTables).toContain("device_auth_tokens"); + expect(manifest.excludedTables).toContain("channel_pairing_requests"); + expect(manifest.excludedTables).toContain("device_pairing_join_codes"); + expect(manifest.tables).not.toHaveProperty("device_auth_tokens"); + expect(manifest.tables).not.toHaveProperty("channel_pairing_requests"); + expect(manifest.tables).not.toHaveProperty("device_pairing_join_codes"); + await expect( + fs.lstat(path.join(dump, "tables", "channel_pairing_requests.jsonl")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + fs.lstat(path.join(dump, "tables", "device_pairing_join_codes.jsonl")), + ).rejects.toMatchObject({ code: "ENOENT" }); + const schema = await fs.readFile(path.join(dump, "schema.sql"), "utf8"); + expect(schema).not.toContain("device_auth_tokens"); + expect(schema).not.toContain("channel_pairing_requests"); + expect(schema).not.toContain("device_pairing_join_codes"); + const restored = await restoreGitBackupDirectory({ + sourcePath: dump, + targetPath: path.join(root, "redacted.sqlite"), + }); + expect(restored.excludedTables).toContain("device_auth_tokens"); + const restoredDatabase = new DatabaseSync(restored.targetPath, { readOnly: true }); + try { + expect( + restoredDatabase.prepare("SELECT COUNT(*) AS count FROM device_auth_tokens").get(), + ).toEqual({ count: 0 }); + expect( + restoredDatabase.prepare("SELECT COUNT(*) AS count FROM channel_pairing_requests").get(), + ).toEqual({ count: 0 }); + } finally { + restoredDatabase.close(); + } + }); + + it("rejects a restored global database without canonical ownership metadata", async () => { + const root = await tempRoot(); + const source = path.join(root, "source.sqlite"); + const dump = path.join(root, "dump"); + const restoredPath = path.join(root, "restored.sqlite"); + await createFormatFixture(source); + const database = new DatabaseSync(source); + try { + database.exec("DROP TABLE schema_meta;"); + } finally { + database.close(); + } + await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: dump, + identity: { role: "global" }, + }); + + await expect( + restoreGitBackupDirectory({ + sourcePath: dump, + targetPath: restoredPath, + expectedIdentity: { role: "global" }, + }), + ).rejects.toThrow(/schema role missing; expected global/u); + await expect(fs.lstat(restoredPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("converges and validates the requested agent database owner", async () => { + const root = await tempRoot(); + const source = path.join(root, "agent.sqlite"); + const dump = path.join(root, "dump"); + const restoredPath = path.join(root, "restored.sqlite"); + createAgentFixture(source, "main"); + await dumpGitBackupDatabase({ + snapshotPath: source, + outputPath: dump, + identity: { role: "agent", agentId: "main" }, + }); + + await restoreGitBackupDirectory({ + sourcePath: dump, + targetPath: restoredPath, + expectedIdentity: { role: "agent", agentId: "main" }, + }); + const restored = new DatabaseSync(restoredPath, { readOnly: true }); + try { + expect( + restored.prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'").get(), + ).toEqual({ role: "agent", agent_id: "main" }); + expect( + restored.prepare("SELECT COUNT(*) AS count FROM session_transcript_index_state").get(), + ).toEqual({ count: 0 }); + } finally { + restored.close(); + } + }); +}); diff --git a/src/snapshot/git-backup.ts b/src/snapshot/git-backup.ts new file mode 100644 index 000000000000..4ede72171fe6 --- /dev/null +++ b/src/snapshot/git-backup.ts @@ -0,0 +1,444 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { canonicalPathFromExistingAncestor, isPathInside } from "../infra/fs-safe.js"; +import { + executeGitCommand as runGit, + requireGitCommand as requireGit, + requireGitCommandBuffer as requireGitBuffer, +} from "../infra/git-exec.js"; +import { + GIT_BACKUP_MANIFEST, + GIT_BACKUP_SCHEMA, + GIT_BACKUP_TABLES, + dumpGitBackupDatabase, + gitBackupScopePath, + parseGitBackupManifest, + restoreGitBackupDirectory, + type GitBackupIdentity, + type GitBackupManifest, + type GitBackupRestoreResult, +} from "./git-backup-codec.js"; +import { ensurePrivateSnapshotRepositoryRoot } from "./local-repository.js"; +import { createOpenClawSnapshotCopy } from "./openclaw-snapshot-copy.js"; +import type { SnapshotDatabaseRef } from "./snapshot-provider.js"; + +const GIT_BACKUP_MATERIALIZE_MAX_BYTES = 1024 * 1024 * 1024; +const GIT_BACKUP_DIAGNOSTIC_MAX_LENGTH = 500; +const GIT_BACKUP_NON_BACKUP_HISTORY_WARNING = + "repository history contains non-backup commits; use a dedicated backup repository"; + +type GitBackupCreateResult = { + repositoryPath: string; + commit?: string; + noChanges: boolean; + pushed: boolean; + pushWarning?: string; + manifests: GitBackupManifest[]; +}; + +function sanitizeGitBackupDiagnostic(value: string): string { + return value.replace(/:\/\/[^@\s]+@/gu, "://***@").slice(0, GIT_BACKUP_DIAGNOSTIC_MAX_LENGTH); +} + +async function assertGitRepository(repositoryPath: string, env?: NodeJS.ProcessEnv): Promise { + const topLevel = await requireGit(repositoryPath, ["rev-parse", "--show-toplevel"], { env }); + const [canonicalTopLevel, canonicalRepository] = await Promise.all([ + fs.realpath(topLevel), + fs.realpath(repositoryPath), + ]); + if (canonicalTopLevel !== canonicalRepository) { + throw new Error(`Backup repository must be the Git worktree root: ${repositoryPath}`); + } +} + +/** Initialize or adopt an operator-owned Git backup repository. */ +export async function initializeGitBackupRepository(params: { + repositoryPath: string; + stateDir: string; + remote?: string; + gitEnv?: NodeJS.ProcessEnv; +}): Promise<{ repositoryPath: string }> { + const repositoryPath = path.resolve(params.repositoryPath); + const stateDir = path.resolve(params.stateDir); + const [canonicalRepositoryPath, canonicalStateDir] = await Promise.all([ + canonicalPathFromExistingAncestor(repositoryPath), + canonicalPathFromExistingAncestor(stateDir), + ]); + if ( + isPathInside(canonicalStateDir, canonicalRepositoryPath) || + isPathInside(canonicalRepositoryPath, canonicalStateDir) + ) { + throw new Error( + `Git backup repository must be outside the OpenClaw state directory: ${stateDir}`, + ); + } + try { + await ensurePrivateSnapshotRepositoryRoot(repositoryPath); + } catch (error) { + throw new Error( + `Git backup repository must be owned by the current user and not writable by other users: ${repositoryPath}. Fix its ownership and run chmod 700 ${repositoryPath}.`, + { cause: error }, + ); + } + const probe = await runGit(repositoryPath, ["rev-parse", "--show-toplevel"], { + env: params.gitEnv, + }); + if (probe.code !== 0) { + await requireGit(repositoryPath, ["init"], { env: params.gitEnv }); + } + await assertGitRepository(repositoryPath, params.gitEnv); + const remote = params.remote?.trim(); + if (remote) { + const existing = await runGit(repositoryPath, ["remote", "get-url", "origin"], { + env: params.gitEnv, + }); + if (existing.code === 0 && existing.stdout.trim() !== remote) { + throw new Error( + `Git backup repository already has a different origin: ${sanitizeGitBackupDiagnostic(existing.stdout.trim())}`, + ); + } + if (existing.code !== 0) { + await requireGit(repositoryPath, ["remote", "add", "origin", remote], { + env: params.gitEnv, + }); + } + } + return { repositoryPath }; +} + +async function isBackupOwnedScope(scopePath: string): Promise { + const identity = await fs + .lstat(scopePath) + .catch((error: unknown) => + (error as NodeJS.ErrnoException).code === "ENOENT" ? undefined : null, + ); + if (identity === undefined) { + return true; + } + if (!identity?.isDirectory()) { + return false; + } + try { + const entries = await fs.readdir(scopePath); + if (entries.length === 0) { + return true; + } + parseGitBackupManifest( + await fs.readFile(path.join(scopePath, GIT_BACKUP_MANIFEST), "utf8"), + scopePath, + ); + return true; + } catch { + return false; + } +} + +async function assertBackupOwnedScope(scopePath: string): Promise { + if (!(await isBackupOwnedScope(scopePath))) { + throw new Error( + `Refusing to replace non-backup-owned path ${scopePath}; the repository must be dedicated to OpenClaw backups.`, + ); + } +} + +async function removeStaleAgentScopes(repositoryPath: string): Promise { + const agentsPath = path.join(repositoryPath, "agents"); + let entries: string[]; + try { + entries = await fs.readdir(agentsPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return; + } + throw error; + } + const scopes = entries.map((entry) => path.join(agentsPath, entry)); + await Promise.all(scopes.map(async (scope) => await assertBackupOwnedScope(scope))); + await Promise.all(scopes.map(async (scope) => await fs.rm(scope, { recursive: true }))); +} + +async function copyStagedScope( + stagingRoot: string, + repositoryPath: string, + identity: GitBackupIdentity, +): Promise { + const relative = gitBackupScopePath(identity); + const source = path.join(stagingRoot, relative); + const target = path.join(repositoryPath, relative); + await assertBackupOwnedScope(target); + await fs.rm(target, { recursive: true, force: true }); + await fs.mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + await fs.cp(source, target, { recursive: true, force: false }); +} + +async function commitGitBackup(params: { + repositoryPath: string; + message: string; + scopes: string[]; + env?: NodeJS.ProcessEnv; +}): Promise { + const email = await runGit(params.repositoryPath, ["config", "--get", "user.email"], { + env: params.env, + }); + const identityArgs = + email.code === 0 && email.stdout.trim() + ? [] + : ["-c", "user.name=OpenClaw", "-c", "user.email=backup@openclaw.local"]; + await requireGit( + params.repositoryPath, + [...identityArgs, "commit", "-m", params.message, "--", ...params.scopes], + { env: params.env }, + ); + return await requireGit(params.repositoryPath, ["rev-parse", "HEAD"], { env: params.env }); +} + +/** Snapshot selected databases, update the deterministic tree, and commit one Git revision. */ +export async function createGitBackup(params: { + repositoryPath: string; + stateDir: string; + databases: Array; + all?: boolean; + excludeSecrets?: boolean; + push?: boolean; + now?: Date; + gitEnv?: NodeJS.ProcessEnv; +}): Promise { + const repositoryPath = path.resolve(params.repositoryPath); + await initializeGitBackupRepository({ + repositoryPath, + stateDir: params.stateDir, + gitEnv: params.gitEnv, + }); + const stagingRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-git-backup-")); + await fs.chmod(stagingRoot, 0o700); + const manifests: GitBackupManifest[] = []; + try { + for (const database of params.databases) { + const outputPath = path.join(stagingRoot, gitBackupScopePath(database.identity)); + await fs.mkdir(path.dirname(outputPath), { recursive: true, mode: 0o700 }); + const copyPath = path.join( + stagingRoot, + `${database.identity.role}-${manifests.length}.sqlite`, + ); + await createOpenClawSnapshotCopy({ database, targetPath: copyPath }); + manifests.push( + await dumpGitBackupDatabase({ + snapshotPath: copyPath, + outputPath, + identity: database.identity, + excludeSecrets: params.excludeSecrets, + }), + ); + await fs.rm(copyPath, { force: true }); + } + if (params.all) { + await removeStaleAgentScopes(repositoryPath); + } + for (const database of params.databases) { + await copyStagedScope(stagingRoot, repositoryPath, database.identity); + } + } finally { + await fs.rm(stagingRoot, { recursive: true, force: true }).catch(() => undefined); + } + // Keep both owned roots present so Git accepts both scoped pathspecs even on a first global-only + // or agent-only backup. Empty directories remain untracked. + await Promise.all( + ["global", "agents"].map(async (scope) => + fs.mkdir(path.join(repositoryPath, scope), { recursive: true, mode: 0o700 }), + ), + ); + await requireGit(repositoryPath, ["add", "-A", "--", "global", "agents"], { + env: params.gitEnv, + }); + const changed = await requireGit( + repositoryPath, + ["status", "--porcelain", "--", "global", "agents"], + { + env: params.gitEnv, + }, + ); + let commit: string | undefined; + if (changed) { + const now = params.now ?? new Date(); + if (!Number.isFinite(now.getTime())) { + throw new Error("Git backup timestamp is invalid."); + } + const stagedBackupPaths = await requireGit( + repositoryPath, + ["diff", "--cached", "--name-only", "--", "global", "agents"], + { env: params.gitEnv }, + ); + const commitScopes = ["global", "agents"].filter((scope) => + stagedBackupPaths.split("\n").some((entry) => entry.startsWith(`${scope}/`)), + ); + commit = await commitGitBackup({ + repositoryPath, + message: `openclaw backup ${now.toISOString()}`, + scopes: commitScopes, + env: params.gitEnv, + }); + } + let pushed = false; + let pushWarning: string | undefined; + if (params.push) { + // Staging is path-scoped, but push ships HEAD's full ancestry. A dedicated + // repository is the supported remote shape. + const nonBackupCommitCount = await requireGit( + repositoryPath, + ["rev-list", "HEAD", "--invert-grep", "--grep=^openclaw backup ", "--count"], + { env: params.gitEnv }, + ); + if (nonBackupCommitCount !== "0") { + pushWarning = GIT_BACKUP_NON_BACKUP_HISTORY_WARNING; + } else { + const pushedResult = await runGit(repositoryPath, ["push", "-u", "origin", "HEAD"], { + env: params.gitEnv, + }); + if (pushedResult.code === 0) { + pushed = true; + } else { + pushWarning = sanitizeGitBackupDiagnostic( + (pushedResult.stderr || pushedResult.stdout).trim() || "git push failed", + ); + } + } + } + return { + repositoryPath, + ...(commit ? { commit } : {}), + noChanges: !changed, + pushed, + ...(pushWarning ? { pushWarning } : {}), + manifests, + }; +} + +async function resolveGitCommit(repositoryPath: string, ref?: string): Promise { + return await requireGit(repositoryPath, [ + "rev-parse", + "--verify", + `${ref?.trim() || "HEAD"}^{commit}`, + ]); +} + +/** Materialize one database scope from a Git ref into a private temporary directory. */ +async function materializeGitBackupRef(params: { + repositoryPath: string; + identity: GitBackupIdentity; + ref?: string; +}): Promise<{ commit: string; path: string; cleanup: () => Promise }> { + const repositoryPath = path.resolve(params.repositoryPath); + await assertGitRepository(repositoryPath); + const commit = await resolveGitCommit(repositoryPath, params.ref); + const scope = gitBackupScopePath(params.identity).split(path.sep).join("/"); + const files = ( + await requireGit(repositoryPath, ["ls-tree", "-r", "--name-only", commit, "--", scope]) + ) + .split("\n") + .filter(Boolean); + const required = new Set([`${scope}/${GIT_BACKUP_MANIFEST}`, `${scope}/${GIT_BACKUP_SCHEMA}`]); + if ([...required].some((entry) => !files.includes(entry))) { + throw new Error(`Git backup ref ${commit} does not contain ${scope}.`); + } + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-git-restore-")); + await fs.chmod(root, 0o700); + const outputPath = path.join(root, scope); + try { + for (const file of files) { + if ( + file !== `${scope}/${GIT_BACKUP_MANIFEST}` && + file !== `${scope}/${GIT_BACKUP_SCHEMA}` && + !file.startsWith(`${scope}/${GIT_BACKUP_TABLES}/`) + ) { + throw new Error(`Git backup ref contains an unexpected file: ${file}`); + } + const relative = file.slice(scope.length + 1); + const destination = path.join(outputPath, relative); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + // Table dumps can be tens of megabytes on real agent databases; the + // 1MB exec default would truncate them into a hash-mismatch failure. + await fs.writeFile( + destination, + await requireGitBuffer(repositoryPath, ["show", `${commit}:${file}`], { + maxOutputBytes: GIT_BACKUP_MATERIALIZE_MAX_BYTES, + }), + { mode: 0o600 }, + ); + } + return { + commit, + path: outputPath, + cleanup: async () => await fs.rm(root, { recursive: true, force: true }), + }; + } catch (error) { + await fs.rm(root, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +/** Restore one database from a Git ref to a caller-selected fresh path. */ +export async function restoreGitBackupRef(params: { + repositoryPath: string; + identity: GitBackupIdentity; + ref?: string; + targetPath: string; +}): Promise { + const materialized = await materializeGitBackupRef(params); + try { + return { + ...(await restoreGitBackupDirectory({ + sourcePath: materialized.path, + targetPath: params.targetPath, + expectedIdentity: params.identity, + })), + commit: materialized.commit, + }; + } finally { + await materialized.cleanup(); + } +} + +/** Verify a Git snapshot by restoring it privately and comparing every table digest. */ +export async function verifyGitBackupRef(params: { + repositoryPath: string; + identity: GitBackupIdentity; + ref?: string; +}): Promise { + const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-git-verify-")); + await fs.chmod(scratch, 0o700); + try { + return await restoreGitBackupRef({ + ...params, + targetPath: path.join(scratch, "database.sqlite"), + }); + } finally { + await fs.rm(scratch, { recursive: true, force: true }).catch(() => undefined); + } +} + +/** Return bounded Git backup log entries for CLI rendering. */ +export async function readGitBackupLog(params: { + repositoryPath: string; + limit: number; +}): Promise> { + await assertGitRepository(params.repositoryPath); + const result = await runGit(params.repositoryPath, [ + "log", + `--max-count=${params.limit}`, + "--pretty=format:%H%x09%cI%x09%s", + ]); + if (result.code !== 0) { + if (result.stderr.includes("does not have any commits yet")) { + return []; + } + throw new Error((result.stderr || result.stdout).trim()); + } + return result.stdout + .split("\n") + .filter(Boolean) + .map((line) => { + const [commit = "", date = "", ...message] = line.split("\t"); + return { commit, date, message: message.join("\t") }; + }); +} diff --git a/src/snapshot/local-repository.ts b/src/snapshot/local-repository.ts index 0d1656417ad6..3b335ea89698 100644 --- a/src/snapshot/local-repository.ts +++ b/src/snapshot/local-repository.ts @@ -31,28 +31,21 @@ import { createPrivateSqliteDirectory, createPrivateSqliteTempDirectory, } from "../infra/sqlite-private-directory.js"; -import { - createVerifiedSqliteSnapshot, - publishVerifiedSqliteFile, - type SqliteSnapshotValidator, -} from "../infra/sqlite-snapshot.js"; +import { publishVerifiedSqliteFile } from "../infra/sqlite-snapshot.js"; import { readSqliteUserVersion } from "../infra/sqlite-user-version.js"; import { runExec } from "../process/exec.js"; -import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; -import { assertOpenClawAgentDatabaseForMaintenance } from "../state/openclaw-agent-db.js"; -import { assertOpenClawStateDatabaseForMaintenance } from "../state/openclaw-state-db.js"; import { - sanitizeOpenClawGlobalStateSnapshot, - sanitizeOpenClawStateLeaseRows, -} from "../state/openclaw-state-snapshot-sanitizer.js"; -import { - containsAsciiControlCharacter, copySnapshotArtifact, hashSnapshotArtifact, readSnapshotManifest, type SnapshotArtifactDigest, writeSnapshotManifest, } from "./manifest.js"; +import { + buildSnapshotValidator, + createOpenClawSnapshotCopy, + normalizeSnapshotIdentity, +} from "./openclaw-snapshot-copy.js"; import { SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_SQLITE_FILENAME, @@ -286,17 +279,9 @@ class LocalSqliteSnapshotProvider implements SqliteSnapshotProvider { applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE); await assertPrivateStagingDirectory(stagingIdentity, stagingDir); await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); - const result = await createVerifiedSqliteSnapshot({ - sourcePath, + const result = await createOpenClawSnapshotCopy({ + database: { path: sourcePath, identity }, targetPath: artifactPath, - requireNonEmptySource: identity.role !== "generic", - transform: - identity.role === "global" - ? sanitizeOpenClawGlobalStateSnapshot - : identity.role === "agent" - ? sanitizeOpenClawStateLeaseRows - : undefined, - validate: buildDatabaseValidator(identity), }); applyPrivateModeSync(artifactPath, SNAPSHOT_FILE_MODE); const artifact = await hashSnapshotArtifact(stagingDir); @@ -726,24 +711,6 @@ async function verifySnapshotDatabaseFile( assertArtifactMatchesManifest(artifactPath, verifiedArtifact, manifest); } -function normalizeSnapshotIdentity(identity: SnapshotDatabaseIdentity): SnapshotDatabaseIdentity { - if (identity.role === "global") { - return identity; - } - if (identity.role === "agent") { - const agentId = normalizeAgentId(identity.agentId); - if (!isValidAgentId(identity.agentId) || agentId !== identity.agentId) { - throw new Error(`SQLite snapshot agent id must be canonical: ${identity.agentId}`); - } - return { role: "agent", agentId }; - } - const id = identity.id.trim(); - if (!id || id !== identity.id || id.length > 256 || containsAsciiControlCharacter(id)) { - throw new Error("SQLite snapshot generic database id is invalid."); - } - return { role: "generic", id }; -} - function buildDatabaseManifest( identity: SnapshotDatabaseIdentity, sourcePath: string, @@ -759,27 +726,10 @@ function buildDatabaseManifest( return { role: "generic", id: identity.id, basename, userVersion }; } -function buildDatabaseValidator( - identity: SnapshotDatabaseIdentity | SnapshotDatabaseManifest, -): SqliteSnapshotValidator { - if (identity.role === "global") { - return (database, pathname) => - assertOpenClawStateDatabaseForMaintenance(database, { pathname }); - } - if (identity.role === "agent") { - return (database, pathname) => - assertOpenClawAgentDatabaseForMaintenance(database, { - agentId: identity.agentId, - pathname, - }); - } - return () => undefined; -} - function buildManifestDatabaseValidator( manifest: SnapshotDatabaseManifest, -): SqliteSnapshotValidator { - const validateOwner = buildDatabaseValidator(manifest); +): import("../infra/sqlite-snapshot.js").SqliteSnapshotValidator { + const validateOwner = buildSnapshotValidator(manifest); return (database, pathname) => { validateOwner(database, pathname); const userVersion = readSqliteUserVersion(database); @@ -1278,6 +1228,19 @@ async function assertTrustedStagingRoot( return trustedRootPath; } +/** Create or strictly admit a Git repository through the local snapshot root trust policy. */ +export async function ensurePrivateSnapshotRepositoryRoot(rootPath: string): Promise { + try { + return await assertTrustedStagingRoot(await fs.lstat(rootPath), rootPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + const receipt = await ensurePrivateDirectory(rootPath, "Git backup repository"); + return await assertTrustedStagingRoot(receipt.identity, rootPath); +} + async function assertPrivateStagingDirectory( expectedIdentity: Stats, directoryPath: string, diff --git a/src/snapshot/openclaw-snapshot-copy.ts b/src/snapshot/openclaw-snapshot-copy.ts new file mode 100644 index 000000000000..703f4402b1d6 --- /dev/null +++ b/src/snapshot/openclaw-snapshot-copy.ts @@ -0,0 +1,71 @@ +import { + createVerifiedSqliteSnapshot, + type SqliteSnapshotValidator, +} from "../infra/sqlite-snapshot.js"; +import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; +import { assertOpenClawAgentDatabaseForMaintenance } from "../state/openclaw-agent-db.js"; +import { assertOpenClawStateDatabaseForMaintenance } from "../state/openclaw-state-db.js"; +import { + sanitizeOpenClawGlobalStateSnapshot, + sanitizeOpenClawStateLeaseRows, +} from "../state/openclaw-state-snapshot-sanitizer.js"; +import { containsAsciiControlCharacter } from "./manifest.js"; +import type { SnapshotDatabaseIdentity, SnapshotDatabaseRef } from "./snapshot-provider.js"; + +export function normalizeSnapshotIdentity( + identity: SnapshotDatabaseIdentity, +): SnapshotDatabaseIdentity { + if (identity.role === "global") { + return identity; + } + if (identity.role === "agent") { + const agentId = normalizeAgentId(identity.agentId); + if (!isValidAgentId(identity.agentId) || agentId !== identity.agentId) { + throw new Error(`SQLite snapshot agent id must be canonical: ${identity.agentId}`); + } + return { role: "agent", agentId }; + } + const id = identity.id.trim(); + if (!id || id !== identity.id || id.length > 256 || containsAsciiControlCharacter(id)) { + throw new Error("SQLite snapshot generic database id is invalid."); + } + return { role: "generic", id }; +} + +export function buildSnapshotValidator( + identity: SnapshotDatabaseIdentity, +): SqliteSnapshotValidator { + if (identity.role === "global") { + return (database, pathname) => + assertOpenClawStateDatabaseForMaintenance(database, { pathname }); + } + if (identity.role === "agent") { + return (database, pathname) => + assertOpenClawAgentDatabaseForMaintenance(database, { + agentId: identity.agentId, + pathname, + }); + } + return () => undefined; +} + +/** Produce the canonical sanitized, compact, verified copy used by every snapshot provider. */ +export async function createOpenClawSnapshotCopy(params: { + database: SnapshotDatabaseRef; + targetPath: string; +}): Promise<{ identity: SnapshotDatabaseIdentity; path: string; userVersion: number }> { + const identity = normalizeSnapshotIdentity(params.database.identity); + const result = await createVerifiedSqliteSnapshot({ + sourcePath: params.database.path, + targetPath: params.targetPath, + requireNonEmptySource: identity.role !== "generic", + transform: + identity.role === "global" + ? sanitizeOpenClawGlobalStateSnapshot + : identity.role === "agent" + ? sanitizeOpenClawStateLeaseRows + : undefined, + validate: buildSnapshotValidator(identity), + }); + return { identity, ...result }; +} diff --git a/src/state/backup-run-records.test.ts b/src/state/backup-run-records.test.ts new file mode 100644 index 000000000000..82b003884db3 --- /dev/null +++ b/src/state/backup-run-records.test.ts @@ -0,0 +1,174 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + buildBackupStatusValue, + noteBackupDoctorHint, + readBackupFreshness, +} from "../commands/backup-health.js"; +import { recordBackupRunOutcome } from "./backup-run-records.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { + closeOpenClawStateDatabaseForTest, + runOpenClawStateWriteTransaction, +} from "./openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; + +const roots: string[] = []; +const mocks = vi.hoisted(() => ({ note: vi.fn() })); + +vi.mock("../../packages/terminal-core/src/note.js", () => ({ note: mocks.note })); + +async function testEnv(options?: { bootstrap?: boolean }): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-backup-runs-test-")); + roots.push(root); + const env = { ...process.env, OPENCLAW_STATE_DIR: path.join(root, "state") }; + if (options?.bootstrap) { + // Recording is non-creating by contract, so the fixture bootstraps the + // state database the way a real gateway host already has. + runOpenClawStateWriteTransaction(() => undefined, { env }); + } + return env; +} + +afterEach(async () => { + vi.restoreAllMocks(); + mocks.note.mockReset(); + closeOpenClawStateDatabaseForTest(); + await Promise.all( + roots.splice(0).map(async (root) => await fs.rm(root, { recursive: true, force: true })), + ); +}); + +describe("backup run records", () => { + it("records archive and Git outcomes and prunes the operational log to 200 rows", async () => { + const env = await testEnv({ bootstrap: true }); + recordBackupRunOutcome({ + env, + archivePath: "/backups/archive.tar.gz", + status: "failed", + kind: "archive", + error: "archive failed", + createdAt: 1, + }); + for (let index = 2; index <= 202; index += 1) { + recordBackupRunOutcome({ + env, + archivePath: "/backups/git", + status: "ok", + kind: "git", + target: `commit-${index}`, + pushFailed: index === 202, + createdAt: index, + }); + } + const rows = withExistingOpenClawStateDatabaseReadOnly( + ({ db }) => + db + .prepare( + "SELECT created_at, status, manifest_json FROM backup_runs ORDER BY created_at ASC", + ) + .all() as Array<{ created_at: number; status: string; manifest_json: string }>, + { env }, + ); + expect(rows).toHaveLength(200); + expect(rows?.[0]?.created_at).toBe(3); + expect(rows?.at(-1)).toMatchObject({ created_at: 202, status: "ok" }); + expect(JSON.parse(rows?.at(-1)?.manifest_json ?? "{}")).toMatchObject({ + kind: "git", + target: "commit-202", + pushFailed: true, + }); + expect(readBackupFreshness(env)).toMatchObject({ + latest: { createdAt: 202, pushFailed: true }, + latestOk: { createdAt: 202, pushFailed: true }, + }); + }); + + it("treats an older same-version database without backup_runs as no recorded backups", async () => { + const env = await testEnv({ bootstrap: true }); + withExistingOpenClawStateDatabaseReadOnly(() => undefined, { env }); + closeOpenClawStateDatabaseForTest(); + const { DatabaseSync } = await import("node:sqlite"); + const raw = new DatabaseSync(resolveOpenClawStateSqlitePath(env)); + raw.exec("DROP TABLE backup_runs"); + raw.close(); + expect(readBackupFreshness(env)).toEqual({}); + }); + + it("keeps absent status reads read-only and formats none, failed, fresh, and stale states", async () => { + const env = await testEnv(); + expect(readBackupFreshness(env)).toEqual({}); + await expect(fs.access(resolveOpenClawStateSqlitePath(env))).rejects.toMatchObject({ + code: "ENOENT", + }); + const formatTimeAgo = (ageMs: number) => `${ageMs / 3_600_000}h ago`; + expect(buildBackupStatusValue({ freshness: {}, now: 10, formatTimeAgo })).toBe("none recorded"); + const failed = { + id: "failed", + createdAt: 1, + archivePath: "/backup", + status: "failed" as const, + kind: "archive" as const, + }; + expect( + buildBackupStatusValue({ + freshness: { latest: failed }, + now: 3 * 24 * 3_600_000 + 1, + formatTimeAgo, + }), + ).toBe("last attempt failed 72h ago (archive)"); + noteBackupDoctorHint(env); + expect(mocks.note).toHaveBeenCalledWith( + expect.stringContaining("No successful backup is recorded."), + "Backups", + ); + + // Recording is non-creating; bootstrap the state database before the + // recording phase the way a real gateway host already has. + runOpenClawStateWriteTransaction(() => undefined, { env }); + vi.spyOn(Date, "now").mockReturnValue(1_000); + recordBackupRunOutcome({ + env, + archivePath: "/backup", + status: "ok", + kind: "git", + createdAt: 1, + }); + mocks.note.mockClear(); + noteBackupDoctorHint(env); + expect(mocks.note).not.toHaveBeenCalled(); + + vi.mocked(Date.now).mockReturnValue(1 + 15 * 24 * 3_600_000); + noteBackupDoctorHint(env); + expect(mocks.note).toHaveBeenCalledWith( + expect.stringContaining("more than 14 days old"), + "Backups", + ); + + recordBackupRunOutcome({ + env, + archivePath: "/backups/git", + status: "ok", + kind: "git", + pushFailed: true, + createdAt: 2, + }); + const pushFailed = readBackupFreshness(env); + expect( + buildBackupStatusValue({ + freshness: pushFailed, + now: 3_600_002, + formatTimeAgo, + }), + ).toBe("last ok 1h ago (git, push failing)"); + mocks.note.mockClear(); + vi.mocked(Date.now).mockReturnValue(3_600_002); + noteBackupDoctorHint(env); + expect(mocks.note).toHaveBeenCalledWith( + expect.stringMatching(/configured Git remote.*\/backups\/git/su), + "Backups", + ); + }); +}); diff --git a/src/state/backup-run-records.ts b/src/state/backup-run-records.ts new file mode 100644 index 000000000000..4cc994fa9053 --- /dev/null +++ b/src/state/backup-run-records.ts @@ -0,0 +1,154 @@ +import { randomUUID } from "node:crypto"; +import { existsSync } from "node:fs"; +import type { DatabaseSync } from "node:sqlite"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateDatabase } from "./openclaw-state-db.generated.js"; +import { runOpenClawStateWriteTransaction } from "./openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; + +type BackupRunDatabase = Pick; + +type BackupRunKind = "archive" | "sqlite-snapshot" | "git"; + +export type BackupRunRecord = { + id: string; + createdAt: number; + archivePath: string; + status: "ok" | "failed"; + kind: BackupRunKind; + target?: string; + error?: string; + pushFailed?: true; +}; + +function boundedText(value: string | undefined, maxLength: number): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed.slice(0, maxLength) : undefined; +} + +function parseBackupRun(row: { + id: string; + created_at: number; + archive_path: string; + status: string; + manifest_json: string; +}): BackupRunRecord | undefined { + if (row.status !== "ok" && row.status !== "failed") { + return undefined; + } + let manifest: unknown; + try { + manifest = JSON.parse(row.manifest_json) as unknown; + } catch { + return undefined; + } + if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) { + return undefined; + } + const value = manifest as Record; + if (value.kind !== "archive" && value.kind !== "sqlite-snapshot" && value.kind !== "git") { + return undefined; + } + return { + id: row.id, + createdAt: row.created_at, + archivePath: row.archive_path, + status: row.status, + kind: value.kind, + ...(typeof value.target === "string" ? { target: value.target } : {}), + ...(typeof value.error === "string" ? { error: value.error } : {}), + ...(value.pushFailed === true ? { pushFailed: true } : {}), + }; +} + +/** Record one best-effort backup outcome in the shared bounded operational log. */ +export function recordBackupRunOutcome(params: { + archivePath: string; + status: "ok" | "failed"; + kind: BackupRunKind; + target?: string; + error?: string; + pushFailed?: boolean; + createdAt?: number; + env?: NodeJS.ProcessEnv; +}): void { + // Best-effort log only: never bootstrap an absent state database to record an + // outcome, or a failed backup on a fresh host would create a blank DB that a + // retry then treats as real backup input. + if (!existsSync(resolveOpenClawStateSqlitePath(params.env ?? process.env))) { + return; + } + const manifest = JSON.stringify({ + kind: params.kind, + ...(boundedText(params.target, 512) ? { target: boundedText(params.target, 512) } : {}), + ...(boundedText(params.error, 1_200) ? { error: boundedText(params.error, 1_200) } : {}), + ...(params.pushFailed === true ? { pushFailed: true } : {}), + }); + runOpenClawStateWriteTransaction( + ({ db }) => { + const kysely = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + kysely.insertInto("backup_runs").values({ + id: randomUUID(), + created_at: params.createdAt ?? Date.now(), + archive_path: params.archivePath, + status: params.status, + manifest_json: manifest, + }), + ); + // This is a bounded operational log. Hourly scheduled backups must not grow it forever. + executeSqliteQuerySync( + db, + kysely + .deleteFrom("backup_runs") + .where( + "id", + "in", + kysely + .selectFrom("backup_runs") + .select("id") + .orderBy("created_at", "desc") + .orderBy("id", "desc") + .limit(2_147_483_647) + .offset(200), + ), + ); + }, + { env: params.env }, + ); +} + +function readBackupRun(database: DatabaseSync, status?: "ok"): BackupRunRecord | undefined { + // backup_runs is same-version additive: an older v6 database may not have it + // until a writable open converges the schema. Read-only freshness paths must + // treat that as "no recorded backups", never as an error. + if (!tableExists(database, "backup_runs")) { + return undefined; + } + const kysely = getNodeSqliteKysely(database); + let query = kysely.selectFrom("backup_runs").selectAll(); + if (status) { + query = query.where("status", "=", status); + } + const row = executeSqliteQueryTakeFirstSync( + database, + query.orderBy("created_at", "desc").orderBy("id", "desc").limit(1), + ); + return row ? parseBackupRun(row) : undefined; +} + +/** Read the newest recorded backup attempt from an already-open database. */ +export function readLatestBackupRun(database: DatabaseSync): BackupRunRecord | undefined { + return readBackupRun(database); +} + +/** Read the newest successful backup from an already-open database. */ +export function readLatestSuccessfulBackupRun(database: DatabaseSync): BackupRunRecord | undefined { + return readBackupRun(database, "ok"); +} diff --git a/src/state/openclaw-agent-db-schema.ts b/src/state/openclaw-agent-db-schema.ts index 4d6512789fdf..a8d6f6e08c7e 100644 --- a/src/state/openclaw-agent-db-schema.ts +++ b/src/state/openclaw-agent-db-schema.ts @@ -697,13 +697,25 @@ function ensureAgentSchema( updated_at: now, }) .onConflict((conflict) => - conflict.column("meta_key").doUpdateSet({ - role: "agent", - schema_version: targetVersion, - agent_id: agentId, - app_version: VERSION, - updated_at: now, - }), + conflict + .column("meta_key") + .doUpdateSet({ + role: "agent", + schema_version: targetVersion, + agent_id: agentId, + app_version: VERSION, + updated_at: now, + }) + // updated_at records when schema metadata last changed, not when + // the database was last opened; unconditional bumps make every + // open dirty the row and defeat no-change backup detection. + .where((eb) => + eb.or([ + eb("schema_meta.schema_version", "!=", targetVersion), + eb("schema_meta.app_version", "!=", VERSION), + eb("schema_meta.agent_id", "!=", agentId), + ]), + ), ), ); assertAgentSchemaVersion(db, { agentId, pathname, version: targetVersion }); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 7174b628711c..97f63c95614d 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -391,13 +391,25 @@ function ensureSchema(db: DatabaseSync, pathname: string, env: NodeJS.ProcessEnv updated_at: now, }) .onConflict((conflict) => - conflict.column("meta_key").doUpdateSet({ - role: "global", - schema_version: OPENCLAW_STATE_SCHEMA_VERSION, - agent_id: null, - app_version: VERSION, - updated_at: now, - }), + conflict + .column("meta_key") + .doUpdateSet({ + role: "global", + schema_version: OPENCLAW_STATE_SCHEMA_VERSION, + agent_id: null, + app_version: VERSION, + updated_at: now, + }) + // updated_at records when schema metadata last changed, not when + // the database was last opened; unconditional bumps make every + // open dirty the row and defeat no-change backup detection. + .where((eb) => + eb.or([ + eb("schema_meta.schema_version", "!=", OPENCLAW_STATE_SCHEMA_VERSION), + eb("schema_meta.app_version", "!=", VERSION), + eb("schema_meta.role", "!=", "global"), + ]), + ), ), ); assertOpenClawStateDatabaseForMaintenance(db, { pathname }); diff --git a/src/state/secret-state-tables.test.ts b/src/state/secret-state-tables.test.ts new file mode 100644 index 000000000000..b025bb60eebb --- /dev/null +++ b/src/state/secret-state-tables.test.ts @@ -0,0 +1,86 @@ +import fs from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +import { AGENT_SECRET_TABLE_NAMES, STATE_SECRET_TABLE_NAMES } from "./secret-state-tables.js"; + +const REVIEWED_SAFE_TABLES = { + exec_approvals_config: + "has_socket_token is a presence bit; snapshot sanitization removes the token value", + operator_approvals: "requested_by_device_token_auth is boolean provenance, not token material", +} as const; + +const EMBEDDED_CREDENTIAL_TABLES = { + // payload_json stores the pairing setup payload, including its live bootstrapToken. + device_pairing_join_codes: "payload_json contains a pairing bootstrapToken", +} as const; + +const CREDENTIAL_COLUMN_SEGMENT = + /(?:^|_)(?:token|secret|private_key|api_key|password|credential)(?:_|$)/u; + +function tablesWithCredentialColumns(sql: string): Map { + const matches = new Map(); + const tablePattern = + /CREATE TABLE IF NOT EXISTS ([A-Za-z_][A-Za-z0-9_]*)\s*\(([\s\S]*?)\)\s*STRICT;/gu; + for (const tableMatch of sql.matchAll(tablePattern)) { + const table = tableMatch[1]; + const body = tableMatch[2]; + if (!table || !body) { + continue; + } + const columns = body + .split("\n") + .map((line) => /^\s*([A-Za-z_][A-Za-z0-9_]*)\s+/u.exec(line)?.[1]) + .filter( + (column): column is string => + typeof column === "string" && + CREDENTIAL_COLUMN_SEGMENT.test(column) && + !column.endsWith("_hash"), + ); + if (columns.length > 0) { + matches.set(table, columns); + } + } + return matches; +} + +describe("secret state table policy", () => { + it("classifies every schema table with credential-suggestive columns", async () => { + const schemas = [ + { + name: "openclaw-state-schema.sql", + sql: await fs.readFile(new URL("./openclaw-state-schema.sql", import.meta.url), "utf8"), + secretTables: new Set(STATE_SECRET_TABLE_NAMES), + }, + { + name: "openclaw-agent-schema.sql", + sql: await fs.readFile(new URL("./openclaw-agent-schema.sql", import.meta.url), "utf8"), + secretTables: new Set(AGENT_SECRET_TABLE_NAMES), + }, + ]; + const reviewedSafeTables = new Set(Object.keys(REVIEWED_SAFE_TABLES)); + const classifiedSafeTables = new Set(); + const missing: string[] = []; + + for (const schema of schemas) { + for (const [table, columns] of tablesWithCredentialColumns(schema.sql)) { + if (schema.secretTables.has(table)) { + continue; + } + if (reviewedSafeTables.has(table)) { + classifiedSafeTables.add(table); + continue; + } + missing.push(`${schema.name}: ${table} (${columns.join(", ")})`); + } + } + + expect(missing, "credential-bearing tables must be redacted or reviewed safe").toEqual([]); + expect([...classifiedSafeTables].toSorted()).toEqual([...reviewedSafeTables].toSorted()); + }); + + it("classifies opaque payload tables that embed credentials", () => { + const secretTables = new Set(STATE_SECRET_TABLE_NAMES); + for (const [table, reason] of Object.entries(EMBEDDED_CREDENTIAL_TABLES)) { + expect(secretTables.has(table), reason).toBe(true); + } + }); +}); diff --git a/src/state/secret-state-tables.ts b/src/state/secret-state-tables.ts new file mode 100644 index 000000000000..3a10d6e5f99b --- /dev/null +++ b/src/state/secret-state-tables.ts @@ -0,0 +1,31 @@ +/** Redaction policy surface: Git snapshots may omit these credential-bearing tables. */ +export const STATE_SECRET_TABLE_NAMES = [ + "audit_identity_keys", + "auth_profile_state", + "auth_profile_stores", + "apns_registrations", + "channel_ingress_events", + "channel_pairing_requests", + "clawhub_promotion_claims", + "device_auth_tokens", + "device_bootstrap_tokens", + "device_identities", + "device_pairing_join_codes", + "device_pairing_paired", + "gateway_origin_device_tokens", + "mcp_oauth_pending_authorizations", + "mcp_oauth_stores", + "native_hook_relay_bridges", + "node_host_config", + "secret_store_entries", + "web_push_subscriptions", + "web_push_vapid_keys", + "worker_environment_credentials", +] as const; + +/** Redaction policy surface for credential-bearing per-agent database tables. */ +export const AGENT_SECRET_TABLE_NAMES = [ + "auth_profile_state", + "auth_profile_store", + "session_suggestions", +] as const;