diff --git a/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift b/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift index 728380bbaea1..229211ab3b91 100644 --- a/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift +++ b/apps/macos/Tests/OpenClawIPCTests/PortGuardianRecordStoreTests.swift @@ -568,7 +568,7 @@ struct PortGuardianRecordStoreTests { let fixture = try Self.fixture() defer { fixture.cleanup() } - for version in [4, 5, 6, 7, 8, 9, 10, 11] { + for version in [4, 5, 6, 7, 8, 9, 10, 11, 12] { let databaseURL = fixture.root.appendingPathComponent("supported-v\(version).sqlite") try Self.seedVersionedPortGuardianDatabase(databaseURL, schemaVersion: version) let store = try PortGuardianRecordStore(databaseURL: databaseURL) @@ -580,7 +580,7 @@ struct PortGuardianRecordStoreTests { #expect(try store.records() == [record]) } - for version in [12, 99] { + for version in [13, 99] { let databaseURL = fixture.root.appendingPathComponent("newer-v\(version).sqlite") try Self.seedVersionedPortGuardianDatabase(databaseURL, schemaVersion: version) #expect(throws: PortGuardianStoreError.self) { diff --git a/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift b/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift index 62f5889c6611..89bcc3a0c70e 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawNativeState/OpenClawNativeStateSQLite.swift @@ -37,7 +37,7 @@ public enum OpenClawNativeStateSQLiteValueType: Equatable, Sendable { /// One recursive connection lock serializes transactions and statement access. public final class OpenClawNativeStateSQLite: @unchecked Sendable { // Keep aligned with OPENCLAW_STATE_SCHEMA_VERSION. Native clients never upgrade this database. - private static let maximumSupportedSchemaVersion: Int64 = 11 + private static let maximumSupportedSchemaVersion: Int64 = 12 private static let defaultBusyTimeoutMilliseconds: Int32 = 5000 private struct SchemaObject: Hashable { diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index 7d95c49c1185..1ec21208fb8d 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -3316,7 +3316,7 @@ src/infra/state-migrations.state-dir.ts 1 src/infra/state-migrations.storage.ts 21 src/infra/state-migrations.task-sidecar-rows.ts 8 src/infra/state-migrations.tui-last-session.ts 2 -src/infra/state-migrations.update-check.ts 3 +src/infra/state-migrations.update-check.ts 2 src/infra/state-migrations.workspace-setup-store.ts 2 src/infra/state-migrations.workspace-setup.ts 1 src/infra/system-presence.ts 2 diff --git a/docs/cli/backup.md b/docs/cli/backup.md index ff4550b4e8a7..440d536ebaa1 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -188,20 +188,23 @@ unrelated files elsewhere in an adopted repository are never staged. - `mcp_oauth_pending_authorizations` - `mcp_oauth_stores` - `native_hook_relay_bridges` -- `node_host_config` - `secret_store_entries` - `web_push_subscriptions` -- `web_push_vapid_keys` - `worker_environment_credentials` +It also omits `config_machine_state` rows whose keys begin with `nodeHost.` or +`webPush.vapidKeys`, while retaining other machine-state rows. + It omits these per-agent tables: - `auth_profile_state` - `auth_profile_store` - `session_suggestions` -Restore reports the omitted tables so a redacted snapshot cannot be mistaken -for a complete credential backup. +The backup manifest records omitted tables in `excludedTables` and omitted +machine-state prefixes in `excludedConfigStateKeyPrefixes`. Restore reports +omitted tables so a redacted snapshot cannot be mistaken for a complete +credential backup. Inspect or verify history without changing the live databases: diff --git a/docs/cli/node.md b/docs/cli/node.md index 1c3a16bf4f93..7d170f448b4a 100644 --- a/docs/cli/node.md +++ b/docs/cli/node.md @@ -242,11 +242,11 @@ identity that the Gateway uses for pairing and routing. This state lives in the OpenClaw state directory (`~/.openclaw` by default, or `$OPENCLAW_STATE_DIR` when set): -| State | Purpose | -| -------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `state/openclaw.sqlite` (`node_host_config`) | Client instance ID, display name, and Gateway connection metadata. The client sends this ID as `instanceId`. | -| `state/openclaw.sqlite` (`device_identities`, `primary`) | Signed Ed25519 keypair and derived device ID. For signed connections, this device ID is the routed node ID and pairing identity. | -| `state/openclaw.sqlite` (`device_auth_tokens`) | Paired device tokens, keyed by cryptographic device ID and role. | +| State | Purpose | +| ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `state/openclaw.sqlite` (`config_machine_state`, key `nodeHost.config`) | Client instance ID, display name, and Gateway connection metadata. The client sends this ID as `instanceId`. | +| `state/openclaw.sqlite` (`device_identities`, `primary`) | Signed Ed25519 keypair and derived device ID. For signed connections, this device ID is the routed node ID and pairing identity. | +| `state/openclaw.sqlite` (`device_auth_tokens`) | Paired device tokens, keyed by cryptographic device ID and role. | `--node-id` changes only the client instance ID in shared SQLite state. It does not change the cryptographic device ID or clear pairing auth. Migrating a retired diff --git a/docs/nodes/index.md b/docs/nodes/index.md index 1ea60995e257..65defb2acc31 100644 --- a/docs/nodes/index.md +++ b/docs/nodes/index.md @@ -185,7 +185,7 @@ If the node retries with changed auth details, re-run `openclaw devices list` an Naming options: -- `--display-name` on `openclaw node run` / `openclaw node install` (persists in the shared `node_host_config` SQLite row alongside the client instance ID and Gateway connection metadata). +- `--display-name` on `openclaw node run` / `openclaw node install` (persists in the shared `nodeHost.config` SQLite machine-state value alongside the client instance ID and Gateway connection metadata). - `openclaw nodes rename --node --name "Build Node"` (gateway override). ### Node-hosted MCP servers @@ -280,7 +280,7 @@ operators can ignore skills from every paired node with The headless node keeps three separate state records in shared SQLite: -- `~/.openclaw/state/openclaw.sqlite` (`node_host_config`): the client instance ID, display name, and Gateway connection metadata. +- `~/.openclaw/state/openclaw.sqlite` (`config_machine_state`, key `nodeHost.config`): the client instance ID, display name, and Gateway connection metadata. - `~/.openclaw/state/openclaw.sqlite` (`device_identities`, key `primary`): the signed device keypair and derived cryptographic device ID. - `~/.openclaw/state/openclaw.sqlite` (`device_auth_tokens`): paired device auth tokens keyed by cryptographic device ID and role. diff --git a/docs/nodes/voicewake.md b/docs/nodes/voicewake.md index fdf980bc9316..20390ef7c88f 100644 --- a/docs/nodes/voicewake.md +++ b/docs/nodes/voicewake.md @@ -14,7 +14,7 @@ Wake words are **one global list owned by the Gateway** — there are no per-nod ## Storage -Wake words and routing rules live in the Gateway state database, `~/.openclaw/state/openclaw.sqlite` by default (override with `OPENCLAW_STATE_DIR`), tables `voicewake_triggers`, `voicewake_routing_config`, `voicewake_routing_routes`. Legacy `settings/voicewake.json` and `settings/voicewake-routing.json` are `openclaw doctor --fix` migration inputs only — runtime never reads them. +Wake words and routing rules live in the Gateway state database, `~/.openclaw/state/openclaw.sqlite` by default (override with `OPENCLAW_STATE_DIR`), under the `config_machine_state` keys `voicewake.triggers` and `voicewake.routing`. Legacy `settings/voicewake.json` and `settings/voicewake-routing.json` are `openclaw doctor --fix` migration inputs only — runtime never reads them. ## Protocol diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index 5a72fe73bf38..f77afd9ad169 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -202,7 +202,7 @@ without exceptions outside doctor/import/export/debug boundaries. - No active session files. - No fake JSONL test fixtures except doctor legacy migration tests. - No raw SQLite access where Kysely is expected. -- No new file-era runtime stores. The current global schema is version `9`, and +- No new file-era runtime stores. The current global schema is version `12`, and the current per-agent schema is version `17`; older supported databases move through the bounded forward migrations listed in [Database schemas](/reference/database-schemas). @@ -309,7 +309,7 @@ The branch already has a real shared SQLite base: - Runtime stores derive selected and inserted row types from those generated Kysely `DB` interfaces instead of shadowing SQLite row shapes by hand. Raw SQL remains limited to schema application, pragmas, and migration-only DDL. -- The global SQLite schema is at `user_version = 9`. The per-agent schema is at +- The global SQLite schema is at `user_version = 12`. The per-agent schema is at version `17`; their openers apply bounded forward migrations from supported older schemas. File-to-database import remains in Doctor code. - Relational ownership is enforced where the ownership boundary is canonical: @@ -325,7 +325,7 @@ The branch already has a real shared SQLite base: `workspace_setup_state`, `workspace_path_aliases`, `workspace_attestations`, `workspace_generated_bootstrap_hashes`, `native_hook_relay_bridges`, `current_conversation_bindings`, `plugin_binding_approvals`, - `tui_last_sessions`, `acp_sessions`, `acp_replay_sessions`, + `acp_sessions`, `acp_replay_sessions`, `acp_replay_events`, `task_runs`, `task_delivery_state`, `flow_runs`, `subagent_runs`, `migration_runs`, and `backup_runs`. - Arbitrary plugin-owned state does not get host-owned typed tables. Installed @@ -425,8 +425,9 @@ The branch already has a real shared SQLite base: account, session, retry, error, platform-send, and recovery state onto the replay JSON. `entry_json` keeps the replay payloads, hooks, and formatting payload, but typed columns are authoritative for hot queue routing/state. -- TUI last-session restore pointers now live in typed shared - `tui_last_sessions` rows keyed by the hashed TUI connection/session scope. +- TUI last-session restore pointers now live in shared `config_machine_state` + rows under `tui.lastSession.`, keyed by the hashed TUI + connection/session scope. Runtime reads and writes only SQLite, atomically upserts each scope, and excludes heartbeat sessions. `openclaw doctor --fix` strictly validates the old TUI JSON file, keeps newer SQLite rows, verifies the canonical result, @@ -521,12 +522,14 @@ The branch already has a real shared SQLite base: - Non-doctor commands do not auto-run legacy config repair. For example, `openclaw update --channel` now fails on invalid legacy config and asks the user to run doctor, rather than silently importing doctor migration code. -- Web push, APNs, Voice Wake, update checks, and config health now use typed shared SQLite - tables for subscriptions, VAPID keys, node registrations, trigger rows, - routing rows, update-notification state, and config health entries instead of - whole opaque JSON blobs. Web Push and APNs writes upsert only the affected - primary-key row; config health reconciles by config path. Their runtime - modules remain separate from Doctor-only legacy JSON import helpers. +- Web push, APNs, and config health use typed shared SQLite tables for + subscriptions, node registrations, and config health entries. VAPID keys, + Voice Wake, and update checks use owner-managed `config_machine_state` values + under `webPush.vapidKeys`, `voicewake.triggers`, `voicewake.routing`, and + `update.checkState`. + Web Push and APNs writes upsert only the affected primary-key row; config + health reconciles by config path. Their runtime modules remain separate from + Doctor-only legacy JSON import helpers. - APNs runtime reads and writes only `apns_registrations`. Explicit `openclaw doctor --fix` strictly imports the retired `push/apns-registrations.json`, preserves existing canonical rows, verifies @@ -534,7 +537,8 @@ The branch already has a real shared SQLite base: Receipt-backed retries perform cleanup only, while `apns_registration_tombstones` cover invalidations before first repair, so stale relay grants or device tokens cannot resurrect. -- Node-host config now uses a typed singleton row in the shared SQLite database. +- Node-host config now uses the `nodeHost.config` machine-state key in the shared + SQLite database. Runtime fails closed while the old `node.json` file or an interrupted claim remains; explicit `openclaw doctor --fix` strictly imports and removes it before normal runtime use. @@ -1119,8 +1123,9 @@ sessionId})`; create, branch, continue, list, and fork flows live in their table and discards its inert rows. Unknown same-named tables or indexes are preserved and the migration is refused. Runtime no longer reads or writes commitment state. Doctor leaves the legacy `commitments.json` source untouched. -- Web Push subscriptions and the generated VAPID identity now use typed shared - `web_push_subscriptions` and `web_push_vapid_keys` rows. Runtime registration, +- Web Push subscriptions use typed shared `web_push_subscriptions` rows, and + generated VAPID identity uses the `webPush.vapidKeys` machine-state key. + Runtime registration, expiry cleanup, and first-use key generation use row-level SQLite transactions. Explicit Doctor repair validates both retired JSON stores, claims them before the SQLite write, imports them atomically, rejects @@ -1311,13 +1316,13 @@ sessionId})`; create, branch, continue, list, and fork flows live in their - PI model discovery now passes canonical credentials into in-memory `pi-coding-agent` auth storage. It no longer creates, scrubs, or writes per-agent `auth.json` during discovery. -- Voice Wake trigger and routing settings now use typed shared SQLite tables - instead of `settings/voicewake.json`, `settings/voicewake-routing.json`, or - opaque generic rows; doctor imports the legacy JSON files and removes them after a - successful migration. -- Update-check state now uses a typed shared `update_check_state` row instead of - `update-check.json` or an opaque generic blob; doctor imports - the legacy JSON file and removes it after a successful migration. +- Voice Wake trigger and routing settings now use `config_machine_state` keys + `voicewake.triggers` and `voicewake.routing` instead of + `settings/voicewake.json` or `settings/voicewake-routing.json`; doctor + imports the legacy JSON files and removes them after a successful migration. +- Update-check state now uses the `config_machine_state` key + `update.checkState` instead of `update-check.json`; doctor imports the legacy + JSON file and removes it after a successful migration. - Config health state now uses typed shared `config_health_entries` rows instead of `logs/config-health.json` or an opaque generic blob; doctor imports the legacy JSON file and removes it after a successful migration. @@ -1481,9 +1486,9 @@ create` validates the written archive by default; `--no-verify` is the ## Target Schema Shape -Keep schemas explicit. Host-owned runtime state uses typed tables. Plugin-owned -opaque state uses `plugin_state_entries` / `plugin_blob_entries`; there is no -generic host `kv` table. +Keep schemas explicit. Host-owned relational state uses typed tables; +owner-managed singleton and workspace snapshots use `config_machine_state`. +Plugin-owned opaque state uses `plugin_state_entries` / `plugin_blob_entries`. Global database: @@ -1491,6 +1496,7 @@ Global database: state_leases(scope, lease_key, owner, expires_at, heartbeat_at, payload_json, created_at, updated_at) exec_approvals_config(config_key, raw_json, socket_path, has_socket_token, default_security, default_ask, default_ask_fallback, auto_allow_skills, agent_count, allowlist_count, updated_at_ms) schema_meta(meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) +config_machine_state(state_key, value_json, updated_at_ms) agent_databases(agent_id, path, schema_version, last_seen_at, size_bytes) task_runs(...) task_delivery_state(...) @@ -1498,16 +1504,13 @@ flow_runs(...) subagent_runs(run_id, child_session_key, requester_session_key, controller_session_key, created_at, ended_at, cleanup_handled, payload_json) current_conversation_bindings(binding_key, binding_id, target_agent_id, target_session_id, target_session_key, channel, account_id, conversation_kind, parent_conversation_id, conversation_id, target_kind, status, bound_at, expires_at, metadata_json, updated_at) plugin_binding_approvals(plugin_root, channel, account_id, plugin_id, plugin_name, approved_at) -tui_last_sessions(scope_key, session_key, updated_at) plugin_state_entries(plugin_id, namespace, entry_key, value_json, created_at, expires_at) plugin_blob_entries(plugin_id, namespace, entry_key, metadata_json, blob, created_at, expires_at) skill_uploads(upload_id, kind, slug, force, size_bytes, sha256, actual_sha256, received_bytes, archive_blob, created_at, expires_at, committed, committed_at, idempotency_key_hash) skill_upload_chunks(upload_id, byte_offset, size_bytes, chunk_blob) web_push_subscriptions(endpoint_hash, subscription_id, endpoint, p256dh, auth, created_at_ms, updated_at_ms) -web_push_vapid_keys(key_id, public_key, private_key, subject, updated_at_ms) apns_registrations(node_id, transport, token, relay_handle, send_grant, installation_id, relay_origin, topic, environment, distribution, token_debug_suffix, updated_at_ms) apns_registration_tombstones(node_id, deleted_at_ms) -node_host_config(config_key, version, node_id, token, display_name, gateway_host, gateway_port, gateway_tls, gateway_tls_fingerprint, gateway_context_path, updated_at_ms) device_identities(identity_key, device_id, public_key_pem, private_key_pem, created_at_ms, updated_at_ms) device_auth_tokens(device_id, role, token, scopes_json, updated_at_ms) macos_port_guardian_records(pid, port, command, mode, timestamp) @@ -1520,10 +1523,6 @@ managed_outgoing_image_records(attachment_id, session_key, agent_id, message_id, gateway_restart_sentinel(sentinel_key, version, kind, status, ts, session_key, thread_id, delivery_channel, delivery_to, delivery_account_id, message, continuation_json, doctor_hint, stats_json, payload_json, updated_at_ms) channel_pairing_requests(channel_key, account_id, request_id, code, created_at, last_seen_at, meta_json) channel_pairing_allow_entries(channel_key, account_id, entry, sort_order, updated_at) -voicewake_triggers(config_key, position, trigger, updated_at_ms) -voicewake_routing_config(config_key, version, default_target_mode, default_target_agent_id, default_target_session_key, updated_at_ms) -voicewake_routing_routes(config_key, position, trigger, target_mode, target_agent_id, target_session_key, updated_at_ms) -update_check_state(state_key, last_checked_at, last_notified_version, last_notified_tag, last_available_version, last_available_tag, auto_install_id, auto_first_seen_version, auto_first_seen_tag, auto_first_seen_at, auto_last_attempt_version, auto_last_attempt_at, auto_last_success_version, auto_last_success_at, updated_at_ms) config_health_entries(config_path, last_known_good_json, last_promoted_good_json, last_observed_suspicious_signature, updated_at_ms) sandbox_registry_entries(registry_kind, container_name, session_key, backend_id, runtime_label, image, created_at_ms, last_used_at_ms, config_label_kind, config_hash, cdp_port, no_vnc_port, entry_json, updated_at) cron_jobs(store_key, job_id, name, description, enabled, delete_after_run, created_at_ms, agent_id, session_key, schedule_kind, schedule_expr, schedule_tz, every_ms, anchor_ms, at, stagger_ms, session_target, wake_mode, payload_kind, payload_message, payload_model, payload_fallbacks_json, payload_thinking, payload_timeout_seconds, payload_allow_unsafe_external_content, payload_external_content_source_json, payload_light_context, payload_tools_allow_json, delivery_mode, delivery_channel, delivery_to, delivery_thread_id, delivery_account_id, delivery_best_effort, failure_delivery_mode, failure_delivery_channel, failure_delivery_to, failure_delivery_account_id, failure_alert_disabled, failure_alert_after, failure_alert_channel, failure_alert_to, failure_alert_cooldown_ms, failure_alert_include_skipped, failure_alert_mode, failure_alert_account_id, next_run_at_ms, running_at_ms, last_run_at_ms, last_run_status, last_error, last_duration_ms, consecutive_errors, consecutive_skipped, schedule_error_count, last_delivery_status, last_delivery_error, last_delivered, last_failure_alert_at_ms, job_json, state_json, runtime_updated_at_ms, schedule_identity, sort_order, updated_at) @@ -1533,6 +1532,13 @@ migration_sources(source_key, migration_kind, source_path, target_table, source_ backup_runs(id, created_at, archive_path, status, manifest_json) ``` +`config_machine_state` owns the `skills.curatorState`, `update.checkState`, +`clawhub.promotionsFeed`, `modelCatalog.remote`, `voicewake.triggers`, +`voicewake.routing`, `onboarding.recommendations.`, +`tui.lastSession.`, `sidebar.sectionOrder`, `nodeHost.config`, and +`webPush.vapidKeys` snapshots. Secret-excluding Git backups omit the `nodeHost.` +and `webPush.vapidKeys` key prefixes without dropping other machine state. + Agent database: ```text @@ -2223,7 +2229,7 @@ Add a repo check that fails new runtime writes to legacy state paths: - `identity/device.json` - `identity/device-auth.json` (retired; Doctor-only import into `device_auth_tokens`) - `push/web-push-subscriptions.json` (retired; Doctor-only import into `web_push_subscriptions`) -- `push/vapid-keys.json` (retired; Doctor-only import into `web_push_vapid_keys`) +- `push/vapid-keys.json` (retired; Doctor-only import into `webPush.vapidKeys`) - `push/apns-registrations.json` (retired; Doctor-only import into `apns_registrations`) - `process-leases.json` - `gateway-instance-id` diff --git a/docs/reference/database-schemas.md b/docs/reference/database-schemas.md index b7d05aa5f5b0..2a4389641ee1 100644 --- a/docs/reference/database-schemas.md +++ b/docs/reference/database-schemas.md @@ -128,6 +128,7 @@ Version 3 was an unshipped development step folded into version 4. | 9 | In-root agent database registry paths stored relative to the state directory | Unreleased | | 10 | Six dead tables retired (agent_model_catalogs, android_notification_recent_packages, command_log_entries, diagnostic_stability_bundles, media_blobs, model_capability_cache) | Unreleased | | 11 | Legacy skill curator lifecycle table and never-read proposal origin-run projection retired | Unreleased | +| 12 | Thirteen singleton/cache tables retired; durable state folded into config_machine_state | Unreleased | ### State schema 11 @@ -188,6 +189,160 @@ The general procedure is: 3. Set `PRAGMA user_version` and `schema_meta.schema_version` to the target version. 4. Run the target release's full database verification before starting the Gateway. +### Example: state schema 12 to 11 + +Schema 12 folded durable state snapshots into `config_machine_state` and retired rebuildable caches plus the write-only cron store epoch table. A schema 11 build still expects the thirteen former tables, so a manual downgrade must recreate their exact schemas and indexes before lowering the version. + +Run equivalent SQL against the global state database after inspecting the exact schema that wrote it: + +```sql +BEGIN IMMEDIATE; + +CREATE TABLE IF NOT EXISTS skill_curator_state ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + last_attempt_at_ms INTEGER NOT NULL, + last_success_at_ms INTEGER, + last_error TEXT, + last_result_json TEXT NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS onboarding_recommendations ( + config_key TEXT NOT NULL PRIMARY KEY, + inventory_hash TEXT NOT NULL, + matches_json TEXT NOT NULL, + offered_at_ms INTEGER NOT NULL, + accepted_at_ms INTEGER, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS voicewake_triggers ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger + ON voicewake_triggers(config_key, trigger); + +CREATE TABLE IF NOT EXISTS voicewake_routing_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + default_target_mode TEXT NOT NULL, + default_target_agent_id TEXT, + default_target_session_key TEXT, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + target_mode TEXT NOT NULL, + target_agent_id TEXT, + target_session_key TEXT, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position), + FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger + ON voicewake_routing_routes(config_key, trigger); + +CREATE TABLE IF NOT EXISTS update_check_state ( + state_key TEXT NOT NULL PRIMARY KEY, + last_checked_at TEXT, + last_notified_version TEXT, + last_notified_tag TEXT, + last_available_version TEXT, + last_available_tag TEXT, + auto_install_id TEXT, + auto_first_seen_version TEXT, + auto_first_seen_tag TEXT, + auto_first_seen_at TEXT, + auto_last_attempt_version TEXT, + auto_last_attempt_at TEXT, + auto_last_success_version TEXT, + auto_last_success_at TEXT, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( + state_key TEXT NOT NULL PRIMARY KEY, + etag TEXT, + payload_json TEXT, + feed_sequence INTEGER, + last_checked_at_ms INTEGER, + notified_slugs_json TEXT NOT NULL DEFAULT '[]', + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS cron_store_epochs ( + store_key TEXT PRIMARY KEY, + store_epoch INTEGER NOT NULL DEFAULT 0 +) STRICT; + +CREATE TABLE IF NOT EXISTS model_catalog_remote ( + id INTEGER PRIMARY KEY CHECK (id = 1), + bundle_json TEXT NOT NULL, + generated_at INTEGER NOT NULL, + min_version TEXT, + source_url TEXT NOT NULL, + etag TEXT, + last_modified TEXT, + checked_at INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS tui_last_sessions ( + scope_key TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + updated_at INTEGER NOT NULL +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key + ON tui_last_sessions(session_key, updated_at DESC, scope_key); + +CREATE TABLE IF NOT EXISTS sidebar_sections ( + section_id TEXT NOT NULL PRIMARY KEY, + position INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS node_host_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + node_id TEXT NOT NULL, + token TEXT, + display_name TEXT, + gateway_host TEXT, + gateway_port INTEGER, + gateway_tls INTEGER, + gateway_tls_fingerprint TEXT, + gateway_context_path TEXT, + gateway_cloudflare_access_json TEXT, + installed_apps_sharing INTEGER NOT NULL DEFAULT 0, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( + key_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + private_key TEXT NOT NULL, + subject TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +) STRICT; + +PRAGMA user_version = 11; +UPDATE schema_meta +SET schema_version = 11, + updated_at = unixepoch('now') * 1000 +WHERE meta_key = 'primary'; + +COMMIT; +``` + +The recreated tables start empty. Migrated voice wake settings, onboarding recommendations, update-check state, sidebar layout, node-host identity, and Web Push signing keys remain readable in `config_machine_state` under `voicewake.triggers`, `voicewake.routing`, `onboarding.recommendations.`, `update.checkState`, `sidebar.sectionOrder`, `nodeHost.config`, and `webPush.vapidKeys`; manually repopulate their former tables if the older build must retain those settings. Node-host identity and Web Push signing keys are sensitive: avoid copying their values into shell history or logs. Skill-curator, promotions-feed, remote-catalog, and TUI last-session caches can be rebuilt. A botched downgrade means restore from the verified backup. + ### Example: state schema 11 to 10 Schema 11 removed the retired skill lifecycle table and the never-read proposal diff --git a/docs/reference/secretref-credential-surface.md b/docs/reference/secretref-credential-surface.md index 78ec8eb7a914..ed41dc8b35a3 100644 --- a/docs/reference/secretref-credential-surface.md +++ b/docs/reference/secretref-credential-surface.md @@ -140,7 +140,8 @@ The lists below are generated from the source target registry and checked agains - `gateway.cloudflareAccess.clientId` - `gateway.cloudflareAccess.clientSecret` -These fields live in the node host's canonical `node_host_config` SQLite row, +These fields live in the node host's canonical `nodeHost.config` SQLite +machine-state value, not `openclaw.json`. They accept the same SecretInput forms and resolve through the configured SecretRef providers when the node starts. The conventional `CF_ACCESS_CLIENT_ID` / `CF_ACCESS_CLIENT_SECRET` fallback persists env refs for diff --git a/docs/web/control-ui.md b/docs/web/control-ui.md index fc20f105a2da..1c091b4dc31f 100644 --- a/docs/web/control-ui.md +++ b/docs/web/control-ui.md @@ -614,12 +614,12 @@ See [Notifications](/web/notifications) for the browser and macOS setup steps. If the page shows **Protocol mismatch** right after an OpenClaw update, first reopen the dashboard with `openclaw dashboard` and hard-refresh. If it still fails, clear site data for the dashboard origin or test in a private browser window; an old tab or browser service-worker cache can keep running a pre-update Control UI bundle against the newer Gateway. -| Surface | What it does | -| -------------------------------------------------- | ---------------------------------------------------------------------------- | -| `ui/public/manifest.webmanifest` | PWA manifest. Browsers offer "Install app" once it is reachable. | -| `ui/public/sw.js` | Service worker that handles `push` events and notification clicks. | -| `state/openclaw.sqlite` → `web_push_vapid_keys` | Auto-generated VAPID keypair used to sign Web Push payloads. | -| `state/openclaw.sqlite` → `web_push_subscriptions` | Persisted browser subscription endpoints, keys, and registration timestamps. | +| Surface | What it does | +| ---------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| `ui/public/manifest.webmanifest` | PWA manifest. Browsers offer "Install app" once it is reachable. | +| `ui/public/sw.js` | Service worker that handles `push` events and notification clicks. | +| `state/openclaw.sqlite` → `config_machine_state` (`webPush.vapidKeys`) | Auto-generated VAPID keypair used to sign Web Push payloads. | +| `state/openclaw.sqlite` → `web_push_subscriptions` | Persisted browser subscription endpoints, keys, and registration timestamps. | Upgrades from the retired `push/vapid-keys.json` and `push/web-push-subscriptions.json` stores are imported by `openclaw doctor --fix`. Stop the Gateway before running that repair so an older process cannot recreate retired state during import. Run the repair before using Web Push after an upgrade; registration, delivery, deletion, and key resolution refuse to proceed while either retired source or an interrupted Doctor claim remains. The Gateway runtime reads and writes SQLite only. diff --git a/extensions/voice-call/doctor-contract-api.ts b/extensions/voice-call/doctor-contract-api.ts index 94ec86500c24..ee88c891fa6f 100644 --- a/extensions/voice-call/doctor-contract-api.ts +++ b/extensions/voice-call/doctor-contract-api.ts @@ -148,6 +148,8 @@ function describeVoiceCallSchemaMigration(migration: OpenClawStateDatabaseSchema return "retired shared-state tables -> removed tables and indexes"; case "state-table-retirement-v11": return "retired skill curator tables -> removed tables and indexes"; + case "singleton-state-foldin-v12": + return "singleton state tables -> shared configuration state"; case "worker-placement-execution-mode-v8": return "cloud worker placements -> execution-mode claims"; case "operator-approvals-system-agent": diff --git a/package.json b/package.json index f866c121d64a..d2b2225d4254 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2026.8.1", "openclaw": { "schemaVersions": { - "state": 11, + "state": 12, "agent": 17 } }, diff --git a/packages/memory-host-sdk/src/host/session-files.path.test.ts b/packages/memory-host-sdk/src/host/session-files.path.test.ts new file mode 100644 index 000000000000..a20a79ca302c --- /dev/null +++ b/packages/memory-host-sdk/src/host/session-files.path.test.ts @@ -0,0 +1,46 @@ +// Memory Host SDK tests cover session transcript path derivation. +import fsSync from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { sessionPathForFile } from "./session-files.js"; + +let tmpDir: string; +let previousStateDir: string | undefined; + +beforeEach(() => { + tmpDir = fsSync.mkdtempSync(path.join(os.tmpdir(), "session-path-test-")); + previousStateDir = process.env.OPENCLAW_STATE_DIR; + Reflect.set(process.env, "OPENCLAW_STATE_DIR", tmpDir); +}); + +afterEach(() => { + if (previousStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", previousStateDir); + } + fsSync.rmSync(tmpDir, { recursive: true, force: true }); +}); + +describe("sessionPathForFile", () => { + it("includes the owning agent id when the transcript lives under an agent sessions dir", () => { + const absPath = path.join( + tmpDir, + "agents", + "main", + "sessions", + "deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z", + ); + + expect(sessionPathForFile(absPath)).toBe( + "sessions/main/deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z", + ); + }); + + it("keeps the legacy basename-only path when the agent owner cannot be derived", () => { + expect(sessionPathForFile(path.join(tmpDir, "loose-session.jsonl"))).toBe( + "sessions/loose-session.jsonl", + ); + }); +}); diff --git a/packages/memory-host-sdk/src/host/session-files.test.ts b/packages/memory-host-sdk/src/host/session-files.test.ts index bcce90dfee63..091b96a17d2f 100644 --- a/packages/memory-host-sdk/src/host/session-files.test.ts +++ b/packages/memory-host-sdk/src/host/session-files.test.ts @@ -26,7 +26,6 @@ import { parseCanonicalSessionSyncTargetFromPath, resolveSessionIdentityForTranscriptFile, resolveSessionFileForSyncTarget, - sessionPathForFile, statSessionEntrySync, type SessionFileEntry, } from "./session-files.js"; @@ -754,28 +753,6 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => { }); }); -describe("sessionPathForFile", () => { - it("includes the owning agent id when the transcript lives under an agent sessions dir", () => { - const absPath = path.join( - tmpDir, - "agents", - "main", - "sessions", - "deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z", - ); - - expect(sessionPathForFile(absPath)).toBe( - "sessions/main/deleted-session.jsonl.deleted.2026-02-16T22-27-33.000Z", - ); - }); - - it("keeps the legacy basename-only path when the agent owner cannot be derived", () => { - expect(sessionPathForFile(path.join(tmpDir, "loose-session.jsonl"))).toBe( - "sessions/loose-session.jsonl", - ); - }); -}); - describe("memory session sync targets", () => { it("parses deprecated canonical OpenClaw transcript paths into sync identity", () => { const sessionFile = path.join(tmpDir, "agents", "main", "sessions", "active.jsonl"); diff --git a/scripts/check-kysely-guardrails.mts b/scripts/check-kysely-guardrails.mts index c3fd36a59f6a..0691a0fc3000 100644 --- a/scripts/check-kysely-guardrails.mts +++ b/scripts/check-kysely-guardrails.mts @@ -66,6 +66,7 @@ const rawSqliteAllowPathGroups = { "src/state/openclaw-state-db-schema-additive.ts", "src/state/openclaw-state-db-schema-helpers.ts", "src/state/openclaw-state-db-schema-repair.ts", + "src/state/openclaw-state-db-schema-v12-foldin.ts", "src/state/openclaw-state-db-startup-checkpoint.ts", "src/state/openclaw-state-db-table-retirements.ts", "src/state/openclaw-state-db-fast-path.ts", @@ -139,7 +140,6 @@ const rawSqliteAllowPathGroups = { "src/plugin-state/plugin-state-store.sqlite.ts", "src/tasks/task-flow-registry.store.sqlite.ts", "src/tasks/task-registry.store.sqlite.ts", - "src/tui/tui-last-session.ts", ], }; diff --git a/src/audit/message-delivery-progress-store.test.ts b/src/audit/message-delivery-progress-store.test.ts index ee81907add75..be4d4b1cf154 100644 --- a/src/audit/message-delivery-progress-store.test.ts +++ b/src/audit/message-delivery-progress-store.test.ts @@ -11,6 +11,7 @@ import { } from "../state/openclaw-state-db.js"; import { STATE_SCHEMA_10_TO_9_DOWNGRADE_SQL } from "../state/openclaw-state-schema-v10-retirement.test-support.js"; import { STATE_SCHEMA_11_TO_10_TABLES_SQL } from "../state/openclaw-state-schema-v11-retirement.test-support.js"; +import { STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL } from "../state/openclaw-state-schema-v12-foldin.test-support.js"; import { recordAuditEvent } from "./audit-event-store.js"; import type { OutboundMessageProgressInput } from "./audit-event-types.js"; import { @@ -145,6 +146,7 @@ describe("outbound message progress companion", () => { expect(opened.db.prepare("PRAGMA user_version").get()).toEqual({ user_version: OPENCLAW_STATE_SCHEMA_VERSION, }); + expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(12); expect(tableExists(opened.db, "outbound_message_progress")).toBe(false); expect(tableExists(opened.db, "outbound_message_execution_bindings")).toBe(false); @@ -275,13 +277,13 @@ describe("outbound message progress companion", () => { ).toBe(true); // This pinned reader predates the Workshop's first-use column and requires present lazy tables // to retain its exact shape; project that unrelated table to the reader's historical contract. - // It is a v9-era build, so both later retirements must be undone before the - // version markers rewind: v11's curator tables, then the documented 10->9 recipe. - openOpenClawStateDatabase(database).db.exec( - "ALTER TABLE skill_workshop_proposals DROP COLUMN claim_released_time;", - ); - openOpenClawStateDatabase(database).db.exec(STATE_SCHEMA_11_TO_10_TABLES_SQL); - openOpenClawStateDatabase(database).db.exec(STATE_SCHEMA_10_TO_9_DOWNGRADE_SQL); + // The v9-era reader needs the v12 singleton fold-in, v11 curator retirement, + // and v10 dead-table retirement projected backward in migration order. + const projectedDatabase = openOpenClawStateDatabase(database).db; + projectedDatabase.exec("ALTER TABLE skill_workshop_proposals DROP COLUMN claim_released_time;"); + projectedDatabase.exec(STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL); + projectedDatabase.exec(STATE_SCHEMA_11_TO_10_TABLES_SQL); + projectedDatabase.exec(STATE_SCHEMA_10_TO_9_DOWNGRADE_SQL); closeOpenClawStateDatabaseForTest(); const repositoryRoot = process.cwd(); diff --git a/src/commands/backup-git.test.ts b/src/commands/backup-git.test.ts index 7380ad01a0d0..fb6518482ca9 100644 --- a/src/commands/backup-git.test.ts +++ b/src/commands/backup-git.test.ts @@ -49,6 +49,7 @@ describe("Git backup command agent selection", () => { mocks.restoreGitBackupRef.mockReset().mockResolvedValue({ commit: "backup-commit", excludedTables: [], + excludedConfigStateKeyPrefixes: [], targetPath: "/tmp/restored.sqlite", }); mocks.verifyGitBackupRef.mockReset().mockResolvedValue({ diff --git a/src/commands/backup-git.ts b/src/commands/backup-git.ts index 5945712c1adf..f16e0af4e9d0 100644 --- a/src/commands/backup-git.ts +++ b/src/commands/backup-git.ts @@ -239,6 +239,11 @@ export async function backupGitRestoreCommand( `Warning: this redacted backup omits tables: ${result.excludedTables.join(", ")}`, ); } + if (result.excludedConfigStateKeyPrefixes.length > 0) { + runtime.error( + `Warning: this redacted backup omits machine-state values under: ${result.excludedConfigStateKeyPrefixes.join(", ")}`, + ); + } } return result; } diff --git a/src/commands/doctor-session-incognito-key-repair-state.ts b/src/commands/doctor-session-incognito-key-repair-state.ts index a7ea275f49d0..f57a66028489 100644 --- a/src/commands/doctor-session-incognito-key-repair-state.ts +++ b/src/commands/doctor-session-incognito-key-repair-state.ts @@ -25,6 +25,18 @@ type SharedStateSchemaDatabase = { type DynamicSharedStateDatabase = Record>; +function listTuiLastSessionStateRows(database: DatabaseSync) { + const db = + getNodeSqliteKysely>(database); + return executeSqliteQuerySync( + database, + db + .selectFrom("config_machine_state") + .select(["state_key", "value_json"]) + .where("state_key", "like", "tui.lastSession.%"), + ).rows; +} + function sqliteSchemaIdentifier(value: string) { return sql.id(value); // kysely-allow-raw -- value comes only from SQLite schema metadata. } @@ -80,6 +92,12 @@ export function collectSharedStateSessionKeys(database: DatabaseSync): Set>(database); + for (const row of listTuiLastSessionStateRows(database)) { + const sessionKey: unknown = JSON.parse(row.value_json); + const renamedKey = typeof sessionKey === "string" ? renames.get(sessionKey) : undefined; + if (renamedKey) { + executeSqliteQuerySync( + database, + stateDb + .updateTable("config_machine_state") + .set({ value_json: JSON.stringify(renamedKey) }) + .where("state_key", "=", row.state_key), + ); + } + } } function collectJsonStringValues(value: unknown, values: Set): void { diff --git a/src/commands/doctor-session-incognito-key-repair.test.ts b/src/commands/doctor-session-incognito-key-repair.test.ts index d29938955695..db84cec3f373 100644 --- a/src/commands/doctor-session-incognito-key-repair.test.ts +++ b/src/commands/doctor-session-incognito-key-repair.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createTempDirTracker } from "../../test/helpers/temp-dir.js"; import { listSessionEntriesCore } from "../config/sessions/session-accessor.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, @@ -109,11 +110,8 @@ describe("doctor reserved incognito session key repair", () => { "INSERT INTO session_watch_cursors (watcher_session_key, target_session_key, updated_at) VALUES (?, ?, 1)", ) .run(oldKey, oldKey); - stateDatabase.db - .prepare( - "INSERT INTO tui_last_sessions (scope_key, session_key, updated_at) VALUES ('main', ?, 1)", - ) - .run(oldKey); + writeConfigMachineState("tui.lastSession.main", oldKey, { env }); + writeConfigMachineState("unrelated.sessionReference", oldKey, { env }); database.db .prepare( "INSERT INTO heartbeat_outcomes (session_key, run_session_key, outcome, summary, occurred_at, updated_at) VALUES (?, ?, 'done', 'done', 1, 1)", @@ -192,9 +190,8 @@ describe("doctor reserved incognito session key repair", () => { .prepare("SELECT watcher_session_key, target_session_key FROM session_watch_cursors") .get(), ).toEqual({ watcher_session_key: newKey, target_session_key: newKey }); - expect(stateDatabase.db.prepare("SELECT session_key FROM tui_last_sessions").get()).toEqual({ - session_key: newKey, - }); + expect(readConfigMachineState("tui.lastSession.main", { env })).toBe(newKey); + expect(readConfigMachineState("unrelated.sessionReference", { env })).toBe(oldKey); expect( stateDatabase.db .prepare("SELECT source_session_key, audience_session_keys_json FROM operator_approvals") diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index 9afbc3884fc2..de3b474ea3ca 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -30,6 +30,7 @@ import { } from "../plugins/installed-plugin-index-store.js"; import type { InstalledPluginInstallRecordInfo } from "../plugins/installed-plugin-index.js"; import { EMPTY_LEGACY_SESSION_SURFACES } from "../plugins/legacy-session-surfaces.types.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest, @@ -2930,11 +2931,14 @@ describe("doctor legacy state migrations", () => { "utf8", ); const { db } = openOpenClawStateDatabase({ env }); - db.prepare( - `INSERT INTO update_check_state ( - state_key, last_checked_at, last_available_version, updated_at_ms - ) VALUES (?, ?, ?, ?)`, - ).run("default", "2026-07-14T00:00:00.000Z", "2026.7.2", 1); + writeConfigMachineState( + "update.checkState", + { + lastCheckedAt: "2026-07-14T00:00:00.000Z", + lastAvailableVersion: "2026.7.2", + }, + { env }, + ); db.prepare( `INSERT INTO config_health_entries ( config_path, last_known_good_json, last_promoted_good_json, @@ -2968,13 +2972,9 @@ describe("doctor legacy state migrations", () => { demo: { source: "npm", spec: "demo@1.0.0", version: "1.0.0" }, }, }); - expect( - db - .prepare( - "SELECT last_available_version FROM update_check_state WHERE state_key = 'default'", - ) - .get(), - ).toMatchObject({ last_available_version: "2026.7.2" }); + expect(readConfigMachineState("update.checkState", { env })).toMatchObject({ + lastAvailableVersion: "2026.7.2", + }); expect( db .prepare("SELECT last_known_good_json FROM config_health_entries WHERE config_path = ?") diff --git a/src/cron/legacy-default-agent-owner-migration.test.ts b/src/cron/legacy-default-agent-owner-migration.test.ts index 197a8cff01ed..bbc8ccb31094 100644 --- a/src/cron/legacy-default-agent-owner-migration.test.ts +++ b/src/cron/legacy-default-agent-owner-migration.test.ts @@ -12,7 +12,6 @@ import { CronService } from "./service.js"; import * as cronStoreModule from "./store.js"; import { cronStoreKey } from "./store/key.js"; import { loadCronRows, replaceCronRows } from "./store/row-codec.js"; -import { ensureCronStoreEpochSchema } from "./store/schema.js"; import type { CronStoreFile } from "./types.js"; afterEach(() => { @@ -33,7 +32,7 @@ function fixture(label: string) { return { env, storePath, storeKey, database }; } -it("preserves undecodable JSON and bumps the epoch once", async () => { +it("preserves undecodable JSON while assigning its owner", async () => { const { env, storePath, storeKey, database } = fixture("openclaw-cron-owner-"); database .prepare("UPDATE cron_jobs SET agent_id = ' ', job_json = ? WHERE store_key = ?") @@ -44,13 +43,6 @@ it("preserves undecodable JSON and bumps the epoch once", async () => { agent_id: "ops", job_json: "{malformed", }); - expect( - ( - database - .prepare("SELECT store_epoch FROM cron_store_epochs WHERE store_key = ?") - .get(storeKey) as { store_epoch: number } - ).store_epoch, - ).toBe(1); }); it("preserves a session-scoped owner stored only in job JSON", async () => { @@ -75,16 +67,6 @@ it("preserves a session-scoped owner stored only in job JSON", async () => { expect(preservedJobJson).not.toHaveProperty("agentId"); }); -it("rolls back the row when the epoch bump fails", async () => { - const { env, storePath, storeKey, database } = fixture("openclaw-cron-atomic-"); - ensureCronStoreEpochSchema(database); - database.exec(`CREATE TRIGGER fail_epoch BEFORE UPDATE OF store_epoch ON cron_store_epochs - BEGIN SELECT RAISE(ABORT, 'synthetic epoch failure'); END`); - - await expect(migrate(storePath, env)).rejects.toThrow("synthetic epoch failure"); - expect(loadCronRows(database, storeKey)[0]?.agent_id).toBeNull(); -}); - it("materializes before scheduler startup", async () => { const { env, storePath } = fixture("openclaw-cron-startup-"); vi.stubEnv("OPENCLAW_STATE_DIR", env.OPENCLAW_STATE_DIR); diff --git a/src/cron/store/row-codec.ts b/src/cron/store/row-codec.ts index b81ac8e14a88..98a67e88578b 100644 --- a/src/cron/store/row-codec.ts +++ b/src/cron/store/row-codec.ts @@ -32,7 +32,7 @@ import { tryParseJsonObject, } from "./scalar-codec.js"; import type { CronJobInsert, CronJobRow } from "./schema.js"; -import { ensureCronStoreEpochSchema, getCronStoreKysely } from "./schema.js"; +import { getCronStoreKysely } from "./schema.js"; import { bindStateColumns, stateFromRow } from "./state-codec.js"; import { bindTriggerColumns, triggerFromRow } from "./trigger-codec.js"; import type { LoadedCronStore } from "./types.js"; @@ -402,25 +402,7 @@ export function loadCronRows(db: DatabaseSync, storeKey: string): CronJobRow[] { ).rows; } -function incrementCronStoreEpoch(db: DatabaseSync, storeKey: string): void { - ensureCronStoreEpochSchema(db); - executeSqliteQuerySync( - db, - getCronStoreKysely(db) - .insertInto("cron_store_epochs") - .values({ store_key: storeKey, store_epoch: 0 }) - .onConflict((conflict) => conflict.column("store_key").doNothing()), - ); - executeSqliteQuerySync( - db, - getCronStoreKysely(db) - .updateTable("cron_store_epochs") - .set((eb) => ({ store_epoch: eb("store_epoch", "+", 1) })) - .where("store_key", "=", storeKey), - ); -} - -/** Materializes retired ownership; the caller's transaction commits row and epoch updates together. */ +/** Materializes retired ownership within the caller's write transaction. */ export function materializeCronRowAgentOwners( db: DatabaseSync, storeKey: string, @@ -457,9 +439,6 @@ export function materializeCronRowAgentOwners( ); rewritten += 1; } - if (rewritten > 0) { - incrementCronStoreEpoch(db, storeKey); - } return rewritten; } diff --git a/src/cron/store/schema.ts b/src/cron/store/schema.ts index 97ad54d90693..a94abc4e33ea 100644 --- a/src/cron/store/schema.ts +++ b/src/cron/store/schema.ts @@ -5,10 +5,7 @@ import { getNodeSqliteKysely } from "../../infra/kysely-sync.js"; import type { DB as OpenClawStateKyselyDatabase } from "../../state/openclaw-state-db.generated.js"; type CronJobsTable = OpenClawStateKyselyDatabase["cron_jobs"]; -type CronStoreDatabase = Pick< - OpenClawStateKyselyDatabase, - "cron_job_scratch" | "cron_jobs" | "cron_store_epochs" ->; +type CronStoreDatabase = Pick; /** Read shape for rows in the cron_jobs SQLite table. */ export type CronJobRow = Selectable; @@ -20,12 +17,3 @@ export type CronJobInsert = Insertable; export function getCronStoreKysely(db: DatabaseSync) { return getNodeSqliteKysely(db); } - -export function ensureCronStoreEpochSchema(db: DatabaseSync): void { - db.exec(/* sqlite-allow-raw: additive schema DDL is outside Kysely's query builder. */ ` - CREATE TABLE IF NOT EXISTS cron_store_epochs ( - store_key TEXT PRIMARY KEY, - store_epoch INTEGER NOT NULL DEFAULT 0 - ) STRICT - `); -} diff --git a/src/gateway/server-methods/skills.proposals.test.ts b/src/gateway/server-methods/skills.proposals.test.ts index 5d97ae49712a..dcd15c7eef8e 100644 --- a/src/gateway/server-methods/skills.proposals.test.ts +++ b/src/gateway/server-methods/skills.proposals.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { readSkillProposalEvents } from "../../skills/workshop/store-evaluation.js"; -import { openOpenClawStateDatabase } from "../../state/openclaw-state-db.js"; +import { writeConfigMachineState } from "../../state/config-machine-state.js"; import { createOpenClawTestState, type OpenClawTestState, @@ -249,20 +249,19 @@ describe("skills proposal gateway handlers", () => { }); it("returns the stored review outcomes from curator status", async () => { - openOpenClawStateDatabase({ env: testState.env }) - .db.prepare( - "INSERT INTO skill_curator_state (id, last_attempt_at_ms, last_success_at_ms, last_error, last_result_json) VALUES (?, ?, ?, ?, ?)", - ) - .run( - 1, - 100, - 100, - null, - JSON.stringify({ + writeConfigMachineState( + "skills.curatorState", + { + lastAttemptAtMs: 100, + lastSuccessAtMs: 100, + lastError: null, + lastResult: { collectionReviews: { workspace: { attemptedAtMs: 100, succeededAtMs: 101 } }, experienceReviews: { workspace: { attemptedAtMs: 102, outcome: "nothing" } }, - }), - ); + }, + }, + { env: testState.env }, + ); await expect(callHandler("skills.curator.status", {})).resolves.toMatchObject({ ok: true, diff --git a/src/gateway/session-groups.test.ts b/src/gateway/session-groups.test.ts index c5192ca53b9c..9f279a848a25 100644 --- a/src/gateway/session-groups.test.ts +++ b/src/gateway/session-groups.test.ts @@ -6,6 +6,7 @@ import type { SessionEntry } from "../config/sessions.js"; import { loadSessionEntry, replaceSessionEntry } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest, @@ -79,44 +80,18 @@ describe("session groups catalog", () => { env, ); expect(listSessionGroups(env).map((group) => group.name)).toEqual(["Alpha", "Beta"]); - expect(listSidebarSectionOrder(env)).toEqual([ + const expectedSectionOrder = [ "work", "catalog:codex", "category:Beta", "category:Alpha", "groups", - ]); + ]; + expect(listSidebarSectionOrder(env)).toEqual(expectedSectionOrder); + expect(readConfigMachineState("sidebar.sectionOrder", { env })).toEqual(expectedSectionOrder); putSessionGroups(["Beta", "Alpha"], undefined, env); - expect(listSidebarSectionOrder(env)).toEqual([ - "work", - "catalog:codex", - "category:Beta", - "category:Alpha", - "groups", - ]); - }); - - it("lazily adds sidebar_sections to a pre-existing current-schema database", () => { - const databasePath = openOpenClawStateDatabase({ env }).path; - closeOpenClawStateDatabaseForTest(); - const { DatabaseSync } = requireNodeSqlite(); - const legacy = new DatabaseSync(databasePath); - legacy.exec("DROP TABLE sidebar_sections;"); - legacy.close(); - - const reopened = openOpenClawStateDatabase({ env }); - expect( - reopened.db - .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") - .get("sidebar_sections"), - ).toBeUndefined(); - expect(listSidebarSectionOrder(env)).toEqual([]); - expect( - reopened.db - .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") - .get("sidebar_sections"), - ).toEqual({ name: "sidebar_sections" }); + expect(listSidebarSectionOrder(env)).toEqual(expectedSectionOrder); }); it("keeps catalog reads and reorders schema-read-only until defaults are used", async () => { diff --git a/src/gateway/session-groups.ts b/src/gateway/session-groups.ts index 35deabd8e582..52c98024df44 100644 --- a/src/gateway/session-groups.ts +++ b/src/gateway/session-groups.ts @@ -7,6 +7,7 @@ import { resolveAllAgentSessionStoreTargetsSync } from "../config/sessions.js"; import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; import { ensureColumn, tableHasColumn } from "../state/openclaw-state-db-schema-helpers.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { @@ -32,7 +33,7 @@ type SessionGroupDefaultsRecord = { type SessionGroupsDatabase = Pick< OpenClawStateKyselyDatabase, - "session_groups" | "sidebar_sections" + "session_groups" | "config_machine_state" >; export class SessionGroupNotFoundError extends Error { @@ -42,14 +43,8 @@ export class SessionGroupNotFoundError extends Error { } } -const ensuredSidebarSectionDatabases = new WeakSet(); const ensuredSessionGroupDefaultsDatabases = new WeakSet(); -const SIDEBAR_SECTIONS_SCHEMA_SQL = ` -CREATE TABLE IF NOT EXISTS sidebar_sections ( - section_id TEXT NOT NULL PRIMARY KEY, - position INTEGER NOT NULL -) STRICT; -`; +const SIDEBAR_SECTION_ORDER_STATE_KEY = "sidebar.sectionOrder"; function dbFor(env: NodeJS.ProcessEnv): DatabaseSync { return openOpenClawStateDatabase({ env }).db; @@ -59,20 +54,43 @@ function kyselyFor(db: DatabaseSync) { return getNodeSqliteKysely(db); } -function ensureSidebarSectionsSchema(env: NodeJS.ProcessEnv): void { - const database = openOpenClawStateDatabase({ env }); - if (ensuredSidebarSectionDatabases.has(database.db)) { +// Config-machine-state helpers open their own transaction; use direct Kysely +// so sidebar edits stay inside the existing session-group write transaction. +function updateSidebarSectionOrder( + db: DatabaseSync, + update: (current: string[] | undefined) => string[] | undefined, +): void { + const kysely = kyselyFor(db); + const row = executeSqliteQuerySync( + db, + kysely + .selectFrom("config_machine_state") + .select("value_json") + .where("state_key", "=", SIDEBAR_SECTION_ORDER_STATE_KEY), + ).rows[0]; + // SAFETY: The sidebar owner and v12 migration store this key only as a string array. + const next = update(row ? (JSON.parse(row.value_json) as string[]) : undefined); + if (!next) { return; } - runOpenClawStateWriteTransaction( - ({ db }) => { - // sqlite-allow-raw -- feature-local additive schema DDL; rows use Kysely below. - db.exec(SIDEBAR_SECTIONS_SCHEMA_SQL); - }, - { env }, - { operationLabel: "session-groups.sidebar-sections.schema.ensure" }, + const valueJson = JSON.stringify(next); + const updatedAtMs = Date.now(); + executeSqliteQuerySync( + db, + kysely + .insertInto("config_machine_state") + .values({ + state_key: SIDEBAR_SECTION_ORDER_STATE_KEY, + value_json: valueJson, + updated_at_ms: updatedAtMs, + }) + .onConflict((conflict) => + conflict.column("state_key").doUpdateSet({ + value_json: valueJson, + updated_at_ms: updatedAtMs, + }), + ), ); - ensuredSidebarSectionDatabases.add(database.db); } function hasSessionGroupDefaultsSchema(db: DatabaseSync): boolean { @@ -164,16 +182,7 @@ export function listSessionGroupDefaults( } export function listSidebarSectionOrder(env: NodeJS.ProcessEnv = process.env): string[] { - ensureSidebarSectionsSchema(env); - const db = dbFor(env); - return executeSqliteQuerySync( - db, - kyselyFor(db) - .selectFrom("sidebar_sections") - .select("section_id") - .orderBy("position", "asc") - .orderBy("section_id", "asc"), - ).rows.map((row) => row.section_id); + return readConfigMachineState(SIDEBAR_SECTION_ORDER_STATE_KEY, { env }) ?? []; } /** Replaces the ordered catalog. Sessions keep their category even when a name is dropped. */ @@ -185,9 +194,6 @@ export function putSessionGroups( const normalized = normalizeGroupNames(names); const normalizedSectionOrder = sectionOrder === undefined ? undefined : normalizeSidebarSectionOrder(sectionOrder, normalized); - if (normalizedSectionOrder) { - ensureSidebarSectionsSchema(env); - } const now = Date.now(); runOpenClawStateWriteTransaction( ({ db }) => { @@ -218,13 +224,7 @@ export function putSessionGroups( ); }); if (normalizedSectionOrder) { - executeSqliteQuerySync(db, kysely.deleteFrom("sidebar_sections")); - normalizedSectionOrder.forEach((sectionId, position) => { - executeSqliteQuerySync( - db, - kysely.insertInto("sidebar_sections").values({ section_id: sectionId, position }), - ); - }); + updateSidebarSectionOrder(db, () => normalizedSectionOrder); // `names` remains authoritative for group-only surfaces such as the Sessions page. // The sidebar stores the caller's cross-section order without silently deriving it. } @@ -277,7 +277,6 @@ export function ensureSessionGroupRegistered( } function renameCatalogEntry(from: string, to: string, env: NodeJS.ProcessEnv): void { - ensureSidebarSectionsSchema(env); runOpenClawStateWriteTransaction( ({ db }) => { const kysely = kyselyFor(db); @@ -301,30 +300,18 @@ function renameCatalogEntry(from: string, to: string, env: NodeJS.ProcessEnv): v ).rows[0]; const sourceSectionId = `category:${from}`; const targetSectionId = `category:${to}`; - const targetSectionExists = executeSqliteQuerySync( - db, - kysely - .selectFrom("sidebar_sections") - .select("section_id") - .where("section_id", "=", targetSectionId) - .limit(1), - ).rows[0]; executeSqliteQuerySync(db, kysely.deleteFrom("session_groups").where("name", "=", from)); - if (targetSectionExists) { + updateSidebarSectionOrder(db, (current) => { + if (!current?.includes(sourceSectionId)) { + return undefined; + } // A target slot already owns the merged group's position; retire the source slot. - executeSqliteQuerySync( - db, - kysely.deleteFrom("sidebar_sections").where("section_id", "=", sourceSectionId), - ); - } else { - executeSqliteQuerySync( - db, - kysely - .updateTable("sidebar_sections") - .set({ section_id: targetSectionId }) - .where("section_id", "=", sourceSectionId), - ); - } + return current.includes(targetSectionId) + ? current.filter((sectionId) => sectionId !== sourceSectionId) + : current.map((sectionId) => + sectionId === sourceSectionId ? targetSectionId : sectionId, + ); + }); if (targetExists) { // Rename into an existing group merges memberships; keep its catalog row. return; @@ -488,14 +475,15 @@ export async function deleteSessionGroup(params: { throw new Error("group delete requires a non-empty name"); } params.assertCurrent?.(); - ensureSidebarSectionsSchema(env); runOpenClawStateWriteTransaction( ({ db }) => { const kysely = kyselyFor(db); executeSqliteQuerySync(db, kysely.deleteFrom("session_groups").where("name", "=", name)); - executeSqliteQuerySync( - db, - kysely.deleteFrom("sidebar_sections").where("section_id", "=", `category:${name}`), + const sectionId = `category:${name}`; + updateSidebarSectionOrder(db, (current) => + current?.includes(sectionId) + ? current.filter((section) => section !== sectionId) + : undefined, ); }, { env }, diff --git a/src/gateway/worker-environments/placement-store.move.test.ts b/src/gateway/worker-environments/placement-store.move.test.ts index 10f4db31bb29..6fd689d854ce 100644 --- a/src/gateway/worker-environments/placement-store.move.test.ts +++ b/src/gateway/worker-environments/placement-store.move.test.ts @@ -4,7 +4,6 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { closeOpenClawStateDatabaseForTest, - OPENCLAW_STATE_SCHEMA_VERSION, openOpenClawStateDatabase, type OpenClawStateDatabase, } from "../../state/openclaw-state-db.js"; @@ -157,9 +156,7 @@ describe("worker session placement moves", () => { }, }); expect(begun.intent.operationId).toMatch(/^move:v1:[A-Za-z0-9_-]{43}$/u); - expect(database.db.prepare("PRAGMA user_version").get()).toEqual({ - user_version: OPENCLAW_STATE_SCHEMA_VERSION, - }); + expect(database.db.prepare("PRAGMA user_version").get()).toEqual({ user_version: 12 }); expect(store.getPlacementMove(SESSION.sessionId)).toEqual(begun.intent); expect(store.getPlacementMoves([SESSION.sessionId, "missing"])).toEqual( new Map([[SESSION.sessionId, begun.intent]]), diff --git a/src/infra/promotions-feed.test.ts b/src/infra/promotions-feed.test.ts index cd52c698097e..2a146bd8f0de 100644 --- a/src/infra/promotions-feed.test.ts +++ b/src/infra/promotions-feed.test.ts @@ -1,17 +1,13 @@ // Covers the promotions feed cache: refresh cadence, 304 revalidation, // sequence monotonicity, notified markers, and claim provenance. import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { - closeOpenClawStateDatabaseForTest, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; +import { updateConfigMachineState } from "../state/config-machine-state.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { useMockHttp } from "../test-utils/mock-http.js"; import { createOpenClawTestState, type OpenClawTestState, } from "../test-utils/openclaw-test-state.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; import { listLivePromotionEntries, markPromotionSlugsNotified, @@ -167,17 +163,10 @@ describe("promotions feed state", () => { reply: { json: feedPayload({ sequence: 5 }), headers: { etag: '"v5"' } }, }); await maybeRefreshPromotionsFeed({ nowMs: NOW, fetchImpl: globalThis.fetch }); - runOpenClawStateWriteTransaction(({ db }) => { - const kysely = - getNodeSqliteKysely>(db); - executeSqliteQuerySync( - db, - kysely - .updateTable("clawhub_promotions_feed_state") - .set({ payload_json: "{invalid" }) - .where("state_key", "=", "default"), - ); - }); + updateConfigMachineState>("clawhub.promotionsFeed", (current) => ({ + ...current, + payloadJson: "{invalid", + })); const state = await maybeRefreshPromotionsFeed({ nowMs: NOW + 60_000, diff --git a/src/infra/promotions-feed.ts b/src/infra/promotions-feed.ts index 1acf58929d6a..7dbfb6936f96 100644 --- a/src/infra/promotions-feed.ts +++ b/src/infra/promotions-feed.ts @@ -1,3 +1,4 @@ +import { readConfigMachineState, updateConfigMachineState } from "../state/config-machine-state.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { openOpenClawStateDatabase, @@ -8,29 +9,30 @@ import { fetchClawHubPromotionsFeed, parseClawHubPromotionsFeed, } from "./clawhub-promotions.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; // Passive-discovery cache for the ClawHub promotions feed. Deliberately a -// separate store from `update_check_state`: promo discovery must never +// separate key from `update.checkState`: promo discovery must never // delay, break, or contend with update checks. The cache is best-effort — // every reader falls back to "no promotions" on any storage or parse error, // and `promos claim` always revalidates against the live API. -const PROMOTIONS_FEED_STATE_KEY = "default"; +const PROMOTIONS_FEED_STATE_KEY = "clawhub.promotionsFeed"; const PROMOTIONS_FEED_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000; // Refreshes run inline from interactive commands, so they get a short // timeout (matching the update check's 2.5s) instead of ClawHub's default // 30s — a blackholed connection must not stall `models list`. const PROMOTIONS_FEED_FETCH_TIMEOUT_MS = 2500; -type PromotionsFeedDatabase = Pick< - OpenClawStateKyselyDatabase, - "clawhub_promotions_feed_state" | "clawhub_promotion_claims" ->; +type PromotionsFeedDatabase = Pick; + +type StoredPromotionsFeedState = { + etag: string | null; + sequence: number | null; + payloadJson: string | null; + lastCheckedAtMs: number | null; + notifiedSlugs: string[]; +}; type PromotionsFeedState = { etag?: string; @@ -56,35 +58,10 @@ type PromotionsFeedStateRead = { payloadInvalid: boolean; }; -function parseSlugListJson(raw: string | null): Set { - if (!raw) { - return new Set(); - } - const parsed = JSON.parse(raw) as unknown; - if (!Array.isArray(parsed)) { - return new Set(); - } - return new Set(parsed.filter((entry): entry is string => typeof entry === "string")); -} - function readPromotionsFeedStateWithMetadata(): PromotionsFeedStateRead { try { - const database = openOpenClawStateDatabase(); - const db = getNodeSqliteKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("clawhub_promotions_feed_state") - .select([ - "etag", - "payload_json", - "feed_sequence", - "last_checked_at_ms", - "notified_slugs_json", - ]) - .where("state_key", "=", PROMOTIONS_FEED_STATE_KEY), - ); - if (!row) { + const stored = readConfigMachineState(PROMOTIONS_FEED_STATE_KEY); + if (!stored) { return { state: { ...EMPTY_STATE, notifiedSlugs: new Set() }, payloadInvalid: false, @@ -93,9 +70,9 @@ function readPromotionsFeedStateWithMetadata(): PromotionsFeedStateRead { let entries: ClawHubPromotionsFeedEntry[] = []; let expiresAtMs: number | undefined; let payloadInvalid = false; - if (row.payload_json) { + if (stored.payloadJson) { try { - const feed = parseClawHubPromotionsFeed(JSON.parse(row.payload_json)); + const feed = parseClawHubPromotionsFeed(JSON.parse(stored.payloadJson)); entries = feed.entries; expiresAtMs = Date.parse(feed.expiresAt); } catch { @@ -104,16 +81,16 @@ function readPromotionsFeedStateWithMetadata(): PromotionsFeedStateRead { } return { state: { - ...(!payloadInvalid && row.etag ? { etag: row.etag } : {}), - ...(!payloadInvalid && typeof row.feed_sequence === "number" - ? { sequence: row.feed_sequence } + ...(!payloadInvalid && stored.etag ? { etag: stored.etag } : {}), + ...(!payloadInvalid && typeof stored.sequence === "number" + ? { sequence: stored.sequence } : {}), ...(!payloadInvalid && expiresAtMs !== undefined ? { expiresAtMs } : {}), entries, - ...(typeof row.last_checked_at_ms === "number" - ? { lastCheckedAtMs: row.last_checked_at_ms } + ...(typeof stored.lastCheckedAtMs === "number" + ? { lastCheckedAtMs: stored.lastCheckedAtMs } : {}), - notifiedSlugs: parseSlugListJson(row.notified_slugs_json), + notifiedSlugs: new Set(stored.notifiedSlugs), }, payloadInvalid, }; @@ -138,41 +115,16 @@ type WritePromotionsFeedStateParams = { }; function writePromotionsFeedState(params: WritePromotionsFeedStateParams): void { - runOpenClawStateWriteTransaction((database) => { - const db = getNodeSqliteKysely(database.db); - const existing = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("clawhub_promotions_feed_state") - .select([ - "etag", - "payload_json", - "feed_sequence", - "last_checked_at_ms", - "notified_slugs_json", - ]) - .where("state_key", "=", PROMOTIONS_FEED_STATE_KEY), - ); - const next = { - etag: params.etag === undefined ? (existing?.etag ?? null) : params.etag, - payload_json: - params.payloadJson === undefined ? (existing?.payload_json ?? null) : params.payloadJson, - feed_sequence: - params.sequence === undefined ? (existing?.feed_sequence ?? null) : params.sequence, - last_checked_at_ms: params.lastCheckedAtMs ?? existing?.last_checked_at_ms ?? null, - notified_slugs_json: params.notifiedSlugs - ? JSON.stringify([...params.notifiedSlugs].toSorted()) - : (existing?.notified_slugs_json ?? "[]"), - updated_at_ms: Date.now(), - }; - executeSqliteQuerySync( - database.db, - db - .insertInto("clawhub_promotions_feed_state") - .values({ state_key: PROMOTIONS_FEED_STATE_KEY, ...next }) - .onConflict((conflict) => conflict.column("state_key").doUpdateSet(next)), - ); - }); + updateConfigMachineState(PROMOTIONS_FEED_STATE_KEY, (existing) => ({ + etag: params.etag === undefined ? (existing?.etag ?? null) : params.etag, + payloadJson: + params.payloadJson === undefined ? (existing?.payloadJson ?? null) : params.payloadJson, + sequence: params.sequence === undefined ? (existing?.sequence ?? null) : params.sequence, + lastCheckedAtMs: params.lastCheckedAtMs ?? existing?.lastCheckedAtMs ?? null, + notifiedSlugs: params.notifiedSlugs + ? [...new Set([...(existing?.notifiedSlugs ?? []), ...params.notifiedSlugs])].toSorted() + : (existing?.notifiedSlugs ?? []), + })); } export function markPromotionSlugsNotified(slugs: Iterable): void { diff --git a/src/infra/push-web-store.ts b/src/infra/push-web-store.ts index 43ea160b6970..c2b24e76132a 100644 --- a/src/infra/push-web-store.ts +++ b/src/infra/push-web-store.ts @@ -1,5 +1,6 @@ // Canonical shared-SQLite store for Web Push subscriptions and VAPID identity. import type { Insertable, Selectable } from "kysely"; +import { readConfigMachineState, updateConfigMachineState } from "../state/config-machine-state.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { openOpenClawStateDatabase, @@ -13,7 +14,7 @@ import { getNodeSqliteKysely, } from "./kysely-sync.js"; -export const WEB_PUSH_VAPID_KEY_ID = "default"; +export const WEB_PUSH_VAPID_STATE_KEY = "webPush.vapidKeys"; export const DEFAULT_WEB_PUSH_VAPID_SUBJECT = "https://openclaw.ai"; const WEB_PUSH_MAX_ENDPOINT_LENGTH = 2048; const WEB_PUSH_MAX_KEY_LENGTH = 512; @@ -42,11 +43,10 @@ export function createWebPushVapidKeyPair( export type WebPushDatabase = Pick< OpenClawStateKyselyDatabase, - "web_push_subscriptions" | "web_push_vapid_keys" + "config_machine_state" | "web_push_subscriptions" >; type WebPushSubscriptionRow = Selectable; type WebPushSubscriptionInsert = Insertable; -type WebPushVapidKeyInsert = Insertable; function webPushStateDatabaseOptions(stateDir?: string): OpenClawStateDatabaseOptions { return stateDir @@ -98,19 +98,6 @@ export function webPushSubscriptionToRow(params: { }; } -export function webPushVapidKeyPairToRow(params: { - keyPair: VapidKeyPair; - nowMs: number; -}): WebPushVapidKeyInsert { - return { - key_id: WEB_PUSH_VAPID_KEY_ID, - public_key: params.keyPair.publicKey, - private_key: params.keyPair.privateKey, - subject: params.keyPair.subject, - updated_at_ms: params.nowMs, - }; -} - export function webPushSubscriptionsEqual( left: WebPushSubscription, right: WebPushSubscription, @@ -230,15 +217,12 @@ export function deleteWebPushSubscriptionIfCurrent(params: { } export function readPersistedVapidKeyPair(stateDir?: string): VapidKeyPair | null { - const database = openOpenClawStateDatabase(webPushStateDatabaseOptions(stateDir)); - const row = executeSqliteQueryTakeFirstSync( - database.db, - getNodeSqliteKysely(database.db) - .selectFrom("web_push_vapid_keys") - .selectAll() - .where("key_id", "=", WEB_PUSH_VAPID_KEY_ID), + return ( + readConfigMachineState( + WEB_PUSH_VAPID_STATE_KEY, + webPushStateDatabaseOptions(stateDir), + ) ?? null ); - return row ? createWebPushVapidKeyPair(row.public_key, row.private_key, row.subject) : null; } /** First committed keypair wins so concurrent gateway bootstraps share one signing identity. */ @@ -247,24 +231,9 @@ export function insertVapidKeyPairIfAbsent(params: { nowMs: number; stateDir?: string; }): VapidKeyPair { - return runOpenClawStateWriteTransaction(({ db }) => { - const stateDb = getNodeSqliteKysely(db); - const existing = executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("web_push_vapid_keys") - .selectAll() - .where("key_id", "=", WEB_PUSH_VAPID_KEY_ID), - ); - if (existing) { - return createWebPushVapidKeyPair(existing.public_key, existing.private_key, existing.subject); - } - executeSqliteQuerySync( - db, - stateDb - .insertInto("web_push_vapid_keys") - .values(webPushVapidKeyPairToRow({ keyPair: params.candidate, nowMs: params.nowMs })), - ); - return params.candidate; - }, webPushStateDatabaseOptions(params.stateDir)); + return updateConfigMachineState( + WEB_PUSH_VAPID_STATE_KEY, + (current) => current ?? params.candidate, + webPushStateDatabaseOptions(params.stateDir), + ); } diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index 8aa6d81d5958..dc0fc5214a7a 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -195,6 +195,8 @@ function describeStateSchemaMigration(migration: OpenClawStateDatabaseSchemaMigr return "retired shared-state tables → removed tables and indexes"; case "state-table-retirement-v11": return "retired skill curator tables → removed tables and indexes"; + case "singleton-state-foldin-v12": + return "singleton state tables → shared configuration state"; case "operator-approvals-system-agent": return "operator approvals → OpenClaw system changes"; case "session-watch-cursor-provenance-v4": diff --git a/src/infra/state-migrations.node-host.test.ts b/src/infra/state-migrations.node-host.test.ts index 2eda2d125c46..af8134bdb0ed 100644 --- a/src/infra/state-migrations.node-host.test.ts +++ b/src/infra/state-migrations.node-host.test.ts @@ -4,24 +4,25 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; -import { loadNodeHostConfig } from "../node-host/config.js"; +import { + loadNodeHostConfig, + NODE_HOST_CONFIG_KEY, + type NodeHostConfig, +} from "../node-host/config.js"; +import { readConfigMachineStateWithMetadata } from "../state/config-machine-state.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, } from "../state/openclaw-state-db.js"; import { acquireGatewayLock } from "./gateway-lock.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; import { detectLegacyNodeHostConfig, migrateLegacyNodeHostConfig, } from "./state-migrations.node-host.js"; -type NodeHostConfigDatabase = Pick; +type NodeHostConfigDatabase = Pick; const fixtureDigest = ["fixture", "digest"].join("-"); describe("legacy node-host Doctor migration", () => { @@ -69,39 +70,34 @@ describe("legacy node-host Doctor migration", () => { displayName?: string; gatewayHost?: string; updatedAtMs: number; - token?: string | null; }): void { const database = openOpenClawStateDatabase({ env: params.env }); executeSqliteQuerySync( database.db, getNodeSqliteKysely(database.db) - .insertInto("node_host_config") + .insertInto("config_machine_state") .values({ - config_key: "current", - version: 1, - node_id: params.nodeId ?? "legacy-node-id", - token: params.token ?? null, - display_name: params.displayName ?? "Legacy Node", - gateway_host: params.gatewayHost ?? "gateway.example", - gateway_port: 18443, - gateway_tls: 0, - gateway_tls_fingerprint: fixtureDigest, - gateway_context_path: "/openclaw-gw", - gateway_cloudflare_access_json: null, + state_key: NODE_HOST_CONFIG_KEY, + value_json: JSON.stringify({ + version: 1, + nodeId: params.nodeId ?? "legacy-node-id", + displayName: params.displayName ?? "Legacy Node", + gateway: { + host: params.gatewayHost ?? "gateway.example", + port: 18443, + tls: false, + tlsFingerprint: fixtureDigest, + contextPath: "/openclaw-gw", + }, + installedAppsSharing: false, + } satisfies NodeHostConfig), updated_at_ms: params.updatedAtMs, }), ); } function readCanonicalRow(env: NodeJS.ProcessEnv) { - const database = openOpenClawStateDatabase({ env }); - return executeSqliteQueryTakeFirstSync( - database.db, - getNodeSqliteKysely(database.db) - .selectFrom("node_host_config") - .selectAll() - .where("config_key", "=", "current"), - ); + return readConfigMachineStateWithMetadata(NODE_HOST_CONFIG_KEY, { env }); } it("detects source and interrupted claim only for explicit Doctor repair", async () => { @@ -142,7 +138,7 @@ describe("legacy node-host Doctor migration", () => { }, installedAppsSharing: false, }); - expect(readCanonicalRow(env)?.token).toBeNull(); + expect(readCanonicalRow(env)?.value).not.toHaveProperty("token"); expect(fs.existsSync(sourcePath)).toBe(false); }); @@ -232,7 +228,6 @@ describe("legacy node-host Doctor migration", () => { displayName: "Newer Canonical", gatewayHost: "newer.example", updatedAtMs: mtimeMs + 1_000, - token: "test-token-placeholder", }); const result = await migrateLegacyNodeHostConfig({ detected: detectLegacyNodeHostConfig({ stateDir, doctorOnlyStateMigrations: true }), @@ -243,10 +238,12 @@ describe("legacy node-host Doctor migration", () => { expect(result.warnings).toEqual([]); expect(result.changes).toContain("Kept newer canonical node-host SQLite state."); expect(readCanonicalRow(env)).toMatchObject({ - display_name: "Newer Canonical", - gateway_host: "newer.example", - token: null, + value: { + displayName: "Newer Canonical", + gateway: { host: "newer.example" }, + }, }); + expect(readCanonicalRow(env)?.value).not.toHaveProperty("token"); expect(fs.existsSync(sourcePath)).toBe(false); }); @@ -267,9 +264,11 @@ describe("legacy node-host Doctor migration", () => { expect(result.warnings).toEqual([]); expect(readCanonicalRow(env)).toMatchObject({ - display_name: "Legacy Node", - gateway_host: "gateway.example", - updated_at_ms: mtimeMs, + value: { + displayName: "Legacy Node", + gateway: { host: "gateway.example" }, + }, + updatedAtMs: mtimeMs, }); }); @@ -344,7 +343,7 @@ describe("legacy node-host Doctor migration", () => { }); expect(first.warnings[0]).toContain("legacy cleanup failed"); expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(true); - expect(readCanonicalRow(env)?.node_id).toBe("legacy-node-id"); + expect(readCanonicalRow(env)?.value.nodeId).toBe("legacy-node-id"); const retry = await migrateLegacyNodeHostConfig({ detected: detectLegacyNodeHostConfig({ stateDir, doctorOnlyStateMigrations: true }), @@ -353,7 +352,7 @@ describe("legacy node-host Doctor migration", () => { }); expect(retry.warnings).toEqual([]); expect(fs.existsSync(`${sourcePath}.doctor-importing`)).toBe(false); - expect(readCanonicalRow(env)?.node_id).toBe("legacy-node-id"); + expect(readCanonicalRow(env)?.value.nodeId).toBe("legacy-node-id"); }); it("refuses symlinked, hardlinked, and oversized sources", async () => { diff --git a/src/infra/state-migrations.node-host.ts b/src/infra/state-migrations.node-host.ts index a89f41908ef0..4463d437408a 100644 --- a/src/infra/state-migrations.node-host.ts +++ b/src/infra/state-migrations.node-host.ts @@ -31,7 +31,7 @@ const LEGACY_NODE_HOST_MAX_BYTES = 64 * 1024; const CONFIG_KEYS = new Set(["version", "nodeId", "token", "displayName", "gateway"]); const GATEWAY_KEYS = new Set(["host", "port", "tls", "tlsFingerprint", "contextPath"]); -type NodeHostConfigDatabase = Pick; +type NodeHostConfigDatabase = Pick; type CanonicalNodeHostState = { config: NodeHostConfig; @@ -141,63 +141,72 @@ function parseLegacyNodeHostConfig(snapshot: LegacySourceSnapshot): CanonicalNod }; } -function nullableNonEmptyString(value: string | null, label: string): string | undefined { - if (value === null) { +function nullableNonEmptyString(value: unknown, label: string): string | undefined { + if (value === null || value === undefined) { return undefined; } - if (!value.trim()) { + if (typeof value !== "string" || !value.trim()) { throw new Error(`invalid node-host SQLite row: ${label} must not be empty`); } return value.trim(); } function rowToCanonicalState(row: { - version: number; - node_id: string; - display_name: string | null; - gateway_host: string | null; - gateway_port: number | null; - gateway_tls: number | null; - gateway_tls_fingerprint: string | null; - gateway_context_path: string | null; - gateway_cloudflare_access_json: string | null; + value_json: string; updated_at_ms: number; }): CanonicalNodeHostState { - if (row.version !== 1 || !row.node_id.trim()) { + const value = JSON.parse(row.value_json) as unknown; + if ( + !isRecord(value) || + value.version !== 1 || + typeof value.nodeId !== "string" || + !value.nodeId.trim() + ) { throw new Error("invalid canonical node-host SQLite identity"); } if (!Number.isSafeInteger(row.updated_at_ms) || row.updated_at_ms < 0) { throw new Error("invalid canonical node-host SQLite timestamp"); } + const storedGateway = value.gateway; + if (storedGateway !== undefined && !isRecord(storedGateway)) { + throw new Error("invalid canonical node-host SQLite gateway"); + } + const gatewayPort = storedGateway?.port; if ( - row.gateway_port !== null && - (!Number.isSafeInteger(row.gateway_port) || row.gateway_port <= 0 || row.gateway_port > 65_535) + gatewayPort !== undefined && + (typeof gatewayPort !== "number" || + !Number.isSafeInteger(gatewayPort) || + gatewayPort <= 0 || + gatewayPort > 65_535) ) { throw new Error("invalid canonical node-host SQLite gateway port"); } - if (row.gateway_tls !== null && row.gateway_tls !== 0 && row.gateway_tls !== 1) { + const gatewayTls = storedGateway?.tls; + if (gatewayTls !== undefined && typeof gatewayTls !== "boolean") { throw new Error("invalid canonical node-host SQLite gateway tls"); } - const cloudflareAccess = - row.gateway_cloudflare_access_json === null - ? undefined - : normalizeNodeHostCloudflareAccessConfig( - JSON.parse(row.gateway_cloudflare_access_json) as unknown, - ); + if (value.installedAppsSharing !== undefined && typeof value.installedAppsSharing !== "boolean") { + throw new Error("invalid canonical node-host SQLite installed-app sharing"); + } + const cloudflareAccess = normalizeNodeHostCloudflareAccessConfig(storedGateway?.cloudflareAccess); const gateway: NodeHostGatewayConfig = { - host: nullableNonEmptyString(row.gateway_host, "gateway_host"), - port: row.gateway_port ?? undefined, - tls: row.gateway_tls === null ? undefined : row.gateway_tls === 1, - tlsFingerprint: nullableNonEmptyString(row.gateway_tls_fingerprint, "gateway_tls_fingerprint"), - contextPath: nullableNonEmptyString(row.gateway_context_path, "gateway_context_path"), + host: nullableNonEmptyString(storedGateway?.host, "gateway_host"), + port: typeof gatewayPort === "number" ? gatewayPort : undefined, + tls: typeof gatewayTls === "boolean" ? gatewayTls : undefined, + tlsFingerprint: nullableNonEmptyString( + storedGateway?.tlsFingerprint, + "gateway_tls_fingerprint", + ), + contextPath: nullableNonEmptyString(storedGateway?.contextPath, "gateway_context_path"), ...(cloudflareAccess ? { cloudflareAccess } : {}), }; return { config: { version: 1, - nodeId: row.node_id.trim(), - displayName: nullableNonEmptyString(row.display_name, "display_name"), + nodeId: value.nodeId.trim(), + displayName: nullableNonEmptyString(value.displayName, "display_name"), gateway: Object.values(gateway).some((entry) => entry !== undefined) ? gateway : undefined, + installedAppsSharing: value.installedAppsSharing === true, }, updatedAtMs: row.updated_at_ms, }; @@ -221,30 +230,21 @@ function writeCanonicalState( db: Parameters[0], state: CanonicalNodeHostState, ): void { - const gateway = state.config.gateway; const row = { - config_key: NODE_HOST_CONFIG_KEY, - version: 1, - node_id: state.config.nodeId, - token: null, - display_name: state.config.displayName ?? null, - gateway_host: gateway?.host ?? null, - gateway_port: gateway?.port ?? null, - gateway_tls: gateway?.tls === undefined ? null : gateway.tls ? 1 : 0, - gateway_tls_fingerprint: gateway?.tlsFingerprint ?? null, - gateway_context_path: gateway?.contextPath ?? null, - gateway_cloudflare_access_json: gateway?.cloudflareAccess - ? JSON.stringify(gateway.cloudflareAccess) - : null, + state_key: NODE_HOST_CONFIG_KEY, + value_json: JSON.stringify({ + ...state.config, + installedAppsSharing: state.config.installedAppsSharing ?? false, + }), updated_at_ms: state.updatedAtMs, }; - const { config_key: _configKey, ...updates } = row; + const { state_key: _stateKey, ...updates } = row; executeSqliteQuerySync( db, getNodeSqliteKysely(db) - .insertInto("node_host_config") + .insertInto("config_machine_state") .values(row) - .onConflict((conflict) => conflict.column("config_key").doUpdateSet(updates)), + .onConflict((conflict) => conflict.column("state_key").doUpdateSet(updates)), ); } @@ -260,9 +260,9 @@ function migrateIntoDatabase(params: { env: NodeJS.ProcessEnv; legacy: Canonical const row = executeSqliteQueryTakeFirstSync( db, stateDb - .selectFrom("node_host_config") + .selectFrom("config_machine_state") .selectAll() - .where("config_key", "=", NODE_HOST_CONFIG_KEY), + .where("state_key", "=", NODE_HOST_CONFIG_KEY), ); const existing = row ? rowToCanonicalState(row) : null; if (existing && existing.config.nodeId !== params.legacy.config.nodeId) { @@ -283,21 +283,26 @@ function migrateIntoDatabase(params: { env: NodeJS.ProcessEnv; legacy: Canonical if ( !existing || !configsEqual(existing.config, expected.config) || - existing.updatedAtMs !== expected.updatedAtMs || - row?.token !== null + existing.updatedAtMs !== expected.updatedAtMs ) { + if (expected === params.legacy && existing?.config.installedAppsSharing) { + expected = { + ...expected, + config: { ...expected.config, installedAppsSharing: true }, + }; + } writeCanonicalState(db, expected); - imported = expected === params.legacy; + imported = expected.updatedAtMs === params.legacy.updatedAtMs; } const verifiedRow = executeSqliteQueryTakeFirstSync( db, stateDb - .selectFrom("node_host_config") + .selectFrom("config_machine_state") .selectAll() - .where("config_key", "=", NODE_HOST_CONFIG_KEY), + .where("state_key", "=", NODE_HOST_CONFIG_KEY), ); - if (!verifiedRow || verifiedRow.token !== null) { + if (!verifiedRow) { throw new Error("SQLite verification failed for node-host config"); } const verified = rowToCanonicalState(verifiedRow); diff --git a/src/infra/state-migrations.onboarding-recommendations.test.ts b/src/infra/state-migrations.onboarding-recommendations.test.ts index 969014efa90e..1b77d705fdaf 100644 --- a/src/infra/state-migrations.onboarding-recommendations.test.ts +++ b/src/infra/state-migrations.onboarding-recommendations.test.ts @@ -1,59 +1,27 @@ import { afterEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { createOnboardingRecommendationsStore } from "../state/onboarding-recommendations.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { - closeOpenClawStateDatabaseForTest, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; import { migrateLegacyOnboardingRecommendationsScope } from "./state-migrations.onboarding-recommendations.js"; -type OnboardingRecommendationsMigrationDatabase = Pick< - OpenClawStateKyselyDatabase, - "onboarding_recommendations" ->; - function insertRecommendationRow(params: { database: { env: NodeJS.ProcessEnv }; configKey: string; inventoryHash: string; }): void { - runOpenClawStateWriteTransaction(({ db: sqlite }) => { - const db = getNodeSqliteKysely(sqlite); - executeSqliteQuerySync( - sqlite, - db.insertInto("onboarding_recommendations").values({ - config_key: params.configKey, - inventory_hash: params.inventoryHash, - matches_json: "[]", - offered_at_ms: 1_000, - accepted_at_ms: 2_000, - updated_at_ms: 2_000, - }), - ); - }, params.database); -} - -function readRecommendationKey( - database: { env: NodeJS.ProcessEnv }, - configKey: string, -): { config_key: string } | undefined { - return runOpenClawStateWriteTransaction(({ db: sqlite }) => { - const db = getNodeSqliteKysely(sqlite); - return executeSqliteQueryTakeFirstSync( - sqlite, - db - .selectFrom("onboarding_recommendations") - .select("config_key") - .where("config_key", "=", configKey), - ); - }, database); + writeConfigMachineState( + `onboarding.recommendations.${params.configKey}`, + { + inventoryHash: params.inventoryHash, + matches: [], + offeredAt: 1_000, + acceptedAt: 2_000, + updatedAt: 2_000, + }, + params.database, + ); } afterEach(() => { @@ -100,7 +68,9 @@ describe("onboarding recommendations scope migration", () => { acceptedAt: 2_000, updatedAt: 2_000, }); - expect(readRecommendationKey(database, "primary")).toBeUndefined(); + expect( + readConfigMachineState("onboarding.recommendations.primary", database), + ).toBeUndefined(); }, ); }); @@ -143,7 +113,9 @@ describe("onboarding recommendations scope migration", () => { warnings: [], }); expect(store.read()).toEqual(scoped); - expect(readRecommendationKey(database, "primary")).toBeUndefined(); + expect( + readConfigMachineState("onboarding.recommendations.primary", database), + ).toBeUndefined(); }, ); }); diff --git a/src/infra/state-migrations.onboarding-recommendations.ts b/src/infra/state-migrations.onboarding-recommendations.ts index 22bfe953f4e5..8046427a99a1 100644 --- a/src/infra/state-migrations.onboarding-recommendations.ts +++ b/src/infra/state-migrations.onboarding-recommendations.ts @@ -13,11 +13,11 @@ import { } from "./kysely-sync.js"; import type { MigrationMessages } from "./state-migrations.types.js"; -const LEGACY_ONBOARDING_RECOMMENDATIONS_KEY = "primary"; +const LEGACY_ONBOARDING_RECOMMENDATIONS_KEY = "onboarding.recommendations.primary"; type OnboardingRecommendationsMigrationDatabase = Pick< OpenClawStateKyselyDatabase, - "onboarding_recommendations" + "config_machine_state" >; /** Move the shipped singleton row into the default workspace during doctor repair. */ @@ -36,6 +36,7 @@ export function migrateLegacyOnboardingRecommendationsScope(params: { ? resolveWorkspaceStateIdentity(resolveAgentWorkspaceDir(params.cfg, migrationAgentId, env)) .workspaceKey : undefined; + const scopedKey = workspaceKey ? `onboarding.recommendations.${workspaceKey}` : undefined; const outcome = runOpenClawStateWriteTransaction( ({ db: writeDatabase }) => { const writeDb = @@ -43,38 +44,38 @@ export function migrateLegacyOnboardingRecommendationsScope(params: { const legacyAtCommit = executeSqliteQueryTakeFirstSync( writeDatabase, writeDb - .selectFrom("onboarding_recommendations") - .select("config_key") - .where("config_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), + .selectFrom("config_machine_state") + .select("state_key") + .where("state_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), ); if (!legacyAtCommit) { return "unchanged" as const; } - if (!workspaceKey) { + if (!scopedKey) { return "deferred" as const; } const scoped = executeSqliteQueryTakeFirstSync( writeDatabase, writeDb - .selectFrom("onboarding_recommendations") - .select("config_key") - .where("config_key", "=", workspaceKey), + .selectFrom("config_machine_state") + .select("state_key") + .where("state_key", "=", scopedKey), ); if (scoped) { executeSqliteQuerySync( writeDatabase, writeDb - .deleteFrom("onboarding_recommendations") - .where("config_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), + .deleteFrom("config_machine_state") + .where("state_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), ); return "removed-legacy" as const; } executeSqliteQuerySync( writeDatabase, writeDb - .updateTable("onboarding_recommendations") - .set({ config_key: workspaceKey }) - .where("config_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), + .updateTable("config_machine_state") + .set({ state_key: scopedKey }) + .where("state_key", "=", LEGACY_ONBOARDING_RECOMMENDATIONS_KEY), ); return "migrated" as const; }, diff --git a/src/infra/state-migrations.runtime-state.ts b/src/infra/state-migrations.runtime-state.ts index 3280c284e2e6..b6d845d73718 100644 --- a/src/infra/state-migrations.runtime-state.ts +++ b/src/infra/state-migrations.runtime-state.ts @@ -18,10 +18,7 @@ import { archiveLegacyImportSource } from "./state-migrations.storage.js"; import type { LegacyStateDetection, MigrationMessages } from "./state-migrations.types.js"; import { normalizeVoiceWakeRoutingConfig } from "./voicewake-routing.js"; -type LegacyVoiceWakeImportDatabase = Pick< - OpenClawStateKyselyDatabase, - "voicewake_routing_config" | "voicewake_routing_routes" | "voicewake_triggers" ->; +type LegacyVoiceWakeImportDatabase = Pick; type LegacyConfigHealthImportDatabase = Pick; type LegacyPluginBindingApprovalsImportDatabase = Pick< OpenClawStateKyselyDatabase, @@ -32,7 +29,8 @@ type LegacyCurrentConversationBindingsImportDatabase = Pick< "current_conversation_bindings" >; -const VOICEWAKE_CONFIG_KEY = "default"; +const VOICEWAKE_TRIGGERS_STATE_KEY = "voicewake.triggers"; +const VOICEWAKE_ROUTING_STATE_KEY = "voicewake.routing"; const DEFAULT_VOICEWAKE_TRIGGERS = ["openclaw", "claude", "computer"]; export function resolveLegacyVoiceWakeTriggersPath(stateDir: string): string { @@ -116,82 +114,28 @@ function normalizeLegacyVoiceWakeTriggers(input: unknown): string[] { return triggers.length > 0 ? triggers : DEFAULT_VOICEWAKE_TRIGGERS; } -function legacyVoiceWakeTriggersMatch( - rows: Array<{ trigger: string }>, - triggers: string[], -): boolean { - return ( - rows.length === triggers.length && rows.every((row, index) => row.trigger === triggers[index]) +function importLegacyVoiceWakeMachineState( + database: DatabaseSync, + key: string, + value: unknown, +): { current: unknown; imported: boolean } { + const db = getNodeSqliteKysely(database); + const existing = executeSqliteQueryTakeFirstSync( + database, + db.selectFrom("config_machine_state").select("value_json").where("state_key", "=", key), ); -} - -function legacyVoiceWakeTargetColumns(target: { - agentId?: string; - mode?: "current"; - sessionKey?: string; -}): { - targetAgentId: string | null; - targetMode: string; - targetSessionKey: string | null; -} { - if (target.agentId) { - return { targetAgentId: target.agentId, targetMode: "agent", targetSessionKey: null }; + if (existing) { + return { current: JSON.parse(existing.value_json), imported: false }; } - if (target.sessionKey) { - return { targetAgentId: null, targetMode: "session", targetSessionKey: target.sessionKey }; - } - return { targetAgentId: null, targetMode: "current", targetSessionKey: null }; -} - -function legacyVoiceWakeTargetColumnsMatch( - left: ReturnType, - right: { - target_agent_id?: string | null; - target_mode?: string | null; - target_session_key?: string | null; - }, -): boolean { - return ( - left.targetAgentId === (right.target_agent_id ?? null) && - left.targetMode === right.target_mode && - left.targetSessionKey === (right.target_session_key ?? null) - ); -} - -function legacyVoiceWakeRoutingMatches( - configRow: { - default_target_agent_id: string | null; - default_target_mode: string; - default_target_session_key: string | null; - }, - routeRows: Array<{ - target_agent_id: string | null; - target_mode: string; - target_session_key: string | null; - trigger: string; - }>, - routingConfig: ReturnType, -): boolean { - const defaultTarget = legacyVoiceWakeTargetColumns(routingConfig.defaultTarget); - if ( - !legacyVoiceWakeTargetColumnsMatch(defaultTarget, { - target_agent_id: configRow.default_target_agent_id, - target_mode: configRow.default_target_mode, - target_session_key: configRow.default_target_session_key, - }) - ) { - return false; - } - return ( - routeRows.length === routingConfig.routes.length && - routeRows.every((row, index) => { - const route = routingConfig.routes[index]; - if (!route || row.trigger !== route.trigger) { - return false; - } - return legacyVoiceWakeTargetColumnsMatch(legacyVoiceWakeTargetColumns(route.target), row); - }) + executeSqliteQuerySync( + database, + db.insertInto("config_machine_state").values({ + state_key: key, + value_json: JSON.stringify(value), + updated_at_ms: Date.now(), + }), ); + return { current: value, imported: true }; } export function migrateLegacyVoiceWakeSettings(params: { @@ -205,19 +149,15 @@ export function migrateLegacyVoiceWakeSettings(params: { normalize: normalizeLegacyVoiceWakeTriggers, shouldMigrate: (triggers) => triggers.length > 0, migrate(db, triggers) { - const stateDb = getNodeSqliteKysely(db); - const existing = executeSqliteQuerySync( + const imported = importLegacyVoiceWakeMachineState( db, - stateDb - .selectFrom("voicewake_triggers") - .select(["trigger"]) - .where("config_key", "=", VOICEWAKE_CONFIG_KEY) - .orderBy("position", "asc"), - ).rows; - if (existing.length > 0) { + VOICEWAKE_TRIGGERS_STATE_KEY, + triggers, + ); + if (!imported.imported) { return { changes: [], - ...(legacyVoiceWakeTriggersMatch(existing, triggers) + ...(JSON.stringify(imported.current) === JSON.stringify(triggers) ? {} : { notices: [ @@ -226,18 +166,6 @@ export function migrateLegacyVoiceWakeSettings(params: { }), }; } - const updatedAtMs = Date.now(); - executeSqliteQuerySync( - db, - stateDb.insertInto("voicewake_triggers").values( - triggers.map((trigger, position) => ({ - config_key: VOICEWAKE_CONFIG_KEY, - position, - trigger, - updated_at_ms: updatedAtMs, - })), - ), - ); return { changes: [ `Migrated ${triggers.length} voice wake ${triggers.length === 1 ? "trigger" : "triggers"} → shared SQLite state`, @@ -253,26 +181,18 @@ export function migrateLegacyVoiceWakeSettings(params: { normalize: normalizeVoiceWakeRoutingConfig, shouldMigrate: Boolean, migrate(db, routingConfig) { - const stateDb = getNodeSqliteKysely(db); - const existing = executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("voicewake_routing_config") - .select(["default_target_agent_id", "default_target_mode", "default_target_session_key"]) - .where("config_key", "=", VOICEWAKE_CONFIG_KEY), - ); - if (existing) { - const routeRows = executeSqliteQuerySync( - db, - stateDb - .selectFrom("voicewake_routing_routes") - .select(["target_agent_id", "target_mode", "target_session_key", "trigger"]) - .where("config_key", "=", VOICEWAKE_CONFIG_KEY) - .orderBy("position", "asc"), - ).rows; + const imported = importLegacyVoiceWakeMachineState(db, VOICEWAKE_ROUTING_STATE_KEY, { + ...routingConfig, + updatedAtMs: Date.now(), + }); + if (!imported.imported) { + const existing = normalizeVoiceWakeRoutingConfig(imported.current); + const matches = + JSON.stringify(existing.defaultTarget) === JSON.stringify(routingConfig.defaultTarget) && + JSON.stringify(existing.routes) === JSON.stringify(routingConfig.routes); return { changes: [], - ...(legacyVoiceWakeRoutingMatches(existing, routeRows, routingConfig) + ...(matches ? {} : { notices: [ @@ -281,38 +201,6 @@ export function migrateLegacyVoiceWakeSettings(params: { }), }; } - const updatedAtMs = Date.now(); - const defaultTarget = legacyVoiceWakeTargetColumns(routingConfig.defaultTarget); - executeSqliteQuerySync( - db, - stateDb.insertInto("voicewake_routing_config").values({ - config_key: VOICEWAKE_CONFIG_KEY, - version: 1, - default_target_mode: defaultTarget.targetMode, - default_target_agent_id: defaultTarget.targetAgentId, - default_target_session_key: defaultTarget.targetSessionKey, - updated_at_ms: updatedAtMs, - }), - ); - if (routingConfig.routes.length > 0) { - executeSqliteQuerySync( - db, - stateDb.insertInto("voicewake_routing_routes").values( - routingConfig.routes.map((route, position) => { - const target = legacyVoiceWakeTargetColumns(route.target); - return { - config_key: VOICEWAKE_CONFIG_KEY, - position, - trigger: route.trigger, - target_mode: target.targetMode, - target_agent_id: target.targetAgentId, - target_session_key: target.targetSessionKey, - updated_at_ms: updatedAtMs, - }; - }), - ), - ); - } return { changes: [ `Migrated voice wake routing config with ${routingConfig.routes.length} ${routingConfig.routes.length === 1 ? "route" : "routes"} → shared SQLite state`, diff --git a/src/infra/state-migrations.test.ts b/src/infra/state-migrations.test.ts index 267f7045413c..758550a4713b 100644 --- a/src/infra/state-migrations.test.ts +++ b/src/infra/state-migrations.test.ts @@ -20,6 +20,7 @@ import type { } from "../plugins/doctor-contract-registry.js"; import { EMPTY_LEGACY_SESSION_SURFACES } from "../plugins/legacy-session-surfaces.types.js"; import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawAgentDatabasesForTest, ensureOpenClawAgentDatabaseSchema, @@ -31,17 +32,12 @@ import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, OPENCLAW_STATE_SCHEMA_VERSION, - runOpenClawStateWriteTransaction, } from "../state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js"; import { acquireGatewayLock } from "./gateway-lock.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; import { loadApnsRegistration } from "./push-apns.js"; import { createWebPushVapidKeyPair, @@ -184,7 +180,6 @@ vi.mock("../plugins/doctor-contract-registry.js", async (importOriginal) => { const tempDirs = createTrackedTempDirs(); const APNS_DEVICE_FIELD = "token"; -type UpdateCheckStateDatabase = Pick; type ConfigHealthDatabase = Pick; type PluginBindingApprovalsDatabase = Pick; type CurrentConversationBindingsDatabase = Pick< @@ -269,26 +264,13 @@ const createTempDir = () => tempDirs.make("openclaw-state-migrations-test-"); function readUpdateCheckState(env: NodeJS.ProcessEnv): | { - last_checked_at: string | null; - last_available_version: string | null; - last_available_tag: string | null; - auto_install_id: string | null; + lastCheckedAt?: string; + lastAvailableVersion?: string; + lastAvailableTag?: string; + autoInstallId?: string; } | undefined { - const { db } = openOpenClawStateDatabase({ env }); - const stateDb = getNodeSqliteKysely(db); - return executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("update_check_state") - .select([ - "last_checked_at", - "last_available_version", - "last_available_tag", - "auto_install_id", - ]) - .where("state_key", "=", "default"), - ); + return readConfigMachineState("update.checkState", { env }); } function readConfigHealthRows(env: NodeJS.ProcessEnv): Array<{ @@ -487,39 +469,14 @@ function seedSchemaOnlyLegacyAgentDatabase( return databasePath; } -type VoiceWakeRoutingTestDatabase = Pick< - OpenClawStateKyselyDatabase, - "voicewake_routing_config" | "voicewake_routing_routes" ->; - function seedCanonicalVoiceWakeRouting(stateDir: string, trigger: string): void { - const updatedAtMs = Date.now(); - runOpenClawStateWriteTransaction( - ({ db }) => { - const routingDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - routingDb.insertInto("voicewake_routing_config").values({ - config_key: "default", - version: 1, - default_target_mode: "current", - default_target_agent_id: null, - default_target_session_key: null, - updated_at_ms: updatedAtMs, - }), - ); - executeSqliteQuerySync( - db, - routingDb.insertInto("voicewake_routing_routes").values({ - config_key: "default", - position: 0, - trigger, - target_mode: "agent", - target_agent_id: "main", - target_session_key: null, - updated_at_ms: updatedAtMs, - }), - ); + writeConfigMachineState( + "voicewake.routing", + { + version: 1, + defaultTarget: { mode: "current" }, + routes: [{ trigger, target: { agentId: "main" } }], + updatedAtMs: Date.now(), }, { env: createEnv(stateDir) }, ); @@ -3750,10 +3707,10 @@ describe("state migrations", () => { expect(result.warnings).toStrictEqual([]); expect(result.changes).toContain("Migrated update-check state → shared SQLite state"); expect(readUpdateCheckState(env)).toMatchObject({ - last_checked_at: "2026-01-17T09:30:00.000Z", - last_available_version: "2.0.0", - last_available_tag: "latest", - auto_install_id: "install-1", + lastCheckedAt: "2026-01-17T09:30:00.000Z", + lastAvailableVersion: "2.0.0", + lastAvailableTag: "latest", + autoInstallId: "install-1", }); await expectMissingPath(sourcePath); await expect(fs.readFile(`${sourcePath}.migrated`, "utf8")).resolves.toContain("2.0.0"); @@ -3772,7 +3729,7 @@ describe("state migrations", () => { expect(conflictResult.notices).toEqual([ expect.stringContaining("Kept shared SQLite update-check state because legacy cache differs"), ]); - expect(readUpdateCheckState(env)?.last_available_version).toBe("2.0.0"); + expect(readUpdateCheckState(env)?.lastAvailableVersion).toBe("2.0.0"); await expectMissingPath(sourcePath); await expect(fs.readFile(`${sourcePath}.migrated.2`, "utf8")).resolves.toContain("3.0.0"); diff --git a/src/infra/state-migrations.tui-last-session.test.ts b/src/infra/state-migrations.tui-last-session.test.ts index 9114e377851a..65ed64d25f2e 100644 --- a/src/infra/state-migrations.tui-last-session.test.ts +++ b/src/infra/state-migrations.tui-last-session.test.ts @@ -3,6 +3,7 @@ import fs from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { readConfigMachineStateWithMetadata } from "../state/config-machine-state.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { closeOpenClawStateDatabaseForTest, @@ -15,7 +16,7 @@ import { migrateLegacyTuiLastSessions, } from "./state-migrations.tui-last-session.js"; -type TuiLastSessionTestDatabase = Pick; +type TuiLastSessionTestDatabase = Pick; const tempDirs = createTempDirTracker(); @@ -58,11 +59,13 @@ function seedPointer(params: { ({ db }) => { executeSqliteQuerySync( db, - getNodeSqliteKysely(db).insertInto("tui_last_sessions").values({ - scope_key: params.scopeKey, - session_key: params.sessionKey, - updated_at: params.updatedAt, - }), + getNodeSqliteKysely(db) + .insertInto("config_machine_state") + .values({ + state_key: `tui.lastSession.${params.scopeKey}`, + value_json: JSON.stringify(params.sessionKey), + updated_at_ms: params.updatedAt, + }), ); }, { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, @@ -105,6 +108,11 @@ describe("legacy TUI last-session migration", () => { await expect(readTuiLastSessionKey({ scopeKey: "terminal", stateDir })).resolves.toBe( "agent:main:tui-123", ); + expect( + readConfigMachineStateWithMetadata("tui.lastSession.terminal", { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toEqual({ value: "agent:main:tui-123", updatedAtMs: 100 }); await expect(readTuiLastSessionKey({ scopeKey: "heartbeat", stateDir })).resolves.toBeNull(); expect(fs.readdirSync(path.dirname(sourcePath))).not.toContain("last-session.json.migrated"); }); diff --git a/src/infra/state-migrations.tui-last-session.ts b/src/infra/state-migrations.tui-last-session.ts index eafa77784ea6..c8f48b1b820d 100644 --- a/src/infra/state-migrations.tui-last-session.ts +++ b/src/infra/state-migrations.tui-last-session.ts @@ -20,7 +20,7 @@ import { } from "./state-migrations.source-snapshot.js"; import type { LegacyStateDetection, MigrationMessages } from "./state-migrations.types.js"; -type TuiLastSessionMigrationDatabase = Pick; +type TuiLastSessionMigrationDatabase = Pick; type LegacyTuiLastSession = { scopeKey: string; @@ -29,6 +29,7 @@ type LegacyTuiLastSession = { }; const LEGACY_RECORD_KEYS = new Set(["sessionKey", "updatedAt"]); +const TUI_LAST_SESSION_STATE_KEY_PREFIX = "tui.lastSession."; function resolveLegacyTuiLastSessionPath(stateDir: string): string { return path.join(stateDir, "tui", "last-session.json"); @@ -105,10 +106,13 @@ function parseLegacyTuiLastSessions(raw: string): LegacyTuiLastSession[] { } function rowMatches( - row: { session_key: string; updated_at: number } | undefined, + row: { value_json: string; updated_at_ms: number } | undefined, expected: LegacyTuiLastSession, ): boolean { - return row?.session_key === expected.sessionKey && row.updated_at === expected.updatedAt; + return ( + row?.value_json === JSON.stringify(expected.sessionKey) && + row.updated_at_ms === expected.updatedAt + ); } /** Import, verify, and remove the retired JSON store during an explicit doctor repair. */ @@ -150,28 +154,31 @@ export function migrateLegacyTuiLastSessions(params: { ({ db }) => { const tuiDb = getNodeSqliteKysely(db); for (const record of activeRecords) { + const stateKey = `${TUI_LAST_SESSION_STATE_KEY_PREFIX}${record.scopeKey}`; const existing = executeSqliteQueryTakeFirstSync( db, tuiDb - .selectFrom("tui_last_sessions") - .select(["session_key", "updated_at"]) - .where("scope_key", "=", record.scopeKey), + .selectFrom("config_machine_state") + .select(["value_json", "updated_at_ms"]) + .where("state_key", "=", stateKey), ); if (!existing) { executeSqliteQuerySync( db, - tuiDb.insertInto("tui_last_sessions").values({ - scope_key: record.scopeKey, - session_key: record.sessionKey, - updated_at: record.updatedAt, + tuiDb.insertInto("config_machine_state").values({ + state_key: stateKey, + value_json: JSON.stringify(record.sessionKey), + updated_at_ms: record.updatedAt, }), ); expectedRows.set(record.scopeKey, record); importedCount += 1; continue; } - if (existing.updated_at === record.updatedAt) { - if (existing.session_key !== record.sessionKey) { + // SAFETY: The TUI owner stores each tui.lastSession value as a JSON string. + const existingSessionKey = JSON.parse(existing.value_json) as string; + if (existing.updated_at_ms === record.updatedAt) { + if (existingSessionKey !== record.sessionKey) { throw new Error( `scope ${record.scopeKey} has divergent JSON and SQLite pointers at the same timestamp`, ); @@ -179,11 +186,11 @@ export function migrateLegacyTuiLastSessions(params: { expectedRows.set(record.scopeKey, record); continue; } - if (existing.updated_at > record.updatedAt) { + if (existing.updated_at_ms > record.updatedAt) { expectedRows.set(record.scopeKey, { scopeKey: record.scopeKey, - sessionKey: existing.session_key, - updatedAt: existing.updated_at, + sessionKey: existingSessionKey, + updatedAt: existing.updated_at_ms, }); supersededCount += 1; continue; @@ -191,9 +198,12 @@ export function migrateLegacyTuiLastSessions(params: { executeSqliteQuerySync( db, tuiDb - .updateTable("tui_last_sessions") - .set({ session_key: record.sessionKey, updated_at: record.updatedAt }) - .where("scope_key", "=", record.scopeKey), + .updateTable("config_machine_state") + .set({ + value_json: JSON.stringify(record.sessionKey), + updated_at_ms: record.updatedAt, + }) + .where("state_key", "=", stateKey), ); expectedRows.set(record.scopeKey, record); importedCount += 1; @@ -216,9 +226,9 @@ export function migrateLegacyTuiLastSessions(params: { const row = executeSqliteQueryTakeFirstSync( database.db, tuiDb - .selectFrom("tui_last_sessions") - .select(["session_key", "updated_at"]) - .where("scope_key", "=", expected.scopeKey), + .selectFrom("config_machine_state") + .select(["value_json", "updated_at_ms"]) + .where("state_key", "=", `${TUI_LAST_SESSION_STATE_KEY_PREFIX}${expected.scopeKey}`), ); if (!rowMatches(row, expected)) { throw new Error(`SQLite verification failed for scope ${expected.scopeKey}`); diff --git a/src/infra/state-migrations.update-check.ts b/src/infra/state-migrations.update-check.ts index 3312c9ecb387..0795ba1d1164 100644 --- a/src/infra/state-migrations.update-check.ts +++ b/src/infra/state-migrations.update-check.ts @@ -1,35 +1,25 @@ import path from "node:path"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { importConfigMachineState, readConfigMachineState } from "../state/config-machine-state.js"; import { migrateLegacyJsonState } from "./state-migrations.runtime-state.js"; import type { LegacyStateDetection, MigrationMessages } from "./state-migrations.types.js"; -type LegacyUpdateCheckImportDatabase = Pick; - -const UPDATE_CHECK_STATE_KEY = "default"; +const UPDATE_CHECK_STATE_KEY = "update.checkState"; const UPDATE_CHECK_STATE_FIELDS = [ - ["lastCheckedAt", "last_checked_at"], - ["lastNotifiedVersion", "last_notified_version"], - ["lastNotifiedTag", "last_notified_tag"], - ["lastAvailableVersion", "last_available_version"], - ["lastAvailableTag", "last_available_tag"], - ["autoInstallId", "auto_install_id"], - ["autoFirstSeenVersion", "auto_first_seen_version"], - ["autoFirstSeenTag", "auto_first_seen_tag"], - ["autoFirstSeenAt", "auto_first_seen_at"], - ["autoLastAttemptVersion", "auto_last_attempt_version"], - ["autoLastAttemptAt", "auto_last_attempt_at"], - ["autoLastSuccessVersion", "auto_last_success_version"], - ["autoLastSuccessAt", "auto_last_success_at"], + "lastCheckedAt", + "lastNotifiedVersion", + "lastNotifiedTag", + "lastAvailableVersion", + "lastAvailableTag", + "autoInstallId", + "autoFirstSeenVersion", + "autoFirstSeenTag", + "autoFirstSeenAt", + "autoLastAttemptVersion", + "autoLastAttemptAt", + "autoLastSuccessVersion", + "autoLastSuccessAt", ] as const; -type LegacyUpdateCheckState = Partial< - Record<(typeof UPDATE_CHECK_STATE_FIELDS)[number][0], string> ->; -type LegacyUpdateCheckRow = Record<(typeof UPDATE_CHECK_STATE_FIELDS)[number][1], string | null>; +type LegacyUpdateCheckState = Partial>; export function resolveLegacyUpdateCheckPath(stateDir: string): string { return path.join(stateDir, "update-check.json"); @@ -38,7 +28,7 @@ export function resolveLegacyUpdateCheckPath(stateDir: string): string { function normalizeLegacyUpdateCheckState(input: unknown): LegacyUpdateCheckState { const record = input && typeof input === "object" ? (input as Record) : {}; return Object.fromEntries( - UPDATE_CHECK_STATE_FIELDS.map(([field]) => { + UPDATE_CHECK_STATE_FIELDS.map((field) => { const value = record[field]; return [field, typeof value === "string" && value.trim().length > 0 ? value : undefined]; }), @@ -46,12 +36,10 @@ function normalizeLegacyUpdateCheckState(input: unknown): LegacyUpdateCheckState } function legacyUpdateCheckStateMatches( - row: LegacyUpdateCheckRow, + existing: LegacyUpdateCheckState, state: LegacyUpdateCheckState, ): boolean { - return UPDATE_CHECK_STATE_FIELDS.every( - ([field, column]) => (state[field] ?? null) === row[column], - ); + return UPDATE_CHECK_STATE_FIELDS.every((field) => state[field] === existing[field]); } export function migrateLegacyUpdateCheckState(params: { @@ -63,19 +51,17 @@ export function migrateLegacyUpdateCheckState(params: { stateDir: params.stateDir, label: "update-check state", normalize: normalizeLegacyUpdateCheckState, - migrate(db, state) { - const stateDb = getNodeSqliteKysely(db); - const existing = executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("update_check_state") - .selectAll() - .where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - if (existing) { + migrate(_db, state) { + const options = { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }; + const result = importConfigMachineState([[UPDATE_CHECK_STATE_KEY, state]], options); + if (result.kept.length > 0) { + const existing = readConfigMachineState( + UPDATE_CHECK_STATE_KEY, + options, + ); return { changes: [], - ...(legacyUpdateCheckStateMatches(existing, state) + ...(existing && legacyUpdateCheckStateMatches(existing, state) ? {} : { notices: [ @@ -84,17 +70,6 @@ export function migrateLegacyUpdateCheckState(params: { }), }; } - const columns = Object.fromEntries( - UPDATE_CHECK_STATE_FIELDS.map(([field, column]) => [column, state[field] ?? null]), - ) as LegacyUpdateCheckRow; - executeSqliteQuerySync( - db, - stateDb.insertInto("update_check_state").values({ - state_key: UPDATE_CHECK_STATE_KEY, - ...columns, - updated_at_ms: Date.now(), - }), - ); return { changes: ["Migrated update-check state → shared SQLite state"] }; }, }); diff --git a/src/infra/state-migrations.web-push.test.ts b/src/infra/state-migrations.web-push.test.ts index c1c311a27cc0..8263bf9d8a69 100644 --- a/src/infra/state-migrations.web-push.test.ts +++ b/src/infra/state-migrations.web-push.test.ts @@ -4,6 +4,7 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { writeConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -17,8 +18,8 @@ import { listWebPushSubscriptions, readPersistedVapidKeyPair, webPushSubscriptionToRow, - webPushVapidKeyPairToRow, DEFAULT_WEB_PUSH_VAPID_SUBJECT, + WEB_PUSH_VAPID_STATE_KEY, type VapidKeyPair, type WebPushDatabase, type WebPushSubscription, @@ -107,13 +108,7 @@ describe("legacy Web Push Doctor migration", () => { } function seedVapid(value: VapidKeyPair): void { - const database = openOpenClawStateDatabase(); - executeSqliteQuerySync( - database.db, - getNodeSqliteKysely(database.db) - .insertInto("web_push_vapid_keys") - .values(webPushVapidKeyPairToRow({ keyPair: value, nowMs: 1 })), - ); + writeConfigMachineState(WEB_PUSH_VAPID_STATE_KEY, value); } it("detects original and interrupted-claim files only for explicit Doctor repair", async () => { diff --git a/src/infra/state-migrations.web-push.ts b/src/infra/state-migrations.web-push.ts index 5fa4f036d280..f164e536fc2e 100644 --- a/src/infra/state-migrations.web-push.ts +++ b/src/infra/state-migrations.web-push.ts @@ -8,12 +8,10 @@ import { getNodeSqliteKysely, } from "./kysely-sync.js"; import { - createWebPushVapidKeyPair, webPushSubscriptionFromRow, webPushSubscriptionToRow, webPushSubscriptionsEqual, - webPushVapidKeyPairToRow, - WEB_PUSH_VAPID_KEY_ID, + WEB_PUSH_VAPID_STATE_KEY, type VapidKeyPair, type WebPushDatabase, type WebPushSubscription, @@ -225,30 +223,28 @@ function migrateIntoDatabase(params: { const existingVapidRow = executeSqliteQueryTakeFirstSync( db, webPushDb - .selectFrom("web_push_vapid_keys") - .selectAll() - .where("key_id", "=", WEB_PUSH_VAPID_KEY_ID), + .selectFrom("config_machine_state") + .select("value_json") + .where("state_key", "=", WEB_PUSH_VAPID_STATE_KEY), ); if (existingVapidRow) { + // SAFETY: The Web Push owner stores only VapidKeyPair objects under this key. + const existingVapidKeys = JSON.parse(existingVapidRow.value_json) as VapidKeyPair; if ( - existingVapidRow.public_key !== params.legacy.vapidKeys.publicKey || - existingVapidRow.private_key !== params.legacy.vapidKeys.privateKey + existingVapidKeys.publicKey !== params.legacy.vapidKeys.publicKey || + existingVapidKeys.privateKey !== params.legacy.vapidKeys.privateKey ) { throw new Error("legacy Web Push VAPID identity conflicts with SQLite"); } - expectedVapidKeys = createWebPushVapidKeyPair( - existingVapidRow.public_key, - existingVapidRow.private_key, - existingVapidRow.subject, - ); + expectedVapidKeys = existingVapidKeys; } else { executeSqliteQuerySync( db, - webPushDb - .insertInto("web_push_vapid_keys") - .values( - webPushVapidKeyPairToRow({ keyPair: params.legacy.vapidKeys, nowMs: params.nowMs }), - ), + webPushDb.insertInto("config_machine_state").values({ + state_key: WEB_PUSH_VAPID_STATE_KEY, + value_json: JSON.stringify(params.legacy.vapidKeys), + updated_at_ms: params.nowMs, + }), ); expectedVapidKeys = params.legacy.vapidKeys; importedVapidKeys = true; @@ -271,15 +267,17 @@ function migrateIntoDatabase(params: { const row = executeSqliteQueryTakeFirstSync( db, webPushDb - .selectFrom("web_push_vapid_keys") - .selectAll() - .where("key_id", "=", WEB_PUSH_VAPID_KEY_ID), + .selectFrom("config_machine_state") + .select("value_json") + .where("state_key", "=", WEB_PUSH_VAPID_STATE_KEY), ); + // SAFETY: This transaction writes or validates this key as a VapidKeyPair above. + const persisted = row ? (JSON.parse(row.value_json) as VapidKeyPair) : undefined; if ( - !row || - row.public_key !== expectedVapidKeys.publicKey || - row.private_key !== expectedVapidKeys.privateKey || - row.subject !== expectedVapidKeys.subject + !persisted || + persisted.publicKey !== expectedVapidKeys.publicKey || + persisted.privateKey !== expectedVapidKeys.privateKey || + persisted.subject !== expectedVapidKeys.subject ) { throw new Error("SQLite verification failed for the Web Push VAPID identity"); } diff --git a/src/infra/update-startup.test.ts b/src/infra/update-startup.test.ts index 3539ece1dc10..bdcbb595dc83 100644 --- a/src/infra/update-startup.test.ts +++ b/src/infra/update-startup.test.ts @@ -3,10 +3,9 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { formatCliCommand } from "../cli/command-format.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { closeOpenClawStateDatabaseForTest, - openOpenClawStateDatabase, runOpenClawStateWriteTransaction, } from "../state/openclaw-state-db.js"; import { @@ -14,11 +13,6 @@ import { type OpenClawTestState, } from "../test-utils/openclaw-test-state.js"; import type { GatewayActiveWorkInspectors } from "./gateway-active-work.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; import { writeUpdateInstallReceiptRowSync } from "./restart-sentinel-store.js"; import type { UpdateCheckResult } from "./update-check.js"; import { parseDevUpdateTargetEnv } from "./update-dev-target.js"; @@ -138,9 +132,8 @@ vi.mock("./update-managed-service-handoff.js", () => ({ startManagedServiceUpdateHandoff: startManagedServiceUpdateHandoffMock, })); -const UPDATE_CHECK_STATE_KEY = "default"; +const UPDATE_CHECK_STATE_KEY = "update.checkState"; -type UpdateCheckStateDatabase = Pick; type PersistedUpdateCheckState = { lastCheckedAt?: string; lastNotifiedVersion?: string; @@ -157,10 +150,6 @@ type PersistedUpdateCheckState = { autoLastSuccessAt?: string; }; -function presentString(value: string | null): string | undefined { - return value ?? undefined; -} - describe("update-startup", () => { let tempDir: string; let testState: OpenClawTestState; @@ -186,63 +175,11 @@ describe("update-startup", () => { } function readPersistedUpdateCheckState(): PersistedUpdateCheckState | null { - const { db } = openOpenClawStateDatabase(); - const stateDb = getNodeSqliteKysely(db); - const row = executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("update_check_state") - .selectAll() - .where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - if (!row) { - return null; - } - return { - lastCheckedAt: presentString(row.last_checked_at), - lastNotifiedVersion: presentString(row.last_notified_version), - lastNotifiedTag: presentString(row.last_notified_tag), - lastAvailableVersion: presentString(row.last_available_version), - lastAvailableTag: presentString(row.last_available_tag), - autoInstallId: presentString(row.auto_install_id), - autoFirstSeenVersion: presentString(row.auto_first_seen_version), - autoFirstSeenTag: presentString(row.auto_first_seen_tag), - autoFirstSeenAt: presentString(row.auto_first_seen_at), - autoLastAttemptVersion: presentString(row.auto_last_attempt_version), - autoLastAttemptAt: presentString(row.auto_last_attempt_at), - autoLastSuccessVersion: presentString(row.auto_last_success_version), - autoLastSuccessAt: presentString(row.auto_last_success_at), - }; + return readConfigMachineState(UPDATE_CHECK_STATE_KEY) ?? null; } function writePersistedUpdateCheckState(state: PersistedUpdateCheckState): void { - runOpenClawStateWriteTransaction(({ db }) => { - const stateDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - stateDb.deleteFrom("update_check_state").where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - executeSqliteQuerySync( - db, - stateDb.insertInto("update_check_state").values({ - state_key: UPDATE_CHECK_STATE_KEY, - last_checked_at: state.lastCheckedAt ?? null, - last_notified_version: state.lastNotifiedVersion ?? null, - last_notified_tag: state.lastNotifiedTag ?? null, - last_available_version: state.lastAvailableVersion ?? null, - last_available_tag: state.lastAvailableTag ?? null, - auto_install_id: state.autoInstallId ?? null, - auto_first_seen_version: state.autoFirstSeenVersion ?? null, - auto_first_seen_tag: state.autoFirstSeenTag ?? null, - auto_first_seen_at: state.autoFirstSeenAt ?? null, - auto_last_attempt_version: state.autoLastAttemptVersion ?? null, - auto_last_attempt_at: state.autoLastAttemptAt ?? null, - auto_last_success_version: state.autoLastSuccessVersion ?? null, - auto_last_success_at: state.autoLastSuccessAt ?? null, - updated_at_ms: Date.now(), - }), - ); - }); + writeConfigMachineState(UPDATE_CHECK_STATE_KEY, state); } beforeEach(async () => { diff --git a/src/infra/update-startup.ts b/src/infra/update-startup.ts index 12ce10758897..c58b412a1366 100644 --- a/src/infra/update-startup.ts +++ b/src/infra/update-startup.ts @@ -19,11 +19,7 @@ import { REMOTE_MODEL_CATALOG_TTL_MS, } from "../model-catalog/remote-refresh.js"; import { runCommandWithTimeout } from "../process/exec.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { - openOpenClawStateDatabase, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; +import { readConfigMachineState, writeConfigMachineState } from "../state/config-machine-state.js"; import { VERSION } from "../version.js"; import { isTruthyEnvValue } from "./env.js"; import type { GatewayActiveWorkInspectors } from "./gateway-active-work.js"; @@ -31,11 +27,6 @@ import { EXTERNAL_SUPERVISOR_UPDATE_REQUIRED_REASON, isGatewayExternallySupervised, } from "./gateway-supervision.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; import { resolveOpenClawPackageRoot } from "./openclaw-root.js"; import { readVerifiedGitUpdateReceipt, type VerifiedGitUpdateReceipt } from "./restart-sentinel.js"; import { @@ -148,7 +139,7 @@ export function resetUpdateAvailableStateForTest(): void { gatewayUpdateCampaign.resetForTest(); } -const UPDATE_CHECK_STATE_KEY = "default"; +const UPDATE_CHECK_STATE_KEY = "update.checkState"; const UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000; const ONE_HOUR_MS = 60 * 60 * 1000; const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000; @@ -159,8 +150,6 @@ const DEV_COMMIT_LIMIT = 5; const DEV_COMMIT_SUBJECT_MAX_LENGTH = 120; const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024; -type UpdateCheckStateDatabase = Pick; - function shouldSkipCheck(allowInTests: boolean): boolean { return !allowInTests && Boolean(process.env.VITEST || process.env.NODE_ENV === "test"); } @@ -196,69 +185,12 @@ function resolveCheckIntervalMs( return UPDATE_CHECK_INTERVAL_MS; } -function presentString(value: string | null): string | undefined { - return value ?? undefined; -} - async function readState(): Promise { - const database = openOpenClawStateDatabase(); - const stateDb = getNodeSqliteKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( - database.db, - stateDb - .selectFrom("update_check_state") - .selectAll() - .where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - if (!row) { - return {}; - } - return { - lastCheckedAt: presentString(row.last_checked_at), - lastNotifiedVersion: presentString(row.last_notified_version), - lastNotifiedTag: presentString(row.last_notified_tag), - lastAvailableVersion: presentString(row.last_available_version), - lastAvailableTag: presentString(row.last_available_tag), - autoInstallId: presentString(row.auto_install_id), - autoFirstSeenVersion: presentString(row.auto_first_seen_version), - autoFirstSeenTag: presentString(row.auto_first_seen_tag), - autoFirstSeenAt: presentString(row.auto_first_seen_at), - autoLastAttemptVersion: presentString(row.auto_last_attempt_version), - autoLastAttemptAt: presentString(row.auto_last_attempt_at), - autoLastSuccessVersion: presentString(row.auto_last_success_version), - autoLastSuccessAt: presentString(row.auto_last_success_at), - }; + return readConfigMachineState(UPDATE_CHECK_STATE_KEY) ?? {}; } async function writeState(state: UpdateCheckState): Promise { - const updatedAtMs = Date.now(); - runOpenClawStateWriteTransaction(({ db }) => { - const stateDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - stateDb.deleteFrom("update_check_state").where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - executeSqliteQuerySync( - db, - stateDb.insertInto("update_check_state").values({ - state_key: UPDATE_CHECK_STATE_KEY, - last_checked_at: state.lastCheckedAt ?? null, - last_notified_version: state.lastNotifiedVersion ?? null, - last_notified_tag: state.lastNotifiedTag ?? null, - last_available_version: state.lastAvailableVersion ?? null, - last_available_tag: state.lastAvailableTag ?? null, - auto_install_id: state.autoInstallId ?? null, - auto_first_seen_version: state.autoFirstSeenVersion ?? null, - auto_first_seen_tag: state.autoFirstSeenTag ?? null, - auto_first_seen_at: state.autoFirstSeenAt ?? null, - auto_last_attempt_version: state.autoLastAttemptVersion ?? null, - auto_last_attempt_at: state.autoLastAttemptAt ?? null, - auto_last_success_version: state.autoLastSuccessVersion ?? null, - auto_last_success_at: state.autoLastSuccessAt ?? null, - updated_at_ms: updatedAtMs, - }), - ); - }); + writeConfigMachineState(UPDATE_CHECK_STATE_KEY, state); } function sameUpdateAvailable(a: UpdateAvailable | null, b: UpdateAvailable | null): boolean { diff --git a/src/infra/voicewake-routing.ts b/src/infra/voicewake-routing.ts index 9a249d1d2279..d35f65fdcbc0 100644 --- a/src/infra/voicewake-routing.ts +++ b/src/infra/voicewake-routing.ts @@ -1,13 +1,7 @@ // Persists and resolves voice wake routing rules. import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { normalizeAgentId } from "../routing/session-key.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; // Voice wake routing maps normalized wake phrases to an agent, session key, or // current session target and persists the mapping under state settings. @@ -28,7 +22,7 @@ export type VoiceWakeRoutingConfig = { updatedAtMs: number; }; -const VOICEWAKE_ROUTING_CONFIG_KEY = "default"; +const VOICEWAKE_ROUTING_STATE_KEY = "voicewake.routing"; const DEFAULT_ROUTING: VoiceWakeRoutingConfig = { version: 1, @@ -37,17 +31,6 @@ const DEFAULT_ROUTING: VoiceWakeRoutingConfig = { updatedAtMs: 0, }; -type VoiceWakeRoutingDatabase = Pick< - OpenClawStateKyselyDatabase, - "voicewake_routing_config" | "voicewake_routing_routes" ->; - -function openStateDatabase(stateDir?: string) { - return openOpenClawStateDatabase({ - env: stateDir ? { ...process.env, OPENCLAW_STATE_DIR: stateDir } : process.env, - }); -} - /** Normalize a voice wake trigger phrase for matching and duplicate checks. */ function normalizeVoiceWakeTriggerWord(value: string): string { return value @@ -127,61 +110,15 @@ export function normalizeVoiceWakeRoutingConfig(input: unknown): VoiceWakeRoutin }; } -function targetFromColumns(params: { - agentId: string | null; - mode: string; - sessionKey: string | null; -}): VoiceWakeRouteTarget { - if (params.mode === "agent" && params.agentId) { - return { agentId: params.agentId }; - } - if (params.mode === "session" && params.sessionKey) { - return { sessionKey: params.sessionKey }; - } - return { mode: "current" }; -} - /** Load persisted voice wake routing config from state. */ export async function loadVoiceWakeRoutingConfig( baseDir?: string, ): Promise { - const database = openStateDatabase(baseDir); - const routingDb = getNodeSqliteKysely(database.db); - const configRow = executeSqliteQueryTakeFirstSync( - database.db, - routingDb - .selectFrom("voicewake_routing_config") - .selectAll() - .where("config_key", "=", VOICEWAKE_ROUTING_CONFIG_KEY), + const config = readConfigMachineState( + VOICEWAKE_ROUTING_STATE_KEY, + baseDir ? { env: { ...process.env, OPENCLAW_STATE_DIR: baseDir } } : {}, ); - if (!configRow) { - return { ...DEFAULT_ROUTING }; - } - const routeRows = executeSqliteQuerySync( - database.db, - routingDb - .selectFrom("voicewake_routing_routes") - .selectAll() - .where("config_key", "=", VOICEWAKE_ROUTING_CONFIG_KEY) - .orderBy("position", "asc"), - ).rows; - return { - version: 1, - defaultTarget: targetFromColumns({ - agentId: configRow.default_target_agent_id, - mode: configRow.default_target_mode, - sessionKey: configRow.default_target_session_key, - }), - routes: routeRows.map((row) => ({ - trigger: row.trigger, - target: targetFromColumns({ - agentId: row.target_agent_id, - mode: row.target_mode, - sessionKey: row.target_session_key, - }), - })), - updatedAtMs: configRow.updated_at_ms, - }; + return config ? normalizeVoiceWakeRoutingConfig(config) : { ...DEFAULT_ROUTING }; } type VoiceWakeResolvedRoute = { mode: "current" } | { agentId: string } | { sessionKey: string }; diff --git a/src/infra/voicewake.ts b/src/infra/voicewake.ts index b8e91c76fdf3..88937fee4bcb 100644 --- a/src/infra/voicewake.ts +++ b/src/infra/voicewake.ts @@ -1,11 +1,9 @@ // Stores voice wake trigger configuration. import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { - openOpenClawStateDatabase, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; + readConfigMachineStateWithMetadata, + writeConfigMachineState, +} from "../state/config-machine-state.js"; // Voice wake config stores trigger words used by local voice integrations. type VoiceWakeConfig = { @@ -14,9 +12,7 @@ type VoiceWakeConfig = { }; const DEFAULT_TRIGGERS = ["openclaw", "claude", "computer"]; -const VOICEWAKE_CONFIG_KEY = "default"; - -type VoiceWakeDatabase = Pick; +const VOICEWAKE_TRIGGERS_STATE_KEY = "voicewake.triggers"; function sanitizeTriggers(triggers: string[] | undefined | null): string[] { const cleaned = (triggers ?? []) @@ -25,10 +21,8 @@ function sanitizeTriggers(triggers: string[] | undefined | null): string[] { return cleaned.length > 0 ? cleaned : DEFAULT_TRIGGERS; } -function openStateDatabase(stateDir?: string) { - return openOpenClawStateDatabase({ - env: stateDir ? { ...process.env, OPENCLAW_STATE_DIR: stateDir } : process.env, - }); +function stateDatabaseOptions(stateDir?: string) { + return stateDir ? { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } } : {}; } /** Return the built-in voice wake trigger list. */ @@ -38,22 +32,16 @@ export function defaultVoiceWakeTriggers() { /** Load persisted voice wake triggers, falling back to defaults. */ export async function loadVoiceWakeConfig(baseDir?: string): Promise { - const database = openStateDatabase(baseDir); - const voicewakeDb = getNodeSqliteKysely(database.db); - const rows = executeSqliteQuerySync( - database.db, - voicewakeDb - .selectFrom("voicewake_triggers") - .select(["trigger", "updated_at_ms"]) - .where("config_key", "=", VOICEWAKE_CONFIG_KEY) - .orderBy("position", "asc"), - ).rows; - if (rows.length === 0) { + const state = readConfigMachineStateWithMetadata( + VOICEWAKE_TRIGGERS_STATE_KEY, + stateDatabaseOptions(baseDir), + ); + if (!state) { return { triggers: defaultVoiceWakeTriggers(), updatedAtMs: 0 }; } return { - triggers: sanitizeTriggers(rows.map((row) => row.trigger)), - updatedAtMs: Math.max(0, ...rows.map((row) => row.updated_at_ms)), + triggers: sanitizeTriggers(state.value), + updatedAtMs: Math.max(0, state.updatedAtMs), }; } @@ -63,30 +51,6 @@ export async function setVoiceWakeTriggers( baseDir?: string, ): Promise { const sanitized = sanitizeTriggers(triggers); - const updatedAtMs = Date.now(); - runOpenClawStateWriteTransaction( - ({ db }) => { - const voicewakeDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - voicewakeDb.deleteFrom("voicewake_triggers").where("config_key", "=", VOICEWAKE_CONFIG_KEY), - ); - executeSqliteQuerySync( - db, - voicewakeDb.insertInto("voicewake_triggers").values( - sanitized.map((trigger, position) => ({ - config_key: VOICEWAKE_CONFIG_KEY, - position, - trigger, - updated_at_ms: updatedAtMs, - })), - ), - ); - }, - baseDir ? { env: { ...process.env, OPENCLAW_STATE_DIR: baseDir } } : {}, - ); - return { - triggers: sanitized, - updatedAtMs, - }; + writeConfigMachineState(VOICEWAKE_TRIGGERS_STATE_KEY, sanitized, stateDatabaseOptions(baseDir)); + return loadVoiceWakeConfig(baseDir); } diff --git a/src/model-catalog/remote-store.test.ts b/src/model-catalog/remote-store.test.ts index 96216a349c23..bf755685d641 100644 --- a/src/model-catalog/remote-store.test.ts +++ b/src/model-catalog/remote-store.test.ts @@ -2,11 +2,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { - closeOpenClawStateDatabaseForTest, - openOpenClawStateDatabase, -} from "../state/openclaw-state-db.js"; +import { readConfigMachineState } from "../state/config-machine-state.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { markRemoteModelCatalogChecked, readRemoteModelCatalog, @@ -22,48 +19,26 @@ afterEach(() => { }); describe("remote model catalog store", () => { - it("lazily adds the cache table to an existing current-schema database", () => { - const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-"))); - roots.push(root); - const options = { path: path.join(root, "state.sqlite") }; - openOpenClawStateDatabase(options); - closeOpenClawStateDatabaseForTest(); - - const { DatabaseSync } = requireNodeSqlite(); - const preCatalog = new DatabaseSync(options.path); - preCatalog.exec("DROP TABLE model_catalog_remote;"); - preCatalog.exec("DROP INDEX idx_task_runs_status;"); - preCatalog.close(); - - const reopened = openOpenClawStateDatabase(options); - expect( - reopened.db - .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") - .get("model_catalog_remote"), - ).toBeUndefined(); - expect( - reopened.db - .prepare("SELECT name FROM sqlite_schema WHERE type = 'index' AND name = ?") - .get("idx_task_runs_status"), - ).toEqual({ name: "idx_task_runs_status" }); - closeOpenClawStateDatabaseForTest(); - - expect(readRemoteModelCatalog(options)).toBeUndefined(); - const upgraded = new DatabaseSync(options.path, { readOnly: true }); - expect( - upgraded - .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") - .get("model_catalog_remote"), - ).toEqual({ name: "model_catalog_remote" }); - upgraded.close(); - }); - - it("lazily ensures twice and upserts the single slot", () => { + it("stores one machine-state snapshot and rejects stale refreshes", () => { const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-"))); roots.push(root); const options = { path: path.join(root, "state.sqlite") }; expect(readRemoteModelCatalog(options)).toBeUndefined(); - expect(readRemoteModelCatalog(options)).toBeUndefined(); + expect( + markRemoteModelCatalogChecked( + 1, + { + expected: { + source_url: "https://catalog.test/one", + generated_at: 1, + etag: null, + last_modified: null, + }, + }, + options, + ), + ).toBe(false); + expect(readConfigMachineState("modelCatalog.remote", options)).toBeUndefined(); writeRemoteModelCatalog( { bundle_json: '{"schemaVersion":1}', @@ -124,7 +99,7 @@ describe("remote model catalog store", () => { { expected: { source_url: "https://catalog.test/two", - generated_at: 2, + generated_at: 3, etag: '"older"', last_modified: null, }, @@ -152,5 +127,14 @@ describe("remote model catalog store", () => { source_url: "https://catalog.test/two", checked_at: 6, }); + expect(readConfigMachineState("modelCatalog.remote", options)).toEqual({ + bundle_json: '{"schemaVersion":1,"updated":true}', + generated_at: 3, + min_version: "2026.7.0", + source_url: "https://catalog.test/two", + etag: '"two"', + last_modified: null, + checked_at: 6, + }); }); }); diff --git a/src/model-catalog/remote-store.ts b/src/model-catalog/remote-store.ts index 864830e9e9af..5b12ed704a54 100644 --- a/src/model-catalog/remote-store.ts +++ b/src/model-catalog/remote-store.ts @@ -1,114 +1,58 @@ -import type { DatabaseSync } from "node:sqlite"; -import type { Selectable } from "kysely"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../infra/kysely-sync.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { - openOpenClawStateDatabase, - runOpenClawStateWriteTransaction, - type OpenClawStateDatabaseOptions, -} from "../state/openclaw-state-db.js"; +import { readConfigMachineState, updateConfigMachineState } from "../state/config-machine-state.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; -type RemoteModelCatalogDatabase = Pick; -type RemoteModelCatalogStoreRow = Selectable; +type RemoteModelCatalogStoreRow = { + id: number; + bundle_json: string; + generated_at: number; + min_version: string | null; + source_url: string; + etag: string | null; + last_modified: string | null; + checked_at: number; +}; + +type RemoteModelCatalogSnapshot = Omit; type RemoteModelCatalogWriteResult = | { status: "written" } | { status: "retained-newer"; row: RemoteModelCatalogStoreRow }; -const ensuredDatabases = new WeakSet(); -const REMOTE_MODEL_CATALOG_SCHEMA_SQL = ` -CREATE TABLE IF NOT EXISTS model_catalog_remote ( - id INTEGER PRIMARY KEY CHECK (id = 1), - bundle_json TEXT NOT NULL, - generated_at INTEGER NOT NULL, - min_version TEXT, - source_url TEXT NOT NULL, - etag TEXT, - last_modified TEXT, - checked_at INTEGER NOT NULL -) STRICT; -`; - -function ensureRemoteModelCatalogSchema(options: OpenClawStateDatabaseOptions = {}): void { - const database = openOpenClawStateDatabase(options); - if (ensuredDatabases.has(database.db)) { - return; - } - runOpenClawStateWriteTransaction( - ({ db }) => { - // sqlite-allow-raw -- feature-local additive schema DDL; catalog rows use Kysely below. - db.exec(REMOTE_MODEL_CATALOG_SCHEMA_SQL); - }, - options, - { operationLabel: "model-catalog.remote.schema.ensure" }, - ); - ensuredDatabases.add(database.db); -} - -function openDatabase(options: OpenClawStateDatabaseOptions) { - ensureRemoteModelCatalogSchema(options); - return openOpenClawStateDatabase(options); -} +const REMOTE_MODEL_CATALOG_STATE_KEY = "modelCatalog.remote"; export function readRemoteModelCatalog( options: OpenClawStateDatabaseOptions = {}, ): RemoteModelCatalogStoreRow | undefined { - const state = openDatabase(options); - const db = getNodeSqliteKysely(state.db); - return executeSqliteQueryTakeFirstSync( - state.db, - db.selectFrom("model_catalog_remote").selectAll().where("id", "=", 1), + const snapshot = readConfigMachineState( + REMOTE_MODEL_CATALOG_STATE_KEY, + options, ); + return snapshot ? { id: 1, ...snapshot } : undefined; } export function writeRemoteModelCatalog( - row: Omit, + row: RemoteModelCatalogSnapshot, options: OpenClawStateDatabaseOptions = {}, ): RemoteModelCatalogWriteResult { - ensureRemoteModelCatalogSchema(options); - return runOpenClawStateWriteTransaction( - ({ db: sqlite }) => { - const db = getNodeSqliteKysely(sqlite); - const current = executeSqliteQueryTakeFirstSync( - sqlite, - db.selectFrom("model_catalog_remote").selectAll().where("id", "=", 1), - ); - // The CLI and Gateway can refresh concurrently in separate processes. - // Compare under BEGIN IMMEDIATE so a slower stale response cannot win the singleton slot. + let result: RemoteModelCatalogWriteResult = { status: "written" }; + updateConfigMachineState( + REMOTE_MODEL_CATALOG_STATE_KEY, + (current) => { + // CLI and Gateway refreshes race across processes; compare inside the write transaction. if ( current && current.source_url === row.source_url && (current.generated_at > row.generated_at || (current.generated_at === row.generated_at && current.bundle_json !== row.bundle_json)) ) { - return { status: "retained-newer", row: current }; + result = { status: "retained-newer", row: { id: 1, ...current } }; + return current; } - executeSqliteQuerySync( - sqlite, - db - .insertInto("model_catalog_remote") - .values({ id: 1, ...row }) - .onConflict((conflict) => - conflict.column("id").doUpdateSet({ - bundle_json: row.bundle_json, - generated_at: row.generated_at, - min_version: row.min_version, - source_url: row.source_url, - etag: row.etag, - last_modified: row.last_modified, - checked_at: row.checked_at, - }), - ), - ); - return { status: "written" }; + return row; }, options, - { operationLabel: "model-catalog.remote.write" }, ); + return result; } export function markRemoteModelCatalogChecked( @@ -123,31 +67,30 @@ export function markRemoteModelCatalogChecked( }, options: OpenClawStateDatabaseOptions = {}, ): boolean { - ensureRemoteModelCatalogSchema(options); - return runOpenClawStateWriteTransaction( - ({ db: sqlite }) => { - const db = getNodeSqliteKysely(sqlite); - let query = db - .updateTable("model_catalog_remote") - .set({ - checked_at: checkedAt, - ...(metadata.etag !== undefined ? { etag: metadata.etag } : {}), - ...(metadata.lastModified !== undefined ? { last_modified: metadata.lastModified } : {}), - }) - .where("id", "=", 1) - .where("source_url", "=", metadata.expected.source_url) - .where("generated_at", "=", metadata.expected.generated_at); - query = - metadata.expected.etag === null - ? query.where("etag", "is", null) - : query.where("etag", "=", metadata.expected.etag); - query = - metadata.expected.last_modified === null - ? query.where("last_modified", "is", null) - : query.where("last_modified", "=", metadata.expected.last_modified); - return executeSqliteQuerySync(sqlite, query).numAffectedRows === 1n; + let matched = false; + updateConfigMachineState( + REMOTE_MODEL_CATALOG_STATE_KEY, + (current) => { + if (!current) { + return undefined; + } + if ( + current.source_url !== metadata.expected.source_url || + current.generated_at !== metadata.expected.generated_at || + current.etag !== metadata.expected.etag || + current.last_modified !== metadata.expected.last_modified + ) { + return current; + } + matched = true; + return { + ...current, + checked_at: checkedAt, + ...(metadata.etag !== undefined ? { etag: metadata.etag } : {}), + ...(metadata.lastModified !== undefined ? { last_modified: metadata.lastModified } : {}), + }; }, options, - { operationLabel: "model-catalog.remote.checked" }, ); + return matched; } diff --git a/src/node-host/config.test.ts b/src/node-host/config.test.ts index 47d9d7a8aa2e..16a968747a36 100644 --- a/src/node-host/config.test.ts +++ b/src/node-host/config.test.ts @@ -4,31 +4,20 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../infra/kysely-sync.js"; -import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; + readConfigMachineState, + readConfigMachineStateWithMetadata, + writeConfigMachineState, +} from "../state/config-machine-state.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { - closeOpenClawStateDatabaseForTest, - openOpenClawStateDatabase, - runOpenClawStateWriteTransaction, -} from "../state/openclaw-state-db.js"; -import { configureNodeHost, loadNodeHostConfig, type NodeHostConfig } from "./config.js"; + configureNodeHost, + loadNodeHostConfig, + NODE_HOST_CONFIG_KEY, + type NodeHostConfig, +} from "./config.js"; const fixtureDigest = ["fixture", "digest"].join("-"); -function readStoredToken(env: NodeJS.ProcessEnv): string | null | undefined { - const database = openOpenClawStateDatabase({ env }); - return executeSqliteQueryTakeFirstSync( - database.db, - getNodeSqliteKysely>(database.db) - .selectFrom("node_host_config") - .select("token") - .where("config_key", "=", "current"), - )?.token; -} - async function runConcurrentImplicitConfigures( stateDir: string, ): Promise<[NodeHostConfig, NodeHostConfig]> { @@ -194,6 +183,13 @@ describe("node-host SQLite config", () => { }, }, }); + expect(readConfigMachineState(NODE_HOST_CONFIG_KEY, { env })).toEqual( + configured, + ); + expect(readConfigMachineStateWithMetadata(NODE_HOST_CONFIG_KEY, { env })?.updatedAtMs).toBe( + 1_234, + ); + expect(readConfigMachineState(NODE_HOST_CONFIG_KEY, { env })).not.toHaveProperty("token"); closeOpenClawStateDatabaseForTest(); await expect(loadNodeHostConfig(env)).resolves.toEqual(configured); await expect(fs.stat(path.join(stateDir, "node.json"))).rejects.toMatchObject({ @@ -223,61 +219,6 @@ describe("node-host SQLite config", () => { await expect(loadNodeHostConfig(env)).resolves.toMatchObject({ installedAppsSharing: true }); }); - it("adds the gateway context-path column to an existing state database", async () => { - const { env } = makeTestEnv(); - const database = openOpenClawStateDatabase({ env }); - database.db.exec(` - ALTER TABLE node_host_config DROP COLUMN gateway_context_path; - PRAGMA user_version = 5; - UPDATE schema_meta SET schema_version = 5 WHERE meta_key = 'primary'; - `); - closeOpenClawStateDatabaseForTest(); - - const configured = await configureNodeHost({ - fallbackDisplayName: "node", - gateway: { contextPath: "/upgraded" }, - env, - nowMs: 1, - candidateNodeId: "upgraded-node", - }); - - expect(configured.gateway?.contextPath).toBe("/upgraded"); - const columns = openOpenClawStateDatabase({ env }) - .db.prepare("PRAGMA table_info(node_host_config)") - .all() as Array<{ name?: unknown }>; - expect(columns).toContainEqual(expect.objectContaining({ name: "gateway_context_path" })); - }); - - it("adds the Cloudflare Access column to an existing state database", async () => { - const { env } = makeTestEnv(); - const database = openOpenClawStateDatabase({ env }); - database.db.exec("ALTER TABLE node_host_config DROP COLUMN gateway_cloudflare_access_json;"); - closeOpenClawStateDatabaseForTest(); - - const configured = await configureNodeHost({ - fallbackDisplayName: "node", - gateway: { - cloudflareAccess: { - clientId: "$CF_ACCESS_CLIENT_ID", - clientSecret: "$CF_ACCESS_CLIENT_SECRET", - }, - }, - env, - nowMs: 1, - }); - - expect(configured.gateway?.cloudflareAccess).toEqual({ - clientId: { source: "env", provider: "default", id: "CF_ACCESS_CLIENT_ID" }, - clientSecret: { source: "env", provider: "default", id: "CF_ACCESS_CLIENT_SECRET" }, - }); - const columns = openOpenClawStateDatabase({ env }) - .db.prepare("PRAGMA table_info(node_host_config)") - .all() as Array<{ name?: unknown }>; - expect(columns).toContainEqual( - expect.objectContaining({ name: "gateway_cloudflare_access_json" }), - ); - }); - it("keeps the first committed implicit node id across processes", async () => { const { env, stateDir } = makeTestEnv(); const [first, second] = await runConcurrentImplicitConfigures(stateDir); @@ -323,30 +264,7 @@ describe("node-host SQLite config", () => { it("rejects corrupt canonical rows instead of rotating identity", async () => { const { env } = makeTestEnv(); - runOpenClawStateWriteTransaction( - ({ db }) => { - executeSqliteQuerySync( - db, - getNodeSqliteKysely>(db) - .insertInto("node_host_config") - .values({ - config_key: "current", - version: 2, - node_id: "stale-node", - token: null, - display_name: null, - gateway_host: null, - gateway_port: null, - gateway_tls: null, - gateway_tls_fingerprint: null, - gateway_context_path: null, - gateway_cloudflare_access_json: null, - updated_at_ms: 1, - }), - ); - }, - { env }, - ); + writeConfigMachineState(NODE_HOST_CONFIG_KEY, { version: 2, nodeId: "stale-node" }, { env }); await expect(loadNodeHostConfig(env)).rejects.toThrow("unsupported version 2"); await expect( @@ -354,39 +272,6 @@ describe("node-host SQLite config", () => { ).rejects.toThrow("unsupported version 2"); }); - it("never reads legacy token material and nulls it on every configure", async () => { - const { env } = makeTestEnv(); - runOpenClawStateWriteTransaction( - ({ db }) => { - executeSqliteQuerySync( - db, - getNodeSqliteKysely>(db) - .insertInto("node_host_config") - .values({ - config_key: "current", - version: 1, - node_id: "node-with-token", - token: "test-token-placeholder", - display_name: null, - gateway_host: null, - gateway_port: null, - gateway_tls: null, - gateway_tls_fingerprint: null, - gateway_context_path: null, - gateway_cloudflare_access_json: null, - updated_at_ms: 1, - }), - ); - }, - { env }, - ); - - await expect(loadNodeHostConfig(env)).resolves.toMatchObject({ nodeId: "node-with-token" }); - expect(readStoredToken(env)).toBe("test-token-placeholder"); - await configureNodeHost({ fallbackDisplayName: "node", gateway: {}, env, nowMs: 2 }); - expect(readStoredToken(env)).toBeNull(); - }); - it.each(["source", "claim", "dangling-source-symlink"] as const)( "blocks runtime while retired state remains: %s", async (kind) => { diff --git a/src/node-host/config.ts b/src/node-host/config.ts index cd3c92a6ea4b..55f28f75df83 100644 --- a/src/node-host/config.ts +++ b/src/node-host/config.ts @@ -2,17 +2,16 @@ import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import type { Insertable, Selectable } from "kysely"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; import { resolveStateDir } from "../config/paths.js"; import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync, getNodeSqliteKysely, } from "../infra/kysely-sync.js"; -import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; +import { readConfigMachineStateWithMetadata } from "../state/config-machine-state.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { - openOpenClawStateDatabase, runOpenClawStateWriteTransaction, type OpenClawStateDatabaseOptions, } from "../state/openclaw-state-db.js"; @@ -42,14 +41,11 @@ export type NodeHostConfig = { installedAppsSharing?: boolean; }; -export const NODE_HOST_CONFIG_KEY = "current"; +export const NODE_HOST_CONFIG_KEY = "nodeHost.config"; export const LEGACY_NODE_HOST_CONFIG_FILE = "node.json"; export const LEGACY_NODE_HOST_CONFIG_CLAIM_SUFFIX = ".doctor-importing"; -type NodeHostConfigDatabase = Pick; -type NodeHostConfigRow = Selectable; -type NodeHostConfigRuntimeRow = Omit; -type NodeHostConfigInsert = Insertable; +type NodeHostConfigDatabase = Pick; function databaseOptions(env: NodeJS.ProcessEnv): OpenClawStateDatabaseOptions { return { env }; @@ -89,10 +85,13 @@ function assertNodeHostLegacyStateMigrated(env: NodeJS.ProcessEnv = process.env) ); } -function optionalNonEmptyString(value: string | null, label: string): string | undefined { - if (value === null) { +function optionalNonEmptyString(value: unknown, label: string): string | undefined { + if (value === null || value === undefined) { return undefined; } + if (typeof value !== "string") { + throw new Error(`invalid node-host SQLite row: ${label} must be a string`); + } const normalized = value.trim(); if (!normalized) { throw new Error(`invalid node-host SQLite row: ${label} must not be empty`); @@ -105,67 +104,70 @@ function optionalInputString(value: string | undefined): string | undefined { return normalized || undefined; } -function parseCloudflareAccessJson( - value: string | null, -): NodeHostCloudflareAccessConfig | undefined { - if (value === null) { - return undefined; - } - try { - return normalizeNodeHostCloudflareAccessConfig(JSON.parse(value) as unknown); - } catch (error) { - throw new Error("invalid node-host SQLite row: gateway_cloudflare_access_json", { - cause: error, - }); - } -} - -function validatePort(value: number | null | undefined, label: string): number | undefined { +function validatePort(value: unknown, label: string): number | undefined { if (value === null || value === undefined) { return undefined; } - if (!Number.isSafeInteger(value) || value <= 0 || value > 65_535) { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0 || value > 65_535) { throw new Error(`invalid node-host ${label}: expected an integer between 1 and 65535`); } return value; } -function rowToNodeHostConfig(row: NodeHostConfigRuntimeRow): NodeHostConfig { - if (row.version !== 1) { - throw new Error(`invalid node-host SQLite row: unsupported version ${String(row.version)}`); +function normalizeStoredNodeHostConfig(value: unknown): NodeHostConfig { + if (!isRecord(value)) { + throw new Error("invalid node-host SQLite row: expected a configuration object"); } - const nodeId = row.node_id.trim(); + if (value.version !== 1) { + throw new Error(`invalid node-host SQLite row: unsupported version ${String(value.version)}`); + } + const nodeId = typeof value.nodeId === "string" ? value.nodeId.trim() : ""; if (!nodeId) { throw new Error("invalid node-host SQLite row: node_id must not be empty"); } - if (!Number.isSafeInteger(row.updated_at_ms) || row.updated_at_ms < 0) { - throw new Error("invalid node-host SQLite row: updated_at_ms must be a non-negative integer"); + const storedGateway = value.gateway; + if (storedGateway !== undefined && !isRecord(storedGateway)) { + throw new Error("invalid node-host SQLite row: gateway must be an object"); } - if (row.gateway_tls !== null && row.gateway_tls !== 0 && row.gateway_tls !== 1) { - throw new Error("invalid node-host SQLite row: gateway_tls must be 0, 1, or null"); + const gatewayTls = storedGateway?.tls; + if (gatewayTls !== undefined && typeof gatewayTls !== "boolean") { + throw new Error("invalid node-host SQLite row: gateway_tls must be a boolean"); } - if (row.installed_apps_sharing !== 0 && row.installed_apps_sharing !== 1) { - throw new Error("invalid node-host SQLite row: installed_apps_sharing must be 0 or 1"); + if (value.installedAppsSharing !== undefined && typeof value.installedAppsSharing !== "boolean") { + throw new Error("invalid node-host SQLite row: installed_apps_sharing must be a boolean"); } - const cloudflareAccess = parseCloudflareAccessJson(row.gateway_cloudflare_access_json); - const gateway: NodeHostGatewayConfig = { - host: optionalNonEmptyString(row.gateway_host, "gateway_host"), - port: validatePort(row.gateway_port, "SQLite gateway_port"), - tls: row.gateway_tls === null ? undefined : row.gateway_tls === 1, - tlsFingerprint: optionalNonEmptyString(row.gateway_tls_fingerprint, "gateway_tls_fingerprint"), - contextPath: optionalNonEmptyString(row.gateway_context_path, "gateway_context_path"), - ...(cloudflareAccess ? { cloudflareAccess } : {}), - }; - const hasGateway = Object.values(gateway).some((value) => value !== undefined); + const gateway = storedGateway + ? normalizeGatewayConfig({ + host: optionalNonEmptyString(storedGateway.host, "gateway_host"), + port: validatePort(storedGateway.port, "SQLite gateway_port"), + tls: typeof gatewayTls === "boolean" ? gatewayTls : undefined, + tlsFingerprint: optionalNonEmptyString( + storedGateway.tlsFingerprint, + "gateway_tls_fingerprint", + ), + contextPath: optionalNonEmptyString(storedGateway.contextPath, "gateway_context_path"), + ...cloudflareAccessEntry( + normalizeNodeHostCloudflareAccessConfig(storedGateway.cloudflareAccess), + ), + }) + : undefined; return { version: 1, nodeId, - displayName: optionalNonEmptyString(row.display_name, "display_name"), - gateway: hasGateway ? gateway : undefined, - installedAppsSharing: row.installed_apps_sharing === 1, + displayName: optionalNonEmptyString(value.displayName, "display_name"), + gateway, + installedAppsSharing: value.installedAppsSharing === true, }; } +// Own-property parity with the retired column reader: an absent Cloudflare +// Access config omits the key entirely so toStrictEqual consumers match. +function cloudflareAccessEntry(cloudflareAccess: NodeHostCloudflareAccessConfig | undefined): { + cloudflareAccess?: NodeHostCloudflareAccessConfig; +} { + return cloudflareAccess ? { cloudflareAccess } : {}; +} + function normalizeGatewayConfig(gateway: NodeHostGatewayConfig): NodeHostGatewayConfig | undefined { const normalized: NodeHostGatewayConfig = { host: optionalInputString(gateway.host), @@ -173,58 +175,23 @@ function normalizeGatewayConfig(gateway: NodeHostGatewayConfig): NodeHostGateway tls: gateway.tls, tlsFingerprint: optionalInputString(gateway.tlsFingerprint), contextPath: optionalInputString(gateway.contextPath), - cloudflareAccess: normalizeNodeHostCloudflareAccessConfig(gateway.cloudflareAccess), + ...cloudflareAccessEntry(normalizeNodeHostCloudflareAccessConfig(gateway.cloudflareAccess)), }; return Object.values(normalized).some((value) => value !== undefined) ? normalized : undefined; } -function configToRow(params: { - config: NodeHostConfig; - updatedAtMs: number; -}): NodeHostConfigInsert { - const gateway = params.config.gateway; - return { - config_key: NODE_HOST_CONFIG_KEY, - version: 1, - node_id: params.config.nodeId, - token: null, - display_name: params.config.displayName ?? null, - gateway_host: gateway?.host ?? null, - gateway_port: gateway?.port ?? null, - gateway_tls: gateway?.tls === undefined ? null : gateway.tls ? 1 : 0, - gateway_tls_fingerprint: gateway?.tlsFingerprint ?? null, - gateway_context_path: gateway?.contextPath ?? null, - gateway_cloudflare_access_json: gateway?.cloudflareAccess - ? JSON.stringify(gateway.cloudflareAccess) - : null, - installed_apps_sharing: params.config.installedAppsSharing ? 1 : 0, - updated_at_ms: params.updatedAtMs, - }; -} - -function readNodeHostConfigRow( - database: Pick, "db">, -): NodeHostConfigRuntimeRow | undefined { - return executeSqliteQueryTakeFirstSync( - database.db, - getNodeSqliteKysely(database.db) - .selectFrom("node_host_config") - .select([ - "config_key", - "version", - "node_id", - "display_name", - "gateway_host", - "gateway_port", - "gateway_tls", - "gateway_tls_fingerprint", - "gateway_context_path", - "gateway_cloudflare_access_json", - "installed_apps_sharing", - "updated_at_ms", - ]) - .where("config_key", "=", NODE_HOST_CONFIG_KEY), +function readNodeHostConfig(env: NodeJS.ProcessEnv): NodeHostConfig | null { + const stored = readConfigMachineStateWithMetadata( + NODE_HOST_CONFIG_KEY, + databaseOptions(env), ); + if (!stored) { + return null; + } + if (!Number.isSafeInteger(stored.updatedAtMs) || stored.updatedAtMs < 0) { + throw new Error("invalid node-host SQLite row: updated_at_ms must be a non-negative integer"); + } + return normalizeStoredNodeHostConfig(stored.value); } /** Load canonical node-host state. Legacy files block the read until Doctor migrates them. */ @@ -232,9 +199,7 @@ export async function loadNodeHostConfig( env: NodeJS.ProcessEnv = process.env, ): Promise { assertNodeHostLegacyStateMigrated(env); - const database = openOpenClawStateDatabase(databaseOptions(env)); - const row = readNodeHostConfigRow(database); - return row ? rowToNodeHostConfig(row) : null; + return readNodeHostConfig(env); } /** Load existing node-host state without creating or joining the writable shared-state lifecycle. */ @@ -242,12 +207,7 @@ export async function loadNodeHostConfigReadOnly( env: NodeJS.ProcessEnv = process.env, ): Promise { assertNodeHostLegacyStateMigrated(env); - return ( - withExistingOpenClawStateDatabaseReadOnly(({ db }) => { - const row = readNodeHostConfigRow({ db }); - return row ? rowToNodeHostConfig(row) : null; - }, databaseOptions(env)) ?? null - ); + return readNodeHostConfig(env); } /** @@ -276,27 +236,40 @@ export async function configureNodeHost(params: { throw new Error("invalid node-host updatedAtMs: expected a non-negative integer"); } - const config = runOpenClawStateWriteTransaction((database) => { - const { db } = database; - const existingRow = readNodeHostConfigRow(database); - const existing = existingRow ? rowToNodeHostConfig(existingRow) : null; - const nodeId = explicitNodeId ?? existing?.nodeId ?? candidateNodeId; - const displayName = explicitDisplayName ?? existing?.displayName ?? fallbackDisplayName; + const config = runOpenClawStateWriteTransaction(({ db }) => { + const stateDb = getNodeSqliteKysely(db); + const stored = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("config_machine_state") + .select("value_json") + .where("state_key", "=", NODE_HOST_CONFIG_KEY), + ); + const existing = stored + ? normalizeStoredNodeHostConfig(JSON.parse(stored.value_json) as unknown) + : undefined; const next: NodeHostConfig = { version: 1, - nodeId, - displayName, + nodeId: explicitNodeId ?? existing?.nodeId ?? candidateNodeId, + displayName: explicitDisplayName ?? existing?.displayName ?? fallbackDisplayName, gateway, installedAppsSharing: params.installedAppsSharing ?? existing?.installedAppsSharing ?? false, }; - const row = configToRow({ config: next, updatedAtMs }); - const { config_key: _configKey, ...updates } = row; + const valueJson = JSON.stringify(next); executeSqliteQuerySync( db, - getNodeSqliteKysely(db) - .insertInto("node_host_config") - .values(row) - .onConflict((conflict) => conflict.column("config_key").doUpdateSet(updates)), + stateDb + .insertInto("config_machine_state") + .values({ + state_key: NODE_HOST_CONFIG_KEY, + value_json: valueJson, + updated_at_ms: updatedAtMs, + }) + .onConflict((conflict) => + conflict + .column("state_key") + .doUpdateSet({ value_json: valueJson, updated_at_ms: updatedAtMs }), + ), ); return next; }, databaseOptions(env)); diff --git a/src/node-host/node-worker-launch-store.test.ts b/src/node-host/node-worker-launch-store.test.ts index db055187c309..1ea4f78f1606 100644 --- a/src/node-host/node-worker-launch-store.test.ts +++ b/src/node-host/node-worker-launch-store.test.ts @@ -312,7 +312,7 @@ describe("node worker launch store container identity", () => { expect(new NodeWorkerLaunchStore({ env }).get("container-launch")).toEqual(receipt); }); - it("lets the same-schema predecessor read and write a populated candidate container journal before candidate reopen", () => { + it("lets the exact v12 predecessor read and write a populated candidate container journal before candidate reopen", () => { const { database, env, store } = fixture(); const databasePath = openOpenClawStateDatabase({ env }).path; const { planHash, supervisor } = claimLaunch(store, "candidate-container-launch"); @@ -330,9 +330,8 @@ describe("node worker launch store container identity", () => { nowMs: NOW_MS, }); expect(hasContainerIdentityTable(database)).toBe(true); - expect(database.prepare("PRAGMA user_version").get()).toEqual({ - user_version: OPENCLAW_STATE_SCHEMA_VERSION, - }); + expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(12); + expect(database.prepare("PRAGMA user_version").get()).toEqual({ user_version: 12 }); closeOpenClawStateDatabaseForTest(); const companionStart = OPENCLAW_STATE_SCHEMA_SQL.indexOf( @@ -360,13 +359,11 @@ describe("node worker launch store container identity", () => { const predecessor = new DatabaseSync(databasePath); try { - expect(predecessor.prepare("PRAGMA user_version").get()).toEqual({ - user_version: OPENCLAW_STATE_SCHEMA_VERSION, - }); + expect(predecessor.prepare("PRAGMA user_version").get()).toEqual({ user_version: 12 }); expect(() => assertSqliteSchemaContains( predecessor, - "predecessor global schema", + "predecessor v12 global schema", predecessorSchema, predecessorCompatibility, ), diff --git a/src/skills/workshop/collection-review-state.ts b/src/skills/workshop/collection-review-state.ts index cf5a52c524f5..10fcd4dccd28 100644 --- a/src/skills/workshop/collection-review-state.ts +++ b/src/skills/workshop/collection-review-state.ts @@ -3,11 +3,11 @@ import path from "node:path"; import { asNullableRecord } from "@openclaw/normalization-core/record-coerce"; import { sha256Hex } from "../../infra/crypto-digest.js"; import { formatErrorMessage } from "../../infra/errors.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../../infra/kysely-sync.js"; + readConfigMachineState, + updateConfigMachineState, +} from "../../state/config-machine-state.js"; import type { DB as OpenClawStateDatabase } from "../../state/openclaw-state-db.generated.js"; import { openOpenClawStateDatabase, @@ -22,15 +22,17 @@ import { type SkillWorkshopStoreOptions, } from "./store-sqlite-schema.js"; -const CURATOR_STATE_ID = 1; const REVIEW_CLAIM_MS = 11 * 60_000; // Bound per-workspace history so unattended weekly maintenance cannot grow state forever. const SKILL_COLLECTION_REVIEW_RETENTION_COUNT = 90; const SKILL_COLLECTION_REVIEW_HISTORY_LIMIT = 20; -type CollectionReviewDatabase = Pick< - OpenClawStateDatabase, - "skill_curator_state" | "skill_workshop_collection_reviews" ->; +type CollectionReviewDatabase = Pick; +type SkillCuratorState = { + lastAttemptAtMs: number; + lastSuccessAtMs: number | null; + lastError: string | null; + lastResult: Record; +}; type SkillCollectionReviewOutcome = { createTime: number; @@ -77,30 +79,17 @@ export async function withSkillCollectionReviewClaim( ); } -function parseReviewState(value: string | null | undefined): Record { - if (!value) { - return {}; - } - try { - return asNullableRecord(JSON.parse(value)) ?? {}; - } catch { - return {}; - } -} - function reviewMap(state: Record, field: string): Record { - // SAFETY: cache-class state written only by recordWorkspaceReview below; invalid outer JSON resets it. + // SAFETY: cache-class state is written only by recordWorkspaceReview below. return (asNullableRecord(state[field]) ?? {}) as Record; } function readReviewState(options: OpenClawStateDatabaseOptions): Record { - const database = openOpenClawStateDatabase(options); - const kysely = getNodeSqliteKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( - database.db, - kysely.selectFrom("skill_curator_state").select("last_result_json").where("id", "=", 1), + return ( + asNullableRecord( + readConfigMachineState("skills.curatorState", options)?.lastResult, + ) ?? {} ); - return parseReviewState(row?.last_result_json); } export function readSkillReviewOutcomes(options: OpenClawStateDatabaseOptions = {}) { @@ -131,9 +120,9 @@ export function recordSkillCollectionReviewStatus( workspaceDir, status, { - last_attempt_at_ms: status.attemptedAtMs, - ...(status.succeededAtMs !== undefined ? { last_success_at_ms: status.succeededAtMs } : {}), - last_error: status.error ?? null, + lastAttemptAtMs: status.attemptedAtMs, + ...(status.succeededAtMs !== undefined ? { lastSuccessAtMs: status.succeededAtMs } : {}), + lastError: status.error ?? null, }, options, ); @@ -151,47 +140,30 @@ function recordWorkspaceReview( field: "collectionReviews" | "experienceReviews", workspaceDir: string, review: SkillCollectionReviewStatus | SkillExperienceReviewStatus, - columns: { - last_attempt_at_ms?: number; - last_success_at_ms?: number; - last_error?: string | null; - }, + columns: Partial>, options: OpenClawStateDatabaseOptions, ): void { - runOpenClawStateWriteTransaction(({ db }) => { - const kysely = getNodeSqliteKysely(db); - const current = executeSqliteQueryTakeFirstSync( - db, - kysely - .selectFrom("skill_curator_state") - .select("last_result_json") - .where("id", "=", CURATOR_STATE_ID), - ); - const state = parseReviewState(current?.last_result_json); - const updatedState = { - ...columns, - last_result_json: JSON.stringify({ - ...state, - [field]: { - ...asNullableRecord(state[field]), - [workspaceKey(workspaceDir)]: review, + updateConfigMachineState( + "skills.curatorState", + (current) => { + const state = asNullableRecord(current?.lastResult) ?? {}; + return { + lastAttemptAtMs: 0, + lastSuccessAtMs: null, + lastError: null, + ...current, + ...columns, + lastResult: { + ...state, + [field]: { + ...asNullableRecord(state[field]), + [workspaceKey(workspaceDir)]: review, + }, }, - }), - }; - executeSqliteQuerySync( - db, - kysely - .insertInto("skill_curator_state") - .values({ - id: CURATOR_STATE_ID, - last_attempt_at_ms: 0, - last_success_at_ms: null, - last_error: null, - ...updatedState, - }) - .onConflict((conflict) => conflict.column("id").doUpdateSet(updatedState)), - ); - }, options); + }; + }, + options, + ); } function parseStoredNames(value: string, field: string): string[] { diff --git a/src/skills/workshop/collection-review.test.ts b/src/skills/workshop/collection-review.test.ts index 5e6dc7029179..9a769fcfe814 100644 --- a/src/skills/workshop/collection-review.test.ts +++ b/src/skills/workshop/collection-review.test.ts @@ -620,7 +620,8 @@ describe("skill collection review", () => { runEmbeddedAgent.mockImplementation(async () => { database.exec(` CREATE TRIGGER reject_collection_review_state - BEFORE UPDATE ON skill_curator_state + BEFORE UPDATE ON config_machine_state + WHEN NEW.state_key = 'skills.curatorState' BEGIN SELECT RAISE(FAIL, 'collection review state unavailable'); END diff --git a/src/skills/workshop/curator.ts b/src/skills/workshop/curator.ts index 7251b7b30532..2cdbb6589d9a 100644 --- a/src/skills/workshop/curator.ts +++ b/src/skills/workshop/curator.ts @@ -5,12 +5,9 @@ import { onTrustedInternalDiagnosticEvent, type DiagnosticSkillUsedEvent, } from "../../infra/diagnostic-events.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../../infra/kysely-sync.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; +import { readConfigMachineState } from "../../state/config-machine-state.js"; import type { DB as OpenClawStateDatabase } from "../../state/openclaw-state-db.generated.js"; import { openOpenClawStateDatabase, @@ -26,16 +23,12 @@ import { import { parseSkillProposalRow } from "./store-sqlite-record.js"; const log = createSubsystemLogger("skills/curator"); -const CURATOR_STATE_ID = 1; export const SKILL_LIFECYCLE_CURATION_RETIRED_MESSAGE = "Skill lifecycle curation is retired. The weekly collection review manages the skill collection; pin, unpin, and restore no longer exist."; type SkillLifecycleState = "active" | "archived" | "stale"; -type CuratorDatabase = Pick< - OpenClawStateDatabase, - "skill_curator_state" | "skill_usage" | "skill_workshop_proposals" ->; +type CuratorDatabase = Pick; type SkillOverlapCandidate = { left: string; right: string; score: number }; export type SkillCuratorStatus = { @@ -103,10 +96,12 @@ export function getSkillCuratorStatus( options: OpenClawStateDatabaseOptions = {}, ): SkillCuratorStatus { const { database, kysely } = curatorDb(options); - const state = executeSqliteQueryTakeFirstSync( - database.db, - kysely.selectFrom("skill_curator_state").selectAll().where("id", "=", CURATOR_STATE_ID), - ); + const state = readConfigMachineState<{ + lastAttemptAtMs: number; + lastSuccessAtMs: number | null; + lastError: string | null; + lastResult: Record; + }>("skills.curatorState", options); const reviewOutcomes = readSkillReviewOutcomes(options); const proposalRows = executeSqliteQuerySync( database.db, @@ -167,9 +162,9 @@ export function getSkillCuratorStatus( }; }); return { - lastAttemptAtMs: state?.last_attempt_at_ms ?? null, - lastSuccessAtMs: state?.last_success_at_ms ?? null, - lastError: state?.last_error ?? null, + lastAttemptAtMs: state?.lastAttemptAtMs ?? null, + lastSuccessAtMs: state?.lastSuccessAtMs ?? null, + lastError: state?.lastError ?? null, collectionReview: reviewOutcomes.collectionReviews, experienceReview: reviewOutcomes.experienceReviews, counts: { active: skills.length, stale: 0, archived: 0 }, diff --git a/src/snapshot/git-backup-codec.ts b/src/snapshot/git-backup-codec.ts index 1c5b3f0dad1c..360407661ec7 100644 --- a/src/snapshot/git-backup-codec.ts +++ b/src/snapshot/git-backup-codec.ts @@ -13,6 +13,7 @@ import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.js"; import { getOpenClawStateRuntimeSchema } from "../state/openclaw-state-schema-compatibility.js"; import { AGENT_SECRET_TABLE_NAMES, + STATE_SECRET_CONFIG_STATE_KEY_PREFIXES, STATE_SECRET_TABLE_NAMES, } from "../state/secret-state-tables.js"; import { hashSnapshotArtifact } from "./manifest.js"; @@ -38,6 +39,7 @@ export type GitBackupManifest = { identity: GitBackupIdentity; userVersion: number; excludedTables: string[]; + excludedConfigStateKeyPrefixes: string[]; tables: Record; }; @@ -53,6 +55,7 @@ export type GitBackupRestoreResult = { targetPath: string; tables: GitBackupTableResult[]; excludedTables: string[]; + excludedConfigStateKeyPrefixes: string[]; }; type SchemaEntry = { @@ -155,7 +158,11 @@ function encodeSqliteValue(value: unknown): unknown { throw new Error(`Git backup cannot encode SQLite value type ${typeof value}.`); } -function serializeTable(database: DatabaseSync, table: string): { content: string; rows: number } { +function serializeTable( + database: DatabaseSync, + table: string, + rowFilter?: (row: Record) => boolean, +): { content: string; rows: number } { const columns = readTableColumns(database, table); if (columns.length === 0) { throw new Error(`Git backup table has no readable columns: ${table}`); @@ -173,6 +180,9 @@ function serializeTable(database: DatabaseSync, table: string): { content: strin const lines: string[] = []; for (const rawRow of statement.iterate()) { const source = rawRow as Record; + if (rowFilter && !rowFilter(source)) { + continue; + } const encoded: Record = {}; for (const column of columns) { encoded[column.name] = encodeSqliteValue(source[column.name]); @@ -215,6 +225,12 @@ export async function dumpGitBackupDatabase(params: { // manifest.excludedTables documents redaction only; operational projection // tables are always omitted and converge on next gateway startup. const excludedTables = [...redacted].filter((table) => existingTables.has(table)).toSorted(); + const excludedConfigStateKeyPrefixes = + identity.role === "global" && + params.excludeSecrets === true && + existingTables.has("config_machine_state") + ? [...STATE_SECRET_CONFIG_STATE_KEY_PREFIXES] + : []; const excluded = new Set([...excludedTables, ...GIT_BACKUP_PROJECTION_TABLES]); const includedSchema = entries.filter( (entry) => !excluded.has(entry.name) && !excluded.has(entry.tableName), @@ -239,7 +255,18 @@ export async function dumpGitBackupDatabase(params: { await fs.mkdir(tablesPath, { recursive: true, mode: 0o700 }); const tables: Record = {}; for (const table of dataTables) { - const serialized = serializeTable(database, table); + const rowFilter = + table === "config_machine_state" && excludedConfigStateKeyPrefixes.length > 0 + ? (row: Record) => { + // Fail closed: a malformed state_key is dropped, never risked into a backup. + const stateKey = row.state_key; + return ( + typeof stateKey === "string" && + !excludedConfigStateKeyPrefixes.some((prefix) => stateKey.startsWith(prefix)) + ); + } + : undefined; + const serialized = serializeTable(database, table, rowFilter); await fs.writeFile(path.join(tablesPath, `${table}.jsonl`), serialized.content, { encoding: "utf8", mode: 0o600, @@ -251,6 +278,7 @@ export async function dumpGitBackupDatabase(params: { identity, userVersion: userVersionRow.user_version, excludedTables, + excludedConfigStateKeyPrefixes, tables, }; await fs.writeFile( @@ -286,12 +314,18 @@ export function parseGitBackupManifest(value: string, source: string): GitBackup (manifest.identity.role !== "global" && manifest.identity.role !== "agent") || !Number.isSafeInteger(manifest.userVersion) || !Array.isArray(manifest.excludedTables) || + (manifest.excludedConfigStateKeyPrefixes !== undefined && + (!Array.isArray(manifest.excludedConfigStateKeyPrefixes) || + manifest.excludedConfigStateKeyPrefixes.some((prefix) => typeof prefix !== "string"))) || !manifest.tables || typeof manifest.tables !== "object" ) { throw new Error(`Git backup manifest has unsupported fields: ${source}`); } - const validated = manifest as GitBackupManifest; + const validated = { + ...manifest, + excludedConfigStateKeyPrefixes: manifest.excludedConfigStateKeyPrefixes ?? [], + } as GitBackupManifest; normalizeIdentity(validated.identity); for (const [table, entry] of Object.entries(validated.tables)) { requireSafeTableName(table); @@ -622,7 +656,14 @@ export async function restoreGitBackupDirectory(params: { guard.assertTargetMatchesExpectedContent(() => assertNoSqliteSidecarsSync(targetPath)); }, }); - return { manifest, targetPath, tables, excludedTables: manifest.excludedTables }; + return { + manifest, + targetPath, + tables, + excludedTables: manifest.excludedTables, + // Older backups predate prefix redaction; absent means nothing was omitted. + excludedConfigStateKeyPrefixes: manifest.excludedConfigStateKeyPrefixes ?? [], + }; } catch (error) { if (database.isOpen) { database.close(); diff --git a/src/snapshot/git-backup.test.ts b/src/snapshot/git-backup.test.ts index c245f5b2c7c0..069d106e0dcb 100644 --- a/src/snapshot/git-backup.test.ts +++ b/src/snapshot/git-backup.test.ts @@ -8,6 +8,7 @@ import { backupGitCreateCommand } from "../commands/backup-git.js"; import { readBackupFreshness } from "../commands/backup-health.js"; import { createTestRuntime } from "../commands/test-runtime-config-helpers.js"; import { executeGitCommand, requireGitCommand as requireGit } from "../infra/git-exec.js"; +import { writeConfigMachineState } from "../state/config-machine-state.js"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; import { @@ -684,6 +685,55 @@ describe("Git-backed SQLite snapshots", () => { } }); + it("redacts secret machine-state keys while retaining ordinary machine state", async () => { + const root = await tempRoot(); + const { stateDir, database } = createStateDatabaseFixture(root); + const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + const nodeSecret = "synthetic-node-host-gateway-secret"; + const pushSecret = "synthetic-web-push-private-key"; + writeConfigMachineState("nodeHost.config", { gateway: { token: nodeSecret } }, { env }); + writeConfigMachineState("nodeHost.otherSecret", { token: nodeSecret }, { env }); + writeConfigMachineState("webPush.vapidKeys", { privateKey: pushSecret }, { env }); + writeConfigMachineState("sidebar.sectionOrder", ["first", "second"], { env }); + closeOpenClawStateDatabaseForTest(); + + const outputPath = path.join(root, "dump"); + const manifest = await dumpGitBackupDatabase({ + snapshotPath: database.path, + outputPath, + identity: { role: "global" }, + excludeSecrets: true, + }); + const rows = await fs.readFile( + path.join(outputPath, "tables", "config_machine_state.jsonl"), + "utf8", + ); + const manifestJson = await fs.readFile(path.join(outputPath, "manifest.json"), "utf8"); + + expect(manifest).toMatchObject({ + excludedConfigStateKeyPrefixes: ["nodeHost.", "webPush.vapidKeys"], + tables: { config_machine_state: { rows: 1 } }, + }); + expect(rows).toContain("sidebar.sectionOrder"); + expect(rows).toContain("first"); + expect(rows).not.toContain("nodeHost."); + expect(rows).not.toContain("webPush.vapidKeys"); + expect(rows).not.toContain(nodeSecret); + expect(rows).not.toContain(pushSecret); + expect(manifestJson).not.toContain(nodeSecret); + expect(manifestJson).not.toContain(pushSecret); + + const restoredPath = path.join(root, "restored.sqlite"); + const restored = await restoreGitBackupDirectory({ + sourcePath: outputPath, + targetPath: restoredPath, + expectedIdentity: { role: "global" }, + }); + // Restore must disclose the intentionally omitted machine-state prefixes so + // operators cannot mistake a redacted restore for a complete one. + expect(restored.excludedConfigStateKeyPrefixes).toEqual(["nodeHost.", "webPush.vapidKeys"]); + }); + it("rejects a restored global database without canonical ownership metadata", async () => { const root = await tempRoot(); const source = path.join(root, "source.sqlite"); diff --git a/src/state/config-machine-state.ts b/src/state/config-machine-state.ts index 94e5cdfdd991..021af3af1d12 100644 --- a/src/state/config-machine-state.ts +++ b/src/state/config-machine-state.ts @@ -31,10 +31,10 @@ function serializeStateValue(value: unknown): string { } // oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- Callers own the JSON shape for open-ended state keys. -export function readConfigMachineState( +export function readConfigMachineStateWithMetadata( key: string, options: OpenClawStateDatabaseOptions = {}, -): T | undefined { +): { value: T; updatedAtMs: number } | undefined { return withExistingOpenClawStateDatabaseReadOnly(({ db: database }) => { if (!tableExists(database, "config_machine_state")) { return undefined; @@ -44,13 +44,23 @@ export function readConfigMachineState( database, db .selectFrom("config_machine_state") - .select("value_json") + .select(["value_json", "updated_at_ms"]) .where("state_key", "=", normalizeStateKey(key)), ); - return row ? (JSON.parse(row.value_json) as T) : undefined; + return row + ? { value: JSON.parse(row.value_json) as T, updatedAtMs: row.updated_at_ms } + : undefined; }, options); } +// oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- Callers own the JSON shape for open-ended state keys. +export function readConfigMachineState( + key: string, + options: OpenClawStateDatabaseOptions = {}, +): T | undefined { + return readConfigMachineStateWithMetadata(key, options)?.value; +} + export function writeConfigMachineState( key: string, value: unknown, @@ -81,8 +91,19 @@ export function writeConfigMachineState( export function updateConfigMachineState( key: string, update: (current: T | undefined) => T, + options?: OpenClawStateDatabaseOptions, +): T; +/** Returning undefined removes the key within the same compare-and-update transaction. */ +export function updateConfigMachineState( + key: string, + update: (current: T | undefined) => T | undefined, + options?: OpenClawStateDatabaseOptions, +): T | undefined; +export function updateConfigMachineState( + key: string, + update: (current: T | undefined) => T | undefined, options: OpenClawStateDatabaseOptions = {}, -): T { +): T | undefined { const stateKey = normalizeStateKey(key); const now = Date.now(); return runOpenClawStateWriteTransaction( @@ -96,6 +117,15 @@ export function updateConfigMachineState( .where("state_key", "=", stateKey), ); const value = update(row ? (JSON.parse(row.value_json) as T) : undefined); + if (value === undefined) { + if (row) { + executeSqliteQuerySync( + database.db, + db.deleteFrom("config_machine_state").where("state_key", "=", stateKey), + ); + } + return undefined; + } const valueJson = serializeStateValue(value); executeSqliteQuerySync( database.db, @@ -113,6 +143,26 @@ export function updateConfigMachineState( ); } +/** Delete one machine-state value, reporting whether a stored value existed. */ +export function deleteConfigMachineState( + key: string, + options: OpenClawStateDatabaseOptions = {}, +): boolean { + const stateKey = normalizeStateKey(key); + return runOpenClawStateWriteTransaction( + (database) => { + const db = getNodeSqliteKysely(database.db); + const result = executeSqliteQuerySync( + database.db, + db.deleteFrom("config_machine_state").where("state_key", "=", stateKey), + ); + return (result.numAffectedRows ?? 0n) > 0n; + }, + options, + { operationLabel: "config-machine-state.delete" }, + ); +} + /** Import retired config values without replacing newer canonical database state. */ export function importConfigMachineState( entries: ReadonlyArray, diff --git a/src/state/onboarding-recommendations.ts b/src/state/onboarding-recommendations.ts index f018f8d9f47d..8f1e999354a2 100644 --- a/src/state/onboarding-recommendations.ts +++ b/src/state/onboarding-recommendations.ts @@ -2,17 +2,11 @@ import { z } from "zod"; import { resolveWorkspaceStateIdentity } from "../agents/workspace-state-identity.js"; import { sha256Hex } from "../infra/crypto-digest.js"; import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../infra/kysely-sync.js"; -import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; -import { tableExists } from "./openclaw-state-db-schema-helpers.js"; -import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; -import { - runOpenClawStateWriteTransaction, - type OpenClawStateDatabaseOptions, -} from "./openclaw-state-db.js"; + deleteConfigMachineState, + readConfigMachineState, + updateConfigMachineState, +} from "./config-machine-state.js"; +import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db.js"; const OnboardingRecommendationMatchSchema = z.object({ appLabel: z.string(), @@ -82,11 +76,6 @@ export type OnboardingRecommendationsStore = { clear: () => boolean; }; -type OnboardingRecommendationsDatabase = Pick< - OpenClawStateKyselyDatabase, - "onboarding_recommendations" ->; - function canonicalInventory( inventory: readonly OnboardingRecommendationInventoryItem[], ): OnboardingRecommendationInventoryItem[] { @@ -113,36 +102,22 @@ function readOnboardingRecommendations( options: OpenClawStateDatabaseOptions = {}, ): OnboardingRecommendationsRecord | null { // CLI reads must not join the Gateway's writable SQLite lifecycle (#101290). + const record = readConfigMachineState(configKey, options); + return record + ? { ...record, matches: OnboardingRecommendationMatchesSchema.parse(record.matches) } + : null; +} + +function matchesExpectedOnboardingRecommendations( + current: OnboardingRecommendationsRecord, + expected: OnboardingRecommendationsRecord, +): boolean { return ( - withExistingOpenClawStateDatabaseReadOnly(({ db: database }) => { - if (!tableExists(database, "onboarding_recommendations")) { - return null; - } - const db = getNodeSqliteKysely(database); - const row = executeSqliteQueryTakeFirstSync( - database, - db - .selectFrom("onboarding_recommendations") - .select([ - "inventory_hash", - "matches_json", - "offered_at_ms", - "accepted_at_ms", - "updated_at_ms", - ]) - .where("config_key", "=", configKey), - ); - if (!row) { - return null; - } - return { - inventoryHash: row.inventory_hash, - matches: OnboardingRecommendationMatchesSchema.parse(JSON.parse(row.matches_json)), - offeredAt: row.offered_at_ms, - acceptedAt: row.accepted_at_ms, - updatedAt: row.updated_at_ms, - }; - }, options) ?? null + current.inventoryHash === expected.inventoryHash && + JSON.stringify(current.matches) === JSON.stringify(expected.matches) && + current.offeredAt === expected.offeredAt && + current.acceptedAt === expected.acceptedAt && + current.updatedAt === expected.updatedAt ); } @@ -155,55 +130,14 @@ function writeOnboardingRecommendationsOffer( const inventoryHash = hashOnboardingRecommendationInventory(params.inventory); const matches = OnboardingRecommendationMatchesSchema.parse(params.matches); const acceptedAt = params.answered ? nowMs : null; - return runOpenClawStateWriteTransaction( - (database) => { - const db = getNodeSqliteKysely(database.db); - const existing = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("onboarding_recommendations") - .select([ - "inventory_hash", - "matches_json", - "offered_at_ms", - "accepted_at_ms", - "updated_at_ms", - ]) - .where("config_key", "=", configKey), - ); + return updateConfigMachineState( + configKey, + (existing) => { // Once the user answers, concurrent or stale offer completions must not // clear acceptance and make later onboarding runs ask again. - if (typeof existing?.accepted_at_ms === "number") { - return { - inventoryHash: existing.inventory_hash, - matches: OnboardingRecommendationMatchesSchema.parse(JSON.parse(existing.matches_json)), - offeredAt: existing.offered_at_ms, - acceptedAt: existing.accepted_at_ms, - updatedAt: existing.updated_at_ms, - }; + if (typeof existing?.acceptedAt === "number") { + return existing; } - executeSqliteQuerySync( - database.db, - db - .insertInto("onboarding_recommendations") - .values({ - config_key: configKey, - inventory_hash: inventoryHash, - matches_json: JSON.stringify(matches), - offered_at_ms: nowMs, - accepted_at_ms: acceptedAt, - updated_at_ms: nowMs, - }) - .onConflict((conflict) => - conflict.column("config_key").doUpdateSet({ - inventory_hash: inventoryHash, - matches_json: JSON.stringify(matches), - offered_at_ms: nowMs, - accepted_at_ms: acceptedAt, - updated_at_ms: nowMs, - }), - ), - ); return { inventoryHash, matches, @@ -213,7 +147,6 @@ function writeOnboardingRecommendationsOffer( }; }, databaseOptions, - { operationLabel: "onboarding.recommendations.write" }, ); } @@ -223,65 +156,25 @@ function acknowledgeOnboardingRecommendations( databaseOptions: OpenClawStateDatabaseOptions = {}, ): OnboardingRecommendationsRecord | null { const nowMs = params.nowMs ?? Date.now(); - return runOpenClawStateWriteTransaction( - (database) => { - const db = getNodeSqliteKysely(database.db); - const existing = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("onboarding_recommendations") - .select([ - "inventory_hash", - "matches_json", - "offered_at_ms", - "accepted_at_ms", - "updated_at_ms", - ]) - .where("config_key", "=", configKey), - ); + let acknowledged: OnboardingRecommendationsRecord | null = null; + updateConfigMachineState( + configKey, + (existing) => { if (!existing) { - return null; + return undefined; } - if ( - params.expected && - (existing.inventory_hash !== params.expected.inventoryHash || - existing.matches_json !== JSON.stringify(params.expected.matches) || - existing.offered_at_ms !== params.expected.offeredAt || - existing.accepted_at_ms !== params.expected.acceptedAt || - existing.updated_at_ms !== params.expected.updatedAt) - ) { - return null; + if (params.expected && !matchesExpectedOnboardingRecommendations(existing, params.expected)) { + return existing; } - if (typeof existing.accepted_at_ms !== "number") { - let update = db - .updateTable("onboarding_recommendations") - .set({ accepted_at_ms: nowMs, updated_at_ms: nowMs }) - .where("config_key", "=", configKey); - if (params.expected) { - update = update - .where("inventory_hash", "=", params.expected.inventoryHash) - .where("matches_json", "=", JSON.stringify(params.expected.matches)) - .where("offered_at_ms", "=", params.expected.offeredAt) - .where("accepted_at_ms", "is", params.expected.acceptedAt) - .where("updated_at_ms", "=", params.expected.updatedAt); - } - const result = executeSqliteQuerySync(database.db, update); - if ((result.numAffectedRows ?? 0n) === 0n) { - return null; - } - } - const acceptedAt = existing.accepted_at_ms ?? nowMs; - return { - inventoryHash: existing.inventory_hash, - matches: OnboardingRecommendationMatchesSchema.parse(JSON.parse(existing.matches_json)), - offeredAt: existing.offered_at_ms, - acceptedAt, - updatedAt: existing.accepted_at_ms == null ? nowMs : existing.updated_at_ms, - }; + acknowledged = + typeof existing.acceptedAt === "number" + ? existing + : { ...existing, acceptedAt: nowMs, updatedAt: nowMs }; + return acknowledged; }, databaseOptions, - { operationLabel: "onboarding.recommendations.acknowledge" }, ); + return acknowledged; } function updatePendingOnboardingRecommendations( @@ -291,59 +184,23 @@ function updatePendingOnboardingRecommendations( ): OnboardingRecommendationsRecord | null { const nowMs = params.nowMs ?? Date.now(); const matches = OnboardingRecommendationMatchesSchema.parse(params.matches); - return runOpenClawStateWriteTransaction( - (database) => { - const db = getNodeSqliteKysely(database.db); - const existing = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("onboarding_recommendations") - .select([ - "inventory_hash", - "matches_json", - "offered_at_ms", - "accepted_at_ms", - "updated_at_ms", - ]) - .where("config_key", "=", configKey), - ); + let updated: OnboardingRecommendationsRecord | null = null; + updateConfigMachineState( + configKey, + (existing) => { if ( !existing || - typeof existing.accepted_at_ms === "number" || - existing.inventory_hash !== params.expected.inventoryHash || - existing.matches_json !== JSON.stringify(params.expected.matches) || - existing.offered_at_ms !== params.expected.offeredAt || - existing.accepted_at_ms !== params.expected.acceptedAt || - existing.updated_at_ms !== params.expected.updatedAt + typeof existing.acceptedAt === "number" || + !matchesExpectedOnboardingRecommendations(existing, params.expected) ) { - return null; + return existing; } - const result = executeSqliteQuerySync( - database.db, - db - .updateTable("onboarding_recommendations") - .set({ matches_json: JSON.stringify(matches), updated_at_ms: nowMs }) - .where("config_key", "=", configKey) - .where("accepted_at_ms", "is", null) - .where("inventory_hash", "=", params.expected.inventoryHash) - .where("matches_json", "=", JSON.stringify(params.expected.matches)) - .where("offered_at_ms", "=", params.expected.offeredAt) - .where("updated_at_ms", "=", params.expected.updatedAt), - ); - if ((result.numAffectedRows ?? 0n) === 0n) { - return null; - } - return { - inventoryHash: existing.inventory_hash, - matches, - offeredAt: existing.offered_at_ms, - acceptedAt: null, - updatedAt: nowMs, - }; + updated = { ...existing, matches, updatedAt: nowMs }; + return updated; }, databaseOptions, - { operationLabel: "onboarding.recommendations.update-pending" }, ); + return updated; } function clearPendingOnboardingRecommendations( @@ -351,43 +208,30 @@ function clearPendingOnboardingRecommendations( params: ClearPendingOnboardingRecommendationsParams, databaseOptions: OpenClawStateDatabaseOptions = {}, ): boolean { - return runOpenClawStateWriteTransaction( - (database) => { - const db = getNodeSqliteKysely(database.db); - const result = executeSqliteQuerySync( - database.db, - db - .deleteFrom("onboarding_recommendations") - .where("config_key", "=", configKey) - .where("accepted_at_ms", "is", null) - .where("inventory_hash", "=", params.expected.inventoryHash) - .where("matches_json", "=", JSON.stringify(params.expected.matches)) - .where("offered_at_ms", "=", params.expected.offeredAt) - .where("updated_at_ms", "=", params.expected.updatedAt), - ); - return (result.numAffectedRows ?? 0n) > 0n; + let cleared = false; + updateConfigMachineState( + configKey, + (existing) => { + if ( + !existing || + existing.acceptedAt !== null || + !matchesExpectedOnboardingRecommendations(existing, params.expected) + ) { + return existing; + } + cleared = true; + return undefined; }, databaseOptions, - { operationLabel: "onboarding.recommendations.clear-pending" }, ); + return cleared; } function clearOnboardingRecommendations( configKey: string, databaseOptions: OpenClawStateDatabaseOptions = {}, ): boolean { - return runOpenClawStateWriteTransaction( - (database) => { - const db = getNodeSqliteKysely(database.db); - const result = executeSqliteQuerySync( - database.db, - db.deleteFrom("onboarding_recommendations").where("config_key", "=", configKey), - ); - return (result.numAffectedRows ?? 0n) > 0n; - }, - databaseOptions, - { operationLabel: "onboarding.recommendations.clear" }, - ); + return deleteConfigMachineState(configKey, databaseOptions); } export function createOnboardingRecommendationsStore(params: { @@ -396,7 +240,7 @@ export function createOnboardingRecommendationsStore(params: { }): OnboardingRecommendationsStore { // Doctor owns the one-time `primary` migration; a runtime fallback would recreate // cross-workspace reads. Every operation stays bound to one canonical workspace key. - const configKey = resolveWorkspaceStateIdentity(params.workspaceDir).workspaceKey; + const configKey = `onboarding.recommendations.${resolveWorkspaceStateIdentity(params.workspaceDir).workspaceKey}`; const database = params.database ?? {}; return { read: () => readOnboardingRecommendations(configKey, database), diff --git a/src/state/openclaw-schema-retirements.json b/src/state/openclaw-schema-retirements.json index 06718ea56581..3ea8f7ed57a7 100644 --- a/src/state/openclaw-schema-retirements.json +++ b/src/state/openclaw-schema-retirements.json @@ -71,6 +71,110 @@ "table": "skill_workshop_proposal_origin_runs", "indexes": [], "note": "Proposal origin runs were a never-read projection of authoritative record_json." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "clawhub_promotions_feed_state", + "indexes": [], + "note": "Folded into config_machine_state key clawhub.promotionsFeed." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "cron_store_epochs", + "indexes": [], + "note": "Retired write-only cron store epoch bookkeeping in state schema 12." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "model_catalog_remote", + "indexes": [], + "note": "Folded into config_machine_state key modelCatalog.remote." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "node_host_config", + "indexes": [], + "note": "Folded into secret-excluded config_machine_state key nodeHost.config." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "onboarding_recommendations", + "indexes": [], + "note": "Folded into config_machine_state keys onboarding.recommendations.." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "sidebar_sections", + "indexes": [], + "note": "Folded into config_machine_state key sidebar.sectionOrder." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "skill_curator_state", + "indexes": [], + "note": "Folded into config_machine_state key skills.curatorState." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "tui_last_sessions", + "indexes": ["idx_tui_last_sessions_session_key"], + "note": "Dropped rebuildable last-session pointers; new pointers use tui.lastSession.." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "update_check_state", + "indexes": [], + "note": "Folded into config_machine_state key update.checkState." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "voicewake_routing_config", + "indexes": [], + "note": "Folded into config_machine_state key voicewake.routing." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "voicewake_routing_routes", + "indexes": ["idx_voicewake_routing_routes_trigger"], + "note": "Folded into config_machine_state key voicewake.routing." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "voicewake_triggers", + "indexes": ["idx_voicewake_triggers_trigger"], + "note": "Folded into config_machine_state key voicewake.triggers." + }, + { + "database": "state", + "status": "completed", + "targetVersion": 12, + "table": "web_push_vapid_keys", + "indexes": [], + "note": "Folded into secret-excluded config_machine_state key webPush.vapidKeys." } ] } diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index d111adf281cd..32cf14ad2ad3 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -1,6 +1,7 @@ import type { DatabaseSync } from "node:sqlite"; import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; +// v12 folds singleton state into config_machine_state and retires write-only cron epochs. // v11 retires the legacy skill curator lifecycle and write-only proposal origin runs. // v10 retires six dead tables that shipped without runtime owners. // v9 stores in-root agent database registry paths relative to the state dir. @@ -8,7 +9,7 @@ import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; // v7 retires the inert shared commitments table. // v6 makes every committed shared-state table part of the canonical runtime schema. // v5 records durable cloud-worker result refs on pending workspace fences. -export const OPENCLAW_STATE_SCHEMA_VERSION = 11; +export const OPENCLAW_STATE_SCHEMA_VERSION = 12; export const OPENCLAW_STATE_STRICT_SCHEMA_VERSION = 3; // Privacy-sensitive feature tables remain absent even in fresh databases until // their feature-local first write. The canonical SQL still owns their shape. @@ -41,9 +42,7 @@ export const LAZY_ADDITIVE_STATE_TABLES = [ ...FIRST_USE_STATE_TABLES, "agent_provenance", "cron_run_receipts", - "cron_store_epochs", "config_revision_keys", - "model_catalog_remote", "secret_store_entries", "projects", "user_preferences", @@ -51,7 +50,6 @@ export const LAZY_ADDITIVE_STATE_TABLES = [ "gateway_origin_device_tokens", "github_publication_requests", "device_pairing_join_codes", - "sidebar_sections", "skill_workshop_proposal_events", "skill_workshop_collection_reviews", "skill_workshop_proposal_rollbacks", @@ -95,6 +93,7 @@ export type OpenClawStateDatabaseSchemaMigration = { | "agent-databases-relative-paths-v9" | "state-table-retirement-v10" | "state-table-retirement-v11" + | "singleton-state-foldin-v12" | "operator-approvals-system-agent" | "session-watch-cursor-provenance-v4" | "strict-tables-v3"; diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index 97d6603829ca..c40f04b693a4 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -50,6 +50,7 @@ const STATE_MIGRATION_ALLOWED_MISSING_TABLES = { 8: STATE_V6_ADDITIVE_TABLES, 9: STATE_V6_ADDITIVE_TABLES, 10: STATE_V6_ADDITIVE_TABLES, + 11: STATE_V6_ADDITIVE_TABLES, } as const satisfies Record; type OpenClawStateMigrationVersion = keyof typeof STATE_MIGRATION_ALLOWED_MISSING_TABLES; @@ -215,6 +216,14 @@ export function assertOpenClawStateDatabaseV10ForMigration( assertOpenClawStateDatabaseVersionForMigration(database, { ...options, version: 10 }); } +/** Require every stable v11 table before singleton state folds into the v12 store. */ +export function assertOpenClawStateDatabaseV11ForMigration( + database: DatabaseSync, + options: { pathname: string }, +): void { + assertOpenClawStateDatabaseVersionForMigration(database, { ...options, version: 11 }); +} + export function markCurrentStateSchemaVersion( db: DatabaseSync, options: { createMetadataIfMissing?: boolean } = {}, diff --git a/src/state/openclaw-state-db-schema-additive.ts b/src/state/openclaw-state-db-schema-additive.ts index 8dc23d84a6d2..c9be677bb228 100644 --- a/src/state/openclaw-state-db-schema-additive.ts +++ b/src/state/openclaw-state-db-schema-additive.ts @@ -380,9 +380,6 @@ export function ensureAdditiveStateColumns(db: DatabaseSync): void { } db.exec("DROP INDEX IF EXISTS idx_diagnostic_events_scope_created;"); ensureColumn(db, "worktrees", "provisioned_paths_json TEXT"); - ensureColumn(db, "node_host_config", "gateway_context_path TEXT"); - ensureColumn(db, "node_host_config", "gateway_cloudflare_access_json TEXT"); - ensureColumn(db, "node_host_config", "installed_apps_sharing INTEGER NOT NULL DEFAULT 0"); ensureColumn(db, "apns_registrations", "relay_origin TEXT"); ensureColumn(db, "device_pairing_pending", "refreshed_at_ms INTEGER"); ensureColumn(db, "device_pairing_pending", "browser_origin TEXT"); diff --git a/src/state/openclaw-state-db-schema-repair.ts b/src/state/openclaw-state-db-schema-repair.ts index fada98fa1c55..84a17441248a 100644 --- a/src/state/openclaw-state-db-schema-repair.ts +++ b/src/state/openclaw-state-db-schema-repair.ts @@ -22,6 +22,7 @@ import { tablePrimaryKeyColumns, } from "./openclaw-state-db-schema-helpers.js"; import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; +import { FOLDED_SINGLETON_STATE_TABLES_V12 } from "./openclaw-state-db-schema-v12-foldin.js"; import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; import { hasRecognizedRetiredCommitmentsSchema, @@ -382,6 +383,12 @@ export function detectOpenClawStateDatabaseSchemaMigrationsFromDatabase( ) { migrations.push({ kind: "state-table-retirement-v11", path: pathname }); } + if ( + userVersion < 12 && + FOLDED_SINGLETON_STATE_TABLES_V12.some((tableName) => tableExists(db, tableName)) + ) { + migrations.push({ kind: "singleton-state-foldin-v12", path: pathname }); + } if (!hasCanonicalAgentDatabasesPrimaryKey(db)) { migrations.push({ kind: "agent-databases-composite-primary-key", path: pathname }); } diff --git a/src/state/openclaw-state-db-schema-v12-foldin.ts b/src/state/openclaw-state-db-schema-v12-foldin.ts new file mode 100644 index 000000000000..30cc191f8c29 --- /dev/null +++ b/src/state/openclaw-state-db-schema-v12-foldin.ts @@ -0,0 +1,206 @@ +import type { DatabaseSync } from "node:sqlite"; +import { tableExists } from "./openclaw-state-db-schema-helpers.js"; + +export const FOLDED_SINGLETON_STATE_TABLES_V12 = [ + "skill_curator_state", + "update_check_state", + "clawhub_promotions_feed_state", + "model_catalog_remote", + "voicewake_triggers", + "voicewake_routing_routes", + "voicewake_routing_config", + "onboarding_recommendations", + "cron_store_epochs", + "tui_last_sessions", + "sidebar_sections", + "node_host_config", + "web_push_vapid_keys", +] as const; + +export function migrateSingletonStateFoldInV12(db: DatabaseSync, previousVersion: number): boolean { + if (previousVersion >= 12) { + return false; + } + // Older schemas can reach this migration before canonical schema creation. + db.exec(` + CREATE TABLE IF NOT EXISTS config_machine_state ( + state_key TEXT NOT NULL PRIMARY KEY, + value_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL + ) STRICT; + `); + const importState = db.prepare( + "INSERT INTO config_machine_state (state_key, value_json, updated_at_ms) VALUES (?, ?, ?) ON CONFLICT(state_key) DO NOTHING", + ); + + if (tableExists(db, "update_check_state")) { + const row = db.prepare("SELECT * FROM update_check_state WHERE state_key = 'default'").get(); + if (row) { + importState.run( + "update.checkState", + JSON.stringify({ + lastCheckedAt: row.last_checked_at ?? undefined, + lastNotifiedVersion: row.last_notified_version ?? undefined, + lastNotifiedTag: row.last_notified_tag ?? undefined, + lastAvailableVersion: row.last_available_version ?? undefined, + lastAvailableTag: row.last_available_tag ?? undefined, + autoInstallId: row.auto_install_id ?? undefined, + autoFirstSeenVersion: row.auto_first_seen_version ?? undefined, + autoFirstSeenTag: row.auto_first_seen_tag ?? undefined, + autoFirstSeenAt: row.auto_first_seen_at ?? undefined, + autoLastAttemptVersion: row.auto_last_attempt_version ?? undefined, + autoLastAttemptAt: row.auto_last_attempt_at ?? undefined, + autoLastSuccessVersion: row.auto_last_success_version ?? undefined, + autoLastSuccessAt: row.auto_last_success_at ?? undefined, + }), + Number(row.updated_at_ms), + ); + } + } + + if (tableExists(db, "voicewake_triggers")) { + const rows = db + .prepare( + "SELECT trigger, updated_at_ms FROM voicewake_triggers WHERE config_key = 'default' ORDER BY position", + ) + .all(); + if (rows.length > 0) { + importState.run( + "voicewake.triggers", + JSON.stringify(rows.map((row) => row.trigger)), + Math.max(...rows.map((row) => Number(row.updated_at_ms))), + ); + } + } + + if (tableExists(db, "voicewake_routing_config")) { + const config = db + .prepare("SELECT * FROM voicewake_routing_config WHERE config_key = 'default'") + .get(); + if (config) { + const routes = tableExists(db, "voicewake_routing_routes") + ? db + .prepare( + "SELECT trigger, target_mode, target_agent_id, target_session_key FROM voicewake_routing_routes WHERE config_key = 'default' ORDER BY position", + ) + .all() + : []; + const targetFromColumns = (mode: unknown, agentId: unknown, sessionKey: unknown) => + mode === "agent" && typeof agentId === "string" && agentId + ? { agentId } + : mode === "session" && typeof sessionKey === "string" && sessionKey + ? { sessionKey } + : { mode: "current" }; + importState.run( + "voicewake.routing", + JSON.stringify({ + version: 1, + defaultTarget: targetFromColumns( + config.default_target_mode, + config.default_target_agent_id, + config.default_target_session_key, + ), + routes: routes.map((route) => ({ + trigger: route.trigger, + target: targetFromColumns( + route.target_mode, + route.target_agent_id, + route.target_session_key, + ), + })), + updatedAtMs: config.updated_at_ms, + }), + Number(config.updated_at_ms), + ); + } + } + + if (tableExists(db, "onboarding_recommendations")) { + const rows = db.prepare("SELECT * FROM onboarding_recommendations").all(); + for (const row of rows) { + importState.run( + `onboarding.recommendations.${String(row.config_key)}`, + JSON.stringify({ + inventoryHash: row.inventory_hash, + matches: JSON.parse(String(row.matches_json)), + offeredAt: row.offered_at_ms, + acceptedAt: row.accepted_at_ms, + updatedAt: row.updated_at_ms, + }), + Number(row.updated_at_ms), + ); + } + } + + if (tableExists(db, "sidebar_sections")) { + const sections = db + .prepare("SELECT section_id FROM sidebar_sections ORDER BY position, section_id") + .all(); + if (sections.length > 0) { + importState.run( + "sidebar.sectionOrder", + JSON.stringify(sections.map((section) => section.section_id)), + Date.now(), + ); + } + } + + if (tableExists(db, "node_host_config")) { + const nodeHost = db + .prepare("SELECT * FROM node_host_config WHERE config_key = 'current'") + .get(); + if (nodeHost) { + const gateway = { + ...(nodeHost.gateway_host == null ? {} : { host: nodeHost.gateway_host }), + ...(nodeHost.gateway_port == null ? {} : { port: nodeHost.gateway_port }), + ...(nodeHost.gateway_tls == null ? {} : { tls: nodeHost.gateway_tls === 1 }), + ...(nodeHost.gateway_tls_fingerprint == null + ? {} + : { tlsFingerprint: nodeHost.gateway_tls_fingerprint }), + ...(nodeHost.gateway_context_path == null + ? {} + : { contextPath: nodeHost.gateway_context_path }), + ...(nodeHost.gateway_cloudflare_access_json == null + ? {} + : { cloudflareAccess: JSON.parse(String(nodeHost.gateway_cloudflare_access_json)) }), + }; + importState.run( + "nodeHost.config", + JSON.stringify({ + version: nodeHost.version, + nodeId: nodeHost.node_id, + ...(nodeHost.display_name == null ? {} : { displayName: nodeHost.display_name }), + ...(Object.keys(gateway).length === 0 ? {} : { gateway }), + installedAppsSharing: nodeHost.installed_apps_sharing === 1, + }), + Number(nodeHost.updated_at_ms), + ); + } + } + + if (tableExists(db, "web_push_vapid_keys")) { + const vapidKeys = db + .prepare("SELECT * FROM web_push_vapid_keys WHERE key_id = 'default'") + .get(); + if (vapidKeys) { + importState.run( + "webPush.vapidKeys", + JSON.stringify({ + publicKey: vapidKeys.public_key, + privateKey: vapidKeys.private_key, + subject: vapidKeys.subject, + }), + Number(vapidKeys.updated_at_ms), + ); + } + } + + let dropped = false; + for (const tableName of FOLDED_SINGLETON_STATE_TABLES_V12) { + if (tableExists(db, tableName)) { + db.exec(`DROP TABLE IF EXISTS ${tableName};`); + dropped = true; + } + } + return dropped; +} diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 2bc52c232dd4..f7919655d6fe 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -343,16 +343,6 @@ export interface ClawhubPromotionClaims { slug: string; } -export interface ClawhubPromotionsFeedState { - etag: string | null; - feed_sequence: number | null; - last_checked_at_ms: number | null; - notified_slugs_json: Generated; - payload_json: string | null; - state_key: string; - updated_at_ms: number; -} - export interface ConfigHealthEntries { config_path: string; last_known_good_json: string | null; @@ -482,11 +472,6 @@ export interface CronRunReceipts { store_key: string; } -export interface CronStoreEpochs { - store_epoch: Generated; - store_key: string; -} - export interface CurrentConversationBindings { account_id: string; binding_id: string; @@ -935,17 +920,6 @@ export interface MigrationSources { target_table: string; } -export interface ModelCatalogRemote { - bundle_json: string; - checked_at: number; - etag: string | null; - generated_at: number; - id: Generated; - last_modified: string | null; - min_version: string | null; - source_url: string; -} - export interface NativeHookRelayBridges { expires_at_ms: number; hostname: string; @@ -956,22 +930,6 @@ export interface NativeHookRelayBridges { updated_at_ms: number; } -export interface NodeHostConfig { - config_key: string; - display_name: string | null; - gateway_cloudflare_access_json: string | null; - gateway_context_path: string | null; - gateway_host: string | null; - gateway_port: number | null; - gateway_tls: number | null; - gateway_tls_fingerprint: string | null; - installed_apps_sharing: Generated; - node_id: string; - token: string | null; - updated_at_ms: number; - version: number; -} - export interface NodeWorkerLaunchContainers { container_json: string | null; launch_id: string; @@ -1014,15 +972,6 @@ export interface OfficialExternalPluginCatalogSnapshots { updated_at_ms: number; } -export interface OnboardingRecommendations { - accepted_at_ms: number | null; - config_key: string; - inventory_hash: string; - matches_json: string; - offered_at_ms: number; - updated_at_ms: number; -} - export interface OperatorApprovalExecutionIdentities { approval_id: string; source_context_id: string; @@ -1246,19 +1195,6 @@ export interface SessionWatchCursors { watcher_session_key: string; } -export interface SidebarSections { - position: number; - section_id: string; -} - -export interface SkillCuratorState { - id: Generated; - last_attempt_at_ms: number; - last_error: string | null; - last_result_json: string; - last_success_at_ms: number | null; -} - export interface SkillUploadChunks { byte_offset: number; chunk_blob: Uint8Array; @@ -1461,30 +1397,6 @@ export interface TaskRuns { tool_use_count: number | null; } -export interface TuiLastSessions { - scope_key: string; - session_key: string; - updated_at: number; -} - -export interface UpdateCheckState { - auto_first_seen_at: string | null; - auto_first_seen_tag: string | null; - auto_first_seen_version: string | null; - auto_install_id: string | null; - auto_last_attempt_at: string | null; - auto_last_attempt_version: string | null; - auto_last_success_at: string | null; - auto_last_success_version: string | null; - last_available_tag: string | null; - last_available_version: string | null; - last_checked_at: string | null; - last_notified_tag: string | null; - last_notified_version: string | null; - state_key: string; - updated_at_ms: number; -} - export interface UserPreferences { pref_key: string; profile_id: string; @@ -1492,32 +1404,6 @@ export interface UserPreferences { value_json: string; } -export interface VoicewakeRoutingConfig { - config_key: string; - default_target_agent_id: string | null; - default_target_mode: string; - default_target_session_key: string | null; - updated_at_ms: number; - version: number; -} - -export interface VoicewakeRoutingRoutes { - config_key: string; - position: number; - target_agent_id: string | null; - target_mode: string; - target_session_key: string | null; - trigger: string; - updated_at_ms: number; -} - -export interface VoicewakeTriggers { - config_key: string; - position: number; - trigger: string; - updated_at_ms: number; -} - export interface WebPushSubscriptions { auth: string; created_at_ms: number; @@ -1528,14 +1414,6 @@ export interface WebPushSubscriptions { updated_at_ms: number; } -export interface WebPushVapidKeys { - key_id: string; - private_key: string; - public_key: string; - subject: string; - updated_at_ms: number; -} - export interface WorkerEnvironmentCredentials { bundle_hash: string; credential_hash: string; @@ -1791,7 +1669,6 @@ export interface DB { claw_package_refs: ClawPackageRefs; claw_workspace_files: ClawWorkspaceFiles; clawhub_promotion_claims: ClawhubPromotionClaims; - clawhub_promotions_feed_state: ClawhubPromotionsFeedState; config_health_entries: ConfigHealthEntries; config_machine_state: ConfigMachineState; config_revision_keys: ConfigRevisionKeys; @@ -1799,7 +1676,6 @@ export interface DB { cron_job_scratch: CronJobScratch; cron_jobs: CronJobs; cron_run_receipts: CronRunReceipts; - cron_store_epochs: CronStoreEpochs; current_conversation_bindings: CurrentConversationBindings; delivery_queue_entries: DeliveryQueueEntries; device_auth_tokens: DeviceAuthTokens; @@ -1832,13 +1708,10 @@ export interface DB { meeting_transcript_utterances: MeetingTranscriptUtterances; migration_runs: MigrationRuns; migration_sources: MigrationSources; - model_catalog_remote: ModelCatalogRemote; native_hook_relay_bridges: NativeHookRelayBridges; - node_host_config: NodeHostConfig; node_worker_launch_containers: NodeWorkerLaunchContainers; node_worker_launches: NodeWorkerLaunches; official_external_plugin_catalog_snapshots: OfficialExternalPluginCatalogSnapshots; - onboarding_recommendations: OnboardingRecommendations; operator_approval_execution_identities: OperatorApprovalExecutionIdentities; operator_approval_standing_grants: OperatorApprovalStandingGrants; operator_approvals: OperatorApprovals; @@ -1857,8 +1730,6 @@ export interface DB { session_state_heads: SessionStateHeads; session_upstream_links: SessionUpstreamLinks; session_watch_cursors: SessionWatchCursors; - sidebar_sections: SidebarSections; - skill_curator_state: SkillCuratorState; skill_upload_chunks: SkillUploadChunks; skill_uploads: SkillUploads; skill_usage: SkillUsage; @@ -1870,14 +1741,8 @@ export interface DB { subagent_runs: SubagentRuns; task_delivery_state: TaskDeliveryState; task_runs: TaskRuns; - tui_last_sessions: TuiLastSessions; - update_check_state: UpdateCheckState; user_preferences: UserPreferences; - voicewake_routing_config: VoicewakeRoutingConfig; - voicewake_routing_routes: VoicewakeRoutingRoutes; - voicewake_triggers: VoicewakeTriggers; web_push_subscriptions: WebPushSubscriptions; - web_push_vapid_keys: WebPushVapidKeys; worker_environment_credentials: WorkerEnvironmentCredentials; worker_environment_ssh_fallback_ports: WorkerEnvironmentSshFallbackPorts; worker_environments: WorkerEnvironments; diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 9edfd8e7912a..8c0a49f1447e 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -28,6 +28,10 @@ import { assertSqliteSchemaContains } from "../infra/sqlite-schema-contract.js"; import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { VERSION } from "../version.js"; +import { + readConfigMachineState, + readConfigMachineStateWithMetadata, +} from "./config-machine-state.js"; import { listOpenClawRegisteredAgentDatabases } from "./openclaw-agent-db-registry.js"; import { FIRST_USE_STATE_TABLES } from "./openclaw-state-db-contract.js"; import { ensureGitHubPublicationSchema } from "./openclaw-state-db-schema-additive.js"; @@ -55,6 +59,7 @@ import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js"; import { STATE_SCHEMA_10_TO_9_DOWNGRADE_SQL } from "./openclaw-state-schema-v10-retirement.test-support.js"; import { STATE_SCHEMA_11_TO_10_TABLES_SQL } from "./openclaw-state-schema-v11-retirement.test-support.js"; +import { STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL } from "./openclaw-state-schema-v12-foldin.test-support.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; import { collectSqliteSchemaShape, @@ -65,7 +70,7 @@ import { type StateDbTestDatabase = Pick< OpenClawStateKyselyDatabase, - "diagnostic_events" | "schema_meta" | "skill_curator_state" | "skill_usage" + "diagnostic_events" | "schema_meta" | "skill_usage" >; const stateDbTempDirs: string[] = []; @@ -277,6 +282,22 @@ const RETIRED_STATE_TABLES_V10 = [ "model_capability_cache", ] as const; +const FOLDED_STATE_TABLES_V12 = [ + "skill_curator_state", + "update_check_state", + "clawhub_promotions_feed_state", + "model_catalog_remote", + "voicewake_triggers", + "voicewake_routing_config", + "voicewake_routing_routes", + "onboarding_recommendations", + "cron_store_epochs", + "tui_last_sessions", + "sidebar_sections", + "node_host_config", + "web_push_vapid_keys", +] as const; + function seedV6CommitmentSchema(database: DatabaseSync): void { database.exec(` CREATE TABLE IF NOT EXISTS commitments ( @@ -1770,13 +1791,14 @@ describe("openclaw state database", () => { ); it.each(["runtime open", "doctor repair"] as const)( - "retires v10 skill curator projections through %s while preserving live skill usage and review state", + "retires v10 skill curator projections through %s while preserving live skill usage and proposal provenance", (migrationPath) => { const stateDir = createTempStateDir(); const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; const databasePath = materializeCurrentStateDatabase(stateDir); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); + legacy.exec(STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL); legacy.exec(STATE_SCHEMA_11_TO_10_TABLES_SQL); legacy.exec(` INSERT INTO skill_workshop_proposals ( @@ -1812,24 +1834,32 @@ describe("openclaw state database", () => { kind: "state-table-retirement-v11", path: databasePath, }); + expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toContainEqual({ + kind: "singleton-state-foldin-v12", + path: databasePath, + }); if (migrationPath === "doctor repair") { expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ - changes: ["Retired legacy skill curator lifecycle and proposal origin-run tables"], + changes: [ + "Retired legacy skill curator lifecycle and proposal origin-run tables", + "Folded singleton state tables into config_machine_state (v12)", + ], warnings: [], }); } const migrated = openOpenClawStateDatabase(options); - expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(11); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(12); expect( migrated.db .prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'") .get(), - ).toEqual({ schema_version: 11 }); + ).toEqual({ schema_version: 12 }); for (const name of [ "skill_lifecycle", "idx_skill_lifecycle_key", "idx_skill_lifecycle_state", "skill_workshop_proposal_origin_runs", + "skill_curator_state", ]) { expect(migrated.db.prepare("SELECT name FROM sqlite_schema WHERE name = ?").get(name)).toBe( undefined, @@ -1839,9 +1869,7 @@ describe("openclaw state database", () => { skill_file: "/skills/archived/SKILL.md", use_count: 4, }); - expect( - migrated.db.prepare("SELECT last_success_at_ms FROM skill_curator_state").get(), - ).toEqual({ last_success_at_ms: 40 }); + expect(readConfigMachineState("skills.curatorState", options)).toBeUndefined(); expect( migrated.db .prepare("SELECT record_json FROM skill_workshop_proposals WHERE proposal_id = ?") @@ -1850,6 +1878,185 @@ describe("openclaw state database", () => { }, ); + it.each(["runtime open", "doctor repair"] as const)( + "folds v11 singleton state into machine-state keys through %s", + (migrationPath) => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = materializeCurrentStateDatabase(stateDir); + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL); + legacy.exec(` + INSERT INTO update_check_state ( + state_key, last_checked_at, last_notified_version, last_notified_tag, + last_available_version, last_available_tag, auto_install_id, + auto_first_seen_version, auto_first_seen_tag, auto_first_seen_at, + auto_last_attempt_version, auto_last_attempt_at, auto_last_success_version, + auto_last_success_at, updated_at_ms + ) VALUES ( + 'default', '2026-08-20T00:00:00.000Z', '2026.8.19', 'stable', + '2026.8.20', 'beta', 'installation-42', + '2026.8.18', 'stable', '2026-08-18T00:00:00.000Z', + '2026.8.19', '2026-08-19T00:00:00.000Z', '2026.8.17', + '2026-08-17T00:00:00.000Z', 200 + ); + INSERT INTO voicewake_triggers (config_key, position, trigger, updated_at_ms) VALUES + ('default', 1, 'second wake word', 101), + ('default', 0, 'first wake word', 100); + INSERT INTO voicewake_routing_config ( + config_key, version, default_target_mode, default_target_agent_id, + default_target_session_key, updated_at_ms + ) VALUES ('default', 1, 'agent', 'assistant', NULL, 300); + INSERT INTO voicewake_routing_routes ( + config_key, position, trigger, target_mode, target_agent_id, + target_session_key, updated_at_ms + ) VALUES ('default', 0, 'route wake word', 'session', NULL, 'agent:main:voice', 300); + INSERT INTO onboarding_recommendations ( + config_key, inventory_hash, matches_json, offered_at_ms, accepted_at_ms, updated_at_ms + ) VALUES + ('workspace-a', 'inventory-a', '[{"candidateId":"first"}]', 400, 401, 402), + ('workspace-b', 'inventory-b', '[{"candidateId":"second"}]', 500, NULL, 501), + ('workspace-existing', 'old-inventory', '[]', 600, NULL, 601); + INSERT INTO config_machine_state (state_key, value_json, updated_at_ms) + VALUES ('onboarding.recommendations.workspace-existing', '{"newer":true}', 999); + INSERT INTO skill_curator_state ( + id, last_attempt_at_ms, last_success_at_ms, last_error, last_result_json + ) VALUES (1, 10, 20, NULL, '{"cached":true}'); + INSERT INTO clawhub_promotions_feed_state ( + state_key, payload_json, updated_at_ms + ) VALUES ('default', '{"cached":true}', 30); + INSERT INTO model_catalog_remote ( + id, bundle_json, generated_at, source_url, checked_at + ) VALUES (1, '{"cached":true}', 40, 'https://example.invalid/catalog', 50); + INSERT INTO cron_store_epochs (store_key, store_epoch) VALUES ('default', 60); + INSERT INTO sidebar_sections (section_id, position) VALUES + ('category:projects', 1), + ('ungrouped', 0); + INSERT INTO node_host_config ( + config_key, version, node_id, token, display_name, gateway_host, + gateway_port, gateway_tls, gateway_tls_fingerprint, gateway_context_path, + gateway_cloudflare_access_json, installed_apps_sharing, updated_at_ms + ) VALUES ( + 'current', 1, 'node-42', 'retired-token', 'Build Node', 'gateway.example', + 443, 1, 'fingerprint-42', '/openclaw-gw', + '{"clientId":"access-id","clientSecret":"access-secret"}', 1, 700 + ); + INSERT INTO web_push_vapid_keys ( + key_id, public_key, private_key, subject, updated_at_ms + ) VALUES ('default', 'public-vapid-key', 'private-vapid-key', 'https://openclaw.ai', 800); + INSERT INTO tui_last_sessions (scope_key, session_key, updated_at) + VALUES ('cached-scope', 'agent:main:cached', 900); + `); + legacy.close(); + + expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toContainEqual({ + kind: "singleton-state-foldin-v12", + path: databasePath, + }); + if (migrationPath === "doctor repair") { + expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ + changes: ["Folded singleton state tables into config_machine_state (v12)"], + warnings: [], + }); + } + + const migrated = openOpenClawStateDatabase(options); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(12); + expect( + migrated.db + .prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'") + .get(), + ).toEqual({ schema_version: 12 }); + for (const tableName of FOLDED_STATE_TABLES_V12) { + expect( + migrated.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get(tableName), + ).toBeUndefined(); + } + expect(readConfigMachineState("update.checkState", options)).toEqual({ + lastCheckedAt: "2026-08-20T00:00:00.000Z", + lastNotifiedVersion: "2026.8.19", + lastNotifiedTag: "stable", + lastAvailableVersion: "2026.8.20", + lastAvailableTag: "beta", + autoInstallId: "installation-42", + autoFirstSeenVersion: "2026.8.18", + autoFirstSeenTag: "stable", + autoFirstSeenAt: "2026-08-18T00:00:00.000Z", + autoLastAttemptVersion: "2026.8.19", + autoLastAttemptAt: "2026-08-19T00:00:00.000Z", + autoLastSuccessVersion: "2026.8.17", + autoLastSuccessAt: "2026-08-17T00:00:00.000Z", + }); + expect(readConfigMachineState("voicewake.triggers", options)).toEqual([ + "first wake word", + "second wake word", + ]); + expect(readConfigMachineState("voicewake.routing", options)).toEqual({ + version: 1, + defaultTarget: { agentId: "assistant" }, + routes: [{ trigger: "route wake word", target: { sessionKey: "agent:main:voice" } }], + updatedAtMs: 300, + }); + expect(readConfigMachineState("onboarding.recommendations.workspace-a", options)).toEqual({ + inventoryHash: "inventory-a", + matches: [{ candidateId: "first" }], + offeredAt: 400, + acceptedAt: 401, + updatedAt: 402, + }); + expect(readConfigMachineState("onboarding.recommendations.workspace-b", options)).toEqual({ + inventoryHash: "inventory-b", + matches: [{ candidateId: "second" }], + offeredAt: 500, + acceptedAt: null, + updatedAt: 501, + }); + expect( + readConfigMachineState("onboarding.recommendations.workspace-existing", options), + ).toEqual({ newer: true }); + expect(readConfigMachineState("sidebar.sectionOrder", options)).toEqual([ + "ungrouped", + "category:projects", + ]); + expect(readConfigMachineStateWithMetadata("nodeHost.config", options)).toEqual({ + value: { + version: 1, + nodeId: "node-42", + displayName: "Build Node", + gateway: { + host: "gateway.example", + port: 443, + tls: true, + tlsFingerprint: "fingerprint-42", + contextPath: "/openclaw-gw", + cloudflareAccess: { clientId: "access-id", clientSecret: "access-secret" }, + }, + installedAppsSharing: true, + }, + updatedAtMs: 700, + }); + expect(readConfigMachineStateWithMetadata("webPush.vapidKeys", options)).toEqual({ + value: { + publicKey: "public-vapid-key", + privateKey: "private-vapid-key", + subject: "https://openclaw.ai", + }, + updatedAtMs: 800, + }); + expect(readConfigMachineState("tui.lastSession.cached-scope", options)).toBeUndefined(); + expect(readConfigMachineState("skills.curatorState", options)).toBeUndefined(); + expect(readConfigMachineState("clawhub.promotionsFeed", options)).toBeUndefined(); + expect(readConfigMachineState("modelCatalog.remote", options)).toBeUndefined(); + expect(detectOpenClawStateDatabaseSchemaMigrations(options)).not.toContainEqual({ + kind: "singleton-state-foldin-v12", + path: databasePath, + }); + }, + ); + it.each(["runtime open", "doctor repair"] as const)( "retires v6 commitments through %s while preserving shared leases", (migrationPath) => { @@ -2068,6 +2275,7 @@ describe("openclaw state database", () => { { kind: "commitments-retirement-v7", path: fixture.databasePath }, { kind: "state-table-retirement-v10", path: fixture.databasePath }, { kind: "state-table-retirement-v11", path: fixture.databasePath }, + { kind: "singleton-state-foldin-v12", path: fixture.databasePath }, { kind: "audit-events-v2", path: fixture.databasePath }, { kind: "strict-tables-v3", path: fixture.databasePath }, ]); @@ -2081,8 +2289,9 @@ describe("openclaw state database", () => { "Retired shared state commitments table and indexes", "Retired six dead shared-state tables (v10)", "Retired legacy skill curator lifecycle and proposal origin-run tables", + "Folded singleton state tables into config_machine_state (v12)", "Migrated shared state audit event ledger → versioned message lifecycle schema", - "Migrated shared state tables to SQLite STRICT typing (63)", + "Migrated shared state tables to SQLite STRICT typing (54)", ], warnings: [], }); @@ -2699,22 +2908,18 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're const legacy = new DatabaseSync(databasePath); legacy .prepare( - `INSERT INTO skill_curator_state ( - id, last_attempt_at_ms, last_success_at_ms, last_error, last_result_json - ) VALUES (1, 10, 20, NULL, '{}')`, + "INSERT INTO auth_profile_stores (store_key, store_json, updated_at) VALUES (?, ?, ?)", ) - .run(); + .run("legacy-store", "{}", 20); legacy.exec(` - ALTER TABLE skill_curator_state RENAME TO skill_curator_state_strict; - CREATE TABLE skill_curator_state ( - id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), - last_attempt_at_ms INTEGER NOT NULL, - last_success_at_ms INTEGER, - last_error TEXT, - last_result_json TEXT NOT NULL + ALTER TABLE auth_profile_stores RENAME TO auth_profile_stores_strict; + CREATE TABLE auth_profile_stores ( + store_key TEXT NOT NULL PRIMARY KEY, + store_json TEXT NOT NULL, + updated_at INTEGER NOT NULL ); - INSERT INTO skill_curator_state SELECT * FROM skill_curator_state_strict; - DROP TABLE skill_curator_state_strict; + INSERT INTO auth_profile_stores SELECT * FROM auth_profile_stores_strict; + DROP TABLE auth_profile_stores_strict; PRAGMA user_version = 2; UPDATE schema_meta SET schema_version = 2 WHERE meta_key = 'primary'; `); @@ -2737,15 +2942,13 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're const migrated = openOpenClawStateDatabase(options); expect( migrated.db - .prepare("SELECT strict FROM pragma_table_list WHERE name = 'skill_curator_state'") + .prepare("SELECT strict FROM pragma_table_list WHERE name = 'auth_profile_stores'") .get(), ).toEqual({ strict: 1 }); - expect(migrated.db.prepare("SELECT * FROM skill_curator_state").get()).toEqual({ - id: 1, - last_attempt_at_ms: 10, - last_success_at_ms: 20, - last_error: null, - last_result_json: "{}", + expect(migrated.db.prepare("SELECT * FROM auth_profile_stores").get()).toEqual({ + store_key: "legacy-store", + store_json: "{}", + updated_at: 20, }); }); @@ -4249,7 +4452,7 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're } }); - it("creates the bounded skill usage and curator state tables", () => { + it("keeps skill usage records scoped to their skill paths", () => { const stateDir = createTempStateDir(); const database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: stateDir } }); const kysely = getNodeSqliteKysely(database.db); @@ -4280,17 +4483,6 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're last_agent_id: "other", }), ); - executeSqliteQuerySync( - database.db, - kysely.insertInto("skill_curator_state").values({ - id: 1, - last_attempt_at_ms: 2, - last_success_at_ms: 2, - last_error: null, - last_result_json: "{}", - }), - ); - expect( executeSqliteQuerySync( database.db, diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 6ecd801ce4ea..f0c2d3db9a65 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -55,12 +55,13 @@ import { } from "./openclaw-state-db-fast-path.js"; import { assertOpenClawStateDatabaseForMaintenance, + assertOpenClawStateDatabaseV10ForMigration, + assertOpenClawStateDatabaseV11ForMigration, assertOpenClawStateDatabaseV5ForMigration, assertOpenClawStateDatabaseV6ForMigration, assertOpenClawStateDatabaseV7ForMigration, assertOpenClawStateDatabaseV8ForMigration, assertOpenClawStateDatabaseV9ForMigration, - assertOpenClawStateDatabaseV10ForMigration, assertSupportedSchemaVersion, markCurrentStateSchemaVersion, resolveDatabasePath, @@ -83,6 +84,7 @@ import { repairAgentDatabasesCompositePrimaryKey, repairLegacyGatewayRestartHandoffsForStrictMigration, } from "./openclaw-state-db-schema-repair.js"; +import { migrateSingletonStateFoldInV12 } from "./openclaw-state-db-schema-v12-foldin.js"; import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; import { withOpenClawStateStartupCheckpointConnection } from "./openclaw-state-db-startup-checkpoint.js"; import { runRetiredStateTableMigrations } from "./openclaw-state-db-table-retirements.js"; @@ -98,16 +100,14 @@ import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibi import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; export { registerOpenClawStateDatabaseLifecycleListener } from "./openclaw-state-db-cache.js"; -const STATE_MIGRATION_ASSERTIONS = new Map< - number, - typeof assertOpenClawStateDatabaseV5ForMigration ->([ +const STATE_MIGRATION_ASSERTIONS = new Map([ [5, assertOpenClawStateDatabaseV5ForMigration], [6, assertOpenClawStateDatabaseV6ForMigration], [7, assertOpenClawStateDatabaseV7ForMigration], [8, assertOpenClawStateDatabaseV8ForMigration], [9, assertOpenClawStateDatabaseV9ForMigration], [10, assertOpenClawStateDatabaseV10ForMigration], + [11, assertOpenClawStateDatabaseV11ForMigration], ]); export { @@ -208,6 +208,9 @@ function repairOpenClawStateDatabaseSchemaWithWriteAccess( } dropLegacyStateTables(db); applied.push(...runRetiredStateTableMigrations(db, previousVersion)); + if (migrateSingletonStateFoldInV12(db, previousVersion)) { + applied.push("Folded singleton state tables into config_machine_state (v12)"); + } if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { applied.push("Migrated cloud worker placements to execution modes"); } @@ -413,6 +416,7 @@ function ensureSchema( } dropLegacyStateTables(db); runRetiredStateTableMigrations(db, previousVersion); + migrateSingletonStateFoldInV12(db, previousVersion); migrateWorkerPlacementExecutionModeSchema(db, previousVersion); const pathMigration: AgentPathSummary = migrateAgentPaths(db, previousVersion, pathname); ensureAdditiveStateColumns(db); diff --git a/src/state/openclaw-state-schema-v12-foldin.test-support.ts b/src/state/openclaw-state-schema-v12-foldin.test-support.ts new file mode 100644 index 000000000000..ecb8bb622fe0 --- /dev/null +++ b/src/state/openclaw-state-schema-v12-foldin.test-support.ts @@ -0,0 +1,139 @@ +// Recreate the exact v11 table contracts so migration and pinned-reader proofs +// can project a current database through the documented 12→11 downgrade. +const FOLDED_STATE_TABLES_V12_FIXTURE_SQL = ` +CREATE TABLE IF NOT EXISTS skill_curator_state ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + last_attempt_at_ms INTEGER NOT NULL, + last_success_at_ms INTEGER, + last_error TEXT, + last_result_json TEXT NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS onboarding_recommendations ( + config_key TEXT NOT NULL PRIMARY KEY, + inventory_hash TEXT NOT NULL, + matches_json TEXT NOT NULL, + offered_at_ms INTEGER NOT NULL, + accepted_at_ms INTEGER, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS voicewake_triggers ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position) +) STRICT; +CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger + ON voicewake_triggers(config_key, trigger); + +CREATE TABLE IF NOT EXISTS voicewake_routing_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + default_target_mode TEXT NOT NULL, + default_target_agent_id TEXT, + default_target_session_key TEXT, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( + config_key TEXT NOT NULL, + position INTEGER NOT NULL, + trigger TEXT NOT NULL, + target_mode TEXT NOT NULL, + target_agent_id TEXT, + target_session_key TEXT, + updated_at_ms INTEGER NOT NULL, + PRIMARY KEY (config_key, position), + FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE +) STRICT; +CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger + ON voicewake_routing_routes(config_key, trigger); + +CREATE TABLE IF NOT EXISTS update_check_state ( + state_key TEXT NOT NULL PRIMARY KEY, + last_checked_at TEXT, + last_notified_version TEXT, + last_notified_tag TEXT, + last_available_version TEXT, + last_available_tag TEXT, + auto_install_id TEXT, + auto_first_seen_version TEXT, + auto_first_seen_tag TEXT, + auto_first_seen_at TEXT, + auto_last_attempt_version TEXT, + auto_last_attempt_at TEXT, + auto_last_success_version TEXT, + auto_last_success_at TEXT, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( + state_key TEXT NOT NULL PRIMARY KEY, + etag TEXT, + payload_json TEXT, + feed_sequence INTEGER, + last_checked_at_ms INTEGER, + notified_slugs_json TEXT NOT NULL DEFAULT '[]', + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS cron_store_epochs ( + store_key TEXT PRIMARY KEY, + store_epoch INTEGER NOT NULL DEFAULT 0 +) STRICT; + +CREATE TABLE IF NOT EXISTS model_catalog_remote ( + id INTEGER PRIMARY KEY CHECK (id = 1), + bundle_json TEXT NOT NULL, + generated_at INTEGER NOT NULL, + min_version TEXT, + source_url TEXT NOT NULL, + etag TEXT, + last_modified TEXT, + checked_at INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS tui_last_sessions ( + scope_key TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + updated_at INTEGER NOT NULL +) STRICT; +CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key + ON tui_last_sessions(session_key, updated_at DESC, scope_key); + +CREATE TABLE IF NOT EXISTS sidebar_sections ( + section_id TEXT NOT NULL PRIMARY KEY, + position INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS node_host_config ( + config_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + node_id TEXT NOT NULL, + token TEXT, + display_name TEXT, + gateway_host TEXT, + gateway_port INTEGER, + gateway_tls INTEGER, + gateway_tls_fingerprint TEXT, + gateway_context_path TEXT, + gateway_cloudflare_access_json TEXT, + installed_apps_sharing INTEGER NOT NULL DEFAULT 0, + updated_at_ms INTEGER NOT NULL +) STRICT; + +CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( + key_id TEXT NOT NULL PRIMARY KEY, + public_key TEXT NOT NULL, + private_key TEXT NOT NULL, + subject TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL +) STRICT; +`; + +export const STATE_SCHEMA_12_TO_11_DOWNGRADE_SQL = `${FOLDED_STATE_TABLES_V12_FIXTURE_SQL} +PRAGMA user_version = 11; +UPDATE schema_meta SET schema_version = 11 WHERE meta_key = 'primary'; +`; diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index e35c9e85c389..47b1cc1062c5 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -49,14 +49,6 @@ CREATE TABLE IF NOT EXISTS skill_usage ( CREATE INDEX IF NOT EXISTS idx_skill_usage_key ON skill_usage(skill_key, skill_file); -CREATE TABLE IF NOT EXISTS skill_curator_state ( - id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), - last_attempt_at_ms INTEGER NOT NULL, - last_success_at_ms INTEGER, - last_error TEXT, - last_result_json TEXT NOT NULL -) STRICT; - CREATE TABLE IF NOT EXISTS skill_workshop_proposals ( proposal_id TEXT NOT NULL PRIMARY KEY, record_json TEXT NOT NULL, @@ -699,15 +691,6 @@ CREATE TABLE IF NOT EXISTS macos_port_guardian_records ( CREATE INDEX IF NOT EXISTS idx_macos_port_guardian_records_port ON macos_port_guardian_records(port, timestamp DESC); -CREATE TABLE IF NOT EXISTS onboarding_recommendations ( - config_key TEXT NOT NULL PRIMARY KEY, - inventory_hash TEXT NOT NULL, - matches_json TEXT NOT NULL, - offered_at_ms INTEGER NOT NULL, - accepted_at_ms INTEGER, - updated_at_ms INTEGER NOT NULL -) STRICT; - CREATE TABLE IF NOT EXISTS workspace_setup_state ( workspace_key TEXT NOT NULL PRIMARY KEY, workspace_path TEXT NOT NULL, @@ -838,14 +821,6 @@ CREATE TABLE IF NOT EXISTS web_push_subscriptions ( CREATE INDEX IF NOT EXISTS idx_web_push_subscriptions_updated ON web_push_subscriptions(updated_at_ms DESC, subscription_id); -CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( - key_id TEXT NOT NULL PRIMARY KEY, - public_key TEXT NOT NULL, - private_key TEXT NOT NULL, - subject TEXT NOT NULL, - updated_at_ms INTEGER NOT NULL -) STRICT; - CREATE TABLE IF NOT EXISTS apns_registrations ( node_id TEXT NOT NULL PRIMARY KEY, transport TEXT NOT NULL, @@ -869,22 +844,6 @@ CREATE TABLE IF NOT EXISTS apns_registration_tombstones ( deleted_at_ms INTEGER NOT NULL ) STRICT; -CREATE TABLE IF NOT EXISTS node_host_config ( - config_key TEXT NOT NULL PRIMARY KEY, - version INTEGER NOT NULL, - node_id TEXT NOT NULL, - token TEXT, - display_name TEXT, - gateway_host TEXT, - gateway_port INTEGER, - gateway_tls INTEGER, - gateway_tls_fingerprint TEXT, - gateway_context_path TEXT, - gateway_cloudflare_access_json TEXT, - installed_apps_sharing INTEGER NOT NULL DEFAULT 0, - updated_at_ms INTEGER NOT NULL -) STRICT; - -- Node-host-owned launch journal. The descriptor and its credential remain -- process memory only; this table records bounded supervision facts. CREATE TABLE IF NOT EXISTS node_worker_launches ( @@ -969,59 +928,6 @@ CREATE TABLE IF NOT EXISTS node_worker_launch_containers ( container_json TEXT ) STRICT; -CREATE TABLE IF NOT EXISTS voicewake_triggers ( - config_key TEXT NOT NULL, - position INTEGER NOT NULL, - trigger TEXT NOT NULL, - updated_at_ms INTEGER NOT NULL, - PRIMARY KEY (config_key, position) -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger - ON voicewake_triggers(config_key, trigger); - -CREATE TABLE IF NOT EXISTS voicewake_routing_config ( - config_key TEXT NOT NULL PRIMARY KEY, - version INTEGER NOT NULL, - default_target_mode TEXT NOT NULL, - default_target_agent_id TEXT, - default_target_session_key TEXT, - updated_at_ms INTEGER NOT NULL -) STRICT; - -CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( - config_key TEXT NOT NULL, - position INTEGER NOT NULL, - trigger TEXT NOT NULL, - target_mode TEXT NOT NULL, - target_agent_id TEXT, - target_session_key TEXT, - updated_at_ms INTEGER NOT NULL, - PRIMARY KEY (config_key, position), - FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger - ON voicewake_routing_routes(config_key, trigger); - -CREATE TABLE IF NOT EXISTS update_check_state ( - state_key TEXT NOT NULL PRIMARY KEY, - last_checked_at TEXT, - last_notified_version TEXT, - last_notified_tag TEXT, - last_available_version TEXT, - last_available_tag TEXT, - auto_install_id TEXT, - auto_first_seen_version TEXT, - auto_first_seen_tag TEXT, - auto_first_seen_at TEXT, - auto_last_attempt_version TEXT, - auto_last_attempt_at TEXT, - auto_last_success_version TEXT, - auto_last_success_at TEXT, - updated_at_ms INTEGER NOT NULL -) STRICT; - CREATE TABLE IF NOT EXISTS config_health_entries ( config_path TEXT NOT NULL PRIMARY KEY, last_known_good_json TEXT, @@ -1030,16 +936,6 @@ CREATE TABLE IF NOT EXISTS config_health_entries ( updated_at_ms INTEGER NOT NULL ) STRICT; -CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( - state_key TEXT NOT NULL PRIMARY KEY, - etag TEXT, - payload_json TEXT, - feed_sequence INTEGER, - last_checked_at_ms INTEGER, - notified_slugs_json TEXT NOT NULL DEFAULT '[]', - updated_at_ms INTEGER NOT NULL -) STRICT; - CREATE TABLE IF NOT EXISTS clawhub_promotion_claims ( slug TEXT NOT NULL PRIMARY KEY, provider TEXT, @@ -1511,11 +1407,6 @@ CREATE TABLE IF NOT EXISTS cron_jobs ( PRIMARY KEY (store_key, job_id) ) STRICT; -CREATE TABLE IF NOT EXISTS cron_store_epochs ( - store_key TEXT PRIMARY KEY, - store_epoch INTEGER NOT NULL DEFAULT 0 -) STRICT; - CREATE INDEX IF NOT EXISTS idx_cron_jobs_store_updated ON cron_jobs(store_key, sort_order ASC, updated_at DESC, job_id); @@ -1792,15 +1683,6 @@ CREATE TABLE IF NOT EXISTS plugin_binding_approvals ( CREATE INDEX IF NOT EXISTS idx_plugin_binding_approvals_plugin ON plugin_binding_approvals(plugin_id, approved_at DESC); -CREATE TABLE IF NOT EXISTS tui_last_sessions ( - scope_key TEXT NOT NULL PRIMARY KEY, - session_key TEXT NOT NULL, - updated_at INTEGER NOT NULL -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key - ON tui_last_sessions(session_key, updated_at DESC, scope_key); - CREATE TABLE IF NOT EXISTS task_delivery_state ( task_id TEXT NOT NULL PRIMARY KEY, requester_origin_json TEXT, @@ -2003,13 +1885,6 @@ CREATE TABLE IF NOT EXISTS session_groups ( worktree INTEGER ) STRICT; --- Gateway-owned sidebar section layout. IDs are ungrouped, groups, work, or --- category:; pinned sessions are ordered separately and never stored. -CREATE TABLE IF NOT EXISTS sidebar_sections ( - section_id TEXT NOT NULL PRIMARY KEY, - position INTEGER NOT NULL -) STRICT; - -- Gateway-owned durable cloud worker lifecycle. Provider-specific execution -- stays in plugins; this table records only core reconciliation facts. CREATE TABLE IF NOT EXISTS worker_environments ( @@ -2563,17 +2438,6 @@ CREATE TABLE IF NOT EXISTS outbound_media_provenance ( created_at_ms INTEGER NOT NULL ) STRICT; -CREATE TABLE IF NOT EXISTS model_catalog_remote ( - id INTEGER PRIMARY KEY CHECK (id = 1), - bundle_json TEXT NOT NULL, - generated_at INTEGER NOT NULL, - min_version TEXT, - source_url TEXT NOT NULL, - etag TEXT, - last_modified TEXT, - checked_at INTEGER NOT NULL -) STRICT; - -- scope_id is non-null because SQLite treats NULLs as distinct in unique indexes/PKs, -- which would allow duplicate team rows. This PK also avoids a rebuild for identity scope. CREATE TABLE IF NOT EXISTS secret_store_entries ( diff --git a/src/state/secret-state-tables.ts b/src/state/secret-state-tables.ts index b21525ef6732..76758ddd618c 100644 --- a/src/state/secret-state-tables.ts +++ b/src/state/secret-state-tables.ts @@ -17,13 +17,14 @@ export const STATE_SECRET_TABLE_NAMES = [ "mcp_oauth_pending_authorizations", "mcp_oauth_stores", "native_hook_relay_bridges", - "node_host_config", "secret_store_entries", "web_push_subscriptions", - "web_push_vapid_keys", "worker_environment_credentials", ] as const; +/** Secret-redacted Git backups must never carry machine-state values under these prefixes. */ +export const STATE_SECRET_CONFIG_STATE_KEY_PREFIXES = ["nodeHost.", "webPush.vapidKeys"] as const; + /** Redaction policy surface for credential-bearing per-agent database tables. */ export const AGENT_SECRET_TABLE_NAMES = [ "auth_profile_state", diff --git a/src/state/user-profiles.test.ts b/src/state/user-profiles.test.ts index 93ef38eb18b6..eae97fb431e0 100644 --- a/src/state/user-profiles.test.ts +++ b/src/state/user-profiles.test.ts @@ -114,6 +114,7 @@ describe("user profiles", () => { openOpenClawStateDatabase(options).db.prepare("PRAGMA user_version").get()?.user_version, ).toBe(versionBefore); expect(versionBefore).toBe(OPENCLAW_STATE_SCHEMA_VERSION); + expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(12); expect(second).toEqual(first); expect(ensureProfileForEmail("ADA@example.com", options)).toEqual(first); expect(listProfiles(options)).toEqual([ diff --git a/src/tui/tui-last-session.test.ts b/src/tui/tui-last-session.test.ts index 9c5b0c651ead..4d435b6ff084 100644 --- a/src/tui/tui-last-session.test.ts +++ b/src/tui/tui-last-session.test.ts @@ -3,6 +3,10 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { + readConfigMachineStateWithMetadata, + writeConfigMachineState, +} from "../state/config-machine-state.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { buildTuiLastSessionScopeKey, @@ -51,6 +55,11 @@ describe("tui last session state", () => { }); await expect(readTuiLastSessionKey({ scopeKey, stateDir })).resolves.toBe("agent:main:tui-123"); + expect( + readConfigMachineStateWithMetadata(`tui.lastSession.${scopeKey}`, { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }), + ).toEqual({ value: "agent:main:tui-123", updatedAtMs: expect.any(Number) }); await expect(fs.stat(path.join(stateDir, "tui", "last-session.json"))).rejects.toMatchObject({ code: "ENOENT", }); @@ -163,17 +172,59 @@ describe("tui last session state", () => { sessionKey: "agent:main:telegram:thread", stateDir, }); + await writeTuiLastSessionKey({ + scopeKey: "other-terminal", + sessionKey: "agent:main:main", + stateDir, + }); + writeConfigMachineState("unrelated.sessionReference", "agent:main:main", { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); expect( clearTuiLastSessionPointers({ stateDir, sessionKeys: new Set(["agent:main:main"]), }), - ).toBe(1); + ).toBe(2); await expect(readTuiLastSessionKey({ scopeKey: "terminal", stateDir })).resolves.toBeNull(); + await expect( + readTuiLastSessionKey({ scopeKey: "other-terminal", stateDir }), + ).resolves.toBeNull(); await expect(readTuiLastSessionKey({ scopeKey: "remote", stateDir })).resolves.toBe( "agent:main:telegram:thread", ); + expect( + readConfigMachineStateWithMetadata("unrelated.sessionReference", { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + })?.value, + ).toBe("agent:main:main"); + }); + + it("keeps a live replacement pointer written after the retired-pointer scan", async () => { + const stateDir = await makeTempStateDir(); + await writeTuiLastSessionKey({ + scopeKey: "terminal", + sessionKey: "agent:main:retired", + stateDir, + }); + // A replacement lands before the delete phase; the in-transaction + // compare-and-delete must preserve it instead of erasing the live pointer. + await writeTuiLastSessionKey({ + scopeKey: "terminal", + sessionKey: "agent:main:live", + stateDir, + }); + + expect( + clearTuiLastSessionPointers({ + stateDir, + sessionKeys: new Set(["agent:main:retired"]), + }), + ).toBe(0); + await expect(readTuiLastSessionKey({ scopeKey: "terminal", stateDir })).resolves.toBe( + "agent:main:live", + ); }); }); diff --git a/src/tui/tui-last-session.ts b/src/tui/tui-last-session.ts index 540ff662dad5..f14f9d2614cd 100644 --- a/src/tui/tui-last-session.ts +++ b/src/tui/tui-last-session.ts @@ -1,20 +1,22 @@ // Stores and resolves the last TUI session per workspace. import { createHash } from "node:crypto"; import { normalizeLowercaseStringOrEmpty as normalizeMarker } from "@openclaw/normalization-core/string-coerce"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "../infra/kysely-sync.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; import { normalizeAgentId, parseAgentSessionKey } from "../routing/session-key.js"; +import { + readConfigMachineStateWithMetadata, + writeConfigMachineState, + updateConfigMachineState, +} from "../state/config-machine-state.js"; import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; -import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; -import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; import type { TuiSessionList } from "./tui-backend.js"; import type { SessionScope } from "./tui-types.js"; -type TuiLastSessionDatabase = Pick; +type TuiLastSessionDatabase = Pick; + +const TUI_LAST_SESSION_STATE_KEY_PREFIX = "tui.lastSession."; function stateDatabaseOptions(stateDir?: string) { return stateDir @@ -62,24 +64,12 @@ export async function readTuiLastSessionKey(params: { scopeKey: string; stateDir?: string; }): Promise { - const options = stateDatabaseOptions(params.stateDir); - // CLI reads must not join the Gateway's writable SQLite lifecycle (#101290). - return ( - withExistingOpenClawStateDatabaseReadOnly(({ db }) => { - if (!tableExists(db, "tui_last_sessions")) { - return null; - } - const row = executeSqliteQueryTakeFirstSync( - db, - getNodeSqliteKysely(db) - .selectFrom("tui_last_sessions") - .select("session_key") - .where("scope_key", "=", params.scopeKey), - ); - const sessionKey = row?.session_key.trim() ?? ""; - return sessionKey && !isHeartbeatSessionKey(sessionKey) ? sessionKey : null; - }, options) ?? null + const sessionKey = readConfigMachineStateWithMetadata( + `${TUI_LAST_SESSION_STATE_KEY_PREFIX}${params.scopeKey}`, + stateDatabaseOptions(params.stateDir), ); + const rememberedKey = sessionKey?.value.trim() ?? ""; + return rememberedKey && !isHeartbeatSessionKey(rememberedKey) ? rememberedKey : null; } /** Writes the remembered session key unless it is empty, unknown, or heartbeat-owned. */ @@ -92,26 +82,11 @@ export async function writeTuiLastSessionKey(params: { if (!sessionKey || sessionKey === "unknown" || isHeartbeatSessionKey(sessionKey)) { return; } - const updatedAt = Date.now(); - runOpenClawStateWriteTransaction(({ db }) => { - const tuiDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - tuiDb - .insertInto("tui_last_sessions") - .values({ - scope_key: params.scopeKey, - session_key: sessionKey, - updated_at: updatedAt, - }) - .onConflict((conflict) => - conflict.column("scope_key").doUpdateSet({ - session_key: sessionKey, - updated_at: updatedAt, - }), - ), - ); - }, stateDatabaseOptions(params.stateDir)); + writeConfigMachineState( + `${TUI_LAST_SESSION_STATE_KEY_PREFIX}${params.scopeKey}`, + sessionKey, + stateDatabaseOptions(params.stateDir), + ); } /** @@ -151,15 +126,49 @@ export function clearTuiLastSessionPointers(params: { if (params.sessionKeys.size === 0) { return 0; } - return runOpenClawStateWriteTransaction(({ db }) => { - const result = executeSqliteQuerySync( + const options = stateDatabaseOptions(params.stateDir); + const matchingKeys = withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const rows = executeSqliteQuerySync( db, getNodeSqliteKysely(db) - .deleteFrom("tui_last_sessions") - .where("session_key", "in", [...params.sessionKeys]), - ); - return Number(result.numAffectedRows ?? 0n); - }, stateDatabaseOptions(params.stateDir)); + .selectFrom("config_machine_state") + .select(["state_key", "value_json"]) + .where("state_key", "like", `${TUI_LAST_SESSION_STATE_KEY_PREFIX}%`), + ).rows; + return rows.flatMap((row) => { + const sessionKey: unknown = JSON.parse(row.value_json); + return typeof sessionKey === "string" && params.sessionKeys.has(sessionKey) + ? [row.state_key] + : []; + }); + }, options); + return (matchingKeys ?? []).reduce( + (cleared, stateKey) => + cleared + Number(clearTuiPointerIfRetired(stateKey, params.sessionKeys, options)), + 0, + ); +} + +// Compare-and-delete inside the write transaction: a live replacement pointer +// written after the read-only scan must survive doctor cleanup. +function clearTuiPointerIfRetired( + stateKey: string, + retiredSessionKeys: ReadonlySet, + options: OpenClawStateDatabaseOptions, +): boolean { + let cleared = false; + updateConfigMachineState( + stateKey, + (current) => { + if (typeof current === "string" && retiredSessionKeys.has(current)) { + cleared = true; + return undefined; + } + return current; + }, + options, + ); + return cleared; } /** Resolves a remembered key to a currently listed session for the active agent. */ diff --git a/test/cli-json-stdout.e2e.test.ts b/test/cli-json-stdout.e2e.test.ts index 1e4c3543d021..86b097d98176 100644 --- a/test/cli-json-stdout.e2e.test.ts +++ b/test/cli-json-stdout.e2e.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { DatabaseSync } from "node:sqlite"; import { withTempHome } from "openclaw/plugin-sdk/test-env"; import { describe, expect, it } from "vitest"; +import { readConfigMachineState } from "../src/state/config-machine-state.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "../src/state/openclaw-state-schema.js"; function runBuiltCli( @@ -700,23 +701,27 @@ describe("cli json stdout contract", () => { ); }); + // Every case opens the state database: config-health observation + // (observeConfigSnapshot -> readConfigHealthStateFromStore) runs on any + // config read whose file exists, so the migration diagnostic always lands + // on stderr; the protected contract is that stdout stays exact. it.each([ { name: "aliases list", args: ["models", "aliases", "list", "--plain"], - opensStateDatabase: false, + opensStateDatabase: true, expectedStdout: "chat anthropic/claude-sonnet-4-6\n", }, { name: "fallbacks list", args: ["models", "fallbacks", "list", "--plain"], - opensStateDatabase: false, + opensStateDatabase: true, expectedStdout: "anthropic/claude-sonnet-4-6\n", }, { name: "image fallbacks list", args: ["models", "image-fallbacks", "list", "--plain"], - opensStateDatabase: false, + opensStateDatabase: true, expectedStdout: "anthropic/claude-sonnet-4-6\n", }, { @@ -775,7 +780,9 @@ describe("cli json stdout contract", () => { expect(result.status, result.stderr).toBe(0); expect(result.stdout).toBe(testCase.expectedStdout); expect(result.stdout).not.toContain(migrationDiagnostic); - expect(result.stderr.includes(migrationDiagnostic)).toBe(testCase.opensStateDatabase); + expect(result.stderr.includes(migrationDiagnostic), result.stderr).toBe( + testCase.opensStateDatabase, + ); }, { prefix: "openclaw-models-plain-stdout-e2e-" }, ); @@ -849,14 +856,11 @@ describe("cli json stdout contract", () => { OPENCLAW_CONFIG_PATH: configPath, OPENCLAW_STATE_DIR: stateDir, }); - const readCatalogRow = () => { - const database = new DatabaseSync(databasePath, { readOnly: true }); - try { - return database.prepare("SELECT * FROM model_catalog_remote WHERE id = 1").get(); - } finally { - database.close(); - } - }; + const readCatalogRow = () => + readConfigMachineState<{ generated_at: number; bundle_json: string }>( + "modelCatalog.remote", + { path: databasePath }, + ); const human = runRefresh(["refresh"], "initial"); expect(human.status, human.stderr).toBe(0); diff --git a/test/scripts/check-native-state-schema-version.test.ts b/test/scripts/check-native-state-schema-version.test.ts index 19f4b3474c30..bbc1f7063617 100644 --- a/test/scripts/check-native-state-schema-version.test.ts +++ b/test/scripts/check-native-state-schema-version.test.ts @@ -7,6 +7,7 @@ import { OPENCLAW_STATE_SCHEMA_VERSION } from "../../src/state/openclaw-state-db describe("native state schema version guard", () => { it("keeps the checked-in Swift and TypeScript contracts aligned", () => { + expect(OPENCLAW_STATE_SCHEMA_VERSION).toBe(12); expect(checkNativeStateSchemaVersion()).toBe(OPENCLAW_STATE_SCHEMA_VERSION); }); diff --git a/test/scripts/lint-suppressions.test.ts b/test/scripts/lint-suppressions.test.ts index dadb1fb5acb9..9bcfe9366ad0 100644 --- a/test/scripts/lint-suppressions.test.ts +++ b/test/scripts/lint-suppressions.test.ts @@ -234,7 +234,7 @@ describe("production lint suppressions", () => { "src/plugins/trusted-tool-policy.ts|typescript/no-unnecessary-type-parameters|1", // Raw PowerShell errors carry the -EncodedCommand argv; only the sanitized cause may escape. "src/secrets/private-plan-file.ts|preserve-caught-error|1", - "src/state/config-machine-state.ts|typescript/no-unnecessary-type-parameters|1", + "src/state/config-machine-state.ts|typescript/no-unnecessary-type-parameters|2", "src/system-agent/setup-inference-activate.ts|no-unsafe-finally|1", "src/system-agent/setup-inference-activate.ts|preserve-caught-error|1", "src/tasks/task-registry.sqlite.shared.ts|typescript/no-unnecessary-type-parameters|1",