diff --git a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 index b1b51969807b..510130bf889e 100644 --- a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 +++ b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 @@ -1 +1 @@ -011b9ec0e0fa64b5a4036648fcd61a50674b813a92280e8e3fe8746f6acdb62d sqlite-session-transcript-schema-baseline.sql +8eca1c5f2bbb9b9d333bb46aeedae23b568c8a8b723ee0fea0ccab651943769f sqlite-session-transcript-schema-baseline.sql diff --git a/package.json b/package.json index e8682b1b1925..fcc4777952dd 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "openclaw": { "schemaVersions": { "state": 5, - "agent": 13 + "agent": 14 } }, "description": "Multi-channel AI gateway with extensible messaging integrations", diff --git a/scripts/bench-transcript-cursors.ts b/scripts/bench-transcript-cursors.ts index ffea591797a4..51afd3d58978 100644 --- a/scripts/bench-transcript-cursors.ts +++ b/scripts/bench-transcript-cursors.ts @@ -47,13 +47,20 @@ function seedTranscript( const now = Date.now(); database.db .prepare( - `INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at) - VALUES (?, ?, 'conversation', ?, ?)`, + `INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) + VALUES (?, ?, '{}', ?)`, + ) + .run(sessionKey, sessionId, now); + database.db + .prepare( + `INSERT INTO session_windows ( + session_id, session_key, session_scope, created_at, updated_at + ) VALUES (?, ?, 'conversation', ?, ?)`, ) .run(sessionId, sessionKey, now, now); database.db .prepare( - `INSERT INTO session_transcript_generations (session_id, generation, updated_at) + `INSERT INTO transcript_rewrite_watermarks (session_id, generation, updated_at) VALUES (?, 'benchmark-generation', ?)`, ) .run(sessionId, now); diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index a0880b735233..dd3422f2db1e 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -42,6 +42,7 @@ const rawSqliteAllowPathGroups = { "src/state/openclaw-agent-db-registry.ts", "src/state/openclaw-agent-db-schema-helpers.ts", "src/state/openclaw-agent-db-schema.ts", + "src/state/openclaw-agent-db-session-nodes-migration.ts", "src/state/openclaw-agent-db-session-migrations.ts", "src/state/openclaw-agent-db-session-provenance.ts", "src/state/openclaw-agent-db.ts", diff --git a/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs b/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs index 3380efab98c0..f54bdf052b79 100644 --- a/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs +++ b/scripts/e2e/lib/codex-npm-plugin-live/assertions.mjs @@ -203,11 +203,11 @@ function readSessionEntry(sessionId) { try { const row = db .prepare( - `SELECT se.session_key, se.entry_json, s.agent_harness_id - FROM sessions AS s - INNER JOIN session_entries AS se ON se.session_id = s.session_id - WHERE s.session_id = ? - ORDER BY se.updated_at DESC, se.session_key + `SELECT sn.session_key, sn.entry_json, sw.agent_harness_id + FROM session_nodes AS sn + INNER JOIN session_windows AS sw ON sw.session_id = sn.current_session_id + WHERE sw.session_id = ? + ORDER BY sn.updated_at DESC, sn.session_key LIMIT 1`, ) .get(sessionId); diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 5f2af47b63ba..a2f8b1cf581c 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -689,14 +689,13 @@ function readMigratedSessionStore(stateDir, targetStorePath) { try { db = new DatabaseSync(dbPath, { readOnly: true }); const hasSessionEntries = db - .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session_entries'") + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session_nodes'") .get(); const rows = hasSessionEntries ? db .prepare( - `SELECT se.session_key AS key, sr.session_id, se.entry_json AS value_json - FROM session_entries AS se - INNER JOIN session_routes AS sr ON sr.session_key = se.session_key`, + `SELECT session_key AS key, current_session_id AS session_id, entry_json AS value_json + FROM session_nodes`, ) .all() : db diff --git a/scripts/e2e/session-runtime-context-docker-client.ts b/scripts/e2e/session-runtime-context-docker-client.ts index 61f3828bd3ec..bbf005534f26 100644 --- a/scripts/e2e/session-runtime-context-docker-client.ts +++ b/scripts/e2e/session-runtime-context-docker-client.ts @@ -217,7 +217,7 @@ async function verifyDoctorRepair(root: string) { let migratedSessionId: string | undefined; try { const row = database - .prepare("SELECT session_id FROM session_routes WHERE session_key = ?") + .prepare("SELECT current_session_id AS session_id FROM session_nodes WHERE session_key = ?") .get("agent:main:qa:docker-runtime-context"); if (typeof row?.session_id === "string") { migratedSessionId = row.session_id; diff --git a/scripts/lib/sqlite-session-schema-baseline.ts b/scripts/lib/sqlite-session-schema-baseline.ts index 78d0856a09fc..651119c37352 100644 --- a/scripts/lib/sqlite-session-schema-baseline.ts +++ b/scripts/lib/sqlite-session-schema-baseline.ts @@ -27,13 +27,15 @@ const DEFAULT_SQL_OUTPUT = ".artifacts/sqlite-session-transcript-schema-baseline const DEFAULT_HASH_OUTPUT = "docs/.generated/sqlite-session-transcript-schema-baseline.sha256"; const TARGET_TABLES = new Set([ - "sessions", - "session_routes", + "session_nodes", + "session_windows", + "session_members", "conversations", "session_conversations", - "session_entries", "transcript_events", + "transcript_rewrite_watermarks", "transcript_event_identities", + "session_transcript_index_state", "session_transcript_active_events", ]); diff --git a/src/agents/acp-parent-stream-store.sqlite.test.ts b/src/agents/acp-parent-stream-store.sqlite.test.ts index cd7bd53e0f2e..25ca3a5b9ad9 100644 --- a/src/agents/acp-parent-stream-store.sqlite.test.ts +++ b/src/agents/acp-parent-stream-store.sqlite.test.ts @@ -21,10 +21,21 @@ describe("ACP parent stream SQLite store", () => { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, }; runOpenClawAgentWriteTransaction((database) => { - const db = getNodeSqliteKysely>(database.db); + const db = getNodeSqliteKysely< + Pick + >(database.db); executeSqliteQuerySync( database.db, - db.insertInto("sessions").values({ + db.insertInto("session_nodes").values({ + session_key: "agent:codex:acp:child", + current_session_id: "session-1", + entry_json: "{}", + updated_at: 1, + }), + ); + executeSqliteQuerySync( + database.db, + db.insertInto("session_windows").values({ session_id: "session-1", session_key: "agent:codex:acp:child", session_scope: "conversation", @@ -52,10 +63,12 @@ describe("ACP parent stream SQLite store", () => { ]); runOpenClawAgentWriteTransaction((database) => { - const db = getNodeSqliteKysely>(database.db); + const db = getNodeSqliteKysely>( + database.db, + ); executeSqliteQuerySync( database.db, - db.deleteFrom("sessions").where("session_id", "=", "session-1"), + db.deleteFrom("session_windows").where("session_id", "=", "session-1"), ); }, options); expect( @@ -71,10 +84,21 @@ describe("ACP parent stream SQLite store", () => { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, }; runOpenClawAgentWriteTransaction((database) => { - const db = getNodeSqliteKysely>(database.db); + const db = getNodeSqliteKysely< + Pick + >(database.db); executeSqliteQuerySync( database.db, - db.insertInto("sessions").values({ + db.insertInto("session_nodes").values({ + session_key: "agent:codex:acp:invalid", + current_session_id: "session-1", + entry_json: "{}", + updated_at: 1, + }), + ); + executeSqliteQuerySync( + database.db, + db.insertInto("session_windows").values({ session_id: "session-1", session_key: "agent:codex:acp:invalid", session_scope: "conversation", diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-phase.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-phase.ts index d0792b3f2c25..d6512495fd57 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-phase.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-phase.ts @@ -163,6 +163,7 @@ export async function runEmbeddedAttemptPromptPhase(input: { claimHeartbeatOutcomeForRun({ agentId: input.context.sessionAgentId, sessionKey: attempt.sessionKey, + storePath: attempt.sessionTarget?.storePath, runId: attempt.runId, }), ) diff --git a/src/boards/board-store.parity.test.ts b/src/boards/board-store.parity.test.ts index 6a189b458e39..6a7eef43ecda 100644 --- a/src/boards/board-store.parity.test.ts +++ b/src/boards/board-store.parity.test.ts @@ -510,7 +510,7 @@ describe("SqliteBoardStore persistence", () => { expect(store.readWidgetMcpApp(sessionKey, "legacy-app")).toBeUndefined(); }); - it("lazily creates board tables for an existing v13 database", () => { + it("lazily creates board tables for an existing v14 database", () => { const stateDir = tempDirs.make("openclaw-board-lazy-schema-"); const env = { OPENCLAW_STATE_DIR: stateDir }; const sessionKey = "agent:main:board"; @@ -521,14 +521,14 @@ describe("SqliteBoardStore persistence", () => { closeOpenClawStateDatabaseForTest(); const { DatabaseSync } = requireNodeSqlite(); - const existingV13 = new DatabaseSync(databasePath); - existingV13.exec(` + const existingV14 = new DatabaseSync(databasePath); + existingV14.exec(` DROP TABLE board_widgets; DROP TABLE board_tabs; - PRAGMA user_version = 13; - UPDATE schema_meta SET schema_version = 13 WHERE meta_key = 'primary'; + PRAGMA user_version = 14; + UPDATE schema_meta SET schema_version = 14 WHERE meta_key = 'primary'; `); - existingV13.close(); + existingV14.close(); const reopened = openOpenClawAgentDatabase({ agentId: "main", env }); expect( @@ -684,6 +684,39 @@ describe("SqliteBoardStore persistence", () => { expect(existsSync(path.join(stateDir, "agents", "attacker-selected"))).toBe(false); }); + it("rejects board writes for transcript-only placeholder nodes", () => { + const stateDir = tempDirs.make("openclaw-board-transcript-only-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const sessionKey = "agent:main:transcript-only"; + const database = openOpenClawAgentDatabase({ agentId: "main", env }); + database.db + .prepare( + `INSERT INTO session_nodes ( + session_key, current_session_id, entry_json, updated_at + ) VALUES (?, 'transcript-only-session', '{}', 1)`, + ) + .run(sessionKey); + database.db + .prepare( + `INSERT INTO session_windows ( + session_id, session_key, session_scope, created_at, updated_at + ) VALUES ('transcript-only-session', ?, 'conversation', 1, 1)`, + ) + .run(sessionKey); + const store = new SqliteBoardStore({ + resolveSession: () => ({ agentId: "main", sessionKey }), + env, + }); + + expect(() => + store.putWidget({ + sessionKey, + name: "status", + content: { kind: "html", html: "no" }, + }), + ).toThrow("board session not found"); + }); + it("canonicalizes aliases before reading and writing board rows", () => { const stateDir = tempDirs.make("openclaw-board-alias-"); const env = { OPENCLAW_STATE_DIR: stateDir }; diff --git a/src/boards/sqlite-board-store.ts b/src/boards/sqlite-board-store.ts index 2f15ac7e6ff9..f320a840cff8 100644 --- a/src/boards/sqlite-board-store.ts +++ b/src/boards/sqlite-board-store.ts @@ -47,7 +47,7 @@ import { type BoardDatabase = Pick< OpenClawAgentKyselyDatabase, - "board_tabs" | "board_widgets" | "session_entries" + "board_tabs" | "board_widgets" | "session_nodes" >; type BoardDatabaseHandle = Pick; @@ -286,16 +286,28 @@ function deleteRemovedTabs( function hasSession(database: BoardDatabaseHandle, sessionKey: string): boolean { const db = getNodeSqliteKysely(database.db); - return Boolean( - executeSqliteQuerySync( - database.db, - db - .selectFrom("session_entries") - .select("session_key") - .where("session_key", "=", sessionKey) - .limit(1), - ).rows[0], - ); + const row = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_nodes") + .select("entry_json") + .where("session_key", "=", sessionKey) + .limit(1), + ).rows[0]; + if (!row) { + return false; + } + try { + const entry = JSON.parse(row.entry_json) as unknown; + return Boolean( + entry && + typeof entry === "object" && + !Array.isArray(entry) && + typeof (entry as { sessionId?: unknown }).sessionId === "string", + ); + } catch { + return false; + } } function emptyBoardSnapshot(sessionKey: string): BoardSnapshot { diff --git a/src/commands/doctor-session-incognito-key-repair.test.ts b/src/commands/doctor-session-incognito-key-repair.test.ts index 89e6471d6649..d05c8deda212 100644 --- a/src/commands/doctor-session-incognito-key-repair.test.ts +++ b/src/commands/doctor-session-incognito-key-repair.test.ts @@ -36,33 +36,26 @@ describe("doctor reserved incognito session key repair", () => { const baseLegacyKey = "agent:main:dashboard:legacy-incognito-collision"; const newKey = `${baseLegacyKey}-1`; try { + const entryJson = JSON.stringify({ + sessionId: "session-old", + parentSessionKey: oldKey, + spawnedBy: oldKey, + completionOwnerSessionKey: oldKey, + forkSource: { sessionKey: oldKey, sessionId: "source" }, + compactionCheckpoints: [{ checkpointId: "checkpoint", sessionKey: oldKey }], + systemPromptReport: { source: "run", generatedAt: 1, sessionKey: oldKey }, + pluginExtensions: { test: { label: oldKey } }, + }); database.db .prepare( - "INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at, parent_session_key, spawned_by) VALUES (?, ?, 'conversation', 1, 1, ?, ?)", + "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at, parent_session_key, spawned_by, fork_source_session_key) VALUES (?, ?, ?, 1, ?, ?, ?)", + ) + .run(oldKey, "session-old", entryJson, oldKey, oldKey, oldKey); + database.db + .prepare( + "INSERT INTO session_windows (session_id, session_key, session_scope, created_at, updated_at, parent_session_key, spawned_by) VALUES (?, ?, 'conversation', 1, 1, ?, ?)", ) .run("session-old", oldKey, oldKey, oldKey); - database.db - .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES (?, ?, ?, 1)", - ) - .run( - oldKey, - "session-old", - JSON.stringify({ - sessionId: "session-old", - parentSessionKey: oldKey, - completionOwnerSessionKey: oldKey, - forkSource: { sessionKey: oldKey, sessionId: "source" }, - compactionCheckpoints: [{ checkpointId: "checkpoint", sessionKey: oldKey }], - systemPromptReport: { source: "run", generatedAt: 1, sessionKey: oldKey }, - pluginExtensions: { test: { label: oldKey } }, - }), - ); - database.db - .prepare( - "INSERT INTO session_routes (session_key, session_id, updated_at) VALUES (?, ?, 1)", - ) - .run(oldKey, "session-old"); database.db .prepare( "INSERT INTO conversations (conversation_id, channel, account_id, kind, peer_id, delivery_target, metadata_json, created_at, updated_at) VALUES ('conversation-1', 'webchat', 'default', 'direct', 'peer', 'peer', '{}', 1, 1)", @@ -85,14 +78,18 @@ describe("doctor reserved incognito session key repair", () => { .run("a".repeat(43), oldKey, JSON.stringify([baseLegacyKey])); secondaryDatabase.db .prepare( - "INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at, parent_session_key, spawned_by) VALUES ('session-work', 'agent:work:dashboard:regular', 'conversation', 1, 1, ?, ?)", + "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at, parent_session_key, spawned_by) VALUES ('agent:work:dashboard:regular', 'session-work', ?, 1, ?, ?)", ) - .run(oldKey, oldKey); + .run( + JSON.stringify({ sessionId: "session-work", completionOwnerSessionKey: oldKey }), + oldKey, + oldKey, + ); secondaryDatabase.db .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES ('agent:work:dashboard:regular', 'session-work', ?, 1)", + "INSERT INTO session_windows (session_id, session_key, session_scope, created_at, updated_at, parent_session_key, spawned_by) VALUES ('session-work', 'agent:work:dashboard:regular', 'conversation', 1, 1, ?, ?)", ) - .run(JSON.stringify({ sessionId: "session-work", completionOwnerSessionKey: oldKey })); + .run(oldKey, oldKey); stateDatabase.db .prepare( "INSERT INTO session_watch_cursors (watcher_session_key, target_session_key, updated_at) VALUES (?, ?, 1)", @@ -118,6 +115,11 @@ describe("doctor reserved incognito session key repair", () => { "INSERT INTO board_widgets (session_key, name, tab_id, content_kind, html, sha256, view_generation, revision, size_w, size_h, position, created_by, created_at, updated_at) VALUES (?, 'widget-1', 'tab-1', 'html', X'00', 'sha', 'view-1', 1, 1, 1, 0, 'user', 1, 1)", ) .run(oldKey); + database.db + .prepare( + "INSERT INTO session_members (session_key, identity_id, added_by, added_at) VALUES (?, 'member-1', 'owner-1', 1)", + ) + .run(oldKey); expect(repairReservedIncognitoSessionKeys({ apply: false, cfg: {}, env })).toEqual({ found: 1, @@ -130,12 +132,21 @@ describe("doctor reserved incognito session key repair", () => { expect( database.db - .prepare("SELECT session_key, parent_session_key, spawned_by FROM sessions") + .prepare( + "SELECT session_key, parent_session_key, spawned_by, fork_source_session_key FROM session_nodes", + ) + .get(), + ).toEqual({ + session_key: newKey, + parent_session_key: newKey, + spawned_by: newKey, + fork_source_session_key: newKey, + }); + expect( + database.db + .prepare("SELECT session_key, parent_session_key, spawned_by FROM session_windows") .get(), ).toEqual({ session_key: newKey, parent_session_key: newKey, spawned_by: newKey }); - expect(database.db.prepare("SELECT session_key FROM session_routes").get()).toEqual({ - session_key: newKey, - }); expect( database.db.prepare("SELECT source_session_key FROM conversation_deliveries").get(), ).toEqual({ source_session_key: newKey }); @@ -148,6 +159,9 @@ describe("doctor reserved incognito session key repair", () => { expect(database.db.prepare("SELECT session_key FROM board_widgets").get()).toEqual({ session_key: newKey, }); + expect(database.db.prepare("SELECT session_key FROM session_members").get()).toEqual({ + session_key: newKey, + }); expect(stateDatabase.db.prepare("SELECT session_key FROM session_state_heads").get()).toEqual( { session_key: newKey, @@ -170,16 +184,18 @@ describe("doctor reserved incognito session key repair", () => { audience_session_keys_json: JSON.stringify([baseLegacyKey]), }); expect( - secondaryDatabase.db.prepare("SELECT parent_session_key, spawned_by FROM sessions").get(), + secondaryDatabase.db + .prepare("SELECT parent_session_key, spawned_by FROM session_windows") + .get(), ).toEqual({ parent_session_key: newKey, spawned_by: newKey }); const secondaryEntry = secondaryDatabase.db - .prepare("SELECT entry_json FROM session_entries") + .prepare("SELECT entry_json FROM session_nodes") .get() as { entry_json: string }; expect(JSON.parse(secondaryEntry.entry_json)).toMatchObject({ completionOwnerSessionKey: newKey, }); const entry = database.db - .prepare("SELECT session_key, entry_json FROM session_entries") + .prepare("SELECT session_key, entry_json FROM session_nodes") .get() as { session_key: string; entry_json: string }; expect(entry.session_key).toBe(newKey); expect(JSON.parse(entry.entry_json)).toMatchObject({ @@ -215,24 +231,24 @@ describe("doctor reserved incognito session key repair", () => { const resumedKey = "agent:main:dashboard:legacy-incognito-interrupted-resumed"; database.db .prepare( - "INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at) VALUES ('session-old', ?, 'conversation', 1, 1)", - ) - .run(oldKey); - database.db - .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES (?, 'session-old', ?, 1)", + "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, 'session-old', ?, 1)", ) .run(oldKey, JSON.stringify({ sessionId: "session-old" })); database.db .prepare( - "INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at) VALUES ('session-new', ?, 'conversation', 1, 1)", + "INSERT INTO session_windows (session_id, session_key, session_scope, created_at, updated_at) VALUES ('session-old', ?, 'conversation', 1, 1)", ) - .run(newCollisionKey); + .run(oldKey); database.db .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES (?, 'session-new', ?, 1)", + "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, 'session-new', ?, 1)", ) .run(newCollisionKey, JSON.stringify({ sessionId: "session-new" })); + database.db + .prepare( + "INSERT INTO session_windows (session_id, session_key, session_scope, created_at, updated_at) VALUES ('session-new', ?, 'conversation', 1, 1)", + ) + .run(newCollisionKey); stateDatabase.db .prepare( "INSERT INTO state_leases (scope, lease_key, owner, payload_json, created_at, updated_at) VALUES ('doctor-session-key-migration', 'reserved-incognito-v1', 'openclaw-doctor', ?, 1, 1)", @@ -259,7 +275,7 @@ describe("doctor reserved incognito session key repair", () => { }); expect( database.db - .prepare("SELECT session_key FROM sessions ORDER BY session_key") + .prepare("SELECT session_key FROM session_nodes ORDER BY session_key") .all() .map((row) => (row as { session_key: string }).session_key), ).toEqual([resumedKey, "agent:main:dashboard:legacy-incognito-new"].toSorted()); diff --git a/src/commands/doctor-session-incognito-key-repair.ts b/src/commands/doctor-session-incognito-key-repair.ts index aebee7a71025..1c106abfbceb 100644 --- a/src/commands/doctor-session-incognito-key-repair.ts +++ b/src/commands/doctor-session-incognito-key-repair.ts @@ -183,13 +183,13 @@ function listReservedIncognitoKeys(database: DatabaseSync): string[] { const keys = new Set(); for (const row of executeSqliteQuerySync( database, - db.selectFrom("sessions").select("session_key"), + db.selectFrom("session_nodes").select("session_key"), ).rows) { keys.add(row.session_key); } for (const row of executeSqliteQuerySync( database, - db.selectFrom("session_entries").select("session_key"), + db.selectFrom("session_windows").select("session_key"), ).rows) { keys.add(row.session_key); } @@ -209,30 +209,31 @@ function collectOccupiedSessionKeys(database: DatabaseSync): Set { collect( executeSqliteQuerySync( database, - db.selectFrom("sessions").select(["session_key", "parent_session_key", "spawned_by"]), + db.selectFrom("session_windows").select(["session_key", "parent_session_key", "spawned_by"]), ).rows.flatMap((row) => [row.session_key, row.parent_session_key, row.spawned_by]), ); + collect( + executeSqliteQuerySync( + database, + db + .selectFrom("session_nodes") + .select(["session_key", "parent_session_key", "spawned_by", "fork_source_session_key"]), + ).rows.flatMap((row) => [ + row.session_key, + row.parent_session_key, + row.spawned_by, + row.fork_source_session_key, + ]), + ); collect( executeSqliteQuerySync( database, db.selectFrom("conversation_deliveries").select("source_session_key"), ).rows.map((row) => row.source_session_key), ); - collect( - executeSqliteQuerySync( - database, - db.selectFrom("session_routes").select("session_key"), - ).rows.map((row) => row.session_key), - ); - collect( - executeSqliteQuerySync( - database, - db.selectFrom("session_entries").select("session_key"), - ).rows.map((row) => row.session_key), - ); for (const row of executeSqliteQuerySync( database, - db.selectFrom("session_entries").select("entry_json"), + db.selectFrom("session_nodes").select("entry_json"), ).rows) { try { collectSessionEntryKeyFields(JSON.parse(row.entry_json), keys); @@ -265,30 +266,45 @@ function updateSessionKeyColumns(database: DatabaseSync, rename: ReservedKeyRena executeSqliteQuerySync(database, query); update( db - .updateTable("sessions") + .updateTable("session_windows") .set({ session_key: rename.to }) .where("session_key", "=", rename.from), ); update( db - .updateTable("sessions") + .updateTable("session_windows") .set({ parent_session_key: rename.to }) .where("parent_session_key", "=", rename.from), ); update( - db.updateTable("sessions").set({ spawned_by: rename.to }).where("spawned_by", "=", rename.from), + db + .updateTable("session_windows") + .set({ spawned_by: rename.to }) + .where("spawned_by", "=", rename.from), ); update( db - .updateTable("session_routes") + .updateTable("session_nodes") .set({ session_key: rename.to }) .where("session_key", "=", rename.from), ); update( db - .updateTable("session_entries") - .set({ session_key: rename.to }) - .where("session_key", "=", rename.from), + .updateTable("session_nodes") + .set({ parent_session_key: rename.to }) + .where("parent_session_key", "=", rename.from), + ); + update( + db + .updateTable("session_nodes") + .set({ spawned_by: rename.to }) + .where("spawned_by", "=", rename.from), + ); + update( + db + .updateTable("session_nodes") + .set({ fork_source_session_key: rename.to }) + .where("fork_source_session_key", "=", rename.from), ); update( db @@ -296,6 +312,12 @@ function updateSessionKeyColumns(database: DatabaseSync, rename: ReservedKeyRena .set({ source_session_key: rename.to }) .where("source_session_key", "=", rename.from), ); + update( + db + .updateTable("session_members") + .set({ session_key: rename.to }) + .where("session_key", "=", rename.from), + ); update( db .updateTable("board_tabs") @@ -329,7 +351,7 @@ function rewriteSessionEntryJsonReferences( const db = getNodeSqliteKysely(database); const rows = executeSqliteQuerySync( database, - db.selectFrom("session_entries").select(["session_key", "entry_json"]), + db.selectFrom("session_nodes").select(["session_key", "entry_json"]), ).rows; for (const row of rows) { let parsed: unknown; @@ -346,7 +368,7 @@ function rewriteSessionEntryJsonReferences( executeSqliteQuerySync( database, db - .updateTable("session_entries") + .updateTable("session_nodes") .set({ entry_json: entryJson }) .where("session_key", "=", row.session_key), ); diff --git a/src/commands/doctor-session-sqlite-readers.ts b/src/commands/doctor-session-sqlite-readers.ts index 4e160faaba0c..1df414b6684f 100644 --- a/src/commands/doctor-session-sqlite-readers.ts +++ b/src/commands/doctor-session-sqlite-readers.ts @@ -133,14 +133,23 @@ export function readOnlySqliteSessionEntries( let database: InstanceType | undefined; try { database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); - const table = database + const nodeTable = database .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") - .get("session_entries"); - if (!table) { + .get("session_nodes"); + const legacyEntryTable = nodeTable + ? undefined + : database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get("session_entries"); + if (!nodeTable && !legacyEntryTable) { return { exists: true, ok: true, summaries: [] }; } const rows = database - .prepare("SELECT session_key, entry_json FROM session_entries ORDER BY session_key ASC") + .prepare( + nodeTable + ? "SELECT session_key, entry_json FROM session_nodes ORDER BY session_key ASC" + : "SELECT session_key, entry_json FROM session_entries ORDER BY session_key ASC", + ) .all() as Array<{ entry_json?: unknown; session_key?: unknown }>; return { exists: true, @@ -293,7 +302,9 @@ export function resolveTargetSqlitePath(target: SessionStoreTarget): string { function parseSqliteSessionEntry(entryJson: string): SessionEntry | undefined { try { const parsed = JSON.parse(entryJson) as unknown; - return isRecord(parsed) ? (parsed as SessionEntry) : undefined; + return isRecord(parsed) && typeof parsed.sessionId === "string" + ? (parsed as SessionEntry) + : undefined; } catch { return undefined; } diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts index a17c20315810..c32436755271 100644 --- a/src/commands/doctor-session-sqlite.test.ts +++ b/src/commands/doctor-session-sqlite.test.ts @@ -32,7 +32,10 @@ import { restoreSessionSqliteMigrationRun, type ActiveSessionSqliteMigrationRun, } from "./doctor-session-sqlite-migration-run.js"; -import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; +import { + readOnlySqliteSessionEntries, + resolveTargetSqlitePath, +} from "./doctor-session-sqlite-readers.js"; import { runDoctorSessionSqlite } from "./doctor-session-sqlite.js"; type SessionSqliteMigrationManifest = ActiveSessionSqliteMigrationRun["manifest"]; @@ -74,6 +77,85 @@ afterEach(() => { }); describe("runDoctorSessionSqlite", () => { + it("reads populated v13 session_entries before migration", () => { + const stateDir = autoCleanupTempDirs.make("openclaw-doctor-v13-reader-"); + const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const target = { agentId: "main", storePath }; + const sqlitePath = resolveTargetSqlitePath(target); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + const sqlite = nodeSqlite.requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + try { + database.exec(` + CREATE TABLE session_entries ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) + VALUES ( + 'agent:main:v13-reader', + 'v13-reader-session', + '{"sessionId":"v13-reader-session","updatedAt":13}', + 13 + ); + PRAGMA user_version = 13; + `); + } finally { + database.close(); + } + + expect(readOnlySqliteSessionEntries(target)).toEqual({ + exists: true, + ok: true, + summaries: [ + { + sessionKey: "agent:main:v13-reader", + entry: { sessionId: "v13-reader-session", updatedAt: 13 }, + }, + ], + }); + }); + + it("excludes v14 transcript-only nodes from doctor entry reads", () => { + const stateDir = autoCleanupTempDirs.make("openclaw-doctor-v14-reader-"); + const storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json"); + const target = { agentId: "main", storePath }; + const sqlitePath = resolveTargetSqlitePath(target); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + const sqlite = nodeSqlite.requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + try { + database.exec(` + CREATE TABLE session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO session_nodes VALUES + ('agent:main:transcript-only', 'transcript-only-session', '{}', 14), + ('agent:main:v14-reader', 'v14-reader-session', + '{"sessionId":"v14-reader-session","updatedAt":14}', 14); + PRAGMA user_version = 14; + `); + } finally { + database.close(); + } + + expect(readOnlySqliteSessionEntries(target)).toEqual({ + exists: true, + ok: true, + summaries: [ + { + sessionKey: "agent:main:v14-reader", + entry: { sessionId: "v14-reader-session", updatedAt: 14 }, + }, + ], + }); + }); + it("dry-runs a legacy store without writing SQLite rows", async () => { const store = createLegacyStore(); @@ -234,7 +316,7 @@ describe("runDoctorSessionSqlite", () => { ).toEqual({ schema_version: OPENCLAW_AGENT_SCHEMA_VERSION }); expect( dormantAfter - .prepare("PRAGMA table_info(sessions)") + .prepare("PRAGMA table_info(session_windows)") .all() .map((column) => (column as { name?: unknown }).name), ).toContain("session_scope"); @@ -367,7 +449,7 @@ describe("runDoctorSessionSqlite", () => { expect( migrated .prepare( - "SELECT session_id, length(generation) AS generation_length FROM session_transcript_generations", + "SELECT session_id, length(generation) AS generation_length FROM transcript_rewrite_watermarks", ) .all(), ).toEqual([{ generation_length: 32, session_id: "session-1" }]); diff --git a/src/config/sessions/conversation-registry.test.ts b/src/config/sessions/conversation-registry.test.ts index 0c6843c9bdf3..7a534d184e2b 100644 --- a/src/config/sessions/conversation-registry.test.ts +++ b/src/config/sessions/conversation-registry.test.ts @@ -163,7 +163,7 @@ describe("conversation registry", () => { ); executeSqliteQuerySync( database.db, - db.deleteFrom("session_entries").where("session_key", "=", staleSessionKey), + db.deleteFrom("session_nodes").where("session_key", "=", staleSessionKey), ); expect( @@ -172,7 +172,7 @@ describe("conversation registry", () => { target: "reef:peer-a", sessionId: "live-session", sessionKey: liveSessionKey, - lastSeenAt: 200, + lastSeenAt: 100, }); }); diff --git a/src/config/sessions/conversation-registry.ts b/src/config/sessions/conversation-registry.ts index c00b7e54e581..bf2694dd45b1 100644 --- a/src/config/sessions/conversation-registry.ts +++ b/src/config/sessions/conversation-registry.ts @@ -8,6 +8,7 @@ import { resolveSqliteReadScope, toDatabaseOptions, } from "./session-accessor.sqlite-scope.js"; +import { parseSqliteSessionEntryJson } from "./session-accessor.sqlite-status.js"; const CONVERSATION_REF_PATTERN = /^conv_[a-f0-9]{32}$/u; @@ -60,6 +61,7 @@ function mapConversationRow(row: { peer_id: string; role: string | null; current_session_id: string | null; + current_entry_json: string | null; current_session_key: string | null; thread_id: string | null; }): ConversationRecord | null { @@ -70,6 +72,10 @@ function mapConversationRow(row: { row.role === "primary" || row.role === "participant" || row.role === "related" ? row.role : undefined; + const currentEntry = row.current_entry_json + ? parseSqliteSessionEntryJson({ entry_json: row.current_entry_json }) + : null; + const hasCurrentBinding = currentEntry?.sessionId === row.current_session_id; return { conversationRef: row.conversation_id, channel: row.channel, @@ -81,9 +87,9 @@ function mapConversationRow(row: { ...(row.native_channel_id ? { nativeChannelId: row.native_channel_id } : {}), ...(row.native_direct_user_id ? { nativeDirectUserId: row.native_direct_user_id } : {}), ...(row.label ? { label: row.label } : {}), - // Only the current session_entries row can bind an address. The joined - // sessions row may be historical after reset, rebind, or deletion. - ...(role && row.current_session_id && row.current_session_key + // Only the current session_nodes row can bind an address. The joined + // window row may be historical after reset, rebind, or deletion. + ...(role && hasCurrentBinding && row.current_session_id && row.current_session_key ? { sessionId: row.current_session_id, sessionKey: row.current_session_key, @@ -109,10 +115,10 @@ function selectConversationRows( let query = db .selectFrom("conversations as c") .leftJoin("session_conversations as sc", "sc.conversation_id", "c.conversation_id") - .leftJoin("sessions as s", "s.session_id", "sc.session_id") - // Historical sessions retain address activity, while session_entries owns + .leftJoin("session_windows as s", "s.session_id", "sc.session_id") + // Historical windows retain address activity, while session_nodes owns // the current session binding after reset/rebind. - .leftJoin("session_entries as se", "se.session_key", "s.session_key") + .leftJoin("session_nodes as sn", "sn.session_key", "s.session_key") .select([ "c.conversation_id", "c.channel", @@ -130,8 +136,9 @@ function selectConversationRows( "sc.role", "sc.first_seen_at", "sc.last_seen_at", - "se.session_id as current_session_id", - "se.session_key as current_session_key", + "sn.current_session_id as current_session_id", + "sn.entry_json as current_entry_json", + "sn.session_key as current_session_key", ]); const channel = normalizeOptionalLowercaseString(options.channel); if (channel) { @@ -148,7 +155,7 @@ function selectConversationRows( database.db, query .orderBy((eb) => eb.fn.coalesce("sc.last_seen_at", "c.updated_at"), "desc") - .orderBy("se.updated_at", "desc"), + .orderBy("sn.updated_at", "desc"), ).rows; const unique = new Map(); for (const row of rows) { diff --git a/src/config/sessions/session-accessor.conformance.test.ts b/src/config/sessions/session-accessor.conformance.test.ts index f7a88069de51..4a0a2081bd6f 100644 --- a/src/config/sessions/session-accessor.conformance.test.ts +++ b/src/config/sessions/session-accessor.conformance.test.ts @@ -442,8 +442,8 @@ describe.each([publicAccessorAdapter, sqliteAdapter])( nowMs, }), ).resolves.toEqual({ - // Only the removed entry's transcript is archived: the orphan's route - // row still targets it, and route-referenced history is retained. + // Only the removed entry's transcript is archived: the orphan's node + // still targets it, and node-referenced history is retained. removedEntries: 2, archivedTranscriptArtifacts: 1, }); @@ -476,19 +476,19 @@ describe.each([publicAccessorAdapter, sqliteAdapter])( const removedRoute = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_routes") - .select("session_id") + .selectFrom("session_nodes") + .select("current_session_id") .where("session_key", "=", "agent:main:lifecycle-cleanup-removed"), ); expect(removedRoute).toBeUndefined(); const freshRoute = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_routes") - .select("session_id") + .selectFrom("session_nodes") + .select("current_session_id") .where("session_key", "=", "agent:main:lifecycle-cleanup-fresh"), ); - expect(freshRoute).toEqual({ session_id: "fresh-lifecycle" }); + expect(freshRoute).toEqual({ current_session_id: "fresh-lifecycle" }); await expect( adapter.loadTranscriptEvents(scopedTranscript("agent:main:regular", "referenced")), ).resolves.not.toEqual([]); @@ -1148,7 +1148,7 @@ describe("sqlite session normalization", () => { fs.rmSync(paths.tempDir, { recursive: true, force: true }); }); - it("maintains normalized session root and route rows", async () => { + it("maintains normalized session node and window rows", async () => { const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; await upsertSqliteSessionEntry( { @@ -1190,7 +1190,7 @@ describe("sqlite session normalization", () => { const session = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select([ "account_id", "agent_harness_id", @@ -1233,12 +1233,12 @@ describe("sqlite session normalization", () => { const route = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_routes") - .select(["session_id", "updated_at"]) + .selectFrom("session_nodes") + .select(["current_session_id", "updated_at"]) .where("session_key", "=", "agent:main:group:example"), ); expect(route).toEqual({ - session_id: "normalized-session", + current_session_id: "normalized-session", updated_at: expect.any(Number), }); }); @@ -1401,7 +1401,7 @@ describe("sqlite session normalization", () => { expect(result.decision?.parentTokens).toBeGreaterThan(100_000); }); - it("does not move current routes back to stale transcript session ids", async () => { + it("does not move current nodes back to stale transcript session ids", async () => { const env = { ...process.env, OPENCLAW_STATE_DIR: paths.stateDir }; const scope = { agentId: "main", @@ -1434,11 +1434,11 @@ describe("sqlite session normalization", () => { const route = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_routes") - .select("session_id") + .selectFrom("session_nodes") + .select("current_session_id") .where("session_key", "=", "agent:main:main"), ); - expect(route).toEqual({ session_id: "current-session" }); + expect(route).toEqual({ current_session_id: "current-session" }); }); it("applies SQLite session-entry maintenance inside entry write transactions", async () => { @@ -1887,17 +1887,15 @@ describe("sqlite session normalization", () => { const row = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("sessions as s") - .innerJoin("session_entries as se", "se.session_id", "s.session_id") - .innerJoin("session_routes as sr", "sr.session_key", "se.session_key") + .selectFrom("session_windows as sw") + .innerJoin("session_nodes as sn", "sn.current_session_id", "sw.session_id") .select([ - "s.created_at as root_created_at", - "s.updated_at as root_updated_at", - "se.entry_json", - "se.updated_at as entry_updated_at", - "sr.updated_at as route_updated_at", + "sw.created_at as window_created_at", + "sw.updated_at as window_updated_at", + "sn.entry_json", + "sn.updated_at as node_updated_at", ]) - .where("s.session_id", "=", "minimal-session"), + .where("sw.session_id", "=", "minimal-session"), ); expect(row).toEqual({ entry_json: JSON.stringify({ @@ -1905,10 +1903,9 @@ describe("sqlite session normalization", () => { sessionStartedAt: 123, updatedAt: 123, }), - entry_updated_at: 123, - root_created_at: 123, - root_updated_at: 123, - route_updated_at: 123, + node_updated_at: 123, + window_created_at: 123, + window_updated_at: 123, }); await upsertSqliteSessionEntry( @@ -1925,7 +1922,7 @@ describe("sqlite session normalization", () => { const upsertRow = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_entries") + .selectFrom("session_nodes") .select(["entry_json", "updated_at"]) .where("session_key", "=", "agent:main:minimal-upsert"), ); diff --git a/src/config/sessions/session-accessor.lifecycle.ts b/src/config/sessions/session-accessor.lifecycle.ts index d13b31e3ab8a..720b0b79a42e 100644 --- a/src/config/sessions/session-accessor.lifecycle.ts +++ b/src/config/sessions/session-accessor.lifecycle.ts @@ -166,7 +166,7 @@ async function applySessionCompactionCheckpointMutation( /** * Forks checkpoint transcript content and persists a new branch entry in one * storage-sized mutation. SQLite adapters implement the transcript row copy - * and `session_entries.entry_json` insert inside the same write transaction. + * and `session_nodes.entry_json` insert inside the same write transaction. */ export async function branchSessionFromCompactionCheckpoint( params: BranchSessionFromCompactionCheckpointParams, @@ -184,7 +184,7 @@ export async function branchSessionFromCompactionCheckpoint( /** * Forks checkpoint transcript content and replaces the current entry in one * storage-sized mutation. SQLite adapters implement the transcript row copy - * and `session_entries.entry_json` update inside the same write transaction. + * and `session_nodes.entry_json` update inside the same write transaction. */ export async function restoreSessionFromCompactionCheckpoint( params: RestoreSessionFromCompactionCheckpointParams, diff --git a/src/config/sessions/session-accessor.sqlite-active-events.ts b/src/config/sessions/session-accessor.sqlite-active-events.ts index d5d319827b5c..bce0534f4d70 100644 --- a/src/config/sessions/session-accessor.sqlite-active-events.ts +++ b/src/config/sessions/session-accessor.sqlite-active-events.ts @@ -32,7 +32,7 @@ import { startSessionTranscriptIndexReconcile } from "./session-transcript-recon type ActiveTranscriptDatabase = Pick< OpenClawAgentKyselyDatabase, | "session_transcript_active_events" - | "session_transcript_generations" + | "transcript_rewrite_watermarks" | "session_transcript_index_state" | "transcript_event_identities" | "transcript_events" @@ -293,7 +293,7 @@ export function readSessionTranscriptVisibleMessageDelta( const generation = executeSqliteQueryTakeFirstSync( projection.database.db, db - .selectFrom("session_transcript_generations") + .selectFrom("transcript_rewrite_watermarks") .select("generation") .where("session_id", "=", projection.resolved.sessionId), )?.generation; diff --git a/src/config/sessions/session-accessor.sqlite-cleanup-race.test.ts b/src/config/sessions/session-accessor.sqlite-cleanup-race.test.ts new file mode 100644 index 000000000000..a9bf3901a0bb --- /dev/null +++ b/src/config/sessions/session-accessor.sqlite-cleanup-race.test.ts @@ -0,0 +1,217 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../../state/openclaw-agent-db.js"; +import { + applySessionEntryLifecycleMutation, + cleanupSessionLifecycleArtifacts, + deleteSessionEntryLifecycle, + loadSessionEntry, + loadTranscriptEvents, + replaceSessionEntry, +} from "./session-accessor.js"; +import { planSqliteSessionLifecycleArtifactCleanup } from "./session-accessor.sqlite-lifecycle-state.js"; +import { replaceSqliteTranscriptEvents } from "./session-accessor.sqlite.js"; +import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; +import type { SessionEntry } from "./types.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("SQLite lifecycle cleanup races", () => { + let tempDir: string; + let storePath: string; + + beforeEach(() => { + tempDir = tempDirs.make("openclaw-session-cleanup-race-"); + storePath = path.join(tempDir, "agents", "main", "sessions", "sessions.json"); + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + }); + + it("revalidates entries before deleting their transcript state", async () => { + const sessionKey = "agent:main:cleanup-race"; + const sessionId = "cleanup-race-session"; + const now = Date.now(); + const event = { + type: "session", + id: sessionId, + content: "cleanup-race-marker transcript", + } as const; + await replaceSessionEntry({ sessionKey, storePath }, { sessionId, updatedAt: now }); + await replaceSqliteTranscriptEvents({ sessionKey, sessionId, storePath }, [event]); + const databasePath = resolveSqliteTargetFromSessionStorePath(storePath, { + agentId: "main", + }).path; + if (!databasePath) { + throw new Error("expected cleanup-race database path"); + } + const database = openOpenClawAgentDatabase({ agentId: "main", path: databasePath }); + const cleanupNow = Date.now() + 60_000; + const planned = planSqliteSessionLifecycleArtifactCleanup(database, { + archiveRemovedEntryTranscripts: true, + archiveDirectory: path.dirname(storePath), + sessionKeySegmentPrefix: "cleanup-race", + transcriptContentMarker: "cleanup-race-marker", + orphanTranscriptMinAgeMs: 0, + nowMs: cleanupNow, + }); + expect(planned.entries).toHaveLength(1); + expect(planned.deletePlans).toHaveLength(1); + + const refreshedEntry = { label: "refreshed", sessionId, updatedAt: now + 1 }; + const originalRenameSync = fs.renameSync; + let refreshed = false; + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((...args) => { + const result = originalRenameSync(...args); + if (!refreshed && String(args[1]).includes(`${sessionId}.jsonl.deleted.`)) { + refreshed = true; + database.db + .prepare("UPDATE session_nodes SET entry_json = ?, updated_at = ? WHERE session_key = ?") + .run(JSON.stringify(refreshedEntry), refreshedEntry.updatedAt, sessionKey); + } + return result; + }); + + try { + await expect( + cleanupSessionLifecycleArtifacts({ + storePath, + sessionKeySegmentPrefix: "cleanup-race", + transcriptContentMarker: "cleanup-race-marker", + orphanTranscriptMinAgeMs: 0, + nowMs: cleanupNow, + }), + ).rejects.toThrow("SQLite lifecycle cleanup entry changed"); + } finally { + renameSpy.mockRestore(); + } + + expect(refreshed).toBe(true); + expect(loadSessionEntry({ sessionKey, storePath })).toEqual(refreshedEntry); + await expect(loadTranscriptEvents({ sessionKey, sessionId, storePath })).resolves.toEqual([ + event, + ]); + }); + + it("retains unplanned historical windows behind a placeholder node", async () => { + const sessionKey = "agent:main:unplanned-history"; + const currentEntry: SessionEntry = { + sessionId: "current-planned-session", + updatedAt: Date.now(), + }; + const currentEvent = { + type: "session", + id: "current-planned-session", + content: "planned current transcript", + } as const; + const historicalEvent = { + type: "session", + id: "unplanned-historical-session", + content: "retained historical transcript", + } as const; + await replaceSessionEntry({ sessionKey, storePath }, currentEntry); + await replaceSqliteTranscriptEvents( + { sessionKey, sessionId: "current-planned-session", storePath }, + [currentEvent], + ); + await replaceSqliteTranscriptEvents( + { sessionKey, sessionId: "unplanned-historical-session", storePath }, + [historicalEvent], + ); + + const result = await applySessionEntryLifecycleMutation({ + storePath, + removals: [ + { + sessionKey, + expectedEntry: currentEntry, + archiveRemovedTranscript: false, + }, + ], + maintenanceOverride: { mode: "enforce" }, + }); + + expect(result.removedSessionKeys).toEqual([sessionKey]); + expect(loadSessionEntry({ sessionKey, storePath })).toBeUndefined(); + await expect( + loadTranscriptEvents({ sessionKey, sessionId: "current-planned-session", storePath }), + ).resolves.toEqual([]); + await expect( + loadTranscriptEvents({ + sessionKey, + sessionId: "unplanned-historical-session", + storePath, + }), + ).resolves.toEqual([historicalEvent]); + + const databasePath = resolveSqliteTargetFromSessionStorePath(storePath, { + agentId: "main", + }).path; + if (!databasePath) { + throw new Error("expected retention database path"); + } + const database = openOpenClawAgentDatabase({ agentId: "main", path: databasePath }); + expect( + database.db + .prepare("SELECT current_session_id, entry_json FROM session_nodes WHERE session_key = ?") + .get(sessionKey), + ).toEqual({ current_session_id: "unplanned-historical-session", entry_json: "{}" }); + }); + + it("rehomes a window retained through a surviving previousSessionId reference", async () => { + const retainedSessionId = "retained-previous-session"; + const survivorKey = "agent:main:window-survivor"; + const now = Date.now(); + const retainedEvent = { + type: "session", + id: retainedSessionId, + content: "retained previous transcript", + } as const; + await replaceSessionEntry( + { sessionKey: "agent:main:window-owner", storePath }, + { sessionId: retainedSessionId, updatedAt: now }, + ); + await replaceSqliteTranscriptEvents( + { sessionKey: "agent:main:window-owner", sessionId: retainedSessionId, storePath }, + [retainedEvent], + ); + await replaceSessionEntry( + { sessionKey: survivorKey, storePath }, + { + previousSessionId: retainedSessionId, + sessionId: "current-survivor-session", + updatedAt: now + 1, + }, + ); + + const deleted = await deleteSessionEntryLifecycle({ + archiveTranscript: true, + storePath, + target: { + canonicalKey: "agent:main:window-owner", + storeKeys: ["agent:main:window-owner"], + }, + }); + + expect(deleted.deleted).toBe(true); + expect(deleted.archivedTranscripts).toEqual([]); + await expect( + loadTranscriptEvents({ sessionKey: survivorKey, sessionId: retainedSessionId, storePath }), + ).resolves.toEqual([retainedEvent]); + const databasePath = resolveSqliteTargetFromSessionStorePath(storePath, { + agentId: "main", + }).path; + const database = openOpenClawAgentDatabase({ agentId: "main", path: databasePath }); + expect( + database.db + .prepare("SELECT session_key FROM session_windows WHERE session_id = ?") + .get(retainedSessionId), + ).toEqual({ session_key: survivorKey }); + }); +}); diff --git a/src/config/sessions/session-accessor.sqlite-delta.ts b/src/config/sessions/session-accessor.sqlite-delta.ts index ed5f06e85237..b54d7c52a434 100644 --- a/src/config/sessions/session-accessor.sqlite-delta.ts +++ b/src/config/sessions/session-accessor.sqlite-delta.ts @@ -128,7 +128,7 @@ function readRawDeltaInTransaction( const state = executeSqliteQueryTakeFirstSync( database, db - .selectFrom("session_transcript_generations") + .selectFrom("transcript_rewrite_watermarks") .select("generation") .where("session_id", "=", scope.sessionId), ); diff --git a/src/config/sessions/session-accessor.sqlite-entry-store.ts b/src/config/sessions/session-accessor.sqlite-entry-store.ts index a91b48d89478..51ec39b6d896 100644 --- a/src/config/sessions/session-accessor.sqlite-entry-store.ts +++ b/src/config/sessions/session-accessor.sqlite-entry-store.ts @@ -3,7 +3,6 @@ import type { Selectable } from "kysely"; import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, } from "../../infra/kysely-sync.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; @@ -12,25 +11,25 @@ import { prepareSessionConversation, upsertConversationIdentity, } from "./session-accessor.sqlite-conversation.js"; -import { normalizeSqliteNumber } from "./session-accessor.sqlite-normalize.js"; +import { + clearSessionMembersForKey, + deleteSessionNodeArtifacts, + rehomeLegacySessionNodeArtifacts, +} from "./session-accessor.sqlite-node-artifacts.js"; import { resolveSessionEntryProvenanceRow } from "./session-accessor.sqlite-provenance.js"; +import { collectSqliteSessionStateIdsForEntry } from "./session-accessor.sqlite-references.js"; import { cloneSessionEntry, getSessionKysely, normalizeSqliteSessionKey, } from "./session-accessor.sqlite-scope.js"; import { + bindSqliteSessionNode, bindSqliteSessionRoot, normalizeSqliteSessionEntryTimestamp, } from "./session-accessor.sqlite-session-row.js"; -import { - normalizeSqliteStatus, - parseSqliteSessionEntryJson as parseSessionEntryRow, -} from "./session-accessor.sqlite-status.js"; -import { - readTranscriptMutationStateInTransaction, - writeSessionRoute, -} from "./session-accessor.sqlite-transcript-state.js"; +import { parseSqliteSessionEntryJson as parseSessionEntryRow } from "./session-accessor.sqlite-status.js"; +import { readTranscriptMutationStateInTransaction } from "./session-accessor.sqlite-transcript-state.js"; import { foldedSessionKeyAliasCandidates, normalizeStoreSessionKey, @@ -38,11 +37,11 @@ import { } from "./store-entry.js"; import type { SessionEntry } from "./types.js"; -// Canonical owner for session_entries row selection, alias snapshots, and writes. +// Canonical owner for session_nodes row selection, alias snapshots, and writes. type OpenClawAgentDatabaseReader = Pick; -type SessionEntryRow = Selectable; +type SessionEntryRow = Selectable; export type ResolvedSessionEntryRow = { entry: SessionEntry; legacyKeys: string[]; @@ -96,7 +95,7 @@ export function readSessionEntryRow( const rows = executeSqliteQuerySync( database.db, db - .selectFrom("session_entries") + .selectFrom("session_nodes") .selectAll() .where("session_key", "in", lookupKeys) .orderBy("session_key", "asc"), @@ -176,7 +175,7 @@ export function collectSessionEntryLookupKeys( const db = getSessionKysely(database.db); const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select("session_key").orderBy("session_key", "asc"), + db.selectFrom("session_nodes").select("session_key").orderBy("session_key", "asc"), ).rows; for (const row of rows) { if (normalizeStoreSessionKey(row.session_key) === normalizedKey) { @@ -193,7 +192,7 @@ export function readExactSessionEntryRow( const db = getSessionKysely(database.db); const row = executeSqliteQueryTakeFirstSync( database.db, - db.selectFrom("session_entries").selectAll().where("session_key", "=", sessionKey), + db.selectFrom("session_nodes").selectAll().where("session_key", "=", sessionKey), ); if (!row) { return undefined; @@ -208,7 +207,7 @@ export function readSqliteSessionEntryStore( const db = getSessionKysely(database.db); const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select(["session_key", "entry_json"]).orderBy("session_key"), + db.selectFrom("session_nodes").select(["session_key", "entry_json"]).orderBy("session_key"), ).rows; const store: Record = {}; for (const row of rows) { @@ -222,12 +221,11 @@ export function readSqliteSessionEntryStore( export function readSqliteSessionEntryCount(database: OpenClawAgentDatabase): number { const db = getSessionKysely(database.db); - const row = executeSqliteQueryTakeFirstSync( + const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select((eb) => eb.fn.countAll().as("entry_count")), - ); - const count = row?.entry_count; - return count === undefined || count === null ? 0 : normalizeSqliteNumber(count); + db.selectFrom("session_nodes").select("entry_json"), + ).rows; + return rows.reduce((count, row) => count + (parseSessionEntryRow(row) ? 1 : 0), 0); } /** Lists persisted session keys without materializing their entry payloads. */ @@ -235,8 +233,11 @@ export function readSqliteSessionEntryKeys(database: OpenClawAgentDatabaseReader const db = getSessionKysely(database.db); return executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select("session_key").orderBy("session_key", "asc"), - ).rows.map((row) => row.session_key); + db + .selectFrom("session_nodes") + .select(["entry_json", "session_key"]) + .orderBy("session_key", "asc"), + ).rows.flatMap((row) => (parseSessionEntryRow(row) ? [row.session_key] : [])); } export function resolveSqliteLifecyclePrimaryEntry( @@ -302,13 +303,99 @@ export function deleteSqliteSessionEntryRows( sessionKey: string, ): void { const db = getSessionKysely(database.db); - executeSqliteQuerySync( + const windows = executeSqliteQuerySync( database.db, - db.deleteFrom("session_routes").where("session_key", "=", sessionKey), + db.selectFrom("session_windows").select("session_id").where("session_key", "=", sessionKey), + ).rows; + const survivingNodes = executeSqliteQuerySync( + database.db, + db + .selectFrom("session_nodes") + .select(["current_session_id", "entry_json", "session_key"]) + .where("session_key", "!=", sessionKey) + .orderBy("session_key", "asc"), + ).rows; + for (const window of windows) { + const survivingNode = survivingNodes.find((node) => { + if (node.current_session_id === window.session_id) { + return true; + } + const entry = parseSessionEntryRow(node); + return entry + ? collectSqliteSessionStateIdsForEntry(entry).includes(window.session_id) + : false; + }); + if (survivingNode) { + executeSqliteQuerySync( + database.db, + db + .updateTable("session_windows") + .set({ session_key: survivingNode.session_key }) + .where("session_id", "=", window.session_id), + ); + } + } + const remainingWindow = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_windows") + .select(["session_id", "updated_at"]) + .where("session_key", "=", sessionKey) + .orderBy("updated_at", "desc") + .orderBy("session_id", "asc") + .limit(1), ); + if (remainingWindow) { + deleteSessionNodeArtifacts(database, sessionKey); + clearSqliteSessionEntryPreservingWindows(database, { + sessionId: remainingWindow.session_id, + sessionKey, + updatedAt: remainingWindow.updated_at, + }); + return; + } executeSqliteQuerySync( database.db, - db.deleteFrom("session_entries").where("session_key", "=", sessionKey), + db.deleteFrom("session_nodes").where("session_key", "=", sessionKey), + ); +} + +/** Remove the logical entry while retaining its node-owned transcript windows. */ +function clearSqliteSessionEntryPreservingWindows( + database: OpenClawAgentDatabase, + params: { sessionId: string; sessionKey: string; updatedAt: number }, +): void { + const db = getSessionKysely(database.db); + const cleared = { + current_session_id: params.sessionId, + entry_json: "{}", + updated_at: params.updatedAt, + status: null, + created_at: null, + created_via: null, + created_actor_type: null, + created_actor_id: null, + parent_session_key: null, + spawned_by: null, + fork_source_session_key: null, + fork_source_session_id: null, + fork_source_entry_id: null, + label: null, + display_name: null, + category: null, + icon: null, + pinned_at: null, + archived_at: null, + last_read_at: null, + last_interaction_at: null, + last_activity_at: null, + } as const; + executeSqliteQuerySync( + database.db, + db + .insertInto("session_nodes") + .values({ session_key: params.sessionKey, ...cleared }) + .onConflict((conflict) => conflict.column("session_key").doUpdateSet(cleared)), ); } @@ -376,6 +463,7 @@ export function deleteLegacySessionEntryRows( database: OpenClawAgentDatabase, legacyKeys: string[], sessionKey: string, + options: { rehomeMembers?: boolean } = {}, ): void { if (legacyKeys.length === 0) { return; @@ -385,32 +473,34 @@ export function deleteLegacySessionEntryRows( if (legacyKey === sessionKey) { continue; } + rehomeSqliteSessionWindows(database, sessionKey, [legacyKey]); + rehomeLegacySessionNodeArtifacts(database, legacyKey, sessionKey, options); executeSqliteQuerySync( database.db, - db.deleteFrom("session_routes").where("session_key", "=", legacyKey), - ); - executeSqliteQuerySync( - database.db, - db.deleteFrom("session_entries").where("session_key", "=", legacyKey), + db.deleteFrom("session_nodes").where("session_key", "=", legacyKey), ); } } -// session_members is an additive sharing surface with a lazy ensure, so a DB -// that predates the feature may lack the table; such a DB also has no members -// to drop. Guard on existence rather than forcing the sharing schema here. -function clearSessionMembersForKey(database: OpenClawAgentDatabase, sessionKey: string): void { - const tableExists = - database.db /* sqlite-allow-raw: sqlite_master table-existence probe for the additive session_members lazy-ensure */ - .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'session_members'") - .get(); - if (!tableExists) { +/** Move retained generations to the canonical node before removing key aliases. */ +export function rehomeSqliteSessionWindows( + database: OpenClawAgentDatabase, + canonicalKey: string, + previousKeys: Iterable, +): void { + const legacyKeys = uniqueStrings([...previousKeys].map((key) => key.trim())).filter( + (key) => key && key !== canonicalKey, + ); + if (legacyKeys.length === 0) { return; } - const db = getNodeSqliteKysely>(database.db); + const db = getSessionKysely(database.db); executeSqliteQuerySync( database.db, - db.deleteFrom("session_members").where("session_key", "=", sessionKey), + db + .updateTable("session_windows") + .set({ session_key: canonicalKey }) + .where("session_key", "in", legacyKeys), ); } @@ -418,11 +508,25 @@ export function writeSessionEntry( database: OpenClawAgentDatabase, sessionKey: string, entry: SessionEntry, + options: { previousEntry?: SessionEntry | null } = {}, ): void { const db = getSessionKysely(database.db); const normalizedEntry = normalizeSqliteSessionEntryTimestamp(entry); const updatedAt = normalizedEntry.updatedAt; - const previousEntry = readExactSessionEntryRow(database, sessionKey)?.entry; + const canonicalPreviousEntry = readExactSessionEntryRow(database, sessionKey)?.entry; + const previousEntry = + options.previousEntry === undefined + ? canonicalPreviousEntry + : (options.previousEntry ?? undefined); + // The lifecycle-selected entry owns visibility copy-forward semantics. + if (previousEntry && previousEntry.sessionId !== normalizedEntry.sessionId) { + delete normalizedEntry.visibility; + } + // Membership belongs to the exact canonical row being overwritten, which + // can differ from the selected alias during canonicalization. + if (canonicalPreviousEntry && canonicalPreviousEntry.sessionId !== normalizedEntry.sessionId) { + clearSessionMembersForKey(database, sessionKey); + } // Registry writes snapshot the current transcript watermark so recovery can // distinguish same-millisecond transcript writes before and after this row. const transcriptObservedAt = @@ -452,14 +556,53 @@ export function writeSessionEntry( entry: normalizedEntry, previousEntry, }); + const sessionNode = bindSqliteSessionNode({ + entry: normalizedEntry, + sessionKey, + updatedAt, + }); executeSqliteQuerySync( database.db, db - .insertInto("sessions") + .insertInto("session_nodes") + .values(sessionNode) + .onConflict((conflict) => + conflict.column("session_key").doUpdateSet({ + current_session_id: sessionNode.current_session_id, + entry_json: sessionNode.entry_json, + updated_at: sessionNode.updated_at, + status: sessionNode.status, + created_at: sessionNode.created_at, + created_via: sessionNode.created_via, + created_actor_type: sessionNode.created_actor_type, + created_actor_id: sessionNode.created_actor_id, + parent_session_key: sessionNode.parent_session_key, + spawned_by: sessionNode.spawned_by, + fork_source_session_key: sessionNode.fork_source_session_key, + fork_source_session_id: sessionNode.fork_source_session_id, + fork_source_entry_id: sessionNode.fork_source_entry_id, + label: sessionNode.label, + display_name: sessionNode.display_name, + category: sessionNode.category, + icon: sessionNode.icon, + pinned_at: sessionNode.pinned_at, + archived_at: sessionNode.archived_at, + last_read_at: sessionNode.last_read_at, + last_interaction_at: sessionNode.last_interaction_at, + last_activity_at: sessionNode.last_activity_at, + }), + ), + ); + executeSqliteQuerySync( + database.db, + db + .insertInto("session_windows") .values(sessionRow) .onConflict((conflict) => conflict.column("session_id").doUpdateSet({ session_key: sessionKey, + previous_session_id: sessionRow.previous_session_id, + reason: sessionRow.reason, session_scope: sessionRow.session_scope, transcript_observed_at: transcriptObservedAt, session_entry_provenance: sessionRow.session_entry_provenance, @@ -491,43 +634,6 @@ export function writeSessionEntry( updatedAt, }); } - writeSessionRoute(database, { - sessionId: sessionRow.session_id, - sessionKey, - updatedAt, - }); - // A canonical key can be reused by a fresh session instance (reset/recreate); - // the entry row is updated in place so the members FK cascade never fires. - // A fresh instance always starts shared with no members, so a changed - // sessionId must reset both — otherwise a copied-forward `visibility` could - // leave the replacement hidden/restricted, and prior members would retain the - // `member` role on a session owned by someone else. - if (previousEntry && previousEntry.sessionId !== normalizedEntry.sessionId) { - // Absent visibility reads as shared; dropping the copied-forward value keeps - // the replacement entry minimal rather than stamping an explicit default. - delete normalizedEntry.visibility; - clearSessionMembersForKey(database, sessionKey); - } - executeSqliteQuerySync( - database.db, - db - .insertInto("session_entries") - .values({ - session_key: sessionKey, - session_id: normalizedEntry.sessionId, - entry_json: JSON.stringify(normalizedEntry), - updated_at: updatedAt, - status: normalizeSqliteStatus(normalizedEntry.status), - }) - .onConflict((conflict) => - conflict.column("session_key").doUpdateSet({ - session_id: normalizedEntry.sessionId, - entry_json: JSON.stringify(normalizedEntry), - updated_at: updatedAt, - status: normalizeSqliteStatus(normalizedEntry.status), - }), - ), - ); } /** Resolves the parent fork decision using SQLite transcript rows when totals are stale. */ diff --git a/src/config/sessions/session-accessor.sqlite-entry.ts b/src/config/sessions/session-accessor.sqlite-entry.ts index 0343e6dfce55..35de584d8a91 100644 --- a/src/config/sessions/session-accessor.sqlite-entry.ts +++ b/src/config/sessions/session-accessor.sqlite-entry.ts @@ -30,12 +30,12 @@ import { collectSessionEntryLookupKeys, createSqliteSessionIdentitySnapshot, deleteLegacySessionEntryRows, - deleteSqliteLifecycleTargetRows, readExactSessionEntryRow, readSessionEntryRow, readSqliteLifecycleTargetSnapshot, readSqliteSessionEntrySelectionSnapshot, readSqliteSessionIdentitySnapshot, + rehomeSqliteSessionWindows, writeSessionEntry, } from "./session-accessor.sqlite-entry-store.js"; import { listSqliteTranscriptInstancesFromDatabase } from "./session-accessor.sqlite-history.js"; @@ -119,7 +119,7 @@ export function resolveSqliteSessionKeyBySessionId( const row = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select("session_key") .where("session_id", "=", resolved.sessionId) .limit(1), @@ -157,8 +157,8 @@ function listSqliteSessionEntriesFromDatabase(database: { db: DatabaseSync }) { const rows = executeSqliteQuerySync( database.db, db - .selectFrom("session_entries") - .select(["session_key", "entry_json", "session_id", "updated_at"]) + .selectFrom("session_nodes") + .select(["session_key", "entry_json", "current_session_id", "updated_at"]) .orderBy("session_key", "asc"), ).rows; return rows @@ -305,11 +305,15 @@ export async function patchSqliteSessionEntry( previous: writeBase, sessionKey: resolved.sessionKey, }); - writeSessionEntry(writeDatabase, resolved.sessionKey, next); + const selectedPreviousEntry = fresh.selected?.entry ?? writeBase; + writeSessionEntry(writeDatabase, resolved.sessionKey, next, { + previousEntry: selectedPreviousEntry, + }); deleteLegacySessionEntryRows( writeDatabase, fresh.selected?.legacyKeys ?? [], resolved.sessionKey, + { rehomeMembers: selectedPreviousEntry.sessionId === next.sessionId }, ); maintenancePlans.push( applySqliteSessionEntryMaintenance(writeDatabase, { @@ -384,8 +388,17 @@ export async function patchSqliteSessionEntryTarget( previous: writeBase, sessionKey: scope.target.canonicalKey, }); - deleteSqliteLifecycleTargetRows(writeDatabase, scope.target); - writeSessionEntry(writeDatabase, scope.target.canonicalKey, next); + const selectedPreviousEntry = fresh.primary?.entry ?? writeBase; + writeSessionEntry(writeDatabase, scope.target.canonicalKey, next, { + previousEntry: selectedPreviousEntry, + }); + rehomeSqliteSessionWindows(writeDatabase, scope.target.canonicalKey, scope.target.storeKeys); + deleteLegacySessionEntryRows( + writeDatabase, + scope.target.storeKeys, + scope.target.canonicalKey, + { rehomeMembers: selectedPreviousEntry.sessionId === next.sessionId }, + ); maintenancePlans.push( applySqliteSessionEntryMaintenance(writeDatabase, { activeSessionKey: scope.target.canonicalKey, diff --git a/src/config/sessions/session-accessor.sqlite-history.ts b/src/config/sessions/session-accessor.sqlite-history.ts index 5c7e586c8453..0e8c4be6ec39 100644 --- a/src/config/sessions/session-accessor.sqlite-history.ts +++ b/src/config/sessions/session-accessor.sqlite-history.ts @@ -16,7 +16,7 @@ export function listSqliteTranscriptInstancesFromDatabase(params: { const rows = executeSqliteQuerySync( params.database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select([ "session_id", "session_key", diff --git a/src/config/sessions/session-accessor.sqlite-lifecycle-state.ts b/src/config/sessions/session-accessor.sqlite-lifecycle-state.ts index b3c67a2d4f1b..a3ab2479cdee 100644 --- a/src/config/sessions/session-accessor.sqlite-lifecycle-state.ts +++ b/src/config/sessions/session-accessor.sqlite-lifecycle-state.ts @@ -26,6 +26,7 @@ import type { } from "./session-accessor.sqlite-lifecycle-types.js"; import { normalizeSqliteNumber } from "./session-accessor.sqlite-normalize.js"; import { loadSqliteTranscriptEventsFromDatabase } from "./session-accessor.sqlite-read.js"; +import { collectSqliteSessionStateIdsForEntry } from "./session-accessor.sqlite-references.js"; import { cloneSessionEntry, getSessionKysely } from "./session-accessor.sqlite-scope.js"; import { parseSqliteSessionEntryJson as parseSessionEntryRow } from "./session-accessor.sqlite-status.js"; import { buildSessionResetBoundaryPlan } from "./session-reset-boundary-event.js"; @@ -118,16 +119,22 @@ function sqliteTranscriptStateHasMarker(params: { return rows.some((row) => row.event_json.includes(params.transcriptContentMarker)); } -/** Session ids protected by live entry state or durable route targets. */ -export function readReferencedSqliteSessionIds(database: OpenClawAgentDatabase): Set { +/** Session ids protected by live node state. */ +export function readReferencedSqliteSessionIds( + database: OpenClawAgentDatabase, + excludedSessionKeys: ReadonlySet = new Set(), +): Set { const db = getSessionKysely(database.db); const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select(["entry_json", "session_id"]), + db.selectFrom("session_nodes").select(["entry_json", "current_session_id", "session_key"]), ).rows; const sessionIds = new Set(); for (const row of rows) { - sessionIds.add(row.session_id); + if (excludedSessionKeys.has(row.session_key)) { + continue; + } + sessionIds.add(row.current_session_id); const entry = parseSessionEntryRow(row); if (!entry) { continue; @@ -136,13 +143,6 @@ export function readReferencedSqliteSessionIds(database: OpenClawAgentDatabase): sessionIds.add(sessionId); } } - const routeRows = executeSqliteQuerySync( - database.db, - db.selectFrom("session_routes").select("session_id"), - ).rows; - for (const row of routeRows) { - sessionIds.add(row.session_id); - } return sessionIds; } @@ -159,14 +159,14 @@ export function readReferencedSqliteSessionIdsAfterTargetMutation( const db = getSessionKysely(database.db); const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select(["entry_json", "session_key", "session_id"]), + db.selectFrom("session_nodes").select(["entry_json", "session_key", "current_session_id"]), ).rows; const sessionIds = new Set(); for (const row of rows) { if (removedKeys.has(row.session_key)) { continue; } - sessionIds.add(row.session_id); + sessionIds.add(row.current_session_id); const entry = parseSessionEntryRow(row); if (!entry) { continue; @@ -175,15 +175,6 @@ export function readReferencedSqliteSessionIdsAfterTargetMutation( sessionIds.add(sessionId); } } - const routeRows = executeSqliteQuerySync( - database.db, - db.selectFrom("session_routes").select(["session_id", "session_key"]), - ).rows; - for (const row of routeRows) { - if (!removedKeys.has(row.session_key)) { - sessionIds.add(row.session_id); - } - } if (nextEntry) { for (const sessionId of collectSqliteSessionStateIdsForEntry(nextEntry)) { sessionIds.add(sessionId); @@ -234,9 +225,10 @@ export function deleteMaterializedSqliteSessionStatePlans( database: OpenClawAgentDatabase, plans: readonly MaterializedSqliteSessionStateDeletePlan[], protectedSessionIds?: ReadonlySet, + excludedSessionKeys?: ReadonlySet, ): SessionLifecycleArchivedTranscript[] { const archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; - const referencedSessionIds = readReferencedSqliteSessionIds(database); + const referencedSessionIds = readReferencedSqliteSessionIds(database, excludedSessionKeys); for (const sessionId of protectedSessionIds ?? []) { referencedSessionIds.add(sessionId); } @@ -301,7 +293,7 @@ export function readSqliteSessionGenerationIdsForKeys( const db = getSessionKysely(database.db); return executeSqliteQuerySync( database.db, - db.selectFrom("sessions").select("session_id").where("session_key", "in", sessionKeys), + db.selectFrom("session_windows").select("session_id").where("session_key", "in", sessionKeys), ).rows.map((row) => row.session_id); } @@ -317,6 +309,7 @@ export async function projectSqliteSessionEntryLifecycleMutation( ): Promise { const store = readSqliteSessionEntryStore(database); const removedEntries: Array<{ archiveTranscript: boolean; entry: SessionEntry }> = []; + const removedKeysToArchive = new Set(); const changedSessionKeys = new Set(); const projectedRemovals: SqliteProjectedLifecycleMutation["removals"] = []; for (const removal of params.removals) { @@ -334,6 +327,9 @@ export async function projectSqliteSessionEntryLifecycleMutation( archiveTranscript: removal.archiveRemovedTranscript === true, entry, }); + if (removal.archiveRemovedTranscript === true) { + removedKeysToArchive.add(sessionKey); + } changedSessionKeys.add(sessionKey); delete store[sessionKey]; } @@ -393,6 +389,24 @@ export async function projectSqliteSessionEntryLifecycleMutation( referencedSessionIds, }), ); + const plannedIds = new Set(deletePlans.map((plan) => plan.sessionId)); + for (const sessionId of readSqliteSessionGenerationIdsForKeys(database, removedKeysToArchive)) { + if (plannedIds.has(sessionId)) { + continue; + } + const plan = planSqliteSessionStateDeleteIfUnreferenced({ + archiveDirectory: params.archiveDirectory, + archiveTranscript: true, + database, + reason: "deleted", + referencedSessionIds, + sessionId, + }); + if (plan) { + deletePlans.push(plan); + plannedIds.add(sessionId); + } + } return { deletePlans, removals: projectedRemovals, upsertedEntries }; } @@ -409,7 +423,7 @@ function collectReferencedSqliteSessionIdsFromStore( return sessionIds; } -// Projected deletes must preserve raw session_entries.session_id references for +// Projected deletes must preserve raw session_nodes.current_session_id references for // remaining rows whose entry_json cannot be parsed into a SessionEntry. export function collectProjectedReferencedSqliteSessionIds(params: { database: OpenClawAgentDatabase; @@ -420,14 +434,14 @@ export function collectProjectedReferencedSqliteSessionIds(params: { const db = getSessionKysely(params.database.db); const rows = executeSqliteQuerySync( params.database.db, - db.selectFrom("session_entries").select(["entry_json", "session_key", "session_id"]), + db.selectFrom("session_nodes").select(["entry_json", "session_key", "current_session_id"]), ).rows; const sessionIds = new Set(); for (const row of rows) { if (excludedSessionKeys.has(row.session_key)) { continue; } - sessionIds.add(row.session_id); + sessionIds.add(row.current_session_id); const entry = parseSessionEntryRow(row); if (!entry) { continue; @@ -439,49 +453,19 @@ export function collectProjectedReferencedSqliteSessionIds(params: { for (const sessionId of collectReferencedSqliteSessionIdsFromStore(params.projectedStore)) { sessionIds.add(sessionId); } - // Routes protect their target session unless the cleanup removes that key's - // route in the same pass; mirroring the post-cleanup state here keeps the - // plan from writing archives for sessions the delete stage will retain. - const routeRows = executeSqliteQuerySync( - params.database.db, - db.selectFrom("session_routes").select(["session_id", "session_key"]), - ).rows; - for (const row of routeRows) { - if (!excludedSessionKeys.has(row.session_key)) { - sessionIds.add(row.session_id); - } - } return sessionIds; } -export function collectSqliteSessionStateIdsForEntry(entry: SessionEntry): string[] { - const sessionIds: string[] = []; - const add = (sessionId: string | undefined) => { - const normalized = sessionId?.trim(); - if (normalized) { - sessionIds.push(normalized); - } - }; - add(entry.sessionId); - for (const sessionId of entry.usageFamilySessionIds ?? []) { - add(sessionId); - } - for (const checkpoint of entry.compactionCheckpoints ?? []) { - add(checkpoint.sessionId); - add(checkpoint.preCompaction.sessionId); - add(checkpoint.postCompaction.sessionId); - } - return uniqueStrings(sessionIds); -} +export { collectSqliteSessionStateIdsForEntry }; function deleteSqliteSessionStateRows(database: OpenClawAgentDatabase, sessionId: string): void { const db = getSessionKysely(database.db); - // The sessions row cascades canonical transcript tables, but FTS is virtual - // and its watermark has no cascade; clear both before dropping the owner row. + // The window row cascades canonical transcript tables, but FTS is virtual; + // clear its projection before dropping the owner row. deleteSessionTranscriptIndexInTransaction(database.db, sessionId); executeSqliteQuerySync( database.db, - db.deleteFrom("sessions").where("session_id", "=", sessionId), + db.deleteFrom("session_windows").where("session_id", "=", sessionId), ); } @@ -500,12 +484,12 @@ function planSqliteOrphanLifecycleTranscriptStateDeletes(params: { const db = getSessionKysely(params.database.db); const rows = executeSqliteQuerySync( params.database.db, - db.selectFrom("sessions").select("session_id").orderBy("session_id", "asc"), + db.selectFrom("session_windows").select("session_id").orderBy("session_id", "asc"), ).rows; const deletePlans: SqliteSessionStateDeletePlan[] = []; - // Orphan transcript state is represented by a sessions row without a live - // session entry. The marker keeps this scoped to the caller-owned lifecycle. + // Orphan transcript state is represented by a historical window that is no + // longer the node's current id. The marker scopes cleanup to this lifecycle. for (const row of rows) { if ( params.referencedSessionIds.has(row.session_id) || @@ -558,8 +542,8 @@ export function planSqliteSessionLifecycleArtifactCleanup( const rows = executeSqliteQuerySync( database.db, db - .selectFrom("session_entries") - .select(["entry_json", "session_key", "session_id"]) + .selectFrom("session_nodes") + .select(["entry_json", "session_key", "current_session_id"]) .orderBy("session_key", "asc"), ).rows; @@ -573,7 +557,7 @@ export function planSqliteSessionLifecycleArtifactCleanup( if ( !sqliteTranscriptStateIsReclaimable({ database, - sessionId: row.session_id, + sessionId: row.current_session_id, nowMs: params.nowMs, orphanTranscriptMinAgeMs: params.orphanTranscriptMinAgeMs, }) @@ -583,7 +567,7 @@ export function planSqliteSessionLifecycleArtifactCleanup( const entry = parseSessionEntryRow(row); for (const sessionId of entry ? collectSqliteSessionStateIdsForEntry(entry) - : [row.session_id]) { + : [row.current_session_id]) { removedSessionIds.add(sessionId); } entries.push({ @@ -630,14 +614,23 @@ export function deletePlannedSqliteLifecycleArtifactEntries( database: OpenClawAgentDatabase, entries: readonly SqliteSessionEntryRemovalPlan[], ): number { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged(database, entries); let removedEntries = 0; for (const planned of entries) { - const current = readExactSessionEntryRow(database, planned.sessionKey)?.entry; - if (!sqliteSessionEntriesEqual(current, planned.expectedEntry)) { - throw new Error(`SQLite lifecycle cleanup entry changed for ${planned.sessionKey}`); - } deleteSqliteSessionEntryRows(database, planned.sessionKey); removedEntries += 1; } return removedEntries; } + +export function assertPlannedSqliteLifecycleArtifactEntriesUnchanged( + database: OpenClawAgentDatabase, + entries: readonly SqliteSessionEntryRemovalPlan[], +): void { + for (const planned of entries) { + const current = readExactSessionEntryRow(database, planned.sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(current, planned.expectedEntry)) { + throw new Error(`SQLite lifecycle cleanup entry changed for ${planned.sessionKey}`); + } + } +} diff --git a/src/config/sessions/session-accessor.sqlite-lifecycle.ts b/src/config/sessions/session-accessor.sqlite-lifecycle.ts index 63430a543de0..3d310f150715 100644 --- a/src/config/sessions/session-accessor.sqlite-lifecycle.ts +++ b/src/config/sessions/session-accessor.sqlite-lifecycle.ts @@ -28,13 +28,16 @@ import { assertSqliteLifecycleTargetSnapshotUnchanged, assertSqliteLifecycleTargetUnchanged, deleteSqliteLifecycleTargetRows, + deleteLegacySessionEntryRows, readSqliteLifecycleTargetSnapshot, + rehomeSqliteSessionWindows, sqliteSessionEntriesEqual, writeSessionEntry, } from "./session-accessor.sqlite-entry-store.js"; import { emitArchivedSqliteTranscriptUpdates } from "./session-accessor.sqlite-events.js"; import { emitCommittedSessionEntryRemovals } from "./session-accessor.sqlite-identity.js"; import { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged, deleteMaterializedSqliteSessionStatePlans, deletePlannedSqliteLifecycleArtifactEntries, planSqliteSessionLifecycleArtifactCleanup, @@ -127,13 +130,16 @@ export async function cleanupSqliteSessionLifecycleArtifacts( let removedEntries = 0; let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; runOpenClawAgentWriteTransaction((transactionDb) => { - removedEntries = deletePlannedSqliteLifecycleArtifactEntries( - transactionDb, - cleanupPlan.entries, - ); + assertPlannedSqliteLifecycleArtifactEntriesUnchanged(transactionDb, cleanupPlan.entries); archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( transactionDb, materializedPlans, + undefined, + new Set(cleanupPlan.entries.map((entry) => entry.sessionKey)), + ); + removedEntries = deletePlannedSqliteLifecycleArtifactEntries( + transactionDb, + cleanupPlan.entries, ); }, toDatabaseOptions(resolved)); emitCommittedSessionEntryRemovals(cleanupPlan.entries); @@ -193,8 +199,20 @@ export async function resetSqliteSessionEntryLifecycle( throw new Error(`Failed to append reset boundary for ${current.key}`); } } - deleteSqliteLifecycleTargetRows(transactionDb, params.target); - writeSessionEntry(transactionDb, params.target.canonicalKey, nextEntry); + writeSessionEntry(transactionDb, params.target.canonicalKey, nextEntry, { + previousEntry: current?.entry ?? null, + }); + rehomeSqliteSessionWindows( + transactionDb, + params.target.canonicalKey, + params.target.storeKeys, + ); + deleteLegacySessionEntryRows( + transactionDb, + params.target.storeKeys, + params.target.canonicalKey, + { rehomeMembers: current?.entry.sessionId === nextEntry.sessionId }, + ); // Reset only advances the live entry and route. Historical rows stay searchable; // disk-budget cleanup owns durable extraction before reclaiming them. }, toDatabaseOptions(resolved)); @@ -414,16 +432,22 @@ async function deleteSqliteSessionEntryLifecycleLocked( if (!shouldDeleteSqliteSessionEntryLifecycle(transactionEntry, params)) { return; } + const archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + undefined, + new Set([ + params.target.canonicalKey, + ...params.target.storeKeys, + ...transactionSnapshot.rows.map((row) => row.sessionKey), + ]), + ); deleteSqliteLifecycleTargetRows(transactionDb, params.target); deleteSessionBoardRows(transactionDb, [ params.target.canonicalKey, ...params.target.storeKeys, ...transactionSnapshot.rows.map((row) => row.sessionKey), ]); - const archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( - transactionDb, - materializedPlans, - ); result = { archivedTranscripts, deleted: true, diff --git a/src/config/sessions/session-accessor.sqlite-maintenance.ts b/src/config/sessions/session-accessor.sqlite-maintenance.ts index 0ae4e1a79c41..bada9d7572f2 100644 --- a/src/config/sessions/session-accessor.sqlite-maintenance.ts +++ b/src/config/sessions/session-accessor.sqlite-maintenance.ts @@ -14,11 +14,13 @@ import type { SessionLifecycleArchivedTranscript } from "./session-accessor.sqli import { readSqliteSessionEntryCount } from "./session-accessor.sqlite-entry-store.js"; import { emitCommittedSessionEntryRemovals } from "./session-accessor.sqlite-identity.js"; import { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged, collectProjectedReferencedSqliteSessionIds, collectSqliteSessionStateIdsForEntry, deleteMaterializedSqliteSessionStatePlans, deletePlannedSqliteLifecycleArtifactEntries, planSqliteSessionStateDeleteIfUnreferenced, + readSqliteSessionGenerationIdsForKeys, } from "./session-accessor.sqlite-lifecycle-state.js"; import type { SqliteSessionEntryMaintenancePlan } from "./session-accessor.sqlite-lifecycle-types.js"; import { @@ -68,8 +70,8 @@ function hasStaleSqliteSessionEntryCandidate( const rows = executeSqliteQuerySync( database.db, db - .selectFrom("session_entries") - .select("session_key") + .selectFrom("session_nodes") + .select(["entry_json", "session_key"]) .where("updated_at", "<", cutoffMs) .where( /* kysely-allow-raw: archivedAt lives inside the canonical JSON entry, not a SQL column. */ @@ -77,7 +79,11 @@ function hasStaleSqliteSessionEntryCandidate( ) .orderBy("updated_at", "asc"), ).rows; - return rows.some((row) => !preserveKeys?.has(normalizeStoreSessionKey(row.session_key))); + return rows.some( + (row) => + parseSessionEntryRow(row) !== null && + !preserveKeys?.has(normalizeStoreSessionKey(row.session_key)), + ); } export function applySqliteSessionEntryMaintenance( @@ -126,7 +132,7 @@ export function applySqliteSessionEntryMaintenance( const db = getSessionKysely(database.db); const rows = executeSqliteQuerySync( database.db, - db.selectFrom("session_entries").select(["session_key", "entry_json"]).orderBy("session_key"), + db.selectFrom("session_nodes").select(["session_key", "entry_json"]).orderBy("session_key"), ).rows; const store: Record = {}; for (const row of rows) { @@ -187,6 +193,9 @@ export function applySqliteSessionEntryMaintenance( preserveKeys, }); } + for (const sessionId of readSqliteSessionGenerationIdsForKeys(database, removedKeys)) { + removedSessionIds.add(sessionId); + } const referencedSessionIds = collectProjectedReferencedSqliteSessionIds({ database, excludedSessionKeys: removedKeys, @@ -227,8 +236,14 @@ export function finalizeSqliteSessionEntryMaintenancePlansBestEffort( const materializedPlans = materializeSqliteSessionStateDeletePlans(stateDeletePlans); let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; runOpenClawAgentWriteTransaction((database) => { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged(database, entryRemovals); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + database, + materializedPlans, + undefined, + new Set(entryRemovals.map((removal) => removal.sessionKey)), + ); deletePlannedSqliteLifecycleArtifactEntries(database, entryRemovals); - archivedTranscripts = deleteMaterializedSqliteSessionStatePlans(database, materializedPlans); }, toDatabaseOptions(scope)); emitCommittedSessionEntryRemovals(entryRemovals); return archivedTranscripts; diff --git a/src/config/sessions/session-accessor.sqlite-node-artifacts.ts b/src/config/sessions/session-accessor.sqlite-node-artifacts.ts new file mode 100644 index 000000000000..895f7d6f3670 --- /dev/null +++ b/src/config/sessions/session-accessor.sqlite-node-artifacts.ts @@ -0,0 +1,201 @@ +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, +} from "../../infra/kysely-sync.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; +import { getSessionKysely } from "./session-accessor.sqlite-scope.js"; + +export function clearSessionMembersForKey( + database: OpenClawAgentDatabase, + sessionKey: string, +): void { + if (!readSessionNodeArtifactTables(database).has("session_members")) { + return; + } + const db = getSessionKysely(database.db); + executeSqliteQuerySync( + database.db, + db.deleteFrom("session_members").where("session_key", "=", sessionKey), + ); +} + +export function rehomeLegacySessionNodeArtifacts( + database: OpenClawAgentDatabase, + legacyKey: string, + canonicalKey: string, + options: { rehomeMembers?: boolean }, +): void { + const db = getSessionKysely(database.db); + const presentTables = readSessionNodeArtifactTables(database); + if (presentTables.has("board_tabs") && presentTables.has("board_widgets")) { + const tabs = executeSqliteQuerySync( + database.db, + db.selectFrom("board_tabs").selectAll().where("session_key", "=", legacyKey), + ).rows; + for (const tab of tabs) { + executeSqliteQuerySync( + database.db, + db + .insertInto("board_tabs") + .values({ ...tab, session_key: canonicalKey }) + .onConflict((conflict) => + conflict + .columns(["session_key", "tab_id"]) + .doUpdateSet({ + title: tab.title, + position: tab.position, + chat_dock: tab.chat_dock, + created_by: tab.created_by, + revision: tab.revision, + }) + .where("revision", "<", tab.revision), + ), + ); + } + const widgets = executeSqliteQuerySync( + database.db, + db.selectFrom("board_widgets").selectAll().where("session_key", "=", legacyKey), + ).rows; + for (const widget of widgets) { + executeSqliteQuerySync( + database.db, + db + .insertInto("board_widgets") + .values({ ...widget, session_key: canonicalKey }) + .onConflict((conflict) => + conflict + .columns(["session_key", "name"]) + .doUpdateSet({ + tab_id: widget.tab_id, + title: widget.title, + content_kind: widget.content_kind, + html: widget.html, + descriptor_json: widget.descriptor_json, + sha256: widget.sha256, + view_generation: widget.view_generation, + revision: widget.revision, + size_w: widget.size_w, + size_h: widget.size_h, + position: widget.position, + manifest: widget.manifest, + grant_state: widget.grant_state, + granted_sha: widget.granted_sha, + created_by: widget.created_by, + created_at: widget.created_at, + updated_at: widget.updated_at, + }) + .where((eb) => + eb.or([ + eb("revision", "<", widget.revision), + eb.and([ + eb("revision", "=", widget.revision), + eb("updated_at", "<", widget.updated_at), + ]), + ]), + ), + ), + ); + } + } + if (presentTables.has("heartbeat_outcomes")) { + const heartbeat = executeSqliteQueryTakeFirstSync( + database.db, + db.selectFrom("heartbeat_outcomes").selectAll().where("session_key", "=", legacyKey), + ); + if (heartbeat) { + executeSqliteQuerySync( + database.db, + db + .insertInto("heartbeat_outcomes") + .values({ ...heartbeat, session_key: canonicalKey }) + .onConflict((conflict) => + conflict + .column("session_key") + .doUpdateSet({ + run_session_key: heartbeat.run_session_key, + outcome: heartbeat.outcome, + summary: heartbeat.summary, + response_reason: heartbeat.response_reason, + priority: heartbeat.priority, + next_check: heartbeat.next_check, + task_names_json: heartbeat.task_names_json, + wake_source: heartbeat.wake_source, + wake_reason: heartbeat.wake_reason, + occurred_at: heartbeat.occurred_at, + context_run_id: heartbeat.context_run_id, + context_claimed_at: heartbeat.context_claimed_at, + updated_at: heartbeat.updated_at, + }) + .where((eb) => + eb.or([ + eb("updated_at", "<", heartbeat.updated_at), + eb.and([ + eb("updated_at", "=", heartbeat.updated_at), + eb("occurred_at", "<", heartbeat.occurred_at), + ]), + ]), + ), + ), + ); + } + } + if (options.rehomeMembers !== false && presentTables.has("session_members")) { + const members = executeSqliteQuerySync( + database.db, + db.selectFrom("session_members").selectAll().where("session_key", "=", legacyKey), + ).rows; + for (const member of members) { + executeSqliteQuerySync( + database.db, + db + .insertInto("session_members") + .values({ ...member, session_key: canonicalKey }) + .onConflict((conflict) => conflict.columns(["session_key", "identity_id"]).doNothing()), + ); + } + } +} + +export function deleteSessionNodeArtifacts( + database: OpenClawAgentDatabase, + sessionKey: string, +): void { + const db = getSessionKysely(database.db); + const presentTables = readSessionNodeArtifactTables(database); + if (presentTables.has("board_tabs") && presentTables.has("board_widgets")) { + executeSqliteQuerySync( + database.db, + db.deleteFrom("board_widgets").where("session_key", "=", sessionKey), + ); + executeSqliteQuerySync( + database.db, + db.deleteFrom("board_tabs").where("session_key", "=", sessionKey), + ); + } + if (presentTables.has("heartbeat_outcomes")) { + executeSqliteQuerySync( + database.db, + db.deleteFrom("heartbeat_outcomes").where("session_key", "=", sessionKey), + ); + } + clearSessionMembersForKey(database, sessionKey); +} + +function readSessionNodeArtifactTables(database: OpenClawAgentDatabase): Set { + const db = getSessionKysely(database.db); + return new Set( + executeSqliteQuerySync( + database.db, + db + .selectFrom("sqlite_schema") + .select("name") + .where("type", "=", "table") + .where("name", "in", [ + "board_tabs", + "board_widgets", + "heartbeat_outcomes", + "session_members", + ]), + ).rows.flatMap((row) => (row.name ? [row.name] : [])), + ); +} diff --git a/src/config/sessions/session-accessor.sqlite-parent-session.ts b/src/config/sessions/session-accessor.sqlite-parent-session.ts index 55fc923fb45f..21eb69f8fc8f 100644 --- a/src/config/sessions/session-accessor.sqlite-parent-session.ts +++ b/src/config/sessions/session-accessor.sqlite-parent-session.ts @@ -12,9 +12,10 @@ import type { SessionParentForkDecision, } from "./session-accessor.sqlite-contract.js"; import { - deleteSqliteLifecycleTargetRows, + deleteLegacySessionEntryRows, normalizeSqliteLifecycleTarget, readSqliteSessionIdentitySnapshot, + rehomeSqliteSessionWindows, resolveSqliteLifecyclePrimaryEntry, writeSessionEntry, } from "./session-accessor.sqlite-entry-store.js"; @@ -227,8 +228,20 @@ export async function forkSqliteSessionEntryFromParentTarget( sessionId: fork.transcript.sessionId, }); previousIdentity = readSqliteSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); - deleteSqliteLifecycleTargetRows(writeDatabase, sessionTarget); - writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next); + writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next, { + previousEntry: freshBase, + }); + rehomeSqliteSessionWindows( + writeDatabase, + sessionTarget.canonicalKey, + sessionTarget.storeKeys, + ); + deleteLegacySessionEntryRows( + writeDatabase, + sessionTarget.storeKeys, + sessionTarget.canonicalKey, + { rehomeMembers: freshBase.sessionId === next.sessionId }, + ); maintenancePlans.push( applySqliteSessionEntryMaintenance(writeDatabase, { activeSessionKey: sessionTarget.canonicalKey, @@ -272,8 +285,20 @@ async function persistSqliteParentForkSkipPatch(params: { let currentIdentity = new Map(); runOpenClawAgentWriteTransaction((database) => { previousIdentity = readSqliteSessionIdentitySnapshot(database, params.sessionTarget.storeKeys); - deleteSqliteLifecycleTargetRows(database, params.sessionTarget); - writeSessionEntry(database, params.sessionTarget.canonicalKey, next); + writeSessionEntry(database, params.sessionTarget.canonicalKey, next, { + previousEntry: params.entry, + }); + rehomeSqliteSessionWindows( + database, + params.sessionTarget.canonicalKey, + params.sessionTarget.storeKeys, + ); + deleteLegacySessionEntryRows( + database, + params.sessionTarget.storeKeys, + params.sessionTarget.canonicalKey, + { rehomeMembers: params.entry.sessionId === next.sessionId }, + ); maintenancePlans.push( applySqliteSessionEntryMaintenance(database, { activeSessionKey: params.sessionTarget.canonicalKey, diff --git a/src/config/sessions/session-accessor.sqlite-projection.ts b/src/config/sessions/session-accessor.sqlite-projection.ts index 52132e90f867..4ba357f709c6 100644 --- a/src/config/sessions/session-accessor.sqlite-projection.ts +++ b/src/config/sessions/session-accessor.sqlite-projection.ts @@ -25,10 +25,12 @@ import type { SessionEntryStatus, } from "./session-accessor.sqlite-contract.js"; import { + deleteLegacySessionEntryRows, deleteSqliteSessionEntryRows, readExactSessionEntryRow, readSqliteSessionEntryCount, readSqliteSessionEntryStore, + rehomeSqliteSessionWindows, sqliteSessionEntriesEqual, writeSessionEntry, } from "./session-accessor.sqlite-entry-store.js"; @@ -39,6 +41,7 @@ import { emitCommittedSessionEntryRemovals, } from "./session-accessor.sqlite-identity.js"; import { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged, collectProjectedReferencedSqliteSessionIds, deleteMaterializedSqliteSessionStatePlans, deletePlannedSqliteLifecycleArtifactEntries, @@ -158,6 +161,7 @@ export async function applySqliteSessionEntryReplacements(params: { transactionDb, replacement.sessionKey, cloneSessionEntry(replacement.entry), + { previousEntry: expectedEntries.get(replacement.sessionKey) ?? null }, ); } maintenancePlans.push( @@ -253,7 +257,9 @@ export async function applySqliteSessionStoreProjection(params: { for (const sessionKey of changedKeys) { const entry = projected[sessionKey]; if (entry) { - writeSessionEntry(transactionDb, sessionKey, cloneSessionEntry(entry)); + writeSessionEntry(transactionDb, sessionKey, cloneSessionEntry(entry), { + previousEntry: before[sessionKey] ?? null, + }); } else { deleteSqliteSessionEntryRows(transactionDb, sessionKey); } @@ -321,19 +327,39 @@ export async function applySqliteSessionEntryLifecycleMutation(params: { captureArtifactCleanupError(error); } runOpenClawAgentWriteTransaction((transactionDb) => { - for (const removal of projected.removals) { + const validatedRemovals = projected.removals.filter((removal) => { const entry = readExactSessionEntryRow(transactionDb, removal.sessionKey)?.entry; if (!sqliteSessionEntriesEqual(entry, removal.expectedEntry)) { + const replacedInSameMutation = projected.upsertedEntries.some( + (upsert) => upsert.sessionKey === removal.sessionKey, + ); throw new Error( - `SQLite session entry changed before lifecycle removal for ${removal.sessionKey}`, + replacedInSameMutation + ? `SQLite session entry has stale lifecycle state for ${removal.sessionKey}` + : `SQLite session entry changed before lifecycle removal for ${removal.sessionKey}`, ); } - if (!shouldRemoveSqliteSessionEntry(entry, removal.removal)) { - continue; + const shouldRemove = shouldRemoveSqliteSessionEntry(entry, removal.removal); + if ( + !shouldRemove && + projected.upsertedEntries.some((upsert) => upsert.sessionKey === removal.sessionKey) + ) { + throw new Error( + `SQLite session entry has stale lifecycle state for ${removal.sessionKey}`, + ); } - deleteSqliteSessionEntryRows(transactionDb, removal.sessionKey); - removedSessionKeys.push(removal.sessionKey); - } + return shouldRemove; + }); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedRemovalPlans, + undefined, + new Set(validatedRemovals.map((removal) => removal.sessionKey)), + ); + const legacyReplacementTargets = new Map< + string, + { canonicalKey: string; rehomeMembers: boolean } + >(); for (const { sessionKey, entry, @@ -341,9 +367,22 @@ export async function applySqliteSessionEntryLifecycleMutation(params: { resetBoundaryPlan, } of projected.upsertedEntries) { const currentEntry = readExactSessionEntryRow(transactionDb, sessionKey)?.entry; - if (!sqliteSessionEntriesEqual(currentEntry, expectedEntry)) { + const sameKeyRemoval = validatedRemovals.find( + (removal) => removal.sessionKey === sessionKey, + ); + const expectedCurrentEntry = expectedEntry ?? sameKeyRemoval?.expectedEntry; + if (!sqliteSessionEntriesEqual(currentEntry, expectedCurrentEntry)) { + if (sameKeyRemoval) { + throw new Error(`SQLite session entry has stale lifecycle state for ${sessionKey}`); + } throw new Error(`SQLite session entry changed before lifecycle upsert for ${sessionKey}`); } + if ( + sameKeyRemoval && + !shouldRemoveSqliteSessionEntry(currentEntry, sameKeyRemoval.removal) + ) { + throw new Error(`SQLite session entry has stale lifecycle state for ${sessionKey}`); + } if (resetBoundaryPlan && expectedEntry?.sessionId) { const events = [...resetBoundaryPlan.seedEvents, resetBoundaryPlan.event]; const appended = appendTranscriptEventsInTransaction( @@ -355,7 +394,53 @@ export async function applySqliteSessionEntryLifecycleMutation(params: { throw new Error(`Failed to append reset boundary for ${sessionKey}`); } } - writeSessionEntry(transactionDb, sessionKey, entry); + writeSessionEntry(transactionDb, sessionKey, entry, { + previousEntry: expectedCurrentEntry ?? null, + }); + const relatedRemovalKeys = validatedRemovals.flatMap((removal) => { + const removedSessionId = removal.expectedEntry.sessionId; + return removal.sessionKey !== sessionKey && + (removedSessionId === entry.sessionId || removedSessionId === entry.previousSessionId) + ? [removal.sessionKey] + : []; + }); + rehomeSqliteSessionWindows(transactionDb, sessionKey, relatedRemovalKeys); + for (const legacyKey of relatedRemovalKeys) { + const removedEntry = validatedRemovals.find( + (removal) => removal.sessionKey === legacyKey, + )?.expectedEntry; + legacyReplacementTargets.set(legacyKey, { + canonicalKey: sessionKey, + rehomeMembers: removedEntry?.sessionId === entry.sessionId, + }); + } + } + const upsertedKeys = new Set(projected.upsertedEntries.map((upsert) => upsert.sessionKey)); + for (const removal of validatedRemovals) { + if (upsertedKeys.has(removal.sessionKey)) { + continue; + } + const entry = readExactSessionEntryRow(transactionDb, removal.sessionKey)?.entry; + if (!sqliteSessionEntriesEqual(entry, removal.expectedEntry)) { + throw new Error( + `SQLite session entry changed before lifecycle removal for ${removal.sessionKey}`, + ); + } + if (!shouldRemoveSqliteSessionEntry(entry, removal.removal)) { + continue; + } + const replacement = legacyReplacementTargets.get(removal.sessionKey); + if (replacement) { + deleteLegacySessionEntryRows( + transactionDb, + [removal.sessionKey], + replacement.canonicalKey, + { rehomeMembers: replacement.rehomeMembers }, + ); + } else { + deleteSqliteSessionEntryRows(transactionDb, removal.sessionKey); + } + removedSessionKeys.push(removal.sessionKey); } maintenancePlans.push( applySqliteSessionEntryMaintenance(transactionDb, { @@ -368,10 +453,6 @@ export async function applySqliteSessionEntryLifecycleMutation(params: { skipMaintenance: params.skipMaintenance, }), ); - archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( - transactionDb, - materializedRemovalPlans, - ); }, toDatabaseOptions(resolved)); emitCommittedLifecycleIdentityMutations({ projected, removedSessionKeys }); const maintenanceArchivedTranscripts = finalizeSqliteSessionEntryMaintenancePlansBestEffort( @@ -457,6 +538,13 @@ export async function purgeSqliteDeletedAgentSessionEntries( let archivedTranscripts: SessionLifecycleArchivedTranscript[] = []; const maintenancePlans: SqliteSessionEntryMaintenancePlan[] = []; runOpenClawAgentWriteTransaction((transactionDb) => { + assertPlannedSqliteLifecycleArtifactEntriesUnchanged(transactionDb, entryRemovals); + archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( + transactionDb, + materializedPlans, + undefined, + new Set(entryRemovals.map((removal) => removal.sessionKey)), + ); deletePlannedSqliteLifecycleArtifactEntries(transactionDb, entryRemovals); maintenancePlans.push( applySqliteSessionEntryMaintenance(transactionDb, { @@ -464,10 +552,6 @@ export async function purgeSqliteDeletedAgentSessionEntries( archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), }), ); - archivedTranscripts = deleteMaterializedSqliteSessionStatePlans( - transactionDb, - materializedPlans, - ); }, toDatabaseOptions(resolved)); emitCommittedSessionEntryRemovals(entryRemovals); archivedTranscripts = [ diff --git a/src/config/sessions/session-accessor.sqlite-provenance.ts b/src/config/sessions/session-accessor.sqlite-provenance.ts index 96970ea6efb8..24b220310de4 100644 --- a/src/config/sessions/session-accessor.sqlite-provenance.ts +++ b/src/config/sessions/session-accessor.sqlite-provenance.ts @@ -34,7 +34,7 @@ export function resolveSessionEntryProvenanceRow const existingRoot = executeSqliteQueryTakeFirstSync( params.database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select([ "session_entry_provenance", "acp_owned", diff --git a/src/config/sessions/session-accessor.sqlite-read.ts b/src/config/sessions/session-accessor.sqlite-read.ts index 758821693d80..6ae18b67a84d 100644 --- a/src/config/sessions/session-accessor.sqlite-read.ts +++ b/src/config/sessions/session-accessor.sqlite-read.ts @@ -157,7 +157,7 @@ export function readSqliteTranscriptStatsSync( const session = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select(["transcript_observed_at", "transcript_updated_at"]) .where("session_id", "=", resolved.sessionId), ); diff --git a/src/config/sessions/session-accessor.sqlite-references.ts b/src/config/sessions/session-accessor.sqlite-references.ts new file mode 100644 index 000000000000..4bf4404be395 --- /dev/null +++ b/src/config/sessions/session-accessor.sqlite-references.ts @@ -0,0 +1,24 @@ +import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; +import type { SessionEntry } from "./types.js"; + +/** Every transcript generation retained by one canonical logical-session record. */ +export function collectSqliteSessionStateIdsForEntry(entry: SessionEntry): string[] { + const sessionIds: string[] = []; + const add = (sessionId: string | undefined) => { + const normalized = sessionId?.trim(); + if (normalized) { + sessionIds.push(normalized); + } + }; + add(entry.sessionId); + add(entry.previousSessionId); + for (const sessionId of entry.usageFamilySessionIds ?? []) { + add(sessionId); + } + for (const checkpoint of entry.compactionCheckpoints ?? []) { + add(checkpoint.sessionId); + add(checkpoint.preCompaction.sessionId); + add(checkpoint.postCompaction.sessionId); + } + return uniqueStrings(sessionIds); +} diff --git a/src/config/sessions/session-accessor.sqlite-reset-window.ts b/src/config/sessions/session-accessor.sqlite-reset-window.ts index 50efeb491451..6ce4f43e88a8 100644 --- a/src/config/sessions/session-accessor.sqlite-reset-window.ts +++ b/src/config/sessions/session-accessor.sqlite-reset-window.ts @@ -13,7 +13,7 @@ import type { SessionTranscriptProjectionState } from "./session-transcript-inde type ResetWindowDatabase = Pick< OpenClawAgentKyselyDatabase, | "session_transcript_active_events" - | "session_transcript_generations" + | "transcript_rewrite_watermarks" | "transcript_event_identities" | "transcript_events" >; @@ -112,7 +112,7 @@ function readTranscriptGeneration(projection: ResetWindowProjection): string | u return executeSqliteQueryTakeFirstSync( projection.database.db, getResetWindowKysely(projection.database) - .selectFrom("session_transcript_generations") + .selectFrom("transcript_rewrite_watermarks") .select("generation") .where("session_id", "=", projection.resolved.sessionId), )?.generation; diff --git a/src/config/sessions/session-accessor.sqlite-scope.ts b/src/config/sessions/session-accessor.sqlite-scope.ts index f4ef8fa35682..29d0f772a4bf 100644 --- a/src/config/sessions/session-accessor.sqlite-scope.ts +++ b/src/config/sessions/session-accessor.sqlite-scope.ts @@ -27,17 +27,22 @@ import type { SessionEntry } from "./types.js"; type SessionSqliteDatabase = Pick< OpenClawAgentKyselyDatabase, + | "board_tabs" + | "board_widgets" | "conversation_deliveries" | "conversations" + | "heartbeat_outcomes" | "session_conversations" - | "session_entries" - | "session_routes" - | "session_transcript_generations" - | "sessions" + | "session_members" + | "session_nodes" + | "session_windows" + | "transcript_rewrite_watermarks" | "trajectory_runtime_events" | "transcript_event_identities" | "transcript_events" ->; +> & { + sqlite_schema: { name: string | null; type: string }; +}; export type ResolvedSqliteScope = { agentId: string; diff --git a/src/config/sessions/session-accessor.sqlite-session-row.ts b/src/config/sessions/session-accessor.sqlite-session-row.ts index b630542c9a6b..88f419099b8d 100644 --- a/src/config/sessions/session-accessor.sqlite-session-row.ts +++ b/src/config/sessions/session-accessor.sqlite-session-row.ts @@ -28,6 +28,8 @@ export function bindSqliteSessionRoot(params: { return { session_id: params.entry.sessionId, session_key: params.sessionKey, + previous_session_id: normalizeSqliteText(params.entry.previousSessionId), + reason: null, session_scope: resolveSqliteSessionScope(params.entry, params.sessionKey), created_at: resolveSqliteSessionCreatedAt(params.entry, updatedAt), updated_at: updatedAt, @@ -48,6 +50,63 @@ export function bindSqliteSessionRoot(params: { }; } +/** Project the canonical entry blob into the logical-node query columns. */ +export function bindSqliteSessionNode(params: { + entry: SessionEntry; + sessionKey: string; + updatedAt: number; +}) { + const actor = params.entry.createdActor; + const legacyActorId = normalizeSqliteText( + (params.entry as SessionEntry & { createdBy?: { id?: unknown } }).createdBy?.id, + ); + return { + session_key: params.sessionKey, + current_session_id: params.entry.sessionId, + entry_json: JSON.stringify(params.entry), + updated_at: params.updatedAt, + status: normalizeSqliteStatus(params.entry.status), + created_at: finiteSqliteNumber(params.entry.createdAt), + created_via: normalizeSqliteCreatedVia(params.entry.createdVia), + created_actor_type: + normalizeSqliteCreatedActorType(actor?.type) ?? (legacyActorId ? "human" : null), + created_actor_id: normalizeSqliteText(actor?.id) ?? legacyActorId, + parent_session_key: + normalizeSqliteText(params.entry.parentSessionKey) ?? + normalizeSqliteText(params.entry.spawnedBy), + spawned_by: normalizeSqliteText(params.entry.spawnedBy), + fork_source_session_key: normalizeSqliteText(params.entry.forkSource?.sessionKey), + fork_source_session_id: normalizeSqliteText(params.entry.forkSource?.sessionId), + fork_source_entry_id: normalizeSqliteText(params.entry.forkSource?.entryId), + label: normalizeSqliteText(params.entry.label), + display_name: normalizeSqliteText(params.entry.displayName), + category: normalizeSqliteText(params.entry.category), + icon: normalizeSqliteText(params.entry.icon), + pinned_at: finiteSqliteNumber(params.entry.pinnedAt), + archived_at: finiteSqliteNumber(params.entry.archivedAt), + last_read_at: finiteSqliteNumber(params.entry.lastReadAt), + last_interaction_at: finiteSqliteNumber(params.entry.lastInteractionAt), + last_activity_at: finiteSqliteNumber(params.entry.lastActivityAt), + }; +} + +function normalizeSqliteCreatedVia(value: SessionEntry["createdVia"]) { + return value === "operator" || + value === "spawn" || + value === "channel" || + value === "cron" || + value === "talk" || + value === "run" || + value === "plugin" || + value === "internal" + ? value + : null; +} + +function normalizeSqliteCreatedActorType(value: unknown) { + return value === "human" || value === "agent" || value === "system" ? value : null; +} + function resolveSqliteSessionScope( entry: Pick, sessionKey: string, diff --git a/src/config/sessions/session-accessor.sqlite-status.ts b/src/config/sessions/session-accessor.sqlite-status.ts index d73ccc88b5cb..c229098717da 100644 --- a/src/config/sessions/session-accessor.sqlite-status.ts +++ b/src/config/sessions/session-accessor.sqlite-status.ts @@ -7,7 +7,7 @@ import type { } from "./session-accessor.sqlite-contract.js"; import type { SessionEntry } from "./types.js"; -type SessionStatusDatabase = Pick; +type SessionStatusDatabase = Pick; export function normalizeSqliteStatus(value: unknown): SessionEntryStatus | null { return value === "running" || @@ -22,9 +22,11 @@ export function normalizeSqliteStatus(value: unknown): SessionEntryStatus | null export function parseSqliteSessionEntryJson(row: { entry_json: string }): SessionEntry | null { try { const parsed = JSON.parse(row.entry_json) as unknown; - return parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as SessionEntry) - : null; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + const entry = parsed as Partial; + return typeof entry.sessionId === "string" ? (entry as SessionEntry) : null; } catch { return null; } @@ -42,8 +44,8 @@ export function readSqliteSessionEntriesByStatus( } const db = getNodeSqliteKysely(database.db); let query = db - .selectFrom("session_entries") - .select(["session_key", "entry_json", "session_id", "updated_at"]) + .selectFrom("session_nodes") + .select(["session_key", "entry_json", "current_session_id", "updated_at"]) .where("status", "in", selectedStatuses); if (selectedSessionKeys) { query = query.where("session_key", "in", selectedSessionKeys); diff --git a/src/config/sessions/session-accessor.sqlite-transcript-state.ts b/src/config/sessions/session-accessor.sqlite-transcript-state.ts index 927ebc2a0609..f448adc0a04f 100644 --- a/src/config/sessions/session-accessor.sqlite-transcript-state.ts +++ b/src/config/sessions/session-accessor.sqlite-transcript-state.ts @@ -21,7 +21,7 @@ export function readTranscriptGenerationInTransaction( return executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("session_transcript_generations") + .selectFrom("transcript_rewrite_watermarks") .select("generation") .where("session_id", "=", sessionId), )?.generation; @@ -37,7 +37,7 @@ export function ensureTranscriptGenerationInTransaction( executeSqliteQuerySync( database.db, db - .insertInto("session_transcript_generations") + .insertInto("transcript_rewrite_watermarks") .values({ session_id: sessionId, generation, updated_at: Date.now() }) .onConflict((conflict) => conflict.column("session_id").doNothing()), ); @@ -54,7 +54,7 @@ export function rotateTranscriptGenerationInTransaction( executeSqliteQuerySync( database.db, db - .insertInto("session_transcript_generations") + .insertInto("transcript_rewrite_watermarks") .values({ session_id: sessionId, generation, updated_at: Date.now() }) .onConflict((conflict) => conflict.column("session_id").doUpdateSet({ generation, updated_at: Date.now() }), @@ -72,10 +72,24 @@ export function ensureTranscriptSessionRoot( executeSqliteQuerySync( database.db, db - .insertInto("sessions") + .insertInto("session_nodes") + .values({ + session_key: scope.sessionKey, + current_session_id: scope.sessionId, + entry_json: "{}", + updated_at: updatedAt, + }) + .onConflict((conflict) => conflict.column("session_key").doNothing()), + ); + executeSqliteQuerySync( + database.db, + db + .insertInto("session_windows") .values({ session_id: scope.sessionId, session_key: scope.sessionKey, + previous_session_id: null, + reason: null, session_scope: "conversation", created_at: updatedAt, updated_at: updatedAt, @@ -87,54 +101,6 @@ export function ensureTranscriptSessionRoot( }), ), ); - writeTranscriptSessionRoute(database, { - sessionId: scope.sessionId, - sessionKey: scope.sessionKey, - updatedAt, - }); -} - -export function writeSessionRoute( - database: OpenClawAgentDatabase, - params: { sessionId: string; sessionKey: string; updatedAt: number }, -): void { - const db = getSessionKysely(database.db); - executeSqliteQuerySync( - database.db, - db - .insertInto("session_routes") - .values({ - session_key: params.sessionKey, - session_id: params.sessionId, - updated_at: params.updatedAt, - }) - .onConflict((conflict) => - conflict.column("session_key").doUpdateSet({ - session_id: params.sessionId, - updated_at: params.updatedAt, - }), - ), - ); -} - -function writeTranscriptSessionRoute( - database: OpenClawAgentDatabase, - params: { sessionId: string; sessionKey: string; updatedAt: number }, -): void { - const db = getSessionKysely(database.db); - const existing = executeSqliteQueryTakeFirstSync( - database.db, - db - .selectFrom("session_routes") - .select("session_id") - .where("session_key", "=", params.sessionKey), - ); - // Late transcript-only appends may create routes, but cannot move a current - // session key back to an older transcript id. - if (existing && existing.session_id !== params.sessionId) { - return; - } - writeSessionRoute(database, params); } export function readNextTranscriptSeq(database: OpenClawAgentDatabase, sessionId: string): number { @@ -164,7 +130,7 @@ export function readTranscriptMutationStateInTransaction( const row = executeSqliteQueryTakeFirstSync( database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select(["transcript_observed_at", "transcript_updated_at"]) .where("session_id", "=", sessionId), ); @@ -195,7 +161,7 @@ export function advanceTranscriptMutationAtInTransaction( executeSqliteQuerySync( database.db, db - .updateTable("sessions") + .updateTable("session_windows") .set({ transcript_updated_at: next }) .where("session_id", "=", sessionId), ); diff --git a/src/config/sessions/session-accessor.test.ts b/src/config/sessions/session-accessor.test.ts index e9ad38f93c8b..4d0aaaef4f47 100644 --- a/src/config/sessions/session-accessor.test.ts +++ b/src/config/sessions/session-accessor.test.ts @@ -47,6 +47,10 @@ import { updateSessionLastRoute, upsertSessionEntry, } from "./session-accessor.js"; +import { + readSqliteSessionEntryCount, + readSqliteSessionEntryKeys, +} from "./session-accessor.sqlite-entry-store.js"; import { appendSqliteTranscriptEventSync, importSqliteSessionRows, @@ -137,6 +141,53 @@ describe("session accessor seam", () => { }); }); + it("excludes transcript-only nodes from logical entry counts and keys", async () => { + await replaceSessionEntry( + { sessionKey: "agent:main:logical-entry", storePath }, + { sessionId: "logical-entry-session", updatedAt: 10 }, + ); + await replaceSqliteTranscriptEvents( + { + agentId: "main", + sessionId: "transcript-only-session", + sessionKey: "agent:main:transcript-only", + storePath, + }, + [{ type: "session", id: "transcript-only-session" }], + ); + const databasePath = expectDefined( + resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" }).path, + "entry count database path", + ); + const database = openOpenClawAgentDatabase({ agentId: "main", path: databasePath }); + + expect(readSqliteSessionEntryCount(database)).toBe(1); + expect(readSqliteSessionEntryKeys(database)).toEqual(["agent:main:logical-entry"]); + }); + + it("retains legacy createdBy actor projections across rewrites", async () => { + const sessionKey = "agent:main:created-by"; + await replaceSessionEntry({ sessionKey, storePath }, { + createdBy: { id: "legacy-human" }, + sessionId: "created-by-session", + updatedAt: 10, + } as SessionEntry & { createdBy: { id: string } }); + await upsertSessionEntry({ sessionKey, storePath }, { label: "rewritten" }); + const databasePath = expectDefined( + resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" }).path, + "createdBy database path", + ); + const database = openOpenClawAgentDatabase({ agentId: "main", path: databasePath }); + + expect( + database.db + .prepare( + "SELECT created_actor_type, created_actor_id FROM session_nodes WHERE session_key = ?", + ) + .get(sessionKey), + ).toEqual({ created_actor_type: "human", created_actor_id: "legacy-human" }); + }); + it("lists retained transcript instances across same-key session rotation", async () => { const scope = { agentId: "main", @@ -223,7 +274,7 @@ describe("session accessor seam", () => { path: databasePath, }); database.db - .prepare("UPDATE sessions SET transcript_updated_at = NULL WHERE session_id = ?") + .prepare("UPDATE session_windows SET transcript_updated_at = NULL WHERE session_id = ?") .run(scope.sessionId); await replaceSessionEntry( @@ -301,7 +352,7 @@ describe("session accessor seam", () => { }); database.db .prepare( - "UPDATE sessions SET session_entry_provenance = 0, plugin_owner_id = NULL WHERE session_id = ?", + "UPDATE session_windows SET session_entry_provenance = 0, plugin_owner_id = NULL WHERE session_id = ?", ) .run("migrated-plugin-session"); @@ -586,6 +637,8 @@ describe("session accessor seam", () => { }); it("patches the freshest target alias and rewrites it to the canonical key", async () => { + const canonicalKey = "agent:main:work"; + const aliasKey = "agent:main:main"; await replaceSessionEntry( { sessionKey: "agent:main:work", @@ -604,8 +657,99 @@ describe("session accessor seam", () => { { sessionId: "legacy-session", updatedAt: 20, + visibility: "read-only", }, ); + await replaceSqliteTranscriptEvents( + { agentId: "main", sessionId: "legacy-history", sessionKey: aliasKey, storePath }, + [{ id: "legacy-history-event", type: "message" }], + ); + const aliasDatabasePath = expectDefined( + resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" }).path, + "alias database path", + ); + const aliasDatabase = openOpenClawAgentDatabase({ agentId: "main", path: aliasDatabasePath }); + aliasDatabase.db + .prepare( + `INSERT INTO session_nodes ( + session_key, current_session_id, entry_json, updated_at + ) VALUES (?, 'canonical-session', ?, 10)`, + ) + .run(canonicalKey, JSON.stringify({ sessionId: "canonical-session", updatedAt: 10 })); + aliasDatabase.db + .prepare( + `INSERT INTO session_windows ( + session_id, session_key, session_scope, created_at, updated_at + ) VALUES ('canonical-session', ?, 'conversation', 10, 10)`, + ) + .run(canonicalKey); + aliasDatabase.db + .prepare( + `INSERT INTO board_tabs ( + session_key, tab_id, title, position, chat_dock, created_by, revision + ) VALUES ('agent:main:main', 'main', 'Alias board', 0, 'right', 'user', 2)`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO board_widgets ( + session_key, name, tab_id, content_kind, html, sha256, view_generation, + revision, size_w, size_h, position, created_by, created_at, updated_at + ) VALUES ( + 'agent:main:main', 'status', 'main', 'html', X'3C703E6F6B3C2F703E', + 'alias-hash', 'view-1', 2, 4, 4, 0, 'user', 20, 20 + )`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO board_tabs ( + session_key, tab_id, title, position, chat_dock, created_by, revision + ) VALUES ('agent:main:work', 'main', 'Stale canonical board', 0, 'right', 'user', 1)`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO board_widgets ( + session_key, name, tab_id, content_kind, html, sha256, view_generation, + revision, size_w, size_h, position, created_by, created_at, updated_at + ) VALUES ( + 'agent:main:work', 'status', 'main', 'html', X'3C703E7374616C653C2F703E', + 'stale-canonical-hash', 'view-stale', 1, 4, 4, 0, 'user', 10, 10 + )`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO heartbeat_outcomes ( + session_key, run_session_key, outcome, summary, occurred_at, updated_at + ) VALUES ( + 'agent:main:work', 'agent:main:work', 'progress', 'stale canonical heartbeat', 10, 10 + )`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO heartbeat_outcomes ( + session_key, run_session_key, outcome, summary, occurred_at, updated_at + ) VALUES ('agent:main:main', 'agent:main:main', 'done', 'alias heartbeat', 20, 20)`, + ) + .run(); + aliasDatabase.db + .prepare( + `INSERT INTO session_members (session_key, identity_id, added_by, added_at) + VALUES (?, ?, ?, ?)`, + ) + .run("agent:main:main", "member-1", "owner-1", 20); + expect( + aliasDatabase.db.prepare("SELECT session_key FROM session_nodes ORDER BY session_key").all(), + ).toEqual([{ session_key: "agent:main:main" }, { session_key: "agent:main:work" }]); + aliasDatabase.db + .prepare( + `INSERT INTO session_members (session_key, identity_id, added_by, added_at) + VALUES (?, ?, ?, ?)`, + ) + .run("agent:main:work", "stale-canonical-member", "stale-owner", 10); const notify = vi.fn(); const unsubscribe = onSessionIdentityMutation(notify); @@ -628,6 +772,7 @@ describe("session accessor seam", () => { expect(patched).toMatchObject({ label: "patched", sessionId: "legacy-session", + visibility: "read-only", }); expect(listSessionEntries({ storePath })).toEqual([ { @@ -638,6 +783,49 @@ describe("session accessor seam", () => { }), }, ]); + expect( + aliasDatabase.db + .prepare("SELECT session_key, title, revision FROM board_tabs ORDER BY session_key") + .all(), + ).toEqual([{ session_key: "agent:main:work", title: "Alias board", revision: 2 }]); + expect( + aliasDatabase.db + .prepare( + "SELECT session_key, name, sha256, revision, updated_at FROM board_widgets ORDER BY session_key", + ) + .all(), + ).toEqual([ + { + session_key: "agent:main:work", + name: "status", + sha256: "alias-hash", + revision: 2, + updated_at: 20, + }, + ]); + expect( + aliasDatabase.db + .prepare("SELECT session_key, summary FROM heartbeat_outcomes ORDER BY session_key") + .all(), + ).toEqual([{ session_key: "agent:main:work", summary: "alias heartbeat" }]); + expect( + aliasDatabase.db + .prepare("SELECT session_key, identity_id FROM session_members ORDER BY session_key") + .all(), + ).toEqual([{ session_key: "agent:main:work", identity_id: "member-1" }]); + expect( + aliasDatabase.db + .prepare("SELECT session_key FROM session_windows WHERE session_id = 'legacy-history'") + .get(), + ).toEqual({ session_key: canonicalKey }); + await expect( + loadTranscriptEvents({ + agentId: "main", + sessionId: "legacy-history", + sessionKey: canonicalKey, + storePath, + }), + ).resolves.toEqual([{ id: "legacy-history-event", type: "message" }]); const sessionKey = "agent:main:other"; const scope = { sessionKey, storePath }; await replaceSessionEntry(scope, { sessionId: "created", updatedAt: 10 }); @@ -654,6 +842,7 @@ describe("session accessor seam", () => { expect(notify.mock.calls.map(([event]) => event.kind)).toEqual([ "move", + "replace", "create", "replace", "reset", @@ -3236,7 +3425,7 @@ describe("session accessor seam", () => { expect(databasePath).toBeDefined(); const readGeneration = () => openOpenClawAgentDatabase({ agentId: scope.agentId, path: databasePath }) - .db.prepare("SELECT generation FROM session_transcript_generations WHERE session_id = ?") + .db.prepare("SELECT generation FROM transcript_rewrite_watermarks WHERE session_id = ?") .get(scope.sessionId) as { generation: string } | undefined; await appendTranscriptMessage(scope, { diff --git a/src/config/sessions/session-history-eviction.test.ts b/src/config/sessions/session-history-eviction.test.ts index 099f530dce36..7d9a49428979 100644 --- a/src/config/sessions/session-history-eviction.test.ts +++ b/src/config/sessions/session-history-eviction.test.ts @@ -259,7 +259,10 @@ describe("SQLite historical session disk budget", () => { const db = getSessionKysely(owner.db); executeSqliteQuerySync( owner.db, - db.updateTable("sessions").set({ updated_at: updatedAt }).where("session_id", "=", sessionId), + db + .updateTable("session_windows") + .set({ updated_at: updatedAt }) + .where("session_id", "=", sessionId), ); } @@ -268,9 +271,12 @@ describe("SQLite historical session disk budget", () => { const db = getSessionKysely(owner.db); executeSqliteQuerySync( owner.db, - db - .insertInto("session_routes") - .values({ session_key: sessionKey, session_id: sessionId, updated_at: Date.now() }), + db.insertInto("session_nodes").values({ + session_key: sessionKey, + current_session_id: sessionId, + entry_json: "{}", + updated_at: Date.now(), + }), ); } @@ -280,7 +286,7 @@ describe("SQLite historical session disk budget", () => { return ( executeSqliteQuerySync( owner.db, - db.selectFrom("sessions").select("session_id").where("session_id", "=", sessionId), + db.selectFrom("session_windows").select("session_id").where("session_id", "=", sessionId), ).rows.length === 1 ); } diff --git a/src/config/sessions/session-history-eviction.ts b/src/config/sessions/session-history-eviction.ts index 74c6a7afc644..a3d26c64e5a2 100644 --- a/src/config/sessions/session-history-eviction.ts +++ b/src/config/sessions/session-history-eviction.ts @@ -137,13 +137,13 @@ export function collectAdmissionProtectedSessionIds(params: { const db = getSessionKysely(params.database.db); const rows = executeSqliteQuerySync( params.database.db, - db.selectFrom("session_entries").select(["entry_json", "session_id", "session_key"]), + db.selectFrom("session_nodes").select(["entry_json", "current_session_id", "session_key"]), ).rows; for (const row of rows) { if (!normalizedAdmissionKeys.has(normalizeStoreSessionKey(row.session_key))) { continue; } - protectedSessionIds.add(row.session_id); + protectedSessionIds.add(row.current_session_id); const entry = parseSqliteSessionEntryJson(row); if (entry) { for (const sessionId of collectSqliteSessionStateIdsForEntry(entry)) { @@ -156,7 +156,7 @@ export function collectAdmissionProtectedSessionIds(params: { // every generation of an admitted key stays off-limits. const generationRows = executeSqliteQuerySync( params.database.db, - db.selectFrom("sessions").select(["session_id", "session_key"]), + db.selectFrom("session_windows").select(["session_id", "session_key"]), ).rows; for (const row of generationRows) { if (normalizedAdmissionKeys.has(normalizeStoreSessionKey(row.session_key))) { @@ -174,7 +174,7 @@ function readHistoricalSessionIds(params: { return executeSqliteQuerySync( params.database.db, db - .selectFrom("sessions") + .selectFrom("session_windows") .select("session_id") .orderBy("updated_at", "asc") .orderBy("session_id", "asc"), @@ -384,7 +384,10 @@ async function enforceSessionHistoryMaintenanceSerialized( deleted = executeSqliteQuerySync( transactionDb.db, - db.selectFrom("sessions").select("session_id").where("session_id", "=", sessionId), + db + .selectFrom("session_windows") + .select("session_id") + .where("session_id", "=", sessionId), ).rows.length === 0; }, toDatabaseOptions(resolved)); if (!deleted) { diff --git a/src/config/sessions/session-sharing-store.test.ts b/src/config/sessions/session-sharing-store.test.ts index 40fa9259387a..5d4a897ecae8 100644 --- a/src/config/sessions/session-sharing-store.test.ts +++ b/src/config/sessions/session-sharing-store.test.ts @@ -4,7 +4,11 @@ import { openOpenClawAgentDatabase, } from "../../state/openclaw-agent-db.js"; import { withTempDir } from "../../test-helpers/temp-dir.js"; -import { loadSessionEntry, upsertSessionEntry } from "./session-accessor.js"; +import { + deleteSessionEntryLifecycle, + loadSessionEntry, + upsertSessionEntry, +} from "./session-accessor.js"; import { addSessionMember, isSessionMember, @@ -131,4 +135,41 @@ describe("session sharing store", () => { expect(isSessionMember(scope, "guest")).toBe(true); }); }); + + it("rejects stale member writes after entry-only deletion leaves a placeholder", async () => { + await withTempDir({ prefix: "openclaw-session-sharing-placeholder-" }, async (dir) => { + const env = { ...process.env, OPENCLAW_STATE_DIR: dir }; + const scope = { agentId: "main", env, sessionKey: "agent:main:main" }; + await upsertSessionEntry(scope, { sessionId: "session-a", updatedAt: 1 }); + expect( + addSessionMember(scope, { identityId: "guest", addedBy: "owner", addedAt: 2 }).inserted, + ).toBe(true); + + await deleteSessionEntryLifecycle({ + agentId: "main", + archiveTranscript: false, + storePath: openOpenClawAgentDatabase({ agentId: "main", env }).path, + target: { canonicalKey: scope.sessionKey, storeKeys: [scope.sessionKey] }, + }); + + expect(loadSessionEntry(scope)).toBeUndefined(); + expect(listSessionMembers(scope)).toEqual([]); + expect(() => + addSessionMember(scope, { + identityId: "stale", + addedBy: "owner", + expectedSessionId: "session-a", + }), + ).toThrow(/session changed/); + expect(() => + addSessionMember(scope, { + identityId: "planted", + addedBy: "owner", + }), + ).toThrow(/session changed/); + + await upsertSessionEntry(scope, { sessionId: "session-b", updatedAt: 3 }); + expect(listSessionMembers(scope)).toEqual([]); + }); + }); }); diff --git a/src/config/sessions/session-sharing-store.ts b/src/config/sessions/session-sharing-store.ts index f9d3a2d43e27..5f1d90727d92 100644 --- a/src/config/sessions/session-sharing-store.ts +++ b/src/config/sessions/session-sharing-store.ts @@ -116,23 +116,36 @@ export function isSessionMember(scope: SessionAccessScope, identityId: string): ); } -// Membership is bound to the session instance. Authorization is rechecked -// before these transactions, but a reset/recreate can replace the row under the -// same key in between; verifying the expected sessionId inside the write -// transaction stops a stale owner from mutating the replacement's members. +// Membership is bound to a live session entry, never a transcript placeholder. +// Authorization is rechecked before these transactions, but a reset/recreate +// can replace the row under the same key in between; the optional expected id +// adds a caller snapshot check after the canonical node/entry check. function assertAuthorizedSessionInstance( database: OpenClawAgentDatabase, sessionKey: string, expectedSessionId: string | undefined, ): void { - if (expectedSessionId === undefined) { - return; - } const row = - database.db /* sqlite-allow-raw: sync TOCTOU re-read of session_id inside a write transaction; Kysely async execution is forbidden in synchronous commit sections */ - .prepare("SELECT session_id FROM session_entries WHERE session_key = ?") - .get(sessionKey) as { session_id?: string } | undefined; - if (row?.session_id !== expectedSessionId) { + database.db /* sqlite-allow-raw: sync TOCTOU re-read of canonical entry identity inside a write transaction; Kysely async execution is forbidden in synchronous commit sections */ + .prepare("SELECT current_session_id, entry_json FROM session_nodes WHERE session_key = ?") + .get(sessionKey) as { current_session_id?: string; entry_json?: string } | undefined; + let entrySessionId: string | undefined; + try { + const entry = row?.entry_json ? (JSON.parse(row.entry_json) as unknown) : undefined; + const candidate = + entry && typeof entry === "object" && !Array.isArray(entry) + ? (entry as { sessionId?: unknown }).sessionId + : undefined; + entrySessionId = typeof candidate === "string" ? candidate : undefined; + } catch { + entrySessionId = undefined; + } + if ( + !row || + entrySessionId === undefined || + row.current_session_id !== entrySessionId || + (expectedSessionId !== undefined && entrySessionId !== expectedSessionId) + ) { throw new Error("session changed before sharing mutation"); } } diff --git a/src/config/sessions/session-transcript-index.ts b/src/config/sessions/session-transcript-index.ts index 18095d9fe8d6..7820a003dcbd 100644 --- a/src/config/sessions/session-transcript-index.ts +++ b/src/config/sessions/session-transcript-index.ts @@ -32,7 +32,7 @@ import { type TranscriptIndexDatabase = Pick< OpenClawAgentKyselyDatabase, - | "sessions" + | "session_windows" | "session_transcript_active_events" | "session_transcript_fts" | "session_transcript_index_state" @@ -426,10 +426,10 @@ export function listSessionsNeedingTranscriptIndexReconcile(db: DatabaseSync): s const rows = executeSqliteQuerySync( db, kysely - .selectFrom("sessions") + .selectFrom("session_windows") .innerJoin("transcript_events as latest", (join) => join - .onRef("latest.session_id", "=", "sessions.session_id") + .onRef("latest.session_id", "=", "session_windows.session_id") .on((eb) => eb( "latest.seq", @@ -437,14 +437,18 @@ export function listSessionsNeedingTranscriptIndexReconcile(db: DatabaseSync): s eb .selectFrom("transcript_events as candidate") .select("candidate.seq") - .whereRef("candidate.session_id", "=", "sessions.session_id") + .whereRef("candidate.session_id", "=", "session_windows.session_id") .orderBy("candidate.seq", "desc") .limit(1), ), ), ) - .leftJoin("session_transcript_index_state as st", "st.session_id", "sessions.session_id") - .select("sessions.session_id") + .leftJoin( + "session_transcript_index_state as st", + "st.session_id", + "session_windows.session_id", + ) + .select("session_windows.session_id") .where((eb) => eb.or([ eb(eb.fn.coalesce("st.needs_rebuild", eb.val(1)), "!=", 0), @@ -453,7 +457,7 @@ export function listSessionsNeedingTranscriptIndexReconcile(db: DatabaseSync): s ) // The transcript PK makes the correlated latest-row lookup one index seek per session. // Grouping transcript_events here made every healthy search rescan the entire history. - .orderBy("sessions.session_id"), + .orderBy("session_windows.session_id"), ).rows; return rows.flatMap((row) => (typeof row.session_id === "string" ? [row.session_id] : [])); } diff --git a/src/config/sessions/session-transcript-projection-rebuild.ts b/src/config/sessions/session-transcript-projection-rebuild.ts index 689bd309d851..ac17e853523e 100644 --- a/src/config/sessions/session-transcript-projection-rebuild.ts +++ b/src/config/sessions/session-transcript-projection-rebuild.ts @@ -18,7 +18,7 @@ import { type TranscriptProjectionDatabase = Pick< OpenClawAgentKyselyDatabase, - "sessions" | "session_transcript_index_state" | "transcript_events" + "session_windows" | "session_transcript_index_state" | "transcript_events" > & { session_transcript_active_events: OpenClawAgentKyselyDatabase["session_transcript_active_events"] & { rowid: Generated; @@ -164,7 +164,7 @@ export function prepareSessionTranscriptProjection( const session = executeSqliteQueryTakeFirstSync( db, kysely - .selectFrom("sessions") + .selectFrom("session_windows") .select("transcript_updated_at") .where("session_id", "=", sessionId), ); @@ -230,7 +230,7 @@ function sourceSnapshotMatches( const session = executeSqliteQueryTakeFirstSync( db, kysely - .selectFrom("sessions") + .selectFrom("session_windows") .select("transcript_updated_at") .where("session_id", "=", plan.sessionId), ); diff --git a/src/config/sessions/session-transcript-search.ts b/src/config/sessions/session-transcript-search.ts index 8fc13a8fce93..d87a2f9f80a9 100644 --- a/src/config/sessions/session-transcript-search.ts +++ b/src/config/sessions/session-transcript-search.ts @@ -76,21 +76,21 @@ export function searchSessionTranscripts(params: { const sessionKeys = params.sessionKeys ?? []; const whereSession = sessionKeys.length > 0 - ? ` AND sessions.session_key IN (${sessionKeys.map(() => "?").join(", ")})` + ? ` AND session_windows.session_key IN (${sessionKeys.map(() => "?").join(", ")})` : ""; // MATCH, snippet(), and bm25() are FTS5 primitives without a Kysely - // representation. session_key lives on the sessions row so key renames + // representation. session_key lives on the window row so key renames // never leave stale keys inside the index. Sessions flagged needs_rebuild // are excluded: their rows may still hold rewound-away branch text that // sessions_history no longer exposes, so they stay hidden until reconcile // rebuilds them (indexing=true tells the caller to retry). const statement = database.db.prepare(/* sqlite-allow-raw: FTS5 MATCH/snippet/bm25 */ ` - SELECT sessions.session_key AS session_key, session_transcript_fts.session_id AS session_id, + SELECT session_windows.session_key AS session_key, session_transcript_fts.session_id AS session_id, message_id, role, timestamp, snippet(session_transcript_fts, 0, '', '', ' … ', 48) AS snippet, bm25(session_transcript_fts) AS rank FROM session_transcript_fts - JOIN sessions ON sessions.session_id = session_transcript_fts.session_id + JOIN session_windows ON session_windows.session_id = session_transcript_fts.session_id WHERE session_transcript_fts MATCH ?${whereSession} AND session_transcript_fts.session_id NOT IN ( SELECT session_id FROM session_transcript_index_state WHERE needs_rebuild != 0 diff --git a/src/gateway/server-methods/sessions-sharing.test.ts b/src/gateway/server-methods/sessions-sharing.test.ts index 66f2c8d4ff9f..d8c94798bd16 100644 --- a/src/gateway/server-methods/sessions-sharing.test.ts +++ b/src/gateway/server-methods/sessions-sharing.test.ts @@ -422,12 +422,20 @@ describe("session sharing handlers", () => { { sessionId: "session-alias-member", updatedAt: 1, visibility: "read-only" }, ); const database = openOpenClawAgentDatabase({ agentId: "ops", env: state.env }); - database.db - .prepare("UPDATE session_entries SET session_key = ? WHERE session_key = ?") - .run(aliasKey, canonicalKey); + database.db.exec("PRAGMA foreign_keys = OFF;"); + try { + database.db + .prepare("UPDATE session_nodes SET session_key = ? WHERE session_key = ?") + .run(aliasKey, canonicalKey); + database.db + .prepare("UPDATE session_windows SET session_key = ? WHERE session_key = ?") + .run(aliasKey, canonicalKey); + } finally { + database.db.exec("PRAGMA foreign_keys = ON;"); + } expect( database.db - .prepare("SELECT session_key FROM session_entries WHERE session_key = ?") + .prepare("SELECT session_key FROM session_nodes WHERE session_key = ?") .get(canonicalKey), ).toBeUndefined(); clearSessionStoreCacheForTest(); diff --git a/src/gateway/server.sessions.create.test.ts b/src/gateway/server.sessions.create.test.ts index fe7b9da48754..6f4b6fe78f57 100644 --- a/src/gateway/server.sessions.create.test.ts +++ b/src/gateway/server.sessions.create.test.ts @@ -89,7 +89,7 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re }); expect( openedIncognitoDatabase.db - .prepare("SELECT session_key FROM session_entries WHERE session_key = ?") + .prepare("SELECT session_key FROM session_nodes WHERE session_key = ?") .get(key), ).toEqual({ session_key: key }); expect(loadSessionEntry({ agentId: "main", sessionKey: key })?.incognito).toBe(true); @@ -101,7 +101,7 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re }); expect( persistentDatabase.db - .prepare("SELECT session_key FROM session_entries WHERE session_key = ?") + .prepare("SELECT session_key FROM session_nodes WHERE session_key = ?") .get(key), ).toBeUndefined(); @@ -178,9 +178,9 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re }); expect( persistentDatabase.db - .prepare("SELECT session_id FROM session_entries WHERE session_key = ?") + .prepare("SELECT current_session_id FROM session_nodes WHERE session_key = ?") .get(durableSubagentKey), - ).toEqual({ session_id: "durable-subagent" }); + ).toEqual({ current_session_id: "durable-subagent" }); const deleted = await directSessionReq<{ archived: string[]; deleted: boolean }>( "sessions.delete", @@ -197,7 +197,7 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re agentId: "main", path: resolveIncognitoOpenClawAgentSqlitePath({ agentId: "main" }), }); - for (const table of ["session_entries", "sessions", "transcript_events"] as const) { + for (const table of ["session_nodes", "session_windows", "transcript_events"] as const) { expect(incognitoDatabase.db.prepare(`SELECT count(*) AS count FROM ${table}`).get()).toEqual({ count: 0, }); @@ -221,7 +221,7 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re expect(resetRematerialized.payload).toMatchObject({ deleted: true }); expect( openedIncognitoDatabase.db - .prepare("SELECT session_key FROM session_entries WHERE session_key = ?") + .prepare("SELECT session_key FROM session_nodes WHERE session_key = ?") .get(key), ).toBeUndefined(); @@ -256,21 +256,21 @@ test("sessions.create keeps incognito rows process-local through list, spawn, re }, }); const durableCollisionKey = "agent:main:dashboard:incognito-durable-collision"; + const durableCollisionUpdatedAt = Date.now(); persistentDatabase.db .prepare( - "INSERT INTO sessions (session_id, session_key, session_scope, created_at, updated_at) VALUES (?, ?, ?, ?, ?)", - ) - .run("durable-collision", durableCollisionKey, "conversation", Date.now(), Date.now()); - persistentDatabase.db - .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES (?, ?, ?, ?)", + "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, 'durable-collision', ?, ?)", ) .run( durableCollisionKey, - "durable-collision", - JSON.stringify({ sessionId: "durable-collision", updatedAt: Date.now() }), - Date.now(), + JSON.stringify({ sessionId: "durable-collision", updatedAt: durableCollisionUpdatedAt }), + durableCollisionUpdatedAt, ); + persistentDatabase.db + .prepare( + "INSERT INTO session_windows (session_id, session_key, session_scope, created_at, updated_at) VALUES ('durable-collision', ?, 'conversation', ?, ?)", + ) + .run(durableCollisionKey, durableCollisionUpdatedAt, durableCollisionUpdatedAt); const rejectedExplicitDashboard = await directSessionReq("sessions.create", { agentId: "main", key: durableCollisionKey, @@ -386,7 +386,7 @@ test("incognito sessions survive non-default-agent webchat reply initialization" }); expect( persistentDatabase.db - .prepare("SELECT session_key FROM session_entries WHERE session_key = ?") + .prepare("SELECT session_key FROM session_nodes WHERE session_key = ?") .get(sessionKey), ).toBeUndefined(); } finally { diff --git a/src/gateway/session-compaction-checkpoints.ts b/src/gateway/session-compaction-checkpoints.ts index 788280d405bf..aaba18777d25 100644 --- a/src/gateway/session-compaction-checkpoints.ts +++ b/src/gateway/session-compaction-checkpoints.ts @@ -739,7 +739,7 @@ function shouldRouteCheckpointSessionMutationToSqlite(params: { * * The branch/restore operations own the transcript fork plus session entry * update so a SQLite implementation can copy transcript rows and update - * `session_entries.entry_json` inside one write transaction. + * `session_nodes.entry_json` inside one write transaction. */ export function createFileBackedCompactionCheckpointStore(): CompactionCheckpointStore { return { diff --git a/src/gateway/sessions-history-http.test.ts b/src/gateway/sessions-history-http.test.ts index 3fd9bae3550a..df6045d633a2 100644 --- a/src/gateway/sessions-history-http.test.ts +++ b/src/gateway/sessions-history-http.test.ts @@ -43,7 +43,7 @@ afterEach(async () => { const AGENT_ID = "main"; type SessionHistoryTestDatabase = Pick< OpenClawAgentKyselyDatabase, - "session_entries" | "session_routes" | "sessions" + "session_nodes" | "session_windows" >; async function createSessionStoreFile(): Promise { @@ -117,7 +117,28 @@ function seedRawSessionRows(params: { executeSqliteQuerySync( database.db, db - .insertInto("sessions") + .insertInto("session_nodes") + .values({ + current_session_id: row.sessionId, + entry_json: JSON.stringify({ + sessionId: row.sessionId, + updatedAt: row.updatedAt, + }), + session_key: row.sessionKey, + updated_at: row.updatedAt, + }) + .onConflict((conflict) => + conflict.column("session_key").doUpdateSet({ + current_session_id: (eb) => eb.ref("excluded.current_session_id"), + entry_json: (eb) => eb.ref("excluded.entry_json"), + updated_at: (eb) => eb.ref("excluded.updated_at"), + }), + ), + ); + executeSqliteQuerySync( + database.db, + db + .insertInto("session_windows") .values({ session_id: row.sessionId, session_key: row.sessionKey, @@ -131,43 +152,6 @@ function seedRawSessionRows(params: { }), ), ); - executeSqliteQuerySync( - database.db, - db - .insertInto("session_routes") - .values({ - session_key: row.sessionKey, - session_id: row.sessionId, - updated_at: row.updatedAt, - }) - .onConflict((conflict) => - conflict.column("session_key").doUpdateSet({ - session_id: (eb) => eb.ref("excluded.session_id"), - updated_at: (eb) => eb.ref("excluded.updated_at"), - }), - ), - ); - executeSqliteQuerySync( - database.db, - db - .insertInto("session_entries") - .values({ - session_id: row.sessionId, - session_key: row.sessionKey, - entry_json: JSON.stringify({ - sessionId: row.sessionId, - updatedAt: row.updatedAt, - }), - updated_at: row.updatedAt, - }) - .onConflict((conflict) => - conflict.column("session_key").doUpdateSet({ - session_id: (eb) => eb.ref("excluded.session_id"), - entry_json: (eb) => eb.ref("excluded.entry_json"), - updated_at: (eb) => eb.ref("excluded.updated_at"), - }), - ), - ); } }, { agentId: AGENT_ID, path: databasePath }, @@ -474,7 +458,7 @@ describe("session history HTTP endpoints", () => { expect(body.messages).toHaveLength(1); expect(body.messages?.[0]?.content?.[0]?.text).toBe("hello from history"); expectOpenClawMetadata(body.messages?.[0]?.["__openclaw"], { - seq: 2, + seq: 1, }); }); }); @@ -614,7 +598,6 @@ describe("session history HTTP endpoints", () => { }); test("prefers the freshest duplicate row for direct history reads", async () => { - testState.agentsConfig = { list: [{ id: "main", default: true }] }; testState.sessionConfig = { mainKey: "work" }; const storePath = await createSessionStoreFile(); await replaceTranscriptEvents( @@ -691,9 +674,9 @@ describe("session history HTTP endpoints", () => { "second message", "third message", ]); - expect(firstBody.messages?.map((message) => message["__openclaw"]?.seq)).toEqual([3, 4]); + expect(firstBody.messages?.map((message) => message["__openclaw"]?.seq)).toEqual([2, 3]); expect(firstBody.hasMore).toBe(true); - expect(firstBody.nextCursor).toBe("3"); + expect(firstBody.nextCursor).toBe("2"); const secondPage = await fetchSessionHistory(harness.port, "agent:main:main", { query: `?limit=2&cursor=${encodeURIComponent(firstBody.nextCursor ?? "")}`, @@ -703,7 +686,7 @@ describe("session history HTTP endpoints", () => { expect(secondBody.items?.map((message) => message.content?.[0]?.text)).toEqual([ "first message", ]); - expect(secondBody.messages?.map((message) => message["__openclaw"]?.seq)).toEqual([2]); + expect(secondBody.messages?.map((message) => message["__openclaw"]?.seq)).toEqual([1]); expect(secondBody.hasMore).toBe(false); expect(secondBody.nextCursor).toBeUndefined(); }); @@ -799,7 +782,7 @@ describe("session history HTTP endpoints", () => { expect(nextData.messages?.[0]?.content?.[0]?.text).toBe("third message"); expectOpenClawMetadata(nextData.messages?.[0]?.["__openclaw"], { id: thirdMessageId, - seq: 4, + seq: 3, }); await stream.reader.cancel(); @@ -825,7 +808,7 @@ describe("session history HTTP endpoints", () => { messages?: Array<{ content?: Array<{ text?: string }>; __openclaw?: { seq?: number } }>; }; expect(refreshData.messages?.[0]?.content?.[0]?.text).toBe("second message"); - expect(refreshData.messages?.[0]?.["__openclaw"]?.seq).toBe(3); + expect(refreshData.messages?.[0]?.["__openclaw"]?.seq).toBe(2); await stream.reader.cancel(); }); @@ -889,7 +872,7 @@ describe("session history HTTP endpoints", () => { expect(body.messages?.[0]?.content?.[0]?.text).toBe("Done."); expectOpenClawMetadata(body.messages?.[0]?.["__openclaw"], { id: visibleMessageId, - seq: 3, + seq: 2, }); }); }); @@ -905,7 +888,7 @@ describe("session history HTTP endpoints", () => { }); await expectMessageEventMatch(stream, { text: "second message", - seq: 3, + seq: 2, id: appendedId, }); }); @@ -1006,7 +989,7 @@ describe("session history HTTP endpoints", () => { await expectMessageEventMatch(stream, { text: "third visible message", - seq: 4, + seq: 3, }); }); }); @@ -1029,7 +1012,7 @@ describe("session history HTTP endpoints", () => { }); await expectMessageEventMatch(stream, { text: "third visible message", - seq: 4, + seq: 3, id: visibleId, }); }); @@ -1047,7 +1030,7 @@ describe("session history HTTP endpoints", () => { await expectMessageEventMatch(stream, { text: "second visible message", - seq: 3, + seq: 2, }); await appendTranscriptMessage({ sessionKey: "agent:main:main", @@ -1065,7 +1048,7 @@ describe("session history HTTP endpoints", () => { }); await expectMessageEventMatch(stream, { text: "third visible message", - seq: 5, + seq: 4, id: thirdId, }); }); @@ -1149,7 +1132,7 @@ describe("session history HTTP endpoints", () => { await expectMessageEventMatch(stream, { text: "bearer sse update", - seq: 3, + seq: 2, id: appendedId, }); diff --git a/src/infra/heartbeat-outcome-store.test.ts b/src/infra/heartbeat-outcome-store.test.ts index d14fe52b8358..e49772cfa007 100644 --- a/src/infra/heartbeat-outcome-store.test.ts +++ b/src/infra/heartbeat-outcome-store.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { upsertSessionEntry } from "../config/sessions/session-accessor.js"; import { closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, @@ -15,10 +16,15 @@ import { const tempDirs: string[] = []; -function createEnv(): NodeJS.ProcessEnv { +async function createEnv(): Promise { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-heartbeat-outcome-")); tempDirs.push(stateDir); - return { OPENCLAW_STATE_DIR: stateDir }; + const env = { OPENCLAW_STATE_DIR: stateDir }; + await upsertSessionEntry( + { agentId: "main", env, sessionKey: "agent:main:main" }, + { sessionId: "heartbeat-outcome-test", updatedAt: 1 }, + ); + return env; } afterEach(() => { @@ -30,8 +36,8 @@ afterEach(() => { }); describe("heartbeat outcome store", () => { - it("keeps one bounded typed outcome per base session with provenance", () => { - const env = createEnv(); + it("keeps one bounded typed outcome per base session with provenance", async () => { + const env = await createEnv(); persistHeartbeatOutcome({ agentId: "main", sessionKey: "agent:main:main", @@ -75,8 +81,8 @@ describe("heartbeat outcome store", () => { ); }); - it("replaces older state and ignores visible or no-change responses", () => { - const env = createEnv(); + it("replaces older state and ignores visible or no-change responses", async () => { + const env = await createEnv(); const base = { agentId: "main", sessionKey: "agent:main:main", @@ -119,8 +125,8 @@ describe("heartbeat outcome store", () => { ).toEqual({ count: 1 }); }); - it("injects once per user run, keeps retries, and resets after a new heartbeat", () => { - const env = createEnv(); + it("injects once per user run, keeps retries, and resets after a new heartbeat", async () => { + const env = await createEnv(); const base = { agentId: "main", sessionKey: "agent:main:main", diff --git a/src/infra/heartbeat-outcome-store.ts b/src/infra/heartbeat-outcome-store.ts index 38b3419b88c1..d6d5d3b27332 100644 --- a/src/infra/heartbeat-outcome-store.ts +++ b/src/infra/heartbeat-outcome-store.ts @@ -1,6 +1,10 @@ import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { Insertable, Selectable } from "kysely"; import type { HeartbeatToolResponse } from "../auto-reply/heartbeat-tool-response.js"; +import { + resolveSqliteScope, + toDatabaseOptions, +} from "../config/sessions/session-accessor.sqlite-scope.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js"; import { runOpenClawAgentWriteTransaction } from "../state/openclaw-agent-db.js"; import type { HeartbeatWakeSource } from "./heartbeat-wake.js"; @@ -88,6 +92,7 @@ function rowToOutcome(row: HeartbeatOutcomeRow): PersistedHeartbeatOutcome | und export function persistHeartbeatOutcome(params: { agentId: string; sessionKey: string; + storePath?: string; runSessionKey: string; response: HeartbeatToolResponse; taskNames?: readonly string[]; @@ -147,7 +152,7 @@ export function persistHeartbeatOutcome(params: { ), ); }, - { agentId: params.agentId, env: params.env }, + toDatabaseOptions(resolveSqliteScope(params)), { operationLabel: "heartbeat.outcome.persist" }, ); } @@ -156,6 +161,7 @@ export function persistHeartbeatOutcome(params: { export function claimHeartbeatOutcomeForRun(params: { agentId: string; sessionKey: string; + storePath?: string; runId: string; env?: NodeJS.ProcessEnv; }): PersistedHeartbeatOutcome | undefined { @@ -187,7 +193,7 @@ export function claimHeartbeatOutcomeForRun(params: { } return rowToOutcome(row); }, - { agentId: params.agentId, env: params.env }, + toDatabaseOptions(resolveSqliteScope(params)), { operationLabel: "heartbeat.outcome.claim" }, ); } diff --git a/src/infra/heartbeat-runner.tool-response.test.ts b/src/infra/heartbeat-runner.tool-response.test.ts index eba754fcd481..03a12964daa6 100644 --- a/src/infra/heartbeat-runner.tool-response.test.ts +++ b/src/infra/heartbeat-runner.tool-response.test.ts @@ -313,7 +313,12 @@ describe("runHeartbeatOnce heartbeat response tool", () => { }); expect( - claimHeartbeatOutcomeForRun({ agentId: "main", sessionKey, runId: "user-run" }), + claimHeartbeatOutcomeForRun({ + agentId: "main", + sessionKey, + storePath, + runId: "user-run", + }), ).toMatchObject({ outcome: "progress", summary: "Deployment completed; smoke test pending.", diff --git a/src/infra/heartbeat-runner.ts b/src/infra/heartbeat-runner.ts index 1e246ea6c00f..388bdee7636a 100644 --- a/src/infra/heartbeat-runner.ts +++ b/src/infra/heartbeat-runner.ts @@ -1865,6 +1865,7 @@ export async function runHeartbeatOnce(opts: { persistHeartbeatOutcome({ agentId, sessionKey, + storePath, runSessionKey, response: heartbeatToolResponse, taskNames: dueHeartbeatTasks.map((task) => task.name), diff --git a/src/state/openclaw-agent-db-contract.ts b/src/state/openclaw-agent-db-contract.ts index 2a136c3b5948..4691196cc76a 100644 --- a/src/state/openclaw-agent-db-contract.ts +++ b/src/state/openclaw-agent-db-contract.ts @@ -2,7 +2,8 @@ import type { DatabaseSync } from "node:sqlite"; import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db.js"; -// v13 = one durable generation token per raw session transcript. +// v14 = logical session nodes, generation windows, and node-owned artifact FKs. +// v13 = one durable rewrite watermark per raw session transcript. // v12 = session-owned ACP parent-stream events. // v11 = agent-scoped runtime leases, durable delivery operations, canonical // external conversation addresses, and bounded per-session heartbeat outcome context. @@ -14,7 +15,7 @@ import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db.js"; // The v4 session/transcript flip and main's v2 memory-identity // change is folded in structure-gated migrations, so v2 main DBs and // pre-merge v4 flip DBs both converge on this schema. -export const OPENCLAW_AGENT_SCHEMA_VERSION = 13; +export const OPENCLAW_AGENT_SCHEMA_VERSION = 14; /** Open per-agent SQLite database handle plus lifecycle maintenance. */ export type OpenClawAgentDatabase = { diff --git a/src/state/openclaw-agent-db-schema.ts b/src/state/openclaw-agent-db-schema.ts index b24dd0e1cea2..200d761acf12 100644 --- a/src/state/openclaw-agent-db-schema.ts +++ b/src/state/openclaw-agent-db-schema.ts @@ -30,6 +30,7 @@ import { migrateSessionEntryStatusProjection, readSqliteTableColumns, } from "./openclaw-agent-db-session-migrations.js"; +import { migrateSessionNodesAndWindows } from "./openclaw-agent-db-session-nodes-migration.js"; import { addSessionProvenanceColumns, backfillSessionEntryProvenance, @@ -258,7 +259,7 @@ function migrateSessionTranscriptGenerations(db: DatabaseSync, previousVersion: return; } db.prepare( - `INSERT OR IGNORE INTO session_transcript_generations (session_id, generation, updated_at) + `INSERT OR IGNORE INTO transcript_rewrite_watermarks (session_id, generation, updated_at) SELECT session_id, lower(hex(randomblob(16))), ? FROM transcript_events GROUP BY session_id`, @@ -395,6 +396,19 @@ function backfillOpenClawAgentSchema(db: DatabaseSync, previousVersion: number): if (previousVersion >= 2) { return; } + if (!readSqliteTableColumns(db, "session_entries") || !readSqliteTableColumns(db, "sessions")) { + return; + } + if (!readSqliteTableColumns(db, "session_routes")) { + db.exec(` + CREATE TABLE session_routes ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ); + `); + } db.exec(` INSERT OR REPLACE INTO session_routes (session_key, session_id, updated_at) SELECT se.session_key, se.session_id, se.updated_at @@ -491,10 +505,9 @@ export function assertAgentDatabaseIntegrityBeforeMutation( function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): void { // FK enforcement must be off before BEGIN: PRAGMA foreign_keys is a silent - // no-op inside a transaction, and the v1 sessions rebuild would otherwise - // cascade-delete session_entries when the old parent table drops. The - // connection pragmas restore enforcement for steady-state work below. - db.exec("PRAGMA foreign_keys = OFF;"); + // no-op inside a transaction, and legacy owner-table rebuilds would otherwise + // cascade-delete their children. Steady-state enforcement is restored below. + db.exec("PRAGMA foreign_keys = OFF; PRAGMA legacy_alter_table = OFF;"); try { runSqliteImmediateTransactionSync(db, () => { // Repeat preflight ownership/version gates inside the write transaction; @@ -513,13 +526,19 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): dropLegacySessionTranscriptSearchSchema(db); migrateMemoryIndexSourcesIdentity(db); migrateOpenClawAgentSchema(db); + migrateConversationDeliveryTargetColumn(db); + backfillOpenClawAgentSchema(db, previousVersion); + if (previousVersion < 11) { + backfillSessionConversations(db); + } + backfillSessionEntryProvenance(db, previousVersion); + migrateSessionNodesAndWindows(db, previousVersion); db.exec( previousVersion === OPENCLAW_AGENT_SCHEMA_VERSION ? AGENT_SCHEMA_WITHOUT_LAZY_SURFACES_SQL : OPENCLAW_AGENT_SCHEMA_SQL, ); migrateSessionTranscriptGenerations(db, previousVersion); - migrateConversationDeliveryTargetColumn(db); migrateSessionTranscriptActiveProjection(db, previousVersion); if (previousVersion < 11) { migrateSqliteSchemaToStrictInTransaction(db, OPENCLAW_AGENT_SCHEMA_SQL, { @@ -527,11 +546,6 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): }); } repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_AGENT_CANONICAL_UNIQUE_INDEXES); - backfillOpenClawAgentSchema(db, previousVersion); - if (previousVersion < 11) { - backfillSessionConversations(db); - } - backfillSessionEntryProvenance(db, previousVersion); const kysely = getNodeSqliteKysely(db); db.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION};`); const now = Date.now(); diff --git a/src/state/openclaw-agent-db-session-migrations.ts b/src/state/openclaw-agent-db-session-migrations.ts index 4f1326a11e1b..42c261129fea 100644 --- a/src/state/openclaw-agent-db-session-migrations.ts +++ b/src/state/openclaw-agent-db-session-migrations.ts @@ -129,6 +129,27 @@ function migratedConversation( /** Backfills canonical external addresses once when conversation routing becomes active. */ export function backfillSessionConversations(db: DatabaseSync): void { + if ( + !readSqliteTableColumns(db, "session_entries") || + !readSqliteTableColumns(db, "sessions") || + !readSqliteTableColumns(db, "conversations") + ) { + return; + } + if (!readSqliteTableColumns(db, "session_conversations")) { + db.exec(` + CREATE TABLE session_conversations ( + session_id TEXT NOT NULL, + conversation_id TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'primary' CHECK (role IN ('primary', 'participant', 'related')), + first_seen_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (session_id, conversation_id, role), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, + FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE + ); + `); + } // Earlier schemas did not retain an exact delivery target. Remove their // derived projection, then rebuild only addresses recoverable from sessions. db.exec(` diff --git a/src/state/openclaw-agent-db-session-nodes-migration.ts b/src/state/openclaw-agent-db-session-nodes-migration.ts new file mode 100644 index 000000000000..0ac48a2fba56 --- /dev/null +++ b/src/state/openclaw-agent-db-session-nodes-migration.ts @@ -0,0 +1,423 @@ +import type { DatabaseSync } from "node:sqlite"; +import { readSqliteTableColumns } from "./openclaw-agent-db-session-migrations.js"; + +const SESSION_NODE_SCHEMA_VERSION = 14; + +function migratedColumn( + columns: ReadonlySet, + columnName: string, + fallback: string, +): string { + return columns.has(columnName) ? columnName : fallback; +} + +function jsonText(path: string): string { + return `CASE + WHEN json_valid(entry_json) AND json_type(entry_json, '${path}') = 'text' + THEN NULLIF(trim(CAST(json_extract(entry_json, '${path}') AS TEXT)), '') + ELSE NULL + END`; +} + +function jsonNumber(path: string): string { + return `CASE + WHEN json_valid(entry_json) AND json_type(entry_json, '${path}') IN ('integer', 'real') + THEN CAST(json_extract(entry_json, '${path}') AS INTEGER) + ELSE NULL + END`; +} + +function createSessionNodes(db: DatabaseSync): void { + db.exec(` + CREATE TABLE IF NOT EXISTS session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + created_at INTEGER, + created_via TEXT CHECK (created_via IS NULL OR created_via IN ('operator', 'spawn', 'channel', 'cron', 'talk', 'run', 'plugin', 'internal')), + created_actor_type TEXT CHECK (created_actor_type IS NULL OR created_actor_type IN ('human', 'agent', 'system')), + created_actor_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + fork_source_session_key TEXT, + fork_source_session_id TEXT, + fork_source_entry_id TEXT, + label TEXT, + display_name TEXT, + category TEXT, + icon TEXT, + pinned_at INTEGER, + archived_at INTEGER, + last_read_at INTEGER, + last_interaction_at INTEGER, + last_activity_at INTEGER + ) STRICT; + `); +} + +function backfillSessionNodes(db: DatabaseSync): void { + const entryColumns = readSqliteTableColumns(db, "session_entries"); + if (entryColumns) { + const status = migratedColumn(entryColumns, "status", "NULL"); + db.exec(` + INSERT OR REPLACE INTO session_nodes ( + session_key, current_session_id, entry_json, updated_at, status, + created_at, created_via, created_actor_type, created_actor_id, + parent_session_key, spawned_by, fork_source_session_key, + fork_source_session_id, fork_source_entry_id, label, display_name, + category, icon, pinned_at, archived_at, last_read_at, + last_interaction_at, last_activity_at + ) + SELECT + session_key, + session_id, + entry_json, + updated_at, + ${status}, + ${jsonNumber("$.createdAt")}, + CASE + WHEN json_valid(entry_json) + AND json_extract(entry_json, '$.createdVia') IN + ('operator', 'spawn', 'channel', 'cron', 'talk', 'run', 'plugin', 'internal') + THEN json_extract(entry_json, '$.createdVia') + ELSE NULL + END, + CASE + WHEN json_valid(entry_json) + AND json_extract(entry_json, '$.createdActor.type') IN ('human', 'agent', 'system') + THEN json_extract(entry_json, '$.createdActor.type') + WHEN ${jsonText("$.createdBy.id")} IS NOT NULL THEN 'human' + ELSE NULL + END, + COALESCE(${jsonText("$.createdActor.id")}, ${jsonText("$.createdBy.id")}), + COALESCE(${jsonText("$.parentSessionKey")}, ${jsonText("$.spawnedBy")}), + ${jsonText("$.spawnedBy")}, + ${jsonText("$.forkSource.sessionKey")}, + ${jsonText("$.forkSource.sessionId")}, + ${jsonText("$.forkSource.entryId")}, + ${jsonText("$.label")}, + ${jsonText("$.displayName")}, + ${jsonText("$.category")}, + ${jsonText("$.icon")}, + ${jsonNumber("$.pinnedAt")}, + ${jsonNumber("$.archivedAt")}, + ${jsonNumber("$.lastReadAt")}, + ${jsonNumber("$.lastInteractionAt")}, + ${jsonNumber("$.lastActivityAt")} + FROM session_entries; + `); + } + + const routeColumns = readSqliteTableColumns(db, "session_routes"); + if (routeColumns) { + db.exec(` + INSERT OR IGNORE INTO session_nodes ( + session_key, current_session_id, entry_json, updated_at + ) + SELECT session_key, session_id, '{}', updated_at + FROM session_routes; + `); + } + + // Legacy history can contain a generation whose key has neither a live entry + // nor a route. It still needs one node owner so the flipped FK can retain it. + db.exec(` + INSERT OR IGNORE INTO session_nodes ( + session_key, current_session_id, entry_json, updated_at + ) + SELECT session_key, session_id, '{}', updated_at + FROM sessions; + `); +} + +function migrateSessionWindows(db: DatabaseSync): void { + const columns = readSqliteTableColumns(db, "sessions"); + if (!columns) { + return; + } + const entryColumns = readSqliteTableColumns(db, "session_entries"); + const routeColumns = readSqliteTableColumns(db, "session_routes"); + const entryOwner = entryColumns + ? `(SELECT se.session_key + FROM session_entries AS se + WHERE se.session_id = session_windows.session_id + ORDER BY CASE WHEN se.session_key = session_windows.session_key THEN 0 ELSE 1 END, + se.updated_at DESC, + se.session_key ASC + LIMIT 1)` + : "NULL"; + const routeOwner = routeColumns + ? `(SELECT sr.session_key + FROM session_routes AS sr + WHERE sr.session_id = session_windows.session_id + ORDER BY CASE WHEN sr.session_key = session_windows.session_key THEN 0 ELSE 1 END, + sr.updated_at DESC, + sr.session_key ASC + LIMIT 1)` + : "NULL"; + const currentEntryJson = entryColumns + ? `(SELECT se.entry_json + FROM session_entries AS se + WHERE se.session_id = session_windows.session_id + ORDER BY CASE WHEN se.session_key = session_windows.session_key THEN 0 ELSE 1 END, + se.updated_at DESC, + se.session_key ASC + LIMIT 1)` + : "NULL"; + + // SQLite rewrites child FK targets on RENAME even while enforcement is off. + // Rebuilding under the renamed owner keeps every transcript child attached. + db.exec("ALTER TABLE sessions RENAME TO session_windows;"); + db.exec(` + DROP TABLE IF EXISTS session_windows_new; + CREATE TABLE session_windows_new ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + previous_session_id TEXT, + reason TEXT CHECK (reason IS NULL OR reason IN ('initial', 'reset', 'rollover', 'fork', 'rewind', 'switch', 'recovery', 'compaction')), + session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + transcript_updated_at INTEGER DEFAULT NULL, + transcript_observed_at INTEGER DEFAULT NULL, + session_entry_provenance INTEGER NOT NULL DEFAULT 0 CHECK (session_entry_provenance IN (0, 1)), + acp_owned INTEGER NOT NULL DEFAULT 0 CHECK (acp_owned IN (0, 1)), + plugin_owner_id TEXT, + hook_external_content_source TEXT CHECK (hook_external_content_source IS NULL OR hook_external_content_source IN ('gmail', 'webhook')), + started_at INTEGER, + ended_at INTEGER, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + chat_type TEXT CHECK (chat_type IS NULL OR chat_type IN ('direct', 'group', 'channel')), + channel TEXT, + account_id TEXT, + primary_conversation_id TEXT, + model_provider TEXT, + model TEXT, + agent_harness_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + display_name TEXT, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE, + FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL + ) STRICT; + INSERT INTO session_windows_new ( + session_id, session_key, previous_session_id, reason, session_scope, + created_at, updated_at, transcript_updated_at, transcript_observed_at, + session_entry_provenance, acp_owned, plugin_owner_id, + hook_external_content_source, started_at, ended_at, status, chat_type, + channel, account_id, primary_conversation_id, model_provider, model, + agent_harness_id, parent_session_key, spawned_by, display_name + ) + SELECT + session_id, + COALESCE(${entryOwner}, ${routeOwner}, session_key), + CASE + WHEN json_valid(${currentEntryJson}) + THEN NULLIF(trim(CAST(json_extract(${currentEntryJson}, '$.previousSessionId') AS TEXT)), '') + ELSE NULL + END, + NULL, + ${migratedColumn(columns, "session_scope", "'conversation'")}, + created_at, + updated_at, + ${migratedColumn(columns, "transcript_updated_at", "NULL")}, + ${migratedColumn(columns, "transcript_observed_at", "NULL")}, + ${migratedColumn(columns, "session_entry_provenance", "0")}, + ${migratedColumn(columns, "acp_owned", "0")}, + ${migratedColumn(columns, "plugin_owner_id", "NULL")}, + ${migratedColumn(columns, "hook_external_content_source", "NULL")}, + ${migratedColumn(columns, "started_at", "NULL")}, + ${migratedColumn(columns, "ended_at", "NULL")}, + ${migratedColumn(columns, "status", "NULL")}, + ${migratedColumn(columns, "chat_type", "NULL")}, + ${migratedColumn(columns, "channel", "NULL")}, + ${migratedColumn(columns, "account_id", "NULL")}, + ${migratedColumn(columns, "primary_conversation_id", "NULL")}, + ${migratedColumn(columns, "model_provider", "NULL")}, + ${migratedColumn(columns, "model", "NULL")}, + ${migratedColumn(columns, "agent_harness_id", "NULL")}, + ${migratedColumn(columns, "parent_session_key", "NULL")}, + ${migratedColumn(columns, "spawned_by", "NULL")}, + ${migratedColumn(columns, "display_name", "NULL")} + FROM session_windows; + DROP TABLE session_windows; + ALTER TABLE session_windows_new RENAME TO session_windows; + `); +} + +function renameTranscriptRewriteWatermarks(db: DatabaseSync): void { + if ( + readSqliteTableColumns(db, "session_transcript_generations") && + !readSqliteTableColumns(db, "transcript_rewrite_watermarks") + ) { + db.exec("ALTER TABLE session_transcript_generations RENAME TO transcript_rewrite_watermarks;"); + } +} + +function rebuildBoardTabs(db: DatabaseSync): void { + const columns = readSqliteTableColumns(db, "board_tabs"); + if (!columns) { + return; + } + if (readSqliteTableColumns(db, "board_widgets")) { + db.exec(` + DELETE FROM board_widgets + WHERE NOT EXISTS ( + SELECT 1 FROM session_nodes WHERE session_nodes.session_key = board_widgets.session_key + ); + `); + } + db.exec(` + DROP TABLE IF EXISTS board_tabs_new; + CREATE TABLE board_tabs_new ( + session_key TEXT NOT NULL, + tab_id TEXT NOT NULL, + title TEXT NOT NULL, + position INTEGER NOT NULL CHECK (position >= 0), + chat_dock TEXT NOT NULL DEFAULT 'right' CHECK (chat_dock IN ('left', 'right', 'bottom', 'hidden')), + created_by TEXT NOT NULL CHECK (created_by IN ('user', 'agent')), + revision INTEGER NOT NULL CHECK (revision >= 0), + PRIMARY KEY (session_key, tab_id), + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE + ) STRICT; + INSERT INTO board_tabs_new ( + session_key, tab_id, title, position, chat_dock, created_by, revision + ) + SELECT b.session_key, b.tab_id, b.title, b.position, b.chat_dock, b.created_by, b.revision + FROM board_tabs AS b + INNER JOIN session_nodes AS n ON n.session_key = b.session_key; + DROP TABLE board_tabs; + ALTER TABLE board_tabs_new RENAME TO board_tabs; + `); +} + +function rebuildHeartbeatOutcomes(db: DatabaseSync): void { + const columns = readSqliteTableColumns(db, "heartbeat_outcomes"); + if (!columns) { + return; + } + db.exec(` + DROP TABLE IF EXISTS heartbeat_outcomes_new; + CREATE TABLE heartbeat_outcomes_new ( + session_key TEXT NOT NULL PRIMARY KEY, + run_session_key TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('progress', 'done', 'blocked', 'needs_attention')), + summary TEXT NOT NULL, + response_reason TEXT, + priority TEXT CHECK (priority IS NULL OR priority IN ('low', 'normal', 'high')), + next_check TEXT, + task_names_json TEXT, + wake_source TEXT, + wake_reason TEXT, + occurred_at INTEGER NOT NULL, + context_run_id TEXT, + context_claimed_at INTEGER, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE + ) STRICT; + INSERT INTO heartbeat_outcomes_new ( + session_key, run_session_key, outcome, summary, response_reason, priority, + next_check, task_names_json, wake_source, wake_reason, occurred_at, + context_run_id, context_claimed_at, updated_at + ) + SELECT + h.session_key, + h.run_session_key, + h.outcome, + h.summary, + ${migratedColumn(columns, "response_reason", "NULL")}, + ${migratedColumn(columns, "priority", "NULL")}, + ${migratedColumn(columns, "next_check", "NULL")}, + ${migratedColumn(columns, "task_names_json", "NULL")}, + ${migratedColumn(columns, "wake_source", "NULL")}, + ${migratedColumn(columns, "wake_reason", "NULL")}, + h.occurred_at, + ${migratedColumn(columns, "context_run_id", "NULL")}, + ${migratedColumn(columns, "context_claimed_at", "NULL")}, + h.updated_at + FROM heartbeat_outcomes AS h + INNER JOIN session_nodes AS n ON n.session_key = h.session_key; + DROP TABLE heartbeat_outcomes; + ALTER TABLE heartbeat_outcomes_new RENAME TO heartbeat_outcomes; + `); +} + +function rebuildSessionMembers(db: DatabaseSync): void { + if (!readSqliteTableColumns(db, "session_members")) { + return; + } + db.exec(` + DROP TABLE IF EXISTS session_members_new; + CREATE TABLE session_members_new ( + session_key TEXT NOT NULL, + identity_id TEXT NOT NULL, + added_by TEXT NOT NULL, + added_at INTEGER NOT NULL, + PRIMARY KEY (session_key, identity_id), + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE + ) STRICT; + INSERT INTO session_members_new (session_key, identity_id, added_by, added_at) + SELECT m.session_key, m.identity_id, m.added_by, m.added_at + FROM session_members AS m + INNER JOIN session_nodes AS n ON n.session_key = m.session_key; + DROP TABLE session_members; + ALTER TABLE session_members_new RENAME TO session_members; + `); +} + +function rebuildTranscriptIndexState(db: DatabaseSync): void { + const columns = readSqliteTableColumns(db, "session_transcript_index_state"); + if (!columns) { + return; + } + db.exec(` + DROP TABLE IF EXISTS session_transcript_index_state_new; + CREATE TABLE session_transcript_index_state_new ( + session_id TEXT NOT NULL PRIMARY KEY, + indexed_seq INTEGER NOT NULL, + leaf_event_id TEXT, + needs_rebuild INTEGER NOT NULL DEFAULT 0, + active_event_count INTEGER NOT NULL DEFAULT 0, + active_message_count INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES session_windows(session_id) ON DELETE CASCADE + ) STRICT; + INSERT INTO session_transcript_index_state_new ( + session_id, indexed_seq, leaf_event_id, needs_rebuild, + active_event_count, active_message_count, updated_at + ) + SELECT + i.session_id, + i.indexed_seq, + ${migratedColumn(columns, "leaf_event_id", "NULL")}, + ${migratedColumn(columns, "needs_rebuild", "0")}, + ${migratedColumn(columns, "active_event_count", "0")}, + ${migratedColumn(columns, "active_message_count", "0")}, + i.updated_at + FROM session_transcript_index_state AS i + INNER JOIN session_windows AS w ON w.session_id = i.session_id; + DROP TABLE session_transcript_index_state; + ALTER TABLE session_transcript_index_state_new RENAME TO session_transcript_index_state; + `); +} + +/** Replace split entry/route roots with logical nodes and generation windows. */ +export function migrateSessionNodesAndWindows(db: DatabaseSync, previousVersion: number): void { + if (previousVersion >= SESSION_NODE_SCHEMA_VERSION || !readSqliteTableColumns(db, "sessions")) { + return; + } + createSessionNodes(db); + backfillSessionNodes(db); + migrateSessionWindows(db); + renameTranscriptRewriteWatermarks(db); + rebuildBoardTabs(db); + rebuildHeartbeatOutcomes(db); + rebuildSessionMembers(db); + rebuildTranscriptIndexState(db); + db.exec(` + DROP TABLE IF EXISTS session_routes; + DROP TABLE IF EXISTS session_entries; + `); +} diff --git a/src/state/openclaw-agent-db-session-provenance.ts b/src/state/openclaw-agent-db-session-provenance.ts index bc9cc60d2d4a..6d0aa9e69461 100644 --- a/src/state/openclaw-agent-db-session-provenance.ts +++ b/src/state/openclaw-agent-db-session-provenance.ts @@ -48,6 +48,15 @@ export function backfillSessionEntryProvenance(db: DatabaseSync, previousVersion if (previousVersion >= 8) { return; } + const hasSessionEntries = db + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'session_entries'") + .get(); + const hasSessions = db + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'sessions'") + .get(); + if (!hasSessionEntries || !hasSessions) { + return; + } const rows = db .prepare( `SELECT se.session_id, se.entry_json diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 82cf9bf6db16..2e35d6f3765a 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -185,14 +185,6 @@ export interface SessionConversations { session_id: string; } -export interface SessionEntries { - entry_json: string; - session_id: string; - session_key: string; - status: string | null; - updated_at: number; -} - export interface SessionMembers { added_at: number; added_by: string; @@ -200,9 +192,29 @@ export interface SessionMembers { session_key: string; } -export interface SessionRoutes { - session_id: string; +export interface SessionNodes { + archived_at: number | null; + category: string | null; + created_actor_id: string | null; + created_actor_type: string | null; + created_at: number | null; + created_via: string | null; + current_session_id: string; + display_name: string | null; + entry_json: string; + fork_source_entry_id: string | null; + fork_source_session_id: string | null; + fork_source_session_key: string | null; + icon: string | null; + label: string | null; + last_activity_at: number | null; + last_interaction_at: number | null; + last_read_at: number | null; + parent_session_key: string | null; + pinned_at: number | null; session_key: string; + spawned_by: string | null; + status: string | null; updated_at: number; } @@ -251,12 +263,6 @@ export interface SessionTranscriptFtsIdx { term: string; } -export interface SessionTranscriptGenerations { - generation: string; - session_id: string; - updated_at: number; -} - export interface SessionTranscriptIndexState { active_event_count: Generated; active_message_count: Generated; @@ -267,7 +273,7 @@ export interface SessionTranscriptIndexState { updated_at: number; } -export interface Sessions { +export interface SessionWindows { account_id: string | null; acp_owned: Generated; agent_harness_id: string | null; @@ -281,7 +287,9 @@ export interface Sessions { model_provider: string | null; parent_session_key: string | null; plugin_owner_id: string | null; + previous_session_id: string | null; primary_conversation_id: string | null; + reason: string | null; session_entry_provenance: Generated; session_id: string; session_key: string; @@ -330,6 +338,12 @@ export interface TranscriptEvents { session_id: string; } +export interface TranscriptRewriteWatermarks { + generation: string; + session_id: string; + updated_at: number; +} + export interface DB { acp_parent_stream_events: AcpParentStreamEvents; auth_profile_state: AuthProfileState; @@ -347,9 +361,8 @@ export interface DB { memory_index_state: MemoryIndexState; schema_meta: SchemaMeta; session_conversations: SessionConversations; - session_entries: SessionEntries; session_members: SessionMembers; - session_routes: SessionRoutes; + session_nodes: SessionNodes; session_transcript_active_events: SessionTranscriptActiveEvents; session_transcript_fts: SessionTranscriptFts; session_transcript_fts_config: SessionTranscriptFtsConfig; @@ -357,11 +370,11 @@ export interface DB { session_transcript_fts_data: SessionTranscriptFtsData; session_transcript_fts_docsize: SessionTranscriptFtsDocsize; session_transcript_fts_idx: SessionTranscriptFtsIdx; - session_transcript_generations: SessionTranscriptGenerations; session_transcript_index_state: SessionTranscriptIndexState; - sessions: Sessions; + session_windows: SessionWindows; state_leases: StateLeases; trajectory_runtime_events: TrajectoryRuntimeEvents; transcript_event_identities: TranscriptEventIdentities; transcript_events: TranscriptEvents; + transcript_rewrite_watermarks: TranscriptRewriteWatermarks; } diff --git a/src/state/openclaw-agent-db.incognito.test.ts b/src/state/openclaw-agent-db.incognito.test.ts index fa4b1ebe0256..926146bd76d3 100644 --- a/src/state/openclaw-agent-db.incognito.test.ts +++ b/src/state/openclaw-agent-db.incognito.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js"; import { withOpenClawAgentDatabaseReadOnly } from "./openclaw-agent-db-readonly.js"; import { closeOpenClawAgentDatabasesForTest, @@ -66,7 +67,7 @@ describe("incognito agent database", () => { fs.rmSync(sentinel); const database = openOpenClawAgentDatabase({ agentId: "main", env, path: sentinel }); - expect(database.db.prepare("SELECT count(*) AS count FROM sessions").get()).toEqual({ + expect(database.db.prepare("SELECT count(*) AS count FROM session_nodes").get()).toEqual({ count: 0, }); expect(fs.existsSync(sentinel)).toBe(false); @@ -94,10 +95,12 @@ describe("incognito agent database", () => { ).toEqual({ found: true, value: true }); expect( first.db - .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'sessions'") + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'session_nodes'") .get(), - ).toEqual({ name: "sessions" }); - expect(first.db.prepare("PRAGMA user_version").get()).toEqual({ user_version: 13 }); + ).toEqual({ name: "session_nodes" }); + expect(first.db.prepare("PRAGMA user_version").get()).toEqual({ + user_version: OPENCLAW_AGENT_SCHEMA_VERSION, + }); expect(fs.existsSync(sentinel)).toBe(false); expect(fs.existsSync(path.dirname(sentinel))).toBe(false); }); diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 609d3266dfd6..2ebbc7362ade 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -70,6 +70,141 @@ function createTempStateDir(): string { return makeTempDir(agentDbTempDirs, "openclaw-agent-db-"); } +function downgradeCurrentAgentDatabaseToV13(databasePath: string): void { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec(` + PRAGMA foreign_keys = OFF; + PRAGMA legacy_alter_table = OFF; + DROP INDEX IF EXISTS idx_agent_session_windows_updated_at; + DROP INDEX IF EXISTS idx_agent_session_windows_created_at; + DROP INDEX IF EXISTS idx_agent_session_windows_conversation; + ALTER TABLE session_windows RENAME TO sessions; + CREATE TABLE sessions_legacy ( + session_id TEXT NOT NULL PRIMARY KEY, + session_key TEXT NOT NULL, + session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + transcript_updated_at INTEGER DEFAULT NULL, + transcript_observed_at INTEGER DEFAULT NULL, + session_entry_provenance INTEGER NOT NULL DEFAULT 0 CHECK (session_entry_provenance IN (0, 1)), + acp_owned INTEGER NOT NULL DEFAULT 0 CHECK (acp_owned IN (0, 1)), + plugin_owner_id TEXT, + hook_external_content_source TEXT CHECK (hook_external_content_source IS NULL OR hook_external_content_source IN ('gmail', 'webhook')), + started_at INTEGER, + ended_at INTEGER, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + chat_type TEXT CHECK (chat_type IS NULL OR chat_type IN ('direct', 'group', 'channel')), + channel TEXT, + account_id TEXT, + primary_conversation_id TEXT, + model_provider TEXT, + model TEXT, + agent_harness_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + display_name TEXT, + FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL + ) STRICT; + INSERT INTO sessions_legacy ( + session_id, session_key, session_scope, created_at, updated_at, + transcript_updated_at, transcript_observed_at, session_entry_provenance, + acp_owned, plugin_owner_id, hook_external_content_source, started_at, + ended_at, status, chat_type, channel, account_id, primary_conversation_id, + model_provider, model, agent_harness_id, parent_session_key, spawned_by, + display_name + ) + SELECT + session_id, session_key, session_scope, created_at, updated_at, + transcript_updated_at, transcript_observed_at, session_entry_provenance, + acp_owned, plugin_owner_id, hook_external_content_source, started_at, + ended_at, status, chat_type, channel, account_id, primary_conversation_id, + model_provider, model, agent_harness_id, parent_session_key, spawned_by, + display_name + FROM sessions; + DROP TABLE sessions; + ALTER TABLE sessions_legacy RENAME TO sessions; + CREATE INDEX idx_agent_sessions_updated_at ON sessions(updated_at DESC, session_id); + CREATE INDEX idx_agent_sessions_created_at ON sessions(created_at DESC, session_id); + CREATE INDEX idx_agent_sessions_conversation + ON sessions(primary_conversation_id, updated_at DESC, session_id) + WHERE primary_conversation_id IS NOT NULL; + CREATE TABLE session_routes ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ) STRICT; + CREATE INDEX idx_agent_session_routes_session_id ON session_routes(session_id); + CREATE TABLE session_entries ( + session_key TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + ) STRICT; + CREATE INDEX idx_agent_session_entries_updated_at + ON session_entries(updated_at DESC, session_key); + CREATE INDEX idx_agent_session_entries_session_updated + ON session_entries(session_id, updated_at DESC, session_key); + CREATE INDEX idx_agent_session_entries_status + ON session_entries(status, session_key) WHERE status IS NOT NULL; + CREATE TABLE session_members_v13 ( + session_key TEXT NOT NULL, + identity_id TEXT NOT NULL, + added_by TEXT NOT NULL, + added_at INTEGER NOT NULL, + PRIMARY KEY (session_key, identity_id), + FOREIGN KEY (session_key) REFERENCES session_entries(session_key) ON DELETE CASCADE + ) STRICT; + INSERT INTO session_members_v13 SELECT * FROM session_members; + DROP TABLE session_members; + ALTER TABLE session_members_v13 RENAME TO session_members; + CREATE INDEX idx_agent_session_members_identity + ON session_members(identity_id, session_key); + ALTER TABLE transcript_rewrite_watermarks RENAME TO session_transcript_generations; + CREATE TABLE board_tabs_v13 ( + session_key TEXT NOT NULL, + tab_id TEXT NOT NULL, + title TEXT NOT NULL, + position INTEGER NOT NULL CHECK (position >= 0), + chat_dock TEXT NOT NULL DEFAULT 'right' CHECK (chat_dock IN ('left', 'right', 'bottom', 'hidden')), + created_by TEXT NOT NULL CHECK (created_by IN ('user', 'agent')), + revision INTEGER NOT NULL CHECK (revision >= 0), + PRIMARY KEY (session_key, tab_id) + ) STRICT; + INSERT INTO board_tabs_v13 SELECT * FROM board_tabs; + DROP TABLE board_tabs; + ALTER TABLE board_tabs_v13 RENAME TO board_tabs; + DROP TABLE heartbeat_outcomes; + CREATE TABLE heartbeat_outcomes ( + session_key TEXT NOT NULL PRIMARY KEY, + run_session_key TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('progress', 'done', 'blocked', 'needs_attention')), + summary TEXT NOT NULL, + response_reason TEXT, + priority TEXT CHECK (priority IS NULL OR priority IN ('low', 'normal', 'high')), + next_check TEXT, + task_names_json TEXT, + wake_source TEXT, + wake_reason TEXT, + occurred_at INTEGER NOT NULL, + context_run_id TEXT, + context_claimed_at INTEGER, + updated_at INTEGER NOT NULL + ) STRICT; + DROP TABLE session_nodes; + PRAGMA user_version = 13; + UPDATE schema_meta SET schema_version = 13 WHERE meta_key = 'primary'; + `); + } finally { + database.close(); + } +} + function readRegisteredAgentDatabaseLastSeenAt(params: { agentId: string; env?: NodeJS.ProcessEnv; @@ -238,7 +373,9 @@ function createTranscriptIdempotencyIndexDrift( DROP INDEX idx_agent_transcript_message_idempotency; CREATE UNIQUE INDEX idx_agent_transcript_message_idempotency ON transcript_event_identities(session_id, event_id); - INSERT INTO sessions ( + INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) + VALUES ('agent:worker-1:session-1', 'session-1', '{}', 1); + INSERT INTO session_windows ( session_id, session_key, session_scope, created_at, updated_at ) VALUES ( 'session-1', 'agent:worker-1:session-1', 'conversation', 1, 1 @@ -649,12 +786,12 @@ describe("openclaw agent database", () => { closeOpenClawAgentDatabasesForTest(); const { DatabaseSync } = requireNodeSqlite(); const database = new DatabaseSync(databasePath); - database.exec("DROP TABLE session_entries;"); + database.exec("DROP TABLE session_nodes;"); database.close(); expect( withOpenClawAgentDatabaseReadOnly( - ({ db }) => db.prepare("SELECT * FROM session_entries").all(), + ({ db }) => db.prepare("SELECT * FROM session_nodes").all(), options, ), ).toEqual({ found: false, reason: "table-missing" }); @@ -740,12 +877,13 @@ describe("openclaw agent database", () => { }); it("opens a v13 database that already contains additive board storage", () => { - expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(13); + expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(14); const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); const databasePath = opened.path; closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const existingV13 = new DatabaseSync(databasePath); @@ -774,13 +912,204 @@ describe("openclaw agent database", () => { ).toEqual({ schema_version: OPENCLAW_AGENT_SCHEMA_VERSION }); }); + it("migrates v13 session entries, routes, and generations into nodes and windows", () => { + const stateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + INSERT INTO sessions (session_id, session_key, created_at, updated_at) + VALUES + ('window-old', 'agent:worker-1:rich', 10, 20), + ('window-current', 'agent:worker-1:rich', 30, 40), + ('window-created-by', 'agent:worker-1:created-by', 50, 60), + ('window-transcript-only', 'agent:worker-1:transcript-only', 70, 80); + INSERT INTO session_routes (session_key, session_id, updated_at) + VALUES + ('agent:worker-1:rich', 'window-current', 40), + ('agent:worker-1:created-by', 'window-created-by', 60), + ('agent:worker-1:transcript-only', 'window-transcript-only', 80); + INSERT INTO session_entries (session_key, session_id, entry_json, updated_at, status) + VALUES ( + 'agent:worker-1:rich', + 'window-current', + '{"sessionId":"window-current","updatedAt":40,"status":"done","createdAt":11,"createdVia":"spawn","createdActor":{"type":"agent","id":"agent:worker-1:parent"},"spawnedBy":"agent:worker-1:parent","forkSource":{"sessionKey":"agent:worker-1:source","sessionId":"source-window","entryId":"source-entry"},"label":"Rich label","displayName":"Rich display","category":"work","icon":"hammer","pinnedAt":12,"archivedAt":13,"lastReadAt":14,"lastInteractionAt":15,"lastActivityAt":16,"previousSessionId":"window-old"}', + 40, + 'done' + ), ( + 'agent:worker-1:created-by', + 'window-created-by', + '{"sessionId":"window-created-by","updatedAt":60,"createdBy":{"id":"legacy-human","label":"Legacy"}}', + 60, + NULL + ); + INSERT INTO transcript_events (session_id, seq, event_json, created_at) + VALUES + ('window-old', 0, '{"type":"session","id":"window-old"}', 10), + ('window-current', 0, '{"type":"session","id":"window-current"}', 30), + ('window-transcript-only', 0, '{"type":"session","id":"window-transcript-only"}', 70); + INSERT INTO session_transcript_generations (session_id, generation, updated_at) + VALUES ('window-current', 'rewrite-token', 40); + INSERT INTO session_transcript_index_state ( + session_id, indexed_seq, leaf_event_id, needs_rebuild, + active_event_count, active_message_count, updated_at + ) VALUES ('window-current', 0, NULL, 0, 1, 0, 40); + INSERT INTO heartbeat_outcomes ( + session_key, run_session_key, outcome, summary, occurred_at, updated_at + ) VALUES ( + 'agent:worker-1:rich', 'agent:worker-1:rich', 'done', 'complete', 40, 40 + ); + INSERT INTO session_members (session_key, identity_id, added_by, added_at) + VALUES ('agent:worker-1:rich', 'member-1', 'owner-1', 40); + INSERT INTO board_tabs ( + session_key, tab_id, title, position, chat_dock, created_by, revision + ) VALUES ('agent:worker-1:rich', 'main', 'Main', 0, 'right', 'user', 1); + INSERT INTO board_widgets ( + session_key, name, tab_id, content_kind, html, sha256, view_generation, + revision, size_w, size_h, position, created_by, created_at, updated_at + ) VALUES ( + 'agent:worker-1:rich', 'summary', 'main', 'html', X'3C703E6F6B3C2F703E', + 'hash', 'view-1', 1, 4, 4, 0, 'user', 40, 40 + ); + PRAGMA user_version = 13; + UPDATE schema_meta SET schema_version = 13 WHERE meta_key = 'primary'; + `); + legacy.close(); + + const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + expect( + migrated.db + .prepare( + `SELECT + current_session_id, created_at, created_via, created_actor_type, + created_actor_id, parent_session_key, spawned_by, + fork_source_session_key, fork_source_session_id, fork_source_entry_id, + label, display_name, category, icon, pinned_at, archived_at, + last_read_at, last_interaction_at, last_activity_at, status + FROM session_nodes WHERE session_key = 'agent:worker-1:rich'`, + ) + .get(), + ).toEqual({ + current_session_id: "window-current", + created_at: 11, + created_via: "spawn", + created_actor_type: "agent", + created_actor_id: "agent:worker-1:parent", + parent_session_key: "agent:worker-1:parent", + spawned_by: "agent:worker-1:parent", + fork_source_session_key: "agent:worker-1:source", + fork_source_session_id: "source-window", + fork_source_entry_id: "source-entry", + label: "Rich label", + display_name: "Rich display", + category: "work", + icon: "hammer", + pinned_at: 12, + archived_at: 13, + last_read_at: 14, + last_interaction_at: 15, + last_activity_at: 16, + status: "done", + }); + expect( + migrated.db + .prepare( + "SELECT created_actor_type, created_actor_id FROM session_nodes WHERE session_key = ?", + ) + .get("agent:worker-1:created-by"), + ).toEqual({ created_actor_type: "human", created_actor_id: "legacy-human" }); + expect( + migrated.db + .prepare("SELECT entry_json FROM session_nodes WHERE session_key = ?") + .get("agent:worker-1:transcript-only"), + ).toEqual({ entry_json: "{}" }); + expect( + migrated.db + .prepare( + "SELECT session_id, session_key, previous_session_id, reason FROM session_windows ORDER BY session_id", + ) + .all(), + ).toEqual([ + { + session_id: "window-created-by", + session_key: "agent:worker-1:created-by", + previous_session_id: null, + reason: null, + }, + { + session_id: "window-current", + session_key: "agent:worker-1:rich", + previous_session_id: "window-old", + reason: null, + }, + { + session_id: "window-old", + session_key: "agent:worker-1:rich", + previous_session_id: null, + reason: null, + }, + { + session_id: "window-transcript-only", + session_key: "agent:worker-1:transcript-only", + previous_session_id: null, + reason: null, + }, + ]); + expect( + migrated.db + .prepare("SELECT generation FROM transcript_rewrite_watermarks WHERE session_id = ?") + .get("window-current"), + ).toEqual({ generation: "rewrite-token" }); + expect( + migrated.db + .prepare("SELECT identity_id, added_by FROM session_members WHERE session_key = ?") + .get("agent:worker-1:rich"), + ).toEqual({ identity_id: "member-1", added_by: "owner-1" }); + expect( + migrated.db + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name IN ('session_entries', 'session_routes', 'session_transcript_generations')", + ) + .all(), + ).toEqual([]); + expect(migrated.db.prepare("PRAGMA integrity_check").get()).toEqual({ + integrity_check: "ok", + }); + expect(migrated.db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); + + migrated.db + .prepare("DELETE FROM session_nodes WHERE session_key = ?") + .run("agent:worker-1:rich"); + for (const table of [ + "session_windows", + "transcript_events", + "transcript_rewrite_watermarks", + "session_transcript_index_state", + "session_members", + "heartbeat_outcomes", + "board_tabs", + "board_widgets", + ]) { + expect(migrated.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get(), table).toEqual({ + count: table === "session_windows" ? 2 : table === "transcript_events" ? 1 : 0, + }); + } + }); + it("keeps additive heartbeat repair while upgrading schema version 12", () => { - expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(13); + expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(14); const stateDir = createTempStateDir(); const env = { OPENCLAW_STATE_DIR: stateDir }; const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); const databasePath = opened.path; closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const existingV12 = new DatabaseSync(databasePath); @@ -812,6 +1141,7 @@ describe("openclaw agent database", () => { const databasePath = opened.path; closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); @@ -831,7 +1161,7 @@ describe("openclaw agent database", () => { const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); const generations = migrated.db .prepare( - "SELECT session_id, generation FROM session_transcript_generations ORDER BY session_id", + "SELECT session_id, generation FROM transcript_rewrite_watermarks ORDER BY session_id", ) .all() as Array<{ generation: string; session_id: string }>; @@ -841,7 +1171,7 @@ describe("openclaw agent database", () => { expect( migrated.db .prepare("SELECT strict FROM pragma_table_list WHERE name = ?") - .get("session_transcript_generations"), + .get("transcript_rewrite_watermarks"), ).toEqual({ strict: 1 }); expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); expect( @@ -863,6 +1193,7 @@ describe("openclaw agent database", () => { .run("last-good", '{"profile":"primary"}', 10); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); @@ -908,6 +1239,7 @@ describe("openclaw agent database", () => { .run("last-good", '{"profile":"primary"}', 10); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); @@ -941,9 +1273,7 @@ describe("openclaw agent database", () => { .get(), ).toEqual({ name: "acp_parent_stream_events" }); expect( - migrated.db - .prepare("SELECT session_id, generation FROM session_transcript_generations") - .get(), + migrated.db.prepare("SELECT session_id, generation FROM transcript_rewrite_watermarks").get(), ).toMatchObject({ session_id: "with-transcript", generation: expect.stringMatching(/^[0-9a-f]{32}$/), @@ -1825,31 +2155,32 @@ describe("openclaw agent database", () => { const env = { OPENCLAW_STATE_DIR: stateDir }; const database = openOpenClawAgentDatabase({ agentId: "worker-1", env }); const databasePath = database.path; - database.db - .prepare( - `INSERT INTO sessions ( - session_id, session_key, session_scope, created_at, updated_at, status - ) VALUES (?, ?, 'conversation', ?, ?, ?)`, - ) - .run("shared-session", "agent:worker-1:running", 10, 10, "done"); - database.db - .prepare( - `INSERT INTO session_entries ( - session_key, session_id, entry_json, updated_at, status - ) VALUES (?, ?, ?, ?, ?)`, - ) - .run( - "agent:worker-1:running", - "shared-session", - JSON.stringify({ sessionId: "shared-session", status: "running", updatedAt: 10 }), - 10, - "running", - ); closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); try { + legacy + .prepare( + `INSERT INTO sessions ( + session_id, session_key, session_scope, created_at, updated_at, status + ) VALUES (?, ?, 'conversation', ?, ?, ?)`, + ) + .run("shared-session", "agent:worker-1:running", 10, 10, "done"); + legacy + .prepare( + `INSERT INTO session_entries ( + session_key, session_id, entry_json, updated_at, status + ) VALUES (?, ?, ?, ?, ?)`, + ) + .run( + "agent:worker-1:running", + "shared-session", + JSON.stringify({ sessionId: "shared-session", status: "running", updatedAt: 10 }), + 10, + "running", + ); legacy.exec(` DROP INDEX idx_agent_session_entries_status; ALTER TABLE session_entries DROP COLUMN status; @@ -1862,14 +2193,14 @@ describe("openclaw agent database", () => { const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); expect( migrated.db - .prepare("SELECT status FROM session_entries WHERE session_key = ?") + .prepare("SELECT status FROM session_nodes WHERE session_key = ?") .get("agent:worker-1:running"), ).toEqual({ status: "running" }); expect( migrated.db .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND name = ?") - .get("idx_agent_session_entries_status"), - ).toEqual({ name: "idx_agent_session_entries_status" }); + .get("idx_agent_session_nodes_status"), + ).toEqual({ name: "idx_agent_session_nodes_status" }); expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); }); @@ -1879,6 +2210,7 @@ describe("openclaw agent database", () => { const database = openOpenClawAgentDatabase({ agentId: "worker-1", env }); const databasePath = database.path; closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const legacy = new DatabaseSync(databasePath); @@ -1902,7 +2234,7 @@ describe("openclaw agent database", () => { WHERE type = 'index' AND name IN ( 'idx_agent_session_entries_session_id', - 'idx_agent_session_entries_session_updated', + 'idx_agent_session_nodes_current_session_id', 'idx_agent_transcript_event_sequence' ) ORDER BY name`, @@ -1911,7 +2243,7 @@ describe("openclaw agent database", () => { .map((row) => (row as { name: string }).name); expect(indexNames).toEqual([ - "idx_agent_session_entries_session_updated", + "idx_agent_session_nodes_current_session_id", "idx_agent_transcript_event_sequence", ]); const transcriptIndex = migrated.db @@ -2032,27 +2364,19 @@ describe("openclaw agent database", () => { "agent", "openclaw-agent.sqlite", ); - fs.mkdirSync(path.dirname(databasePath), { recursive: true }); - const currentSchema = fs.readFileSync( - new URL("./openclaw-agent-schema.sql", import.meta.url), - "utf8", - ); - const previousSchema = currentSchema.replace( - [ - " transcript_updated_at INTEGER DEFAULT NULL,\n", - " transcript_observed_at INTEGER DEFAULT NULL,\n", - " session_entry_provenance INTEGER NOT NULL DEFAULT 0 CHECK (session_entry_provenance IN (0, 1)),\n", - " acp_owned INTEGER NOT NULL DEFAULT 0 CHECK (acp_owned IN (0, 1)),\n", - " plugin_owner_id TEXT,\n", - " hook_external_content_source TEXT CHECK (hook_external_content_source IS NULL OR hook_external_content_source IN ('gmail', 'webhook')),\n", - ].join(""), - "", - ); - expect(previousSchema).not.toBe(currentSchema); + openOpenClawAgentDatabase({ agentId: "worker-1", env: { OPENCLAW_STATE_DIR: stateDir } }); + closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const db = new DatabaseSync(databasePath); - db.exec(previousSchema); db.exec(` + ALTER TABLE sessions DROP COLUMN transcript_updated_at; + ALTER TABLE sessions DROP COLUMN transcript_observed_at; + ALTER TABLE sessions DROP COLUMN session_entry_provenance; + ALTER TABLE sessions DROP COLUMN acp_owned; + ALTER TABLE sessions DROP COLUMN plugin_owner_id; + ALTER TABLE sessions DROP COLUMN hook_external_content_source; + DELETE FROM schema_meta; INSERT INTO schema_meta (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) VALUES ('primary', 'agent', 4, 'worker-1', NULL, 1, 1); @@ -2081,7 +2405,7 @@ describe("openclaw agent database", () => { agentId: "worker-1", env: { OPENCLAW_STATE_DIR: stateDir }, }); - const columns = database.db.prepare("PRAGMA table_info(sessions)").all() as Array<{ + const columns = database.db.prepare("PRAGMA table_info(session_windows)").all() as Array<{ name?: unknown; }>; @@ -2098,7 +2422,7 @@ describe("openclaw agent database", () => { expect( database.db .prepare( - "SELECT transcript_observed_at, transcript_updated_at FROM sessions WHERE session_id = ?", + "SELECT transcript_observed_at, transcript_updated_at FROM session_windows WHERE session_id = ?", ) .get("session-1"), ).toEqual({ @@ -2108,7 +2432,7 @@ describe("openclaw agent database", () => { expect( database.db .prepare( - "SELECT transcript_observed_at, transcript_updated_at FROM sessions WHERE session_id = ?", + "SELECT transcript_observed_at, transcript_updated_at FROM session_windows WHERE session_id = ?", ) .get("session-2"), ).toEqual({ @@ -2118,7 +2442,7 @@ describe("openclaw agent database", () => { expect( database.db .prepare( - "SELECT session_entry_provenance, acp_owned, plugin_owner_id, hook_external_content_source FROM sessions WHERE session_id = ?", + "SELECT session_entry_provenance, acp_owned, plugin_owner_id, hook_external_content_source FROM session_windows WHERE session_id = ?", ) .get("session-1"), ).toEqual({ @@ -2139,26 +2463,18 @@ describe("openclaw agent database", () => { "agent", "openclaw-agent.sqlite", ); - fs.mkdirSync(path.dirname(databasePath), { recursive: true }); - const currentSchema = fs.readFileSync( - new URL("./openclaw-agent-schema.sql", import.meta.url), - "utf8", - ); - const v7Schema = currentSchema - .replace( - [ - " session_entry_provenance INTEGER NOT NULL DEFAULT 0 CHECK (session_entry_provenance IN (0, 1)),\n", - " acp_owned INTEGER NOT NULL DEFAULT 0 CHECK (acp_owned IN (0, 1)),\n", - " plugin_owner_id TEXT,\n", - " hook_external_content_source TEXT CHECK (hook_external_content_source IS NULL OR hook_external_content_source IN ('gmail', 'webhook')),\n", - ].join(""), - "", - ) - .replace(" delivery_target TEXT NOT NULL,\n", ""); + openOpenClawAgentDatabase({ agentId: "worker-1", env: { OPENCLAW_STATE_DIR: stateDir } }); + closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const db = new DatabaseSync(databasePath); - db.exec(v7Schema); db.exec(` + ALTER TABLE sessions DROP COLUMN session_entry_provenance; + ALTER TABLE sessions DROP COLUMN acp_owned; + ALTER TABLE sessions DROP COLUMN plugin_owner_id; + ALTER TABLE sessions DROP COLUMN hook_external_content_source; + ALTER TABLE conversations DROP COLUMN delivery_target; + DELETE FROM schema_meta; INSERT INTO schema_meta (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) VALUES ('primary', 'agent', 7, 'worker-1', NULL, 1, 1); @@ -2184,7 +2500,7 @@ describe("openclaw agent database", () => { }); expect( database.db - .prepare("SELECT session_entry_provenance, acp_owned, plugin_owner_id FROM sessions") + .prepare("SELECT session_entry_provenance, acp_owned, plugin_owner_id FROM session_windows") .get(), ).toEqual({ session_entry_provenance: 1, @@ -2217,18 +2533,16 @@ describe("openclaw agent database", () => { "agent", "openclaw-agent.sqlite", ); - fs.mkdirSync(path.dirname(databasePath), { recursive: true }); - const currentSchema = fs.readFileSync( - new URL("./openclaw-agent-schema.sql", import.meta.url), - "utf8", - ); + openOpenClawAgentDatabase({ agentId: "worker-1", env: { OPENCLAW_STATE_DIR: stateDir } }); + closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const db = new DatabaseSync(databasePath); - db.exec(currentSchema); db.exec(` DROP TABLE session_transcript_active_events; ALTER TABLE session_transcript_index_state DROP COLUMN active_event_count; ALTER TABLE session_transcript_index_state DROP COLUMN active_message_count; + DELETE FROM schema_meta; INSERT INTO schema_meta (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) VALUES ('primary', 'agent', 9, 'worker-1', NULL, 1, 1); @@ -2280,7 +2594,7 @@ describe("openclaw agent database", () => { expect( database.db .prepare( - "SELECT length(generation) AS generation_length FROM session_transcript_generations WHERE session_id = ?", + "SELECT length(generation) AS generation_length FROM transcript_rewrite_watermarks WHERE session_id = ?", ) .get("session-1"), ).toEqual({ generation_length: 32 }); @@ -2296,17 +2610,15 @@ describe("openclaw agent database", () => { "agent", "openclaw-agent.sqlite", ); - fs.mkdirSync(path.dirname(databasePath), { recursive: true }); - const currentSchema = fs.readFileSync( - new URL("./openclaw-agent-schema.sql", import.meta.url), - "utf8", - ); + openOpenClawAgentDatabase({ agentId: "worker-1", env: { OPENCLAW_STATE_DIR: stateDir } }); + closeOpenClawAgentDatabasesForTest(); + downgradeCurrentAgentDatabaseToV13(databasePath); const { DatabaseSync } = requireNodeSqlite(); const db = new DatabaseSync(databasePath); - db.exec(currentSchema); db.exec(` DROP TABLE conversation_deliveries; ALTER TABLE conversations DROP COLUMN delivery_target; + DELETE FROM schema_meta; INSERT INTO schema_meta (meta_key, role, schema_version, agent_id, app_version, created_at, updated_at) VALUES ('primary', 'agent', 10, 'worker-1', NULL, 1, 1); @@ -2471,7 +2783,7 @@ describe("openclaw agent database", () => { spawned_by, started_at, status - FROM sessions + FROM session_windows WHERE session_id = ? `, ) @@ -2492,13 +2804,15 @@ describe("openclaw agent database", () => { status: "done", }); const route = database.db - .prepare("SELECT session_id, updated_at FROM session_routes WHERE session_key = ?") + .prepare("SELECT current_session_id, updated_at FROM session_nodes WHERE session_key = ?") .get("agent:worker-1:group:example"); expect(route).toEqual({ - session_id: "session-1", + current_session_id: "session-1", updated_at: 20, }); - const sessionForeignKeys = database.db.prepare("PRAGMA foreign_key_list(sessions)").all() as + const sessionForeignKeys = database.db + .prepare("PRAGMA foreign_key_list(session_windows)") + .all() as | Array<{ from?: unknown; on_delete?: unknown; table?: unknown; to?: unknown }> | undefined; expect(sessionForeignKeys).toContainEqual( @@ -2616,18 +2930,18 @@ describe("openclaw agent database", () => { corrupted.exec("PRAGMA foreign_keys = OFF;"); corrupted .prepare( - "INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) VALUES (?, ?, ?, ?)", + "INSERT INTO session_windows (session_id, session_key, created_at, updated_at) VALUES (?, ?, ?, ?)", ) - .run("orphan", "missing-session", "{}", 1); + .run("orphan-window", "missing-node", 1, 1); expect(corrupted.prepare("PRAGMA quick_check").get()).toEqual({ quick_check: "ok" }); expect(corrupted.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok", }); expect(corrupted.prepare("PRAGMA foreign_key_check").get()).toEqual({ - table: "session_entries", + table: "session_windows", rowid: 1, - parent: "sessions", - fkid: 0, + parent: "session_nodes", + fkid: 1, }); } finally { corrupted.close(); @@ -2639,10 +2953,10 @@ describe("openclaw agent database", () => { const after = new DatabaseSync(databasePath, { readOnly: true }); try { expect(after.prepare("PRAGMA foreign_key_check").get()).toEqual({ - table: "session_entries", + table: "session_windows", rowid: 1, - parent: "sessions", - fkid: 0, + parent: "session_nodes", + fkid: 1, }); } finally { after.close(); diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index e42619bf88ee..f5ce2dc4e551 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -3,7 +3,11 @@ * Please do not edit it manually. */ -export const OPENCLAW_AGENT_SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS schema_meta ( +export const OPENCLAW_AGENT_SCHEMA_SQL = `-- Session storage doctrine: session_nodes.entry_json is the canonical logical-session +-- record. Promoted session_nodes columns are query indexes projected only by the +-- session entry writer; session_windows and their children own transcript generations. + +CREATE TABLE IF NOT EXISTS schema_meta ( meta_key TEXT NOT NULL PRIMARY KEY, role TEXT NOT NULL, schema_version INTEGER NOT NULL, @@ -32,9 +36,60 @@ CREATE INDEX IF NOT EXISTS idx_agent_state_leases_expiry CREATE INDEX IF NOT EXISTS idx_agent_state_leases_owner ON state_leases(owner, updated_at DESC); -CREATE TABLE IF NOT EXISTS sessions ( +CREATE TABLE IF NOT EXISTS session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + created_at INTEGER, + created_via TEXT CHECK (created_via IS NULL OR created_via IN ('operator', 'spawn', 'channel', 'cron', 'talk', 'run', 'plugin', 'internal')), + created_actor_type TEXT CHECK (created_actor_type IS NULL OR created_actor_type IN ('human', 'agent', 'system')), + created_actor_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + fork_source_session_key TEXT, + fork_source_session_id TEXT, + fork_source_entry_id TEXT, + label TEXT, + display_name TEXT, + category TEXT, + icon TEXT, + pinned_at INTEGER, + archived_at INTEGER, + last_read_at INTEGER, + last_interaction_at INTEGER, + last_activity_at INTEGER +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_updated_at + ON session_nodes(updated_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_last_interaction_at + ON session_nodes(last_interaction_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_parent_session_key + ON session_nodes(parent_session_key, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_spawned_by + ON session_nodes(spawned_by, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_status + ON session_nodes(status, session_key) + WHERE status IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_archived_at + ON session_nodes(archived_at, session_key) + WHERE archived_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_current_session_id + ON session_nodes(current_session_id); + +CREATE TABLE IF NOT EXISTS session_windows ( session_id TEXT NOT NULL PRIMARY KEY, session_key TEXT NOT NULL, + previous_session_id TEXT, + reason TEXT CHECK (reason IS NULL OR reason IN ('initial', 'reset', 'rollover', 'fork', 'rewind', 'switch', 'recovery', 'compaction')), session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -57,29 +112,20 @@ CREATE TABLE IF NOT EXISTS sessions ( parent_session_key TEXT, spawned_by TEXT, display_name TEXT, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE, FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL ) STRICT; -CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at - ON sessions(updated_at DESC, session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_updated_at + ON session_windows(updated_at DESC, session_id); -CREATE INDEX IF NOT EXISTS idx_agent_sessions_created_at - ON sessions(created_at DESC, session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_created_at + ON session_windows(created_at DESC, session_id); -CREATE INDEX IF NOT EXISTS idx_agent_sessions_conversation - ON sessions(primary_conversation_id, updated_at DESC, session_id) +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_conversation + ON session_windows(primary_conversation_id, updated_at DESC, session_id) WHERE primary_conversation_id IS NOT NULL; -CREATE TABLE IF NOT EXISTS session_routes ( - session_key TEXT NOT NULL PRIMARY KEY, - session_id TEXT NOT NULL, - updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id - ON session_routes(session_id); - CREATE TABLE IF NOT EXISTS conversations ( conversation_id TEXT NOT NULL PRIMARY KEY, channel TEXT NOT NULL, @@ -152,7 +198,7 @@ CREATE TABLE IF NOT EXISTS session_conversations ( first_seen_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL, PRIMARY KEY (session_id, conversation_id, role), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE, FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE ) STRICT; @@ -163,37 +209,17 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_session_conversations_primary ON session_conversations(session_id) WHERE role = 'primary'; -CREATE TABLE IF NOT EXISTS session_entries ( - session_key TEXT NOT NULL PRIMARY KEY, - session_id TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at INTEGER NOT NULL, - status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at - ON session_entries(updated_at DESC, session_key); - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_updated - ON session_entries(session_id, updated_at DESC, session_key); - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_status - ON session_entries(status, session_key) - WHERE status IS NOT NULL; - CREATE TABLE IF NOT EXISTS session_members ( session_key TEXT NOT NULL, identity_id TEXT NOT NULL, added_by TEXT NOT NULL, added_at INTEGER NOT NULL, PRIMARY KEY (session_key, identity_id), - FOREIGN KEY (session_key) REFERENCES session_entries(session_key) ON DELETE CASCADE + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_members_identity ON session_members(identity_id, session_key); - CREATE TABLE IF NOT EXISTS board_tabs ( session_key TEXT NOT NULL, tab_id TEXT NOT NULL, @@ -202,7 +228,8 @@ CREATE TABLE IF NOT EXISTS board_tabs ( chat_dock TEXT NOT NULL DEFAULT 'right' CHECK (chat_dock IN ('left', 'right', 'bottom', 'hidden')), created_by TEXT NOT NULL CHECK (created_by IN ('user', 'agent')), revision INTEGER NOT NULL CHECK (revision >= 0), - PRIMARY KEY (session_key, tab_id) + PRIMARY KEY (session_key, tab_id), + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS board_widgets ( @@ -251,7 +278,8 @@ CREATE TABLE IF NOT EXISTS heartbeat_outcomes ( occurred_at INTEGER NOT NULL, context_run_id TEXT, context_claimed_at INTEGER, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS transcript_events ( @@ -260,14 +288,14 @@ CREATE TABLE IF NOT EXISTS transcript_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; -CREATE TABLE IF NOT EXISTS session_transcript_generations ( +CREATE TABLE IF NOT EXISTS transcript_rewrite_watermarks ( session_id TEXT NOT NULL PRIMARY KEY, generation TEXT NOT NULL, updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( @@ -277,7 +305,7 @@ CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run @@ -291,7 +319,7 @@ CREATE TABLE IF NOT EXISTS acp_parent_stream_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, run_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run @@ -400,7 +428,8 @@ CREATE TABLE IF NOT EXISTS session_transcript_index_state ( needs_rebuild INTEGER NOT NULL DEFAULT 0, active_event_count INTEGER NOT NULL DEFAULT 0, active_message_count INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES session_windows(session_id) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS session_transcript_active_events ( diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 31bff0891040..6e0a6fcb05a8 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -1,3 +1,7 @@ +-- Session storage doctrine: session_nodes.entry_json is the canonical logical-session +-- record. Promoted session_nodes columns are query indexes projected only by the +-- session entry writer; session_windows and their children own transcript generations. + CREATE TABLE IF NOT EXISTS schema_meta ( meta_key TEXT NOT NULL PRIMARY KEY, role TEXT NOT NULL, @@ -27,9 +31,60 @@ CREATE INDEX IF NOT EXISTS idx_agent_state_leases_expiry CREATE INDEX IF NOT EXISTS idx_agent_state_leases_owner ON state_leases(owner, updated_at DESC); -CREATE TABLE IF NOT EXISTS sessions ( +CREATE TABLE IF NOT EXISTS session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL, + status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), + created_at INTEGER, + created_via TEXT CHECK (created_via IS NULL OR created_via IN ('operator', 'spawn', 'channel', 'cron', 'talk', 'run', 'plugin', 'internal')), + created_actor_type TEXT CHECK (created_actor_type IS NULL OR created_actor_type IN ('human', 'agent', 'system')), + created_actor_id TEXT, + parent_session_key TEXT, + spawned_by TEXT, + fork_source_session_key TEXT, + fork_source_session_id TEXT, + fork_source_entry_id TEXT, + label TEXT, + display_name TEXT, + category TEXT, + icon TEXT, + pinned_at INTEGER, + archived_at INTEGER, + last_read_at INTEGER, + last_interaction_at INTEGER, + last_activity_at INTEGER +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_updated_at + ON session_nodes(updated_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_last_interaction_at + ON session_nodes(last_interaction_at DESC, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_parent_session_key + ON session_nodes(parent_session_key, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_spawned_by + ON session_nodes(spawned_by, session_key); + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_status + ON session_nodes(status, session_key) + WHERE status IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_archived_at + ON session_nodes(archived_at, session_key) + WHERE archived_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_current_session_id + ON session_nodes(current_session_id); + +CREATE TABLE IF NOT EXISTS session_windows ( session_id TEXT NOT NULL PRIMARY KEY, session_key TEXT NOT NULL, + previous_session_id TEXT, + reason TEXT CHECK (reason IS NULL OR reason IN ('initial', 'reset', 'rollover', 'fork', 'rewind', 'switch', 'recovery', 'compaction')), session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')), created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, @@ -52,29 +107,20 @@ CREATE TABLE IF NOT EXISTS sessions ( parent_session_key TEXT, spawned_by TEXT, display_name TEXT, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE, FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL ) STRICT; -CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at - ON sessions(updated_at DESC, session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_updated_at + ON session_windows(updated_at DESC, session_id); -CREATE INDEX IF NOT EXISTS idx_agent_sessions_created_at - ON sessions(created_at DESC, session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_created_at + ON session_windows(created_at DESC, session_id); -CREATE INDEX IF NOT EXISTS idx_agent_sessions_conversation - ON sessions(primary_conversation_id, updated_at DESC, session_id) +CREATE INDEX IF NOT EXISTS idx_agent_session_windows_conversation + ON session_windows(primary_conversation_id, updated_at DESC, session_id) WHERE primary_conversation_id IS NOT NULL; -CREATE TABLE IF NOT EXISTS session_routes ( - session_key TEXT NOT NULL PRIMARY KEY, - session_id TEXT NOT NULL, - updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id - ON session_routes(session_id); - CREATE TABLE IF NOT EXISTS conversations ( conversation_id TEXT NOT NULL PRIMARY KEY, channel TEXT NOT NULL, @@ -147,7 +193,7 @@ CREATE TABLE IF NOT EXISTS session_conversations ( first_seen_at INTEGER NOT NULL, last_seen_at INTEGER NOT NULL, PRIMARY KEY (session_id, conversation_id, role), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE, FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE ) STRICT; @@ -158,37 +204,17 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_session_conversations_primary ON session_conversations(session_id) WHERE role = 'primary'; -CREATE TABLE IF NOT EXISTS session_entries ( - session_key TEXT NOT NULL PRIMARY KEY, - session_id TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at INTEGER NOT NULL, - status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -) STRICT; - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at - ON session_entries(updated_at DESC, session_key); - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_updated - ON session_entries(session_id, updated_at DESC, session_key); - -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_status - ON session_entries(status, session_key) - WHERE status IS NOT NULL; - CREATE TABLE IF NOT EXISTS session_members ( session_key TEXT NOT NULL, identity_id TEXT NOT NULL, added_by TEXT NOT NULL, added_at INTEGER NOT NULL, PRIMARY KEY (session_key, identity_id), - FOREIGN KEY (session_key) REFERENCES session_entries(session_key) ON DELETE CASCADE + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_members_identity ON session_members(identity_id, session_key); - CREATE TABLE IF NOT EXISTS board_tabs ( session_key TEXT NOT NULL, tab_id TEXT NOT NULL, @@ -197,7 +223,8 @@ CREATE TABLE IF NOT EXISTS board_tabs ( chat_dock TEXT NOT NULL DEFAULT 'right' CHECK (chat_dock IN ('left', 'right', 'bottom', 'hidden')), created_by TEXT NOT NULL CHECK (created_by IN ('user', 'agent')), revision INTEGER NOT NULL CHECK (revision >= 0), - PRIMARY KEY (session_key, tab_id) + PRIMARY KEY (session_key, tab_id), + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS board_widgets ( @@ -246,7 +273,8 @@ CREATE TABLE IF NOT EXISTS heartbeat_outcomes ( occurred_at INTEGER NOT NULL, context_run_id TEXT, context_claimed_at INTEGER, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_key) REFERENCES session_nodes(session_key) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS transcript_events ( @@ -255,14 +283,14 @@ CREATE TABLE IF NOT EXISTS transcript_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; -CREATE TABLE IF NOT EXISTS session_transcript_generations ( +CREATE TABLE IF NOT EXISTS transcript_rewrite_watermarks ( session_id TEXT NOT NULL PRIMARY KEY, generation TEXT NOT NULL, updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( @@ -272,7 +300,7 @@ CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run @@ -286,7 +314,7 @@ CREATE TABLE IF NOT EXISTS acp_parent_stream_events ( event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, run_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES "session_windows"(session_id) ON DELETE CASCADE ) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run @@ -395,7 +423,8 @@ CREATE TABLE IF NOT EXISTS session_transcript_index_state ( needs_rebuild INTEGER NOT NULL DEFAULT 0, active_event_count INTEGER NOT NULL DEFAULT 0, active_message_count INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL + updated_at INTEGER NOT NULL, + FOREIGN KEY (session_id) REFERENCES session_windows(session_id) ON DELETE CASCADE ) STRICT; CREATE TABLE IF NOT EXISTS session_transcript_active_events ( diff --git a/src/state/sqlite-query-plan.test.ts b/src/state/sqlite-query-plan.test.ts index e3b87a530dd7..d4b80c442bf1 100644 --- a/src/state/sqlite-query-plan.test.ts +++ b/src/state/sqlite-query-plan.test.ts @@ -171,23 +171,23 @@ describe("sqlite hot query plans", () => { }); expectPlanUsesIndex({ db: database.db, - indexName: "idx_agent_session_entries_session_updated", + indexName: "idx_agent_session_nodes_current_session_id", params: ["session-1"], sql: ` SELECT session_key - FROM session_entries - WHERE session_id = ? + FROM session_nodes + WHERE current_session_id = ? ORDER BY updated_at DESC, session_key ASC LIMIT 1 `, }); expectPlanUsesIndex({ db: database.db, - indexName: "idx_agent_session_entries_status", + indexName: "idx_agent_session_nodes_status", params: ["running"], sql: ` SELECT session_key, entry_json - FROM session_entries + FROM session_nodes WHERE status = ? `, }); @@ -211,11 +211,11 @@ describe("sqlite hot query plans", () => { expectPlanIncludes({ db: database.db, - expected: "sqlite_autoindex_session_transcript_generations_1", + expected: "sqlite_autoindex_transcript_rewrite_watermarks_1", params: ["session-1"], sql: ` SELECT generation - FROM session_transcript_generations + FROM transcript_rewrite_watermarks WHERE session_id = ? `, }); diff --git a/src/trajectory/runtime-store.sqlite.test.ts b/src/trajectory/runtime-store.sqlite.test.ts index 99b8a9a7bbc1..39438c6c436e 100644 --- a/src/trajectory/runtime-store.sqlite.test.ts +++ b/src/trajectory/runtime-store.sqlite.test.ts @@ -92,7 +92,7 @@ describe("SQLite trajectory runtime store", () => { ]); const database = openOpenClawAgentDatabase({ agentId: "main", path: sqlitePath() }); - database.db.prepare("DELETE FROM sessions WHERE session_id = ?").run("session-1"); + database.db.prepare("DELETE FROM session_windows WHERE session_id = ?").run("session-1"); await expect( loadSqliteTrajectoryRuntimeEvents({ sessionId: "session-1", storePath }), diff --git a/test/helpers/sqlite-sessions-transcripts-flip-proof.ts b/test/helpers/sqlite-sessions-transcripts-flip-proof.ts index 5bfb3a747c68..2d9f68d0cdfe 100644 --- a/test/helpers/sqlite-sessions-transcripts-flip-proof.ts +++ b/test/helpers/sqlite-sessions-transcripts-flip-proof.ts @@ -2361,8 +2361,8 @@ function readSqliteEvidence(dbPath: string, trackedSessionKeys: readonly string[ return { exists: true, path: dbPath, - sessionEntries: scalarNumber(db, "SELECT COUNT(*) AS count FROM session_entries"), - sessions: scalarNumber(db, "SELECT COUNT(*) AS count FROM sessions"), + sessionEntries: scalarNumber(db, "SELECT COUNT(*) AS count FROM session_nodes"), + sessions: scalarNumber(db, "SELECT COUNT(*) AS count FROM session_windows"), trajectoryRuntimeEvents: scalarNumber( db, "SELECT COUNT(*) AS count FROM trajectory_runtime_events", @@ -2381,8 +2381,8 @@ function readTrackedEntries( ): SqliteSessionEntryEvidence[] { const rows = db .prepare( - `SELECT session_key AS sessionKey, session_id AS sessionId, entry_json AS entryJson - FROM session_entries + `SELECT session_key AS sessionKey, current_session_id AS sessionId, entry_json AS entryJson + FROM session_nodes ORDER BY session_key ASC`, ) .all() as Array<{ entryJson?: unknown; sessionId?: unknown; sessionKey?: unknown }>; diff --git a/test/scripts/codex-install-assertions.test.ts b/test/scripts/codex-install-assertions.test.ts index 9d12d488aa7d..866522f99dc7 100644 --- a/test/scripts/codex-install-assertions.test.ts +++ b/test/scripts/codex-install-assertions.test.ts @@ -262,19 +262,19 @@ function writeSessionStoreSqlite(params: { const db = new DatabaseSync(dbPath); try { db.exec(` - CREATE TABLE sessions ( + CREATE TABLE session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE session_windows ( session_id TEXT NOT NULL PRIMARY KEY, session_key TEXT NOT NULL, agent_harness_id TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE session_entries ( - session_key TEXT NOT NULL PRIMARY KEY, - session_id TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at INTEGER NOT NULL - ); CREATE TABLE transcript_events ( session_id TEXT NOT NULL, seq INTEGER NOT NULL, @@ -285,12 +285,12 @@ function writeSessionStoreSqlite(params: { `); const now = Date.now(); db.prepare( - `INSERT INTO sessions ( + `INSERT INTO session_windows ( session_id, session_key, agent_harness_id, created_at, updated_at ) VALUES (?, ?, ?, ?, ?)`, ).run(params.sessionId, params.sessionKey, "codex", now, now); db.prepare( - `INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) + `INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, ?, ?, ?)`, ).run( params.sessionKey, diff --git a/test/scripts/sqlite-session-schema-baseline.test.ts b/test/scripts/sqlite-session-schema-baseline.test.ts index bce03112fdea..676c82d31ead 100644 --- a/test/scripts/sqlite-session-schema-baseline.test.ts +++ b/test/scripts/sqlite-session-schema-baseline.test.ts @@ -15,7 +15,8 @@ describe("SQLite sessions/transcripts schema baseline", () => { const rendered = renderSqliteSessionSchemaBaseline(sourceSql); - expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS sessions"); + expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS session_nodes"); + expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS session_windows"); expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS transcript_events"); expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS transcript_event_identities"); expect(rendered.sql).toContain("CREATE TABLE IF NOT EXISTS session_transcript_active_events"); @@ -27,18 +28,18 @@ describe("SQLite sessions/transcripts schema baseline", () => { it("automatically includes new indexes declared on target tables", () => { const rendered = renderSqliteSessionSchemaBaseline(` - CREATE TABLE IF NOT EXISTS sessions ( - session_id TEXT NOT NULL PRIMARY KEY + CREATE TABLE IF NOT EXISTS session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY ); - CREATE INDEX IF NOT EXISTS idx_agent_sessions_custom - ON sessions(session_id); + CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_custom + ON session_nodes(session_key); CREATE INDEX IF NOT EXISTS idx_agent_cache_custom ON cache_entries(scope); `); - expect(rendered.sql).toContain("CREATE INDEX IF NOT EXISTS idx_agent_sessions_custom"); + expect(rendered.sql).toContain("CREATE INDEX IF NOT EXISTS idx_agent_session_nodes_custom"); expect(rendered.sql).not.toContain("idx_agent_cache_custom"); }); @@ -50,8 +51,8 @@ describe("SQLite sessions/transcripts schema baseline", () => { await writeFile( schemaPath, ` - CREATE TABLE IF NOT EXISTS sessions ( - session_id TEXT NOT NULL PRIMARY KEY + CREATE TABLE IF NOT EXISTS session_nodes ( + session_key TEXT NOT NULL PRIMARY KEY ); `, ); diff --git a/test/scripts/upgrade-survivor-assertions.test.ts b/test/scripts/upgrade-survivor-assertions.test.ts index adc971d2313b..50793214c9e9 100644 --- a/test/scripts/upgrade-survivor-assertions.test.ts +++ b/test/scripts/upgrade-survivor-assertions.test.ts @@ -21,44 +21,33 @@ function writeMigratedSessionState(stateDir: string): void { const db = new DatabaseSync(join(agentDbDir, "openclaw-agent.sqlite")); try { db.exec(` - CREATE TABLE sessions ( + CREATE TABLE session_nodes ( + session_key TEXT PRIMARY KEY, + current_session_id TEXT NOT NULL, + entry_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE session_windows ( session_id TEXT PRIMARY KEY, session_key TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); - CREATE TABLE session_routes ( - session_key TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE - ); - CREATE TABLE session_entries ( - session_key TEXT PRIMARY KEY, - session_id TEXT NOT NULL, - entry_json TEXT NOT NULL, - updated_at INTEGER NOT NULL, - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE - ); CREATE TABLE transcript_events ( session_id TEXT NOT NULL, seq INTEGER NOT NULL, event_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), - FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE + FOREIGN KEY (session_id) REFERENCES session_windows(session_id) ON DELETE CASCADE ); `); const insertSession = db.prepare(` - INSERT INTO sessions (session_id, session_key, created_at, updated_at) + INSERT INTO session_windows (session_id, session_key, created_at, updated_at) VALUES (?, ?, ?, ?) `); - const insertRoute = db.prepare(` - INSERT INTO session_routes (session_key, session_id, updated_at) - VALUES (?, ?, ?) - `); const insertEntry = db.prepare(` - INSERT INTO session_entries (session_key, session_id, entry_json, updated_at) + INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, ?, ?, ?) `); const insertTranscript = db.prepare(` @@ -88,7 +77,6 @@ function writeMigratedSessionState(stateDir: string): void { ]; for (const { entry, sessionId, sessionKey } of migratedSessions) { insertSession.run(sessionId, sessionKey, 1710000000000, 1710000000000); - insertRoute.run(sessionKey, sessionId, 1710000000000); insertEntry.run(sessionKey, sessionId, JSON.stringify(entry), 1710000000000); insertTranscript.run( sessionId,