From c50237e37dff697dfad850ef18f3b8bd233de7d5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 31 Jul 2026 20:36:00 -0700 Subject: [PATCH] perf: count large histories before Gateway prewarm (#117118) * perf(gateway): count before sidebar prewarm * fix(sessions): narrow count row before normalization * fix(gateway): align sidebar prewarm admission targets * test(gateway): prove large prewarm stays optional * fix(gateway): budget repeated shared-store prewarm --- src/config/sessions/combined-store-gateway.ts | 182 ++++++++++++------ src/config/sessions/session-accessor.entry.ts | 2 + .../sessions/session-accessor.sqlite-entry.ts | 16 ++ .../sessions/session-accessor.sqlite.ts | 1 + src/config/sessions/session-accessor.test.ts | 20 ++ src/config/sessions/session-accessor.ts | 1 + .../server-startup-handler-prewarm.test.ts | 44 +++-- src/gateway/server-startup-handler-prewarm.ts | 54 ++++-- ...essions.list-store-materialization.test.ts | 56 ++++++ src/gateway/session-utils.subagent.test.ts | 28 +++ 10 files changed, 309 insertions(+), 95 deletions(-) diff --git a/src/config/sessions/combined-store-gateway.ts b/src/config/sessions/combined-store-gateway.ts index 01bec1ec03b1..1e1827442ccb 100644 --- a/src/config/sessions/combined-store-gateway.ts +++ b/src/config/sessions/combined-store-gateway.ts @@ -16,7 +16,11 @@ import { import { listOpenIncognitoAgentDatabases } from "../../state/openclaw-agent-db.js"; import type { OpenClawConfig } from "../types.openclaw.js"; import { resolveStorePath } from "./paths.js"; -import { listSessionEntries, listSessionEntriesReadOnly } from "./session-accessor.js"; +import { + countSessionEntryRowsReadOnly, + listSessionEntries, + listSessionEntriesReadOnly, +} from "./session-accessor.js"; import type { SessionEntryListScope } from "./session-accessor.types.js"; import { canonicalSessionKeyMigrationRequiredError } from "./session-canonical-key.js"; import { resolveDeliveryProvenCanonicalSessionKey } from "./store-entry.js"; @@ -32,6 +36,23 @@ import type { SessionEntry } from "./types.js"; type GatewaySessionEntryProjection = NonNullable; +type GatewaySessionStoreOptions = { + agentId?: string; + configuredAgentsOnly?: boolean; + includeIncognito?: boolean; + projection?: SessionEntryListScope["projection"]; +}; + +type ResolvedGatewaySessionStoreTargets = { + configuredAgentIds?: ReadonlySet; + defaultAgentId: string; + diagnostics: string[]; + durableTargets: Array<{ agentId: string; storePath: string }>; + incognitoTargets: Array<{ agentId: string; storePath: string }>; + requestedAgentId?: string; + storeConfig?: string; +}; + // Template-backed stores need per-agent scans before they can be merged for Gateway views. function isStorePathTemplate(store?: string): boolean { return typeof store === "string" && store.includes("{agentId}"); @@ -97,20 +118,13 @@ function mergeSessionEntryIntoCombined(params: { } function mergeOpenIncognitoStores(params: { - allowedAgentIds?: ReadonlySet; cfg: OpenClawConfig; combined: Record; - agentId?: string; projection: GatewaySessionEntryProjection; + targets: Array<{ agentId: string; storePath: string }>; }): string[] { const storePaths: string[] = []; - for (const target of listOpenIncognitoAgentDatabases()) { - if (params.allowedAgentIds && !params.allowedAgentIds.has(target.agentId)) { - continue; - } - if (params.agentId && target.agentId !== params.agentId) { - continue; - } + for (const target of params.targets) { const store = loadGatewayStoreEntries({ agentId: target.agentId, includeOpenDatabases: true, @@ -138,27 +152,12 @@ function mergeOpenIncognitoStores(params: { return storePaths; } -/** Loads and canonicalizes session entries for gateway views across one or more agent stores. */ -export function loadCombinedSessionStoreForGateway( +function resolveGatewaySessionStoreTargets( cfg: OpenClawConfig, - opts: { - agentId?: string; - configuredAgentsOnly?: boolean; - includeIncognito?: boolean; - projection?: SessionEntryListScope["projection"]; - } = {}, -): { - diagnostics?: string[]; - durableStorePath?: string; - storePath: string; - store: Record; -} { + opts: GatewaySessionStoreOptions, +): ResolvedGatewaySessionStoreTargets { const storeConfig = cfg.session?.store; - const projection = opts.projection ?? "full"; const diagnostics: string[] = []; - // Exclusion happens before path aggregation; filtering rows afterward would - // still leak a live incognito handle by changing the projected store path. - const includeIncognito = opts.includeIncognito !== false; const defaultAgentId = normalizeAgentId(resolveDefaultAgentId(cfg)); const requestedAgentId = typeof opts.agentId === "string" && opts.agentId.trim() @@ -171,6 +170,13 @@ export function loadCombinedSessionStoreForGateway( const allowedIncognitoAgentIds = requestedAgentId ? new Set([requestedAgentId]) : configuredAgentIds; + const incognitoTargets = + opts.includeIncognito === false + ? [] + : listOpenIncognitoAgentDatabases().filter( + (target) => !allowedIncognitoAgentIds || allowedIncognitoAgentIds.has(target.agentId), + ); + if (storeConfig && !isStorePathTemplate(storeConfig)) { const ownerIds = [ ...new Set([ @@ -181,10 +187,7 @@ export function loadCombinedSessionStoreForGateway( ...(requestedAgentId ? [requestedAgentId] : []), ]), ]; - const combined: Record = {}; - // Runtime session access is SQLite-only: a fixed literal is a naming seed whose - // resolved database is partitioned per owner. Legacy flat JSON is migration-only. - const ownerTargets = dedupeSessionStoreTargetsBySqliteTarget( + const durableTargets = dedupeSessionStoreTargetsBySqliteTarget( ownerIds.map((agentId) => ({ agentId, storePath: resolveStorePath(storeConfig, { agentId }), @@ -194,7 +197,81 @@ export function loadCombinedSessionStoreForGateway( onDiagnostic: (diagnostic) => diagnostics.push(diagnostic.message), }, ); - for (const { agentId, storePath } of ownerTargets) { + return { + configuredAgentIds, + defaultAgentId, + diagnostics, + durableTargets, + incognitoTargets, + requestedAgentId, + storeConfig, + }; + } + + const durableTargets = requestedAgentId + ? resolveAgentSessionStoreTargetsSync(cfg, requestedAgentId) + : opts.configuredAgentsOnly === true + ? resolveSessionStoreTargets(cfg, { allAgents: true }) + : resolveAllAgentSessionStoreTargetsSync(cfg); + return { + configuredAgentIds, + defaultAgentId, + diagnostics, + durableTargets, + incognitoTargets, + requestedAgentId, + storeConfig, + }; +} + +/** Checks whether Gateway prewarm can project the selected stores within a bounded row budget. */ +export function canPrewarmCombinedSessionStoresForGateway( + cfg: OpenClawConfig, + params: { agentIds: readonly string[]; maxRows: number }, +): boolean { + const defaultAgentId = normalizeAgentId(resolveDefaultAgentId(cfg)); + let totalRows = 0; + for (const agentId of params.agentIds) { + const resolved = resolveGatewaySessionStoreTargets(cfg, { agentId }); + const projectionTargets = dedupeSessionStoreTargetsBySqliteTarget( + [...resolved.durableTargets, ...resolved.incognitoTargets], + { defaultAgentId }, + ); + for (const target of projectionTargets) { + totalRows += countSessionEntryRowsReadOnly(target); + if (totalRows > params.maxRows) { + return false; + } + } + } + return true; +} + +/** Loads and canonicalizes session entries for gateway views across one or more agent stores. */ +export function loadCombinedSessionStoreForGateway( + cfg: OpenClawConfig, + opts: GatewaySessionStoreOptions = {}, +): { + diagnostics?: string[]; + durableStorePath?: string; + storePath: string; + store: Record; +} { + const projection = opts.projection ?? "full"; + // Count admission and projection share this exact target set. Otherwise an optional + // prewarm can approve one database and synchronously materialize another. + const { + configuredAgentIds, + defaultAgentId, + diagnostics, + durableTargets, + incognitoTargets, + requestedAgentId, + storeConfig, + } = resolveGatewaySessionStoreTargets(cfg, opts); + if (storeConfig && !isStorePathTemplate(storeConfig)) { + const combined: Record = {}; + for (const { agentId, storePath } of durableTargets) { const store = loadGatewayStoreEntries({ agentId, projection, storePath }); for (const { sessionKey: key, entry } of store) { const canonicalKey = resolveStoredSessionKeyForAgentStore({ @@ -226,15 +303,12 @@ export function loadCombinedSessionStoreForGateway( } } const durableStorePath = resolveStorePath(storeConfig, { agentId: defaultAgentId }); - const incognitoStorePaths = includeIncognito - ? mergeOpenIncognitoStores({ - ...(allowedIncognitoAgentIds ? { allowedAgentIds: allowedIncognitoAgentIds } : {}), - cfg, - combined, - ...(requestedAgentId ? { agentId: requestedAgentId } : {}), - projection, - }) - : []; + const incognitoStorePaths = mergeOpenIncognitoStores({ + cfg, + combined, + projection, + targets: incognitoTargets, + }); return { diagnostics, durableStorePath, @@ -242,13 +316,8 @@ export function loadCombinedSessionStoreForGateway( store: combined, }; } - const targets = requestedAgentId - ? resolveAgentSessionStoreTargetsSync(cfg, requestedAgentId) - : opts.configuredAgentsOnly === true - ? resolveSessionStoreTargets(cfg, { allAgents: true }) - : resolveAllAgentSessionStoreTargetsSync(cfg); const combined: Record = {}; - for (const target of targets) { + for (const target of durableTargets) { const agentId = target.agentId; const storePath = target.storePath; const store = loadGatewayStoreEntries({ agentId, projection, storePath }); @@ -282,17 +351,14 @@ export function loadCombinedSessionStoreForGateway( } } - const incognitoStorePaths = includeIncognito - ? mergeOpenIncognitoStores({ - ...(allowedIncognitoAgentIds ? { allowedAgentIds: allowedIncognitoAgentIds } : {}), - cfg, - combined, - ...(requestedAgentId ? { agentId: requestedAgentId } : {}), - projection, - }) - : []; + const incognitoStorePaths = mergeOpenIncognitoStores({ + cfg, + combined, + projection, + targets: incognitoTargets, + }); - const durableStorePaths = targets.map((target) => target.storePath); + const durableStorePaths = durableTargets.map((target) => target.storePath); const durableStorePath = resolveCombinedStorePath(durableStorePaths, storeConfig); const storePath = resolveCombinedStorePath( [...durableStorePaths, ...incognitoStorePaths], diff --git a/src/config/sessions/session-accessor.entry.ts b/src/config/sessions/session-accessor.entry.ts index 0eb2d32dd95f..fd8e4b063445 100644 --- a/src/config/sessions/session-accessor.entry.ts +++ b/src/config/sessions/session-accessor.entry.ts @@ -10,6 +10,7 @@ import { resolveAgentMainSessionKey } from "./main-session.js"; import { resolveStorePath } from "./paths.js"; import { clearPluginOwnedSessionState } from "./plugin-host-cleanup.js"; import { + countSqliteSessionEntryRowsReadOnly as countSessionEntryRowsReadOnly, copySqliteSessionOwnedStateForCanonicalRepair as copySessionOwnedStateForCanonicalRepair, listSqliteSessionGenerationIdsForCanonicalRepair as listSessionGenerationIdsForCanonicalRepair, listSqliteSessionChildEntriesReadOnly as listSessionChildEntriesReadOnly, @@ -58,6 +59,7 @@ export { clearPluginOwnedSessionState }; // SQLite is the only runtime session store. Re-export its canonical entry // operations directly instead of maintaining a second pass-through layer. export { + countSessionEntryRowsReadOnly, copySessionOwnedStateForCanonicalRepair, listSessionGenerationIdsForCanonicalRepair, listSessionChildEntriesReadOnly, diff --git a/src/config/sessions/session-accessor.sqlite-entry.ts b/src/config/sessions/session-accessor.sqlite-entry.ts index beabd365aedc..d12f1f94b158 100644 --- a/src/config/sessions/session-accessor.sqlite-entry.ts +++ b/src/config/sessions/session-accessor.sqlite-entry.ts @@ -281,6 +281,22 @@ export function listSqliteSessionEntriesReadOnly( return result.found ? result.value : []; } +/** Counts durable session rows without materializing entry JSON or warming the entry cache. */ +export function countSqliteSessionEntryRowsReadOnly(scope: SessionEntryListScope = {}): number { + const resolved = resolveSqliteScope({ ...scope, sessionKey: "" }); + const result = withOpenClawAgentDatabaseReadOnly((database) => { + const db = getSessionKysely(database.db); + const row = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_nodes") + .select((expression) => expression.fn.countAll().as("count")), + ); + return row ? normalizeSqliteNumber(row.count) : 0; + }, toDatabaseOptions(resolved)); + return result.found ? result.value : 0; +} + function listSqliteSessionEntriesFromDatabase( database: Pick, resolved: ResolvedSqliteScope, diff --git a/src/config/sessions/session-accessor.sqlite.ts b/src/config/sessions/session-accessor.sqlite.ts index 6a1b80373e36..324ae6189c97 100644 --- a/src/config/sessions/session-accessor.sqlite.ts +++ b/src/config/sessions/session-accessor.sqlite.ts @@ -1,5 +1,6 @@ // Stable SQLite accessor surface. Domain owners live in the focused modules below. export { + countSqliteSessionEntryRowsReadOnly, listSqliteSessionEntries, listSqliteSessionChildEntriesReadOnly, listSqliteSessionEntriesReadOnly, diff --git a/src/config/sessions/session-accessor.test.ts b/src/config/sessions/session-accessor.test.ts index ba9bd9f84b88..7b64da17cc4c 100644 --- a/src/config/sessions/session-accessor.test.ts +++ b/src/config/sessions/session-accessor.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { withTestTimeout } from "../../../test/helpers/promise.js"; @@ -26,6 +27,7 @@ import { appendTranscriptMessage, applySessionEntryLifecycleMutation, commitReplySessionInitialization, + countSessionEntryRowsReadOnly, createSessionEntryWithTranscript, deleteSessionEntryLifecycle, findTranscriptEvent, @@ -225,6 +227,24 @@ describe("session accessor seam", () => { expect(readSqliteSessionEntryCount(database)).toBe(1); expect(readSqliteSessionEntryKeys(database)).toEqual(["agent:main:logical-entry"]); + expect(countSessionEntryRowsReadOnly({ agentId: "main", storePath })).toBe(2); + }); + + it("counts rows on a cold handle without parsing invalid entry JSON", async () => { + await replaceSessionEntry( + { sessionKey: "agent:main:cold-count", storePath }, + { sessionId: "cold-count-session", updatedAt: 10 }, + ); + const databasePath = expectDefined( + resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" }).path, + "cold count database path", + ); + closeOpenClawAgentDatabasesForTest(); + const database = new DatabaseSync(databasePath); + database.prepare("UPDATE session_nodes SET entry_valid = 0").run(); + database.close(); + + expect(countSessionEntryRowsReadOnly({ agentId: "main", storePath })).toBe(1); }); it("retains legacy createdBy actor projections across rewrites", async () => { diff --git a/src/config/sessions/session-accessor.ts b/src/config/sessions/session-accessor.ts index dcfe5cdeef3b..2ae27420ac88 100644 --- a/src/config/sessions/session-accessor.ts +++ b/src/config/sessions/session-accessor.ts @@ -119,6 +119,7 @@ export type { UpdateSessionLastRouteParams, } from "./session-accessor.entry-mutation.js"; export { + countSessionEntryRowsReadOnly, copySessionOwnedStateForCanonicalRepair, listSessionGenerationIdsForCanonicalRepair, clearPluginOwnedSessionState, diff --git a/src/gateway/server-startup-handler-prewarm.test.ts b/src/gateway/server-startup-handler-prewarm.test.ts index b955dce9f62f..2a4d170285b0 100644 --- a/src/gateway/server-startup-handler-prewarm.test.ts +++ b/src/gateway/server-startup-handler-prewarm.test.ts @@ -3,19 +3,16 @@ import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js" const mocks = vi.hoisted(() => ({ events: [] as string[], - sessionEntryCounts: new Map(), + canPrewarmCombinedSessionStoresForGateway: vi.fn(() => { + mocks.events.push("sessions.count"); + return true; + }), loadCombinedSessionStoreForGateway: vi.fn((_cfg: unknown, options: { agentId: string }) => { mocks.events.push(`sessions.load.${options.agentId}`); - const entryCount = mocks.sessionEntryCounts.get(options.agentId) ?? 0; return { durableStorePath: `/state/${options.agentId}.sqlite`, storePath: `/state/${options.agentId}.sqlite`, - store: Object.fromEntries( - Array.from({ length: entryCount }, (_, index) => [ - `agent:${options.agentId}:fixture-${index}`, - { sessionId: `session-${index}`, updatedAt: index }, - ]), - ), + store: {}, }; }), listSessionsFromStoreAsync: vi.fn(async (params: { opts: { agentId: string } }) => { @@ -32,6 +29,7 @@ const mocks = vi.hoisted(() => ({ })); vi.mock("../config/sessions/combined-store-gateway.js", () => ({ + canPrewarmCombinedSessionStoresForGateway: mocks.canPrewarmCombinedSessionStoresForGateway, loadCombinedSessionStoreForGateway: mocks.loadCombinedSessionStoreForGateway, })); @@ -51,7 +49,11 @@ const { scheduleGatewayHandlerPrewarm } = await import("./server-startup-handler beforeEach(() => { mocks.events.length = 0; - mocks.sessionEntryCounts.clear(); + mocks.canPrewarmCombinedSessionStoresForGateway.mockClear(); + mocks.canPrewarmCombinedSessionStoresForGateway.mockImplementation(() => { + mocks.events.push("sessions.count"); + return true; + }); mocks.loadCombinedSessionStoreForGateway.mockClear(); mocks.listSessionsFromStoreAsync.mockClear(); mocks.listManagedPlugins.mockClear(); @@ -79,6 +81,7 @@ describe("scheduleGatewayHandlerPrewarm", () => { await vi.runAllTimersAsync(); expect(mocks.events).toEqual([ + "sessions.count", "sessions.load.main", "sessions.rows.main", "sessions.load.research", @@ -120,6 +123,10 @@ describe("scheduleGatewayHandlerPrewarm", () => { agentId: "research", limitPerHost: 40, }); + expect(mocks.canPrewarmCombinedSessionStoresForGateway).toHaveBeenCalledWith(cfg, { + agentIds: ["main", "research"], + maxRows: 2_000, + }); sidecar.stop(); }); @@ -203,25 +210,26 @@ describe("scheduleGatewayHandlerPrewarm", () => { it("skips optional catalog prewarm when the combined session stores are large", async () => { vi.useFakeTimers(); - mocks.sessionEntryCounts.set("main", 2_001); + const info = vi.fn(); + mocks.canPrewarmCombinedSessionStoresForGateway.mockImplementation(() => { + mocks.events.push("sessions.count"); + return false; + }); const cfg = { agents: { list: [{ id: "main", default: true }, { id: "research" }] }, } as never; scheduleGatewayHandlerPrewarm({ cfgAtStart: cfg, - log: { warn: vi.fn() }, + log: { info, warn: vi.fn() }, }); await vi.runAllTimersAsync(); - expect(mocks.events).toEqual([ - "sessions.load.main", - "sessions.rows.main", - "sessions.load.research", - "sessions.rows.research", - "plugins", - ]); + expect(mocks.events).toEqual(["sessions.count", "plugins"]); expect(mocks.prewarmSessionCatalogList).not.toHaveBeenCalled(); + expect(info).toHaveBeenCalledWith( + "skipping optional dashboard session prewarm: combined stores exceed 2000 rows", + ); }); it("stops before scheduling another event-loop turn", async () => { diff --git a/src/gateway/server-startup-handler-prewarm.ts b/src/gateway/server-startup-handler-prewarm.ts index 3092ed95aefc..ccd6ac534bd3 100644 --- a/src/gateway/server-startup-handler-prewarm.ts +++ b/src/gateway/server-startup-handler-prewarm.ts @@ -4,7 +4,7 @@ import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-w const SIDEBAR_SESSION_LIST_LIMIT = 60; const SIDEBAR_CATALOG_LIMIT_PER_HOST = 40; -const SIDEBAR_CATALOG_PREWARM_MAX_SESSION_ENTRIES = 2_000; +const SIDEBAR_PREWARM_MAX_SESSION_ENTRIES = 2_000; type StartupTrace = { measure: (name: string, run: () => T | Promise) => Promise; @@ -19,10 +19,7 @@ type GatewayHandlerPrewarmHandle = { stop: () => void; }; -async function prewarmGatewaySessionListData( - cfg: OpenClawConfig, - agentId: string, -): Promise { +async function prewarmGatewaySessionListData(cfg: OpenClawConfig, agentId: string): Promise { const [{ loadCombinedSessionStoreForGateway }, { listSessionsFromStoreAsync }] = await Promise.all([ import("../config/sessions/combined-store-gateway.js"), @@ -46,19 +43,43 @@ async function prewarmGatewaySessionListData( limit: SIDEBAR_SESSION_LIST_LIMIT, }, }); - return Object.keys(store).length; } -function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmItem[] { +function dashboardDataPrewarmItems( + cfg: OpenClawConfig, + log: { info?: (msg: string) => void }, +): GatewayHandlerPrewarmItem[] { const agentIds = listAgentIds(cfg); - let loadedSessionStores = 0; - let totalSessionEntries = 0; + let sessionDataPrewarmChecked = false; + let sessionDataPrewarmAllowed = false; + const shouldPrewarmSessionData = async () => { + if (sessionDataPrewarmChecked) { + return sessionDataPrewarmAllowed; + } + sessionDataPrewarmChecked = true; + const { canPrewarmCombinedSessionStoresForGateway } = + await import("../config/sessions/combined-store-gateway.js"); + sessionDataPrewarmAllowed = canPrewarmCombinedSessionStoresForGateway(cfg, { + agentIds, + maxRows: SIDEBAR_PREWARM_MAX_SESSION_ENTRIES, + }); + if (!sessionDataPrewarmAllowed) { + log.info?.( + `skipping optional dashboard session prewarm: combined stores exceed ${SIDEBAR_PREWARM_MAX_SESSION_ENTRIES} rows`, + ); + } + return sessionDataPrewarmAllowed; + }; return [ ...agentIds.map((agentId) => ({ name: `sessions.${agentId}`, load: async () => { - totalSessionEntries += await prewarmGatewaySessionListData(cfg, agentId); - loadedSessionStores += 1; + // A count-only query keeps unusually large stores off the synchronous JSON projection + // path. Request-time session and catalog handlers remain authoritative when skipped. + if (!(await shouldPrewarmSessionData())) { + return; + } + await prewarmGatewaySessionListData(cfg, agentId); }, })), { @@ -71,12 +92,7 @@ function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmIt ...agentIds.map((agentId) => ({ name: `session-catalog.${agentId}`, load: async () => { - // Catalog providers may project every OpenClaw session before returning their bounded - // page. Keep that optional cold-cache work off the event loop for unusually large stores. - if ( - loadedSessionStores !== agentIds.length || - totalSessionEntries > SIDEBAR_CATALOG_PREWARM_MAX_SESSION_ENTRIES - ) { + if (!(await shouldPrewarmSessionData())) { return; } const { prewarmSessionCatalogList } = await import("./server-methods/session-catalog.js"); @@ -93,13 +109,13 @@ function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmIt export function scheduleGatewayHandlerPrewarm(params: { cfgAtStart: OpenClawConfig; startupTrace?: StartupTrace; - log: { warn: (msg: string) => void }; + log: { info?: (msg: string) => void; warn: (msg: string) => void }; items?: readonly GatewayHandlerPrewarmItem[]; waitForPostReadyWork?: () => Promise; }): GatewayHandlerPrewarmHandle { // Frequent updater restarts make cold dashboard data the remaining slow tier. // Keep cheap session reads first, process-stable plugin data second, and provider catalogs last. - const items = params.items ?? dashboardDataPrewarmItems(params.cfgAtStart); + const items = params.items ?? dashboardDataPrewarmItems(params.cfgAtStart, params.log); let stopped = false; let nextIndex = 0; let currentItemName = "unknown"; diff --git a/src/gateway/server.sessions.list-store-materialization.test.ts b/src/gateway/server.sessions.list-store-materialization.test.ts index a488f7ab69b3..b358bf964bf2 100644 --- a/src/gateway/server.sessions.list-store-materialization.test.ts +++ b/src/gateway/server.sessions.list-store-materialization.test.ts @@ -175,6 +175,62 @@ test("startup prewarm fills session snapshot and title caches before the first l } }); +test("startup skips a large session prewarm while request-time listing remains available", async () => { + const { storePath } = await createSessionStoreDir(); + await writeSessionStore({ + entries: Object.fromEntries( + Array.from({ length: 2_001 }, (_, index) => [ + `agent:main:large-${index}`, + sessionStoreEntry(`large-${index}`, { updatedAt: 1_781_000_000_000 - index }), + ]), + ), + }); + const info = vi.fn(); + const listSpy = vi.spyOn(sessionAccessor, "listSessionEntriesReadOnly"); + let sidecar: ReturnType | undefined; + vi.useFakeTimers(); + try { + let resolveSessionPrewarm!: () => void; + const sessionPrewarm = new Promise((resolve) => { + resolveSessionPrewarm = resolve; + }); + sidecar = scheduleGatewayHandlerPrewarm({ + cfgAtStart: { + agents: { list: [{ id: "main", default: true }] }, + session: { store: storePath }, + } as never, + log: { info, warn: vi.fn() }, + startupTrace: { + measure: async (name, run) => { + try { + return await run(); + } finally { + if (name === "post-ready.gateway-data.sessions.main") { + resolveSessionPrewarm(); + } + } + }, + }, + }); + + await vi.advanceTimersToNextTimerAsync(); + await sessionPrewarm; + sidecar.stop(); + expect(info).toHaveBeenCalledWith( + "skipping optional dashboard session prewarm: combined stores exceed 2000 rows", + ); + expect(listSpy).not.toHaveBeenCalled(); + + vi.useRealTimers(); + const result = await directSessionReq("sessions.list", LIST_PARAMS); + expect(result.ok).toBe(true); + } finally { + sidecar?.stop(); + vi.useRealTimers(); + listSpy.mockRestore(); + } +}); + test("sessions.list projects out prompt snapshots without changing full entry reads", async () => { await createSessionStoreDir(); await writeSessionStore({ diff --git a/src/gateway/session-utils.subagent.test.ts b/src/gateway/session-utils.subagent.test.ts index 2902117becc4..8c889a772ee5 100644 --- a/src/gateway/session-utils.subagent.test.ts +++ b/src/gateway/session-utils.subagent.test.ts @@ -15,6 +15,7 @@ import { import type { SubagentRunRecord } from "../agents/subagent-registry.types.js"; import type { OpenClawConfig } from "../config/config.js"; import type { SessionEntry } from "../config/sessions.js"; +import { canPrewarmCombinedSessionStoresForGateway } from "../config/sessions/combined-store-gateway.js"; import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; import { registerAgentRunContext, resetAgentEventsForTest } from "../infra/agent-events.js"; import { @@ -1407,6 +1408,13 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" "main", ); + expect( + canPrewarmCombinedSessionStoresForGateway(cfg, { + agentIds: ["main", "ops"], + maxRows: 1, + }), + ).toBe(false); + const { diagnostics, store } = loadCombinedSessionStoreForGateway(cfg); expect(store["agent:main:main"]?.sessionId).toBe("s-main-unscoped"); expect(store["agent:ops:main"]).toBeUndefined(); @@ -1493,6 +1501,19 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" { incognito: true, sessionId: "s-incognito-dynamic", updatedAt: 500 }, "dynamic", ); + await seedSessionEntry( + resolveIncognitoOpenClawAgentSqlitePath({ agentId: "ops" }), + "dashboard:incognito-ops", + { incognito: true, sessionId: "s-incognito-ops", updatedAt: 600 }, + "ops", + ); + + expect( + canPrewarmCombinedSessionStoresForGateway(cfg, { + agentIds: ["ops"], + maxRows: 4, + }), + ).toBe(false); const { store } = loadCombinedSessionStoreForGateway(cfg); expect(store["agent:ops:main"]?.sessionId).toBe("s-ops"); @@ -1590,6 +1611,13 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)" const { store, storePath } = loadCombinedSessionStoreForGateway(cfg, { agentId: "codex" }); + expect( + canPrewarmCombinedSessionStoresForGateway(cfg, { + agentIds: ["codex"], + maxRows: 0, + }), + ).toBe(false); + expect(path.resolve(storePath)).toBe(path.resolve(codexStorePath)); expect(store["agent:codex:acp-task"]?.sessionId).toBe("s-codex"); expect(store["agent:main:main"]).toBeUndefined();