mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-23 10:55:31 -06:00
perf: count large histories before Gateway prewarm (#117118)
* perf(gateway): count before sidebar prewarm * fix(sessions): narrow count row before normalization * fix(gateway): align sidebar prewarm admission targets * test(gateway): prove large prewarm stays optional * fix(gateway): budget repeated shared-store prewarm
This commit is contained in:
committed by
GitHub
parent
982add9618
commit
c50237e37d
@@ -16,7 +16,11 @@ import {
|
||||
import { listOpenIncognitoAgentDatabases } from "../../state/openclaw-agent-db.js";
|
||||
import type { OpenClawConfig } from "../types.openclaw.js";
|
||||
import { resolveStorePath } from "./paths.js";
|
||||
import { listSessionEntries, listSessionEntriesReadOnly } from "./session-accessor.js";
|
||||
import {
|
||||
countSessionEntryRowsReadOnly,
|
||||
listSessionEntries,
|
||||
listSessionEntriesReadOnly,
|
||||
} from "./session-accessor.js";
|
||||
import type { SessionEntryListScope } from "./session-accessor.types.js";
|
||||
import { canonicalSessionKeyMigrationRequiredError } from "./session-canonical-key.js";
|
||||
import { resolveDeliveryProvenCanonicalSessionKey } from "./store-entry.js";
|
||||
@@ -32,6 +36,23 @@ import type { SessionEntry } from "./types.js";
|
||||
|
||||
type GatewaySessionEntryProjection = NonNullable<SessionEntryListScope["projection"]>;
|
||||
|
||||
type GatewaySessionStoreOptions = {
|
||||
agentId?: string;
|
||||
configuredAgentsOnly?: boolean;
|
||||
includeIncognito?: boolean;
|
||||
projection?: SessionEntryListScope["projection"];
|
||||
};
|
||||
|
||||
type ResolvedGatewaySessionStoreTargets = {
|
||||
configuredAgentIds?: ReadonlySet<string>;
|
||||
defaultAgentId: string;
|
||||
diagnostics: string[];
|
||||
durableTargets: Array<{ agentId: string; storePath: string }>;
|
||||
incognitoTargets: Array<{ agentId: string; storePath: string }>;
|
||||
requestedAgentId?: string;
|
||||
storeConfig?: string;
|
||||
};
|
||||
|
||||
// Template-backed stores need per-agent scans before they can be merged for Gateway views.
|
||||
function isStorePathTemplate(store?: string): boolean {
|
||||
return typeof store === "string" && store.includes("{agentId}");
|
||||
@@ -97,20 +118,13 @@ function mergeSessionEntryIntoCombined(params: {
|
||||
}
|
||||
|
||||
function mergeOpenIncognitoStores(params: {
|
||||
allowedAgentIds?: ReadonlySet<string>;
|
||||
cfg: OpenClawConfig;
|
||||
combined: Record<string, SessionEntry>;
|
||||
agentId?: string;
|
||||
projection: GatewaySessionEntryProjection;
|
||||
targets: Array<{ agentId: string; storePath: string }>;
|
||||
}): string[] {
|
||||
const storePaths: string[] = [];
|
||||
for (const target of listOpenIncognitoAgentDatabases()) {
|
||||
if (params.allowedAgentIds && !params.allowedAgentIds.has(target.agentId)) {
|
||||
continue;
|
||||
}
|
||||
if (params.agentId && target.agentId !== params.agentId) {
|
||||
continue;
|
||||
}
|
||||
for (const target of params.targets) {
|
||||
const store = loadGatewayStoreEntries({
|
||||
agentId: target.agentId,
|
||||
includeOpenDatabases: true,
|
||||
@@ -138,27 +152,12 @@ function mergeOpenIncognitoStores(params: {
|
||||
return storePaths;
|
||||
}
|
||||
|
||||
/** Loads and canonicalizes session entries for gateway views across one or more agent stores. */
|
||||
export function loadCombinedSessionStoreForGateway(
|
||||
function resolveGatewaySessionStoreTargets(
|
||||
cfg: OpenClawConfig,
|
||||
opts: {
|
||||
agentId?: string;
|
||||
configuredAgentsOnly?: boolean;
|
||||
includeIncognito?: boolean;
|
||||
projection?: SessionEntryListScope["projection"];
|
||||
} = {},
|
||||
): {
|
||||
diagnostics?: string[];
|
||||
durableStorePath?: string;
|
||||
storePath: string;
|
||||
store: Record<string, SessionEntry>;
|
||||
} {
|
||||
opts: GatewaySessionStoreOptions,
|
||||
): ResolvedGatewaySessionStoreTargets {
|
||||
const storeConfig = cfg.session?.store;
|
||||
const projection = opts.projection ?? "full";
|
||||
const diagnostics: string[] = [];
|
||||
// Exclusion happens before path aggregation; filtering rows afterward would
|
||||
// still leak a live incognito handle by changing the projected store path.
|
||||
const includeIncognito = opts.includeIncognito !== false;
|
||||
const defaultAgentId = normalizeAgentId(resolveDefaultAgentId(cfg));
|
||||
const requestedAgentId =
|
||||
typeof opts.agentId === "string" && opts.agentId.trim()
|
||||
@@ -171,6 +170,13 @@ export function loadCombinedSessionStoreForGateway(
|
||||
const allowedIncognitoAgentIds = requestedAgentId
|
||||
? new Set([requestedAgentId])
|
||||
: configuredAgentIds;
|
||||
const incognitoTargets =
|
||||
opts.includeIncognito === false
|
||||
? []
|
||||
: listOpenIncognitoAgentDatabases().filter(
|
||||
(target) => !allowedIncognitoAgentIds || allowedIncognitoAgentIds.has(target.agentId),
|
||||
);
|
||||
|
||||
if (storeConfig && !isStorePathTemplate(storeConfig)) {
|
||||
const ownerIds = [
|
||||
...new Set([
|
||||
@@ -181,10 +187,7 @@ export function loadCombinedSessionStoreForGateway(
|
||||
...(requestedAgentId ? [requestedAgentId] : []),
|
||||
]),
|
||||
];
|
||||
const combined: Record<string, SessionEntry> = {};
|
||||
// Runtime session access is SQLite-only: a fixed literal is a naming seed whose
|
||||
// resolved database is partitioned per owner. Legacy flat JSON is migration-only.
|
||||
const ownerTargets = dedupeSessionStoreTargetsBySqliteTarget(
|
||||
const durableTargets = dedupeSessionStoreTargetsBySqliteTarget(
|
||||
ownerIds.map((agentId) => ({
|
||||
agentId,
|
||||
storePath: resolveStorePath(storeConfig, { agentId }),
|
||||
@@ -194,7 +197,81 @@ export function loadCombinedSessionStoreForGateway(
|
||||
onDiagnostic: (diagnostic) => diagnostics.push(diagnostic.message),
|
||||
},
|
||||
);
|
||||
for (const { agentId, storePath } of ownerTargets) {
|
||||
return {
|
||||
configuredAgentIds,
|
||||
defaultAgentId,
|
||||
diagnostics,
|
||||
durableTargets,
|
||||
incognitoTargets,
|
||||
requestedAgentId,
|
||||
storeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
const durableTargets = requestedAgentId
|
||||
? resolveAgentSessionStoreTargetsSync(cfg, requestedAgentId)
|
||||
: opts.configuredAgentsOnly === true
|
||||
? resolveSessionStoreTargets(cfg, { allAgents: true })
|
||||
: resolveAllAgentSessionStoreTargetsSync(cfg);
|
||||
return {
|
||||
configuredAgentIds,
|
||||
defaultAgentId,
|
||||
diagnostics,
|
||||
durableTargets,
|
||||
incognitoTargets,
|
||||
requestedAgentId,
|
||||
storeConfig,
|
||||
};
|
||||
}
|
||||
|
||||
/** Checks whether Gateway prewarm can project the selected stores within a bounded row budget. */
|
||||
export function canPrewarmCombinedSessionStoresForGateway(
|
||||
cfg: OpenClawConfig,
|
||||
params: { agentIds: readonly string[]; maxRows: number },
|
||||
): boolean {
|
||||
const defaultAgentId = normalizeAgentId(resolveDefaultAgentId(cfg));
|
||||
let totalRows = 0;
|
||||
for (const agentId of params.agentIds) {
|
||||
const resolved = resolveGatewaySessionStoreTargets(cfg, { agentId });
|
||||
const projectionTargets = dedupeSessionStoreTargetsBySqliteTarget(
|
||||
[...resolved.durableTargets, ...resolved.incognitoTargets],
|
||||
{ defaultAgentId },
|
||||
);
|
||||
for (const target of projectionTargets) {
|
||||
totalRows += countSessionEntryRowsReadOnly(target);
|
||||
if (totalRows > params.maxRows) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Loads and canonicalizes session entries for gateway views across one or more agent stores. */
|
||||
export function loadCombinedSessionStoreForGateway(
|
||||
cfg: OpenClawConfig,
|
||||
opts: GatewaySessionStoreOptions = {},
|
||||
): {
|
||||
diagnostics?: string[];
|
||||
durableStorePath?: string;
|
||||
storePath: string;
|
||||
store: Record<string, SessionEntry>;
|
||||
} {
|
||||
const projection = opts.projection ?? "full";
|
||||
// Count admission and projection share this exact target set. Otherwise an optional
|
||||
// prewarm can approve one database and synchronously materialize another.
|
||||
const {
|
||||
configuredAgentIds,
|
||||
defaultAgentId,
|
||||
diagnostics,
|
||||
durableTargets,
|
||||
incognitoTargets,
|
||||
requestedAgentId,
|
||||
storeConfig,
|
||||
} = resolveGatewaySessionStoreTargets(cfg, opts);
|
||||
if (storeConfig && !isStorePathTemplate(storeConfig)) {
|
||||
const combined: Record<string, SessionEntry> = {};
|
||||
for (const { agentId, storePath } of durableTargets) {
|
||||
const store = loadGatewayStoreEntries({ agentId, projection, storePath });
|
||||
for (const { sessionKey: key, entry } of store) {
|
||||
const canonicalKey = resolveStoredSessionKeyForAgentStore({
|
||||
@@ -226,15 +303,12 @@ export function loadCombinedSessionStoreForGateway(
|
||||
}
|
||||
}
|
||||
const durableStorePath = resolveStorePath(storeConfig, { agentId: defaultAgentId });
|
||||
const incognitoStorePaths = includeIncognito
|
||||
? mergeOpenIncognitoStores({
|
||||
...(allowedIncognitoAgentIds ? { allowedAgentIds: allowedIncognitoAgentIds } : {}),
|
||||
cfg,
|
||||
combined,
|
||||
...(requestedAgentId ? { agentId: requestedAgentId } : {}),
|
||||
projection,
|
||||
})
|
||||
: [];
|
||||
const incognitoStorePaths = mergeOpenIncognitoStores({
|
||||
cfg,
|
||||
combined,
|
||||
projection,
|
||||
targets: incognitoTargets,
|
||||
});
|
||||
return {
|
||||
diagnostics,
|
||||
durableStorePath,
|
||||
@@ -242,13 +316,8 @@ export function loadCombinedSessionStoreForGateway(
|
||||
store: combined,
|
||||
};
|
||||
}
|
||||
const targets = requestedAgentId
|
||||
? resolveAgentSessionStoreTargetsSync(cfg, requestedAgentId)
|
||||
: opts.configuredAgentsOnly === true
|
||||
? resolveSessionStoreTargets(cfg, { allAgents: true })
|
||||
: resolveAllAgentSessionStoreTargetsSync(cfg);
|
||||
const combined: Record<string, SessionEntry> = {};
|
||||
for (const target of targets) {
|
||||
for (const target of durableTargets) {
|
||||
const agentId = target.agentId;
|
||||
const storePath = target.storePath;
|
||||
const store = loadGatewayStoreEntries({ agentId, projection, storePath });
|
||||
@@ -282,17 +351,14 @@ export function loadCombinedSessionStoreForGateway(
|
||||
}
|
||||
}
|
||||
|
||||
const incognitoStorePaths = includeIncognito
|
||||
? mergeOpenIncognitoStores({
|
||||
...(allowedIncognitoAgentIds ? { allowedAgentIds: allowedIncognitoAgentIds } : {}),
|
||||
cfg,
|
||||
combined,
|
||||
...(requestedAgentId ? { agentId: requestedAgentId } : {}),
|
||||
projection,
|
||||
})
|
||||
: [];
|
||||
const incognitoStorePaths = mergeOpenIncognitoStores({
|
||||
cfg,
|
||||
combined,
|
||||
projection,
|
||||
targets: incognitoTargets,
|
||||
});
|
||||
|
||||
const durableStorePaths = targets.map((target) => target.storePath);
|
||||
const durableStorePaths = durableTargets.map((target) => target.storePath);
|
||||
const durableStorePath = resolveCombinedStorePath(durableStorePaths, storeConfig);
|
||||
const storePath = resolveCombinedStorePath(
|
||||
[...durableStorePaths, ...incognitoStorePaths],
|
||||
|
||||
@@ -10,6 +10,7 @@ import { resolveAgentMainSessionKey } from "./main-session.js";
|
||||
import { resolveStorePath } from "./paths.js";
|
||||
import { clearPluginOwnedSessionState } from "./plugin-host-cleanup.js";
|
||||
import {
|
||||
countSqliteSessionEntryRowsReadOnly as countSessionEntryRowsReadOnly,
|
||||
copySqliteSessionOwnedStateForCanonicalRepair as copySessionOwnedStateForCanonicalRepair,
|
||||
listSqliteSessionGenerationIdsForCanonicalRepair as listSessionGenerationIdsForCanonicalRepair,
|
||||
listSqliteSessionChildEntriesReadOnly as listSessionChildEntriesReadOnly,
|
||||
@@ -58,6 +59,7 @@ export { clearPluginOwnedSessionState };
|
||||
// SQLite is the only runtime session store. Re-export its canonical entry
|
||||
// operations directly instead of maintaining a second pass-through layer.
|
||||
export {
|
||||
countSessionEntryRowsReadOnly,
|
||||
copySessionOwnedStateForCanonicalRepair,
|
||||
listSessionGenerationIdsForCanonicalRepair,
|
||||
listSessionChildEntriesReadOnly,
|
||||
|
||||
@@ -281,6 +281,22 @@ export function listSqliteSessionEntriesReadOnly(
|
||||
return result.found ? result.value : [];
|
||||
}
|
||||
|
||||
/** Counts durable session rows without materializing entry JSON or warming the entry cache. */
|
||||
export function countSqliteSessionEntryRowsReadOnly(scope: SessionEntryListScope = {}): number {
|
||||
const resolved = resolveSqliteScope({ ...scope, sessionKey: "" });
|
||||
const result = withOpenClawAgentDatabaseReadOnly((database) => {
|
||||
const db = getSessionKysely(database.db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("session_nodes")
|
||||
.select((expression) => expression.fn.countAll<number | bigint>().as("count")),
|
||||
);
|
||||
return row ? normalizeSqliteNumber(row.count) : 0;
|
||||
}, toDatabaseOptions(resolved));
|
||||
return result.found ? result.value : 0;
|
||||
}
|
||||
|
||||
function listSqliteSessionEntriesFromDatabase(
|
||||
database: Pick<OpenClawAgentDatabase, "agentId" | "db" | "path">,
|
||||
resolved: ResolvedSqliteScope,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Stable SQLite accessor surface. Domain owners live in the focused modules below.
|
||||
export {
|
||||
countSqliteSessionEntryRowsReadOnly,
|
||||
listSqliteSessionEntries,
|
||||
listSqliteSessionChildEntriesReadOnly,
|
||||
listSqliteSessionEntriesReadOnly,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { withTestTimeout } from "../../../test/helpers/promise.js";
|
||||
@@ -26,6 +27,7 @@ import {
|
||||
appendTranscriptMessage,
|
||||
applySessionEntryLifecycleMutation,
|
||||
commitReplySessionInitialization,
|
||||
countSessionEntryRowsReadOnly,
|
||||
createSessionEntryWithTranscript,
|
||||
deleteSessionEntryLifecycle,
|
||||
findTranscriptEvent,
|
||||
@@ -225,6 +227,24 @@ describe("session accessor seam", () => {
|
||||
|
||||
expect(readSqliteSessionEntryCount(database)).toBe(1);
|
||||
expect(readSqliteSessionEntryKeys(database)).toEqual(["agent:main:logical-entry"]);
|
||||
expect(countSessionEntryRowsReadOnly({ agentId: "main", storePath })).toBe(2);
|
||||
});
|
||||
|
||||
it("counts rows on a cold handle without parsing invalid entry JSON", async () => {
|
||||
await replaceSessionEntry(
|
||||
{ sessionKey: "agent:main:cold-count", storePath },
|
||||
{ sessionId: "cold-count-session", updatedAt: 10 },
|
||||
);
|
||||
const databasePath = expectDefined(
|
||||
resolveSqliteTargetFromSessionStorePath(storePath, { agentId: "main" }).path,
|
||||
"cold count database path",
|
||||
);
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
const database = new DatabaseSync(databasePath);
|
||||
database.prepare("UPDATE session_nodes SET entry_valid = 0").run();
|
||||
database.close();
|
||||
|
||||
expect(countSessionEntryRowsReadOnly({ agentId: "main", storePath })).toBe(1);
|
||||
});
|
||||
|
||||
it("retains legacy createdBy actor projections across rewrites", async () => {
|
||||
|
||||
@@ -119,6 +119,7 @@ export type {
|
||||
UpdateSessionLastRouteParams,
|
||||
} from "./session-accessor.entry-mutation.js";
|
||||
export {
|
||||
countSessionEntryRowsReadOnly,
|
||||
copySessionOwnedStateForCanonicalRepair,
|
||||
listSessionGenerationIdsForCanonicalRepair,
|
||||
clearPluginOwnedSessionState,
|
||||
|
||||
@@ -3,19 +3,16 @@ import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js"
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
events: [] as string[],
|
||||
sessionEntryCounts: new Map<string, number>(),
|
||||
canPrewarmCombinedSessionStoresForGateway: vi.fn(() => {
|
||||
mocks.events.push("sessions.count");
|
||||
return true;
|
||||
}),
|
||||
loadCombinedSessionStoreForGateway: vi.fn((_cfg: unknown, options: { agentId: string }) => {
|
||||
mocks.events.push(`sessions.load.${options.agentId}`);
|
||||
const entryCount = mocks.sessionEntryCounts.get(options.agentId) ?? 0;
|
||||
return {
|
||||
durableStorePath: `/state/${options.agentId}.sqlite`,
|
||||
storePath: `/state/${options.agentId}.sqlite`,
|
||||
store: Object.fromEntries(
|
||||
Array.from({ length: entryCount }, (_, index) => [
|
||||
`agent:${options.agentId}:fixture-${index}`,
|
||||
{ sessionId: `session-${index}`, updatedAt: index },
|
||||
]),
|
||||
),
|
||||
store: {},
|
||||
};
|
||||
}),
|
||||
listSessionsFromStoreAsync: vi.fn(async (params: { opts: { agentId: string } }) => {
|
||||
@@ -32,6 +29,7 @@ const mocks = vi.hoisted(() => ({
|
||||
}));
|
||||
|
||||
vi.mock("../config/sessions/combined-store-gateway.js", () => ({
|
||||
canPrewarmCombinedSessionStoresForGateway: mocks.canPrewarmCombinedSessionStoresForGateway,
|
||||
loadCombinedSessionStoreForGateway: mocks.loadCombinedSessionStoreForGateway,
|
||||
}));
|
||||
|
||||
@@ -51,7 +49,11 @@ const { scheduleGatewayHandlerPrewarm } = await import("./server-startup-handler
|
||||
|
||||
beforeEach(() => {
|
||||
mocks.events.length = 0;
|
||||
mocks.sessionEntryCounts.clear();
|
||||
mocks.canPrewarmCombinedSessionStoresForGateway.mockClear();
|
||||
mocks.canPrewarmCombinedSessionStoresForGateway.mockImplementation(() => {
|
||||
mocks.events.push("sessions.count");
|
||||
return true;
|
||||
});
|
||||
mocks.loadCombinedSessionStoreForGateway.mockClear();
|
||||
mocks.listSessionsFromStoreAsync.mockClear();
|
||||
mocks.listManagedPlugins.mockClear();
|
||||
@@ -79,6 +81,7 @@ describe("scheduleGatewayHandlerPrewarm", () => {
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(mocks.events).toEqual([
|
||||
"sessions.count",
|
||||
"sessions.load.main",
|
||||
"sessions.rows.main",
|
||||
"sessions.load.research",
|
||||
@@ -120,6 +123,10 @@ describe("scheduleGatewayHandlerPrewarm", () => {
|
||||
agentId: "research",
|
||||
limitPerHost: 40,
|
||||
});
|
||||
expect(mocks.canPrewarmCombinedSessionStoresForGateway).toHaveBeenCalledWith(cfg, {
|
||||
agentIds: ["main", "research"],
|
||||
maxRows: 2_000,
|
||||
});
|
||||
sidecar.stop();
|
||||
});
|
||||
|
||||
@@ -203,25 +210,26 @@ describe("scheduleGatewayHandlerPrewarm", () => {
|
||||
|
||||
it("skips optional catalog prewarm when the combined session stores are large", async () => {
|
||||
vi.useFakeTimers();
|
||||
mocks.sessionEntryCounts.set("main", 2_001);
|
||||
const info = vi.fn();
|
||||
mocks.canPrewarmCombinedSessionStoresForGateway.mockImplementation(() => {
|
||||
mocks.events.push("sessions.count");
|
||||
return false;
|
||||
});
|
||||
const cfg = {
|
||||
agents: { list: [{ id: "main", default: true }, { id: "research" }] },
|
||||
} as never;
|
||||
|
||||
scheduleGatewayHandlerPrewarm({
|
||||
cfgAtStart: cfg,
|
||||
log: { warn: vi.fn() },
|
||||
log: { info, warn: vi.fn() },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(mocks.events).toEqual([
|
||||
"sessions.load.main",
|
||||
"sessions.rows.main",
|
||||
"sessions.load.research",
|
||||
"sessions.rows.research",
|
||||
"plugins",
|
||||
]);
|
||||
expect(mocks.events).toEqual(["sessions.count", "plugins"]);
|
||||
expect(mocks.prewarmSessionCatalogList).not.toHaveBeenCalled();
|
||||
expect(info).toHaveBeenCalledWith(
|
||||
"skipping optional dashboard session prewarm: combined stores exceed 2000 rows",
|
||||
);
|
||||
});
|
||||
|
||||
it("stops before scheduling another event-loop turn", async () => {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-w
|
||||
|
||||
const SIDEBAR_SESSION_LIST_LIMIT = 60;
|
||||
const SIDEBAR_CATALOG_LIMIT_PER_HOST = 40;
|
||||
const SIDEBAR_CATALOG_PREWARM_MAX_SESSION_ENTRIES = 2_000;
|
||||
const SIDEBAR_PREWARM_MAX_SESSION_ENTRIES = 2_000;
|
||||
|
||||
type StartupTrace = {
|
||||
measure: <T>(name: string, run: () => T | Promise<T>) => Promise<T>;
|
||||
@@ -19,10 +19,7 @@ type GatewayHandlerPrewarmHandle = {
|
||||
stop: () => void;
|
||||
};
|
||||
|
||||
async function prewarmGatewaySessionListData(
|
||||
cfg: OpenClawConfig,
|
||||
agentId: string,
|
||||
): Promise<number> {
|
||||
async function prewarmGatewaySessionListData(cfg: OpenClawConfig, agentId: string): Promise<void> {
|
||||
const [{ loadCombinedSessionStoreForGateway }, { listSessionsFromStoreAsync }] =
|
||||
await Promise.all([
|
||||
import("../config/sessions/combined-store-gateway.js"),
|
||||
@@ -46,19 +43,43 @@ async function prewarmGatewaySessionListData(
|
||||
limit: SIDEBAR_SESSION_LIST_LIMIT,
|
||||
},
|
||||
});
|
||||
return Object.keys(store).length;
|
||||
}
|
||||
|
||||
function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmItem[] {
|
||||
function dashboardDataPrewarmItems(
|
||||
cfg: OpenClawConfig,
|
||||
log: { info?: (msg: string) => void },
|
||||
): GatewayHandlerPrewarmItem[] {
|
||||
const agentIds = listAgentIds(cfg);
|
||||
let loadedSessionStores = 0;
|
||||
let totalSessionEntries = 0;
|
||||
let sessionDataPrewarmChecked = false;
|
||||
let sessionDataPrewarmAllowed = false;
|
||||
const shouldPrewarmSessionData = async () => {
|
||||
if (sessionDataPrewarmChecked) {
|
||||
return sessionDataPrewarmAllowed;
|
||||
}
|
||||
sessionDataPrewarmChecked = true;
|
||||
const { canPrewarmCombinedSessionStoresForGateway } =
|
||||
await import("../config/sessions/combined-store-gateway.js");
|
||||
sessionDataPrewarmAllowed = canPrewarmCombinedSessionStoresForGateway(cfg, {
|
||||
agentIds,
|
||||
maxRows: SIDEBAR_PREWARM_MAX_SESSION_ENTRIES,
|
||||
});
|
||||
if (!sessionDataPrewarmAllowed) {
|
||||
log.info?.(
|
||||
`skipping optional dashboard session prewarm: combined stores exceed ${SIDEBAR_PREWARM_MAX_SESSION_ENTRIES} rows`,
|
||||
);
|
||||
}
|
||||
return sessionDataPrewarmAllowed;
|
||||
};
|
||||
return [
|
||||
...agentIds.map((agentId) => ({
|
||||
name: `sessions.${agentId}`,
|
||||
load: async () => {
|
||||
totalSessionEntries += await prewarmGatewaySessionListData(cfg, agentId);
|
||||
loadedSessionStores += 1;
|
||||
// A count-only query keeps unusually large stores off the synchronous JSON projection
|
||||
// path. Request-time session and catalog handlers remain authoritative when skipped.
|
||||
if (!(await shouldPrewarmSessionData())) {
|
||||
return;
|
||||
}
|
||||
await prewarmGatewaySessionListData(cfg, agentId);
|
||||
},
|
||||
})),
|
||||
{
|
||||
@@ -71,12 +92,7 @@ function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmIt
|
||||
...agentIds.map((agentId) => ({
|
||||
name: `session-catalog.${agentId}`,
|
||||
load: async () => {
|
||||
// Catalog providers may project every OpenClaw session before returning their bounded
|
||||
// page. Keep that optional cold-cache work off the event loop for unusually large stores.
|
||||
if (
|
||||
loadedSessionStores !== agentIds.length ||
|
||||
totalSessionEntries > SIDEBAR_CATALOG_PREWARM_MAX_SESSION_ENTRIES
|
||||
) {
|
||||
if (!(await shouldPrewarmSessionData())) {
|
||||
return;
|
||||
}
|
||||
const { prewarmSessionCatalogList } = await import("./server-methods/session-catalog.js");
|
||||
@@ -93,13 +109,13 @@ function dashboardDataPrewarmItems(cfg: OpenClawConfig): GatewayHandlerPrewarmIt
|
||||
export function scheduleGatewayHandlerPrewarm(params: {
|
||||
cfgAtStart: OpenClawConfig;
|
||||
startupTrace?: StartupTrace;
|
||||
log: { warn: (msg: string) => void };
|
||||
log: { info?: (msg: string) => void; warn: (msg: string) => void };
|
||||
items?: readonly GatewayHandlerPrewarmItem[];
|
||||
waitForPostReadyWork?: () => Promise<void>;
|
||||
}): GatewayHandlerPrewarmHandle {
|
||||
// Frequent updater restarts make cold dashboard data the remaining slow tier.
|
||||
// Keep cheap session reads first, process-stable plugin data second, and provider catalogs last.
|
||||
const items = params.items ?? dashboardDataPrewarmItems(params.cfgAtStart);
|
||||
const items = params.items ?? dashboardDataPrewarmItems(params.cfgAtStart, params.log);
|
||||
let stopped = false;
|
||||
let nextIndex = 0;
|
||||
let currentItemName = "unknown";
|
||||
|
||||
@@ -175,6 +175,62 @@ test("startup prewarm fills session snapshot and title caches before the first l
|
||||
}
|
||||
});
|
||||
|
||||
test("startup skips a large session prewarm while request-time listing remains available", async () => {
|
||||
const { storePath } = await createSessionStoreDir();
|
||||
await writeSessionStore({
|
||||
entries: Object.fromEntries(
|
||||
Array.from({ length: 2_001 }, (_, index) => [
|
||||
`agent:main:large-${index}`,
|
||||
sessionStoreEntry(`large-${index}`, { updatedAt: 1_781_000_000_000 - index }),
|
||||
]),
|
||||
),
|
||||
});
|
||||
const info = vi.fn();
|
||||
const listSpy = vi.spyOn(sessionAccessor, "listSessionEntriesReadOnly");
|
||||
let sidecar: ReturnType<typeof scheduleGatewayHandlerPrewarm> | undefined;
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let resolveSessionPrewarm!: () => void;
|
||||
const sessionPrewarm = new Promise<void>((resolve) => {
|
||||
resolveSessionPrewarm = resolve;
|
||||
});
|
||||
sidecar = scheduleGatewayHandlerPrewarm({
|
||||
cfgAtStart: {
|
||||
agents: { list: [{ id: "main", default: true }] },
|
||||
session: { store: storePath },
|
||||
} as never,
|
||||
log: { info, warn: vi.fn() },
|
||||
startupTrace: {
|
||||
measure: async (name, run) => {
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
if (name === "post-ready.gateway-data.sessions.main") {
|
||||
resolveSessionPrewarm();
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await vi.advanceTimersToNextTimerAsync();
|
||||
await sessionPrewarm;
|
||||
sidecar.stop();
|
||||
expect(info).toHaveBeenCalledWith(
|
||||
"skipping optional dashboard session prewarm: combined stores exceed 2000 rows",
|
||||
);
|
||||
expect(listSpy).not.toHaveBeenCalled();
|
||||
|
||||
vi.useRealTimers();
|
||||
const result = await directSessionReq("sessions.list", LIST_PARAMS);
|
||||
expect(result.ok).toBe(true);
|
||||
} finally {
|
||||
sidecar?.stop();
|
||||
vi.useRealTimers();
|
||||
listSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test("sessions.list projects out prompt snapshots without changing full entry reads", async () => {
|
||||
await createSessionStoreDir();
|
||||
await writeSessionStore({
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import type { SubagentRunRecord } from "../agents/subagent-registry.types.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import { canPrewarmCombinedSessionStoresForGateway } from "../config/sessions/combined-store-gateway.js";
|
||||
import { replaceSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import { registerAgentRunContext, resetAgentEventsForTest } from "../infra/agent-events.js";
|
||||
import {
|
||||
@@ -1407,6 +1408,13 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)"
|
||||
"main",
|
||||
);
|
||||
|
||||
expect(
|
||||
canPrewarmCombinedSessionStoresForGateway(cfg, {
|
||||
agentIds: ["main", "ops"],
|
||||
maxRows: 1,
|
||||
}),
|
||||
).toBe(false);
|
||||
|
||||
const { diagnostics, store } = loadCombinedSessionStoreForGateway(cfg);
|
||||
expect(store["agent:main:main"]?.sessionId).toBe("s-main-unscoped");
|
||||
expect(store["agent:ops:main"]).toBeUndefined();
|
||||
@@ -1493,6 +1501,19 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)"
|
||||
{ incognito: true, sessionId: "s-incognito-dynamic", updatedAt: 500 },
|
||||
"dynamic",
|
||||
);
|
||||
await seedSessionEntry(
|
||||
resolveIncognitoOpenClawAgentSqlitePath({ agentId: "ops" }),
|
||||
"dashboard:incognito-ops",
|
||||
{ incognito: true, sessionId: "s-incognito-ops", updatedAt: 600 },
|
||||
"ops",
|
||||
);
|
||||
|
||||
expect(
|
||||
canPrewarmCombinedSessionStoresForGateway(cfg, {
|
||||
agentIds: ["ops"],
|
||||
maxRows: 4,
|
||||
}),
|
||||
).toBe(false);
|
||||
|
||||
const { store } = loadCombinedSessionStoreForGateway(cfg);
|
||||
expect(store["agent:ops:main"]?.sessionId).toBe("s-ops");
|
||||
@@ -1590,6 +1611,13 @@ describe("loadCombinedSessionStoreForGateway includes disk-only agents (#32804)"
|
||||
|
||||
const { store, storePath } = loadCombinedSessionStoreForGateway(cfg, { agentId: "codex" });
|
||||
|
||||
expect(
|
||||
canPrewarmCombinedSessionStoresForGateway(cfg, {
|
||||
agentIds: ["codex"],
|
||||
maxRows: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
|
||||
expect(path.resolve(storePath)).toBe(path.resolve(codexStorePath));
|
||||
expect(store["agent:codex:acp-task"]?.sessionId).toBe("s-codex");
|
||||
expect(store["agent:main:main"]).toBeUndefined();
|
||||
|
||||
Reference in New Issue
Block a user