diff --git a/extensions/memory-core/src/dreaming-ingestion-state.ts b/extensions/memory-core/src/dreaming-ingestion-state.ts new file mode 100644 index 000000000000..9d188c9f7e76 --- /dev/null +++ b/extensions/memory-core/src/dreaming-ingestion-state.ts @@ -0,0 +1,117 @@ +// Memory Core codecs normalize canonical and legacy dreaming ingestion state. +import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { asRecord } from "./dreaming-shared.js"; + +const MEMORY_DAY_RE = /^\d{4}-\d{2}-\d{2}$/; + +export const SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION = 4096; + +export type DailyIngestionFileState = { + mtimeMs: number; + size: number; + lastDreamingDayIngested?: string; +}; + +export type DailyIngestionState = { + version: 1; + files: Record; +}; + +export type SessionIngestionFileState = { + mtimeMs: number; + size: number; + contentHash: string; + lineCount: number; + lastContentLine: number; +}; + +export type SessionIngestionState = { + version: 3; + files: Record; + seenMessages: Record; +}; + +export function normalizeMemoryDay(value: unknown): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const day = value.trim(); + return MEMORY_DAY_RE.test(day) ? day : undefined; +} + +export function normalizeDailyIngestionState(raw: unknown): DailyIngestionState { + const record = asRecord(raw); + const filesRaw = asRecord(record?.files); + if (!filesRaw) { + return { version: 1, files: {} }; + } + const files: Record = {}; + for (const [key, value] of Object.entries(filesRaw)) { + const file = asRecord(value); + if (!file || typeof key !== "string" || key.trim().length === 0) { + continue; + } + const mtimeMs = Number(file.mtimeMs); + const size = Number(file.size); + if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) { + continue; + } + const lastDreamingDayIngested = normalizeMemoryDay(file.lastDreamingDayIngested); + files[key] = { + mtimeMs: Math.floor(mtimeMs), + size: Math.floor(size), + ...(lastDreamingDayIngested ? { lastDreamingDayIngested } : {}), + }; + } + return { version: 1, files }; +} + +export function normalizeSessionIngestionState(raw: unknown): SessionIngestionState { + const record = asRecord(raw); + const filesRaw = asRecord(record?.files); + const files: Record = {}; + if (filesRaw) { + for (const [key, value] of Object.entries(filesRaw)) { + const file = asRecord(value); + if (!file || key.trim().length === 0) { + continue; + } + const mtimeMs = Number(file.mtimeMs); + const size = Number(file.size); + if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) { + continue; + } + const lineCountRaw = Number(file.lineCount); + const lastContentLineRaw = Number(file.lastContentLine); + const lineCount = + Number.isFinite(lineCountRaw) && lineCountRaw >= 0 ? Math.floor(lineCountRaw) : 0; + const lastContentLine = + Number.isFinite(lastContentLineRaw) && lastContentLineRaw >= 0 + ? Math.floor(lastContentLineRaw) + : 0; + files[key] = { + mtimeMs: Math.floor(mtimeMs), + size: Math.floor(size), + contentHash: typeof file.contentHash === "string" ? file.contentHash.trim() : "", + lineCount, + lastContentLine: Math.min(lineCount, lastContentLine), + }; + } + } + const seenMessagesRaw = asRecord(record?.seenMessages); + const seenMessages: Record = {}; + if (seenMessagesRaw) { + for (const [scope, value] of Object.entries(seenMessagesRaw)) { + if (scope.trim().length === 0 || !Array.isArray(value)) { + continue; + } + const unique = normalizeStringEntries([ + ...new Set(value.filter((entry): entry is string => typeof entry === "string")), + ]).slice(-SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION); + if (unique.length > 0) { + seenMessages[scope] = unique; + } + } + } + return { version: 3, files, seenMessages }; +} diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 1b83081225aa..d899b70d58fe 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -22,6 +22,16 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import { appendFailedDreamingEvent } from "./dreaming-events.js"; +import { + normalizeDailyIngestionState, + normalizeMemoryDay, + normalizeSessionIngestionState, + SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION, + type DailyIngestionFileState, + type DailyIngestionState, + type SessionIngestionFileState, + type SessionIngestionState, +} from "./dreaming-ingestion-state.js"; import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js"; import { generateAndAppendDreamNarrative, @@ -29,7 +39,7 @@ import { type NarrativePhaseData, runDetachedDreamNarrative, } from "./dreaming-narrative.js"; -import { asRecord, formatErrorMessage } from "./dreaming-shared.js"; +import { formatErrorMessage } from "./dreaming-shared.js"; import { DREAMING_DAILY_INGESTION_NAMESPACE, DREAMING_SESSION_INGESTION_FILES_NAMESPACE, @@ -70,22 +80,11 @@ type RemDreamingConfig = DreamingPhaseStorageConfig & { limit: number; minPatternStrength: number; }; -const MEMORY_DAY_RE = /^\d{4}-\d{2}-\d{2}$/; const DAILY_MEMORY_FILENAME_RE = /^(\d{4}-\d{2}-\d{2})(?:-[^/]+)?\.md$/i; -export const DAILY_INGESTION_STATE_RELATIVE_PATH = path.join( - "memory", - ".dreams", - "daily-ingestion.json", -); const DAILY_INGESTION_SCORE = 0.62; const DAILY_INGESTION_MAX_SNIPPET_CHARS = 280; const DAILY_INGESTION_MIN_SNIPPET_CHARS = 8; const DAILY_INGESTION_MAX_CHUNK_LINES = 4; -export const SESSION_INGESTION_STATE_RELATIVE_PATH = path.join( - "memory", - ".dreams", - "session-ingestion.json", -); const SESSION_CORPUS_RELATIVE_DIR = path.join("memory", ".dreams", "session-corpus"); const SESSION_INGESTION_SCORE = 0.58; const SESSION_INGESTION_MAX_SNIPPET_CHARS = 280; @@ -93,7 +92,6 @@ const SESSION_INGESTION_MIN_SNIPPET_CHARS = 12; const SESSION_INGESTION_MAX_MESSAGES_PER_SWEEP = 240; const SESSION_INGESTION_MAX_MESSAGES_PER_FILE = 80; const SESSION_INGESTION_MIN_MESSAGES_PER_FILE = 12; -const SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION = 4096; const SESSION_INGESTION_MAX_TRACKED_SCOPES = 2048; const SESSION_CHECKPOINT_TRANSCRIPT_FILENAME_RE = /\.checkpoint\..+\.jsonl$/i; const LIGHT_DIARY_HISTORY_LIMIT = 4; @@ -369,12 +367,6 @@ type DailyMemoryFile = { canonical: boolean; }; -type DailyIngestionFileState = { - mtimeMs: number; - size: number; - lastDreamingDayIngested?: string; -}; - function parseDailyMemoryFileName(fileName: string): DailyMemoryFile | null { const match = fileName.match(DAILY_MEMORY_FILENAME_RE); const day = match?.[1]; @@ -406,52 +398,6 @@ function resolveWorkspaceMemoryRelativePath(workspaceDir: string, filePath: stri return `memory/${path.basename(filePath)}`; } -type DailyIngestionState = { - version: 1; - files: Record; -}; - -export function normalizeDailyIngestionState(raw: unknown): DailyIngestionState { - const record = asRecord(raw); - const filesRaw = asRecord(record?.files); - if (!filesRaw) { - return { - version: 1, - files: {}, - }; - } - const files: Record = {}; - for (const [key, value] of Object.entries(filesRaw)) { - const file = asRecord(value); - if (!file || typeof key !== "string" || key.trim().length === 0) { - continue; - } - const mtimeMs = Number(file.mtimeMs); - const size = Number(file.size); - if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) { - continue; - } - const lastDreamingDayIngested = normalizeMemoryDay(file.lastDreamingDayIngested); - files[key] = { - mtimeMs: Math.floor(mtimeMs), - size: Math.floor(size), - ...(lastDreamingDayIngested ? { lastDreamingDayIngested } : {}), - }; - } - return { - version: 1, - files, - }; -} - -function normalizeMemoryDay(value: unknown): string | undefined { - if (typeof value !== "string") { - return undefined; - } - const day = value.trim(); - return MEMORY_DAY_RE.test(day) ? day : undefined; -} - async function readDailyIngestionState(workspaceDir: string): Promise { const entries = await readMemoryCoreWorkspaceEntries({ namespace: DREAMING_DAILY_INGESTION_NAMESPACE, @@ -474,20 +420,6 @@ async function writeDailyIngestionState( }); } -type SessionIngestionFileState = { - mtimeMs: number; - size: number; - contentHash: string; - lineCount: number; - lastContentLine: number; -}; - -type SessionIngestionState = { - version: 3; - files: Record; - seenMessages: Record; -}; - type SessionIngestionMessage = { day: string; snippet: string; @@ -500,56 +432,6 @@ type SessionIngestionCollectionResult = { changed: boolean; }; -export function normalizeSessionIngestionState(raw: unknown): SessionIngestionState { - const record = asRecord(raw); - const filesRaw = asRecord(record?.files); - const files: Record = {}; - if (filesRaw) { - for (const [key, value] of Object.entries(filesRaw)) { - const file = asRecord(value); - if (!file || key.trim().length === 0) { - continue; - } - const mtimeMs = Number(file.mtimeMs); - const size = Number(file.size); - if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) { - continue; - } - const lineCountRaw = Number(file.lineCount); - const lastContentLineRaw = Number(file.lastContentLine); - const lineCount = - Number.isFinite(lineCountRaw) && lineCountRaw >= 0 ? Math.floor(lineCountRaw) : 0; - const lastContentLine = - Number.isFinite(lastContentLineRaw) && lastContentLineRaw >= 0 - ? Math.floor(lastContentLineRaw) - : 0; - files[key] = { - mtimeMs: Math.floor(mtimeMs), - size: Math.floor(size), - contentHash: typeof file.contentHash === "string" ? file.contentHash.trim() : "", - lineCount, - lastContentLine: Math.min(lineCount, lastContentLine), - }; - } - } - const seenMessagesRaw = asRecord(record?.seenMessages); - const seenMessages: Record = {}; - if (seenMessagesRaw) { - for (const [scope, value] of Object.entries(seenMessagesRaw)) { - if (scope.trim().length === 0 || !Array.isArray(value)) { - continue; - } - const unique = normalizeStringEntries([ - ...new Set(value.filter((entry): entry is string => typeof entry === "string")), - ]).slice(-SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION); - if (unique.length > 0) { - seenMessages[scope] = unique; - } - } - } - return { version: 3, files, seenMessages }; -} - async function readSessionIngestionState(workspaceDir: string): Promise { const [fileEntries, seenChunks] = await Promise.all([ readMemoryCoreWorkspaceEntries({ diff --git a/extensions/memory-core/src/migration/doctor-dreaming-state.ts b/extensions/memory-core/src/migration/doctor-dreaming-state.ts index 4c87bff8fb80..d2cddb862580 100644 --- a/extensions/memory-core/src/migration/doctor-dreaming-state.ts +++ b/extensions/memory-core/src/migration/doctor-dreaming-state.ts @@ -6,11 +6,9 @@ import { legacyStateFileExists, } from "openclaw/plugin-sdk/runtime-doctor"; import { - DAILY_INGESTION_STATE_RELATIVE_PATH, - SESSION_INGESTION_STATE_RELATIVE_PATH, normalizeDailyIngestionState, normalizeSessionIngestionState, -} from "../dreaming-phases.js"; +} from "../dreaming-ingestion-state.js"; import { DREAMING_DAILY_INGESTION_NAMESPACE, DREAMING_SESSION_INGESTION_FILES_NAMESPACE, @@ -38,6 +36,17 @@ type LegacySource = { filePath: string; }; +const LEGACY_DAILY_INGESTION_STATE_RELATIVE_PATH = path.join( + "memory", + ".dreams", + "daily-ingestion.json", +); +const LEGACY_SESSION_INGESTION_STATE_RELATIVE_PATH = path.join( + "memory", + ".dreams", + "session-ingestion.json", +); + async function readJsonFile(filePath: string): Promise { return JSON.parse(await fs.readFile(filePath, "utf8")); } @@ -49,8 +58,8 @@ async function collectLegacySources( const sources: LegacySource[] = []; for (const workspaceDir of resolveConfiguredWorkspaces(config, env)) { const candidates = [ - { label: "daily ingestion", relativePath: DAILY_INGESTION_STATE_RELATIVE_PATH }, - { label: "session ingestion", relativePath: SESSION_INGESTION_STATE_RELATIVE_PATH }, + { label: "daily ingestion", relativePath: LEGACY_DAILY_INGESTION_STATE_RELATIVE_PATH }, + { label: "session ingestion", relativePath: LEGACY_SESSION_INGESTION_STATE_RELATIVE_PATH }, { label: "short-term recall", relativePath: SHORT_TERM_STORE_RELATIVE_PATH }, { label: "phase signals", relativePath: SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH }, ]; diff --git a/extensions/memory-core/src/migration/dreaming-state-comparison.ts b/extensions/memory-core/src/migration/dreaming-state-comparison.ts index a9d7dea070ae..bba876aac15d 100644 --- a/extensions/memory-core/src/migration/dreaming-state-comparison.ts +++ b/extensions/memory-core/src/migration/dreaming-state-comparison.ts @@ -5,7 +5,7 @@ import { isDeepStrictEqual } from "node:util"; import { normalizeDailyIngestionState, normalizeSessionIngestionState, -} from "../dreaming-phases.js"; +} from "../dreaming-ingestion-state.js"; import { DREAMING_DAILY_INGESTION_NAMESPACE, DREAMING_SESSION_INGESTION_FILES_NAMESPACE, diff --git a/extensions/memory-core/src/test-helpers.ts b/extensions/memory-core/src/test-helpers.ts index ea564b4e28fa..77a29a293a22 100644 --- a/extensions/memory-core/src/test-helpers.ts +++ b/extensions/memory-core/src/test-helpers.ts @@ -5,7 +5,10 @@ import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-run import { createPluginStateKeyedStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { afterAll, beforeAll } from "vitest"; -import { normalizeDailyIngestionState, normalizeSessionIngestionState } from "./dreaming-phases.js"; +import { + normalizeDailyIngestionState, + normalizeSessionIngestionState, +} from "./dreaming-ingestion-state.js"; import { configureMemoryCoreDreamingState, DREAMING_DAILY_INGESTION_NAMESPACE,