ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(memory-core): separate dreaming state codec ownership (#113949)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-25 18:22:44 -07:00
committed by GitHub
parent 4596fd2dd2
commit 6d6f36a8a5
5 changed files with 147 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/memory-core/src/dreaming-ingestion-state.ts
+117
View File
@@ -0,0 +1,117 @@
// Memory Core codecs normalize canonical and legacy dreaming ingestion state.
import { normalizeStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime";
import { asRecord } from "./dreaming-shared.js";
const MEMORY_DAY_RE = /^\d{4}-\d{2}-\d{2}$/;
export const SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION = 4096;
export type DailyIngestionFileState = {
mtimeMs: number;
size: number;
lastDreamingDayIngested?: string;
};
export type DailyIngestionState = {
version: 1;
files: Record<string, DailyIngestionFileState>;
};
export type SessionIngestionFileState = {
mtimeMs: number;
size: number;
contentHash: string;
lineCount: number;
lastContentLine: number;
};
export type SessionIngestionState = {
version: 3;
files: Record<string, SessionIngestionFileState>;
seenMessages: Record<string, string[]>;
};
export function normalizeMemoryDay(value: unknown): string | undefined {
if (typeof value !== "string") {
return undefined;
}
const day = value.trim();
return MEMORY_DAY_RE.test(day) ? day : undefined;
}
export function normalizeDailyIngestionState(raw: unknown): DailyIngestionState {
const record = asRecord(raw);
const filesRaw = asRecord(record?.files);
if (!filesRaw) {
return { version: 1, files: {} };
}
const files: Record<string, DailyIngestionFileState> = {};
for (const [key, value] of Object.entries(filesRaw)) {
const file = asRecord(value);
if (!file || typeof key !== "string" || key.trim().length === 0) {
continue;
}
const mtimeMs = Number(file.mtimeMs);
const size = Number(file.size);
if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) {
continue;
}
const lastDreamingDayIngested = normalizeMemoryDay(file.lastDreamingDayIngested);
files[key] = {
mtimeMs: Math.floor(mtimeMs),
size: Math.floor(size),
...(lastDreamingDayIngested ? { lastDreamingDayIngested } : {}),
};
}
return { version: 1, files };
}
export function normalizeSessionIngestionState(raw: unknown): SessionIngestionState {
const record = asRecord(raw);
const filesRaw = asRecord(record?.files);
const files: Record<string, SessionIngestionFileState> = {};
if (filesRaw) {
for (const [key, value] of Object.entries(filesRaw)) {
const file = asRecord(value);
if (!file || key.trim().length === 0) {
continue;
}
const mtimeMs = Number(file.mtimeMs);
const size = Number(file.size);
if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) {
continue;
}
const lineCountRaw = Number(file.lineCount);
const lastContentLineRaw = Number(file.lastContentLine);
const lineCount =
Number.isFinite(lineCountRaw) && lineCountRaw >= 0 ? Math.floor(lineCountRaw) : 0;
const lastContentLine =
Number.isFinite(lastContentLineRaw) && lastContentLineRaw >= 0
? Math.floor(lastContentLineRaw)
: 0;
files[key] = {
mtimeMs: Math.floor(mtimeMs),
size: Math.floor(size),
contentHash: typeof file.contentHash === "string" ? file.contentHash.trim() : "",
lineCount,
lastContentLine: Math.min(lineCount, lastContentLine),
};
}
}
const seenMessagesRaw = asRecord(record?.seenMessages);
const seenMessages: Record<string, string[]> = {};
if (seenMessagesRaw) {
for (const [scope, value] of Object.entries(seenMessagesRaw)) {
if (scope.trim().length === 0 || !Array.isArray(value)) {
continue;
}
const unique = normalizeStringEntries([
...new Set(value.filter((entry): entry is string => typeof entry === "string")),
]).slice(-SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION);
if (unique.length > 0) {
seenMessages[scope] = unique;
}
}
}
return { version: 3, files, seenMessages };
}
extensions/memory-core/src/dreaming-phases.ts
+11 -129
View File
@@ -22,6 +22,16 @@ import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime";
import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import { appendFailedDreamingEvent } from "./dreaming-events.js";
import {
normalizeDailyIngestionState,
normalizeMemoryDay,
normalizeSessionIngestionState,
SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION,
type DailyIngestionFileState,
type DailyIngestionState,
type SessionIngestionFileState,
type SessionIngestionState,
} from "./dreaming-ingestion-state.js";
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
import {
generateAndAppendDreamNarrative,
@@ -29,7 +39,7 @@ import {
type NarrativePhaseData,
runDetachedDreamNarrative,
} from "./dreaming-narrative.js";
import { asRecord, formatErrorMessage } from "./dreaming-shared.js";
import { formatErrorMessage } from "./dreaming-shared.js";
import {
DREAMING_DAILY_INGESTION_NAMESPACE,
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
@@ -70,22 +80,11 @@ type RemDreamingConfig = DreamingPhaseStorageConfig & {
limit: number;
minPatternStrength: number;
};
const MEMORY_DAY_RE = /^\d{4}-\d{2}-\d{2}$/;
const DAILY_MEMORY_FILENAME_RE = /^(\d{4}-\d{2}-\d{2})(?:-[^/]+)?\.md$/i;
export const DAILY_INGESTION_STATE_RELATIVE_PATH = path.join(
"memory",
".dreams",
"daily-ingestion.json",
);
const DAILY_INGESTION_SCORE = 0.62;
const DAILY_INGESTION_MAX_SNIPPET_CHARS = 280;
const DAILY_INGESTION_MIN_SNIPPET_CHARS = 8;
const DAILY_INGESTION_MAX_CHUNK_LINES = 4;
export const SESSION_INGESTION_STATE_RELATIVE_PATH = path.join(
"memory",
".dreams",
"session-ingestion.json",
);
const SESSION_CORPUS_RELATIVE_DIR = path.join("memory", ".dreams", "session-corpus");
const SESSION_INGESTION_SCORE = 0.58;
const SESSION_INGESTION_MAX_SNIPPET_CHARS = 280;
@@ -93,7 +92,6 @@ const SESSION_INGESTION_MIN_SNIPPET_CHARS = 12;
const SESSION_INGESTION_MAX_MESSAGES_PER_SWEEP = 240;
const SESSION_INGESTION_MAX_MESSAGES_PER_FILE = 80;
const SESSION_INGESTION_MIN_MESSAGES_PER_FILE = 12;
const SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION = 4096;
const SESSION_INGESTION_MAX_TRACKED_SCOPES = 2048;
const SESSION_CHECKPOINT_TRANSCRIPT_FILENAME_RE = /\.checkpoint\..+\.jsonl$/i;
const LIGHT_DIARY_HISTORY_LIMIT = 4;
@@ -369,12 +367,6 @@ type DailyMemoryFile = {
canonical: boolean;
};
type DailyIngestionFileState = {
mtimeMs: number;
size: number;
lastDreamingDayIngested?: string;
};
function parseDailyMemoryFileName(fileName: string): DailyMemoryFile | null {
const match = fileName.match(DAILY_MEMORY_FILENAME_RE);
const day = match?.[1];
@@ -406,52 +398,6 @@ function resolveWorkspaceMemoryRelativePath(workspaceDir: string, filePath: stri
return `memory/${path.basename(filePath)}`;
}
type DailyIngestionState = {
version: 1;
files: Record<string, DailyIngestionFileState>;
};
export function normalizeDailyIngestionState(raw: unknown): DailyIngestionState {
const record = asRecord(raw);
const filesRaw = asRecord(record?.files);
if (!filesRaw) {
return {
version: 1,
files: {},
};
}
const files: Record<string, DailyIngestionFileState> = {};
for (const [key, value] of Object.entries(filesRaw)) {
const file = asRecord(value);
if (!file || typeof key !== "string" || key.trim().length === 0) {
continue;
}
const mtimeMs = Number(file.mtimeMs);
const size = Number(file.size);
if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) {
continue;
}
const lastDreamingDayIngested = normalizeMemoryDay(file.lastDreamingDayIngested);
files[key] = {
mtimeMs: Math.floor(mtimeMs),
size: Math.floor(size),
...(lastDreamingDayIngested ? { lastDreamingDayIngested } : {}),
};
}
return {
version: 1,
files,
};
}
function normalizeMemoryDay(value: unknown): string | undefined {
if (typeof value !== "string") {
return undefined;
}
const day = value.trim();
return MEMORY_DAY_RE.test(day) ? day : undefined;
}
async function readDailyIngestionState(workspaceDir: string): Promise<DailyIngestionState> {
const entries = await readMemoryCoreWorkspaceEntries<DailyIngestionFileState>({
namespace: DREAMING_DAILY_INGESTION_NAMESPACE,
@@ -474,20 +420,6 @@ async function writeDailyIngestionState(
});
}
type SessionIngestionFileState = {
mtimeMs: number;
size: number;
contentHash: string;
lineCount: number;
lastContentLine: number;
};
type SessionIngestionState = {
version: 3;
files: Record<string, SessionIngestionFileState>;
seenMessages: Record<string, string[]>;
};
type SessionIngestionMessage = {
day: string;
snippet: string;
@@ -500,56 +432,6 @@ type SessionIngestionCollectionResult = {
changed: boolean;
};
export function normalizeSessionIngestionState(raw: unknown): SessionIngestionState {
const record = asRecord(raw);
const filesRaw = asRecord(record?.files);
const files: Record<string, SessionIngestionFileState> = {};
if (filesRaw) {
for (const [key, value] of Object.entries(filesRaw)) {
const file = asRecord(value);
if (!file || key.trim().length === 0) {
continue;
}
const mtimeMs = Number(file.mtimeMs);
const size = Number(file.size);
if (!Number.isFinite(mtimeMs) || mtimeMs < 0 || !Number.isFinite(size) || size < 0) {
continue;
}
const lineCountRaw = Number(file.lineCount);
const lastContentLineRaw = Number(file.lastContentLine);
const lineCount =
Number.isFinite(lineCountRaw) && lineCountRaw >= 0 ? Math.floor(lineCountRaw) : 0;
const lastContentLine =
Number.isFinite(lastContentLineRaw) && lastContentLineRaw >= 0
? Math.floor(lastContentLineRaw)
: 0;
files[key] = {
mtimeMs: Math.floor(mtimeMs),
size: Math.floor(size),
contentHash: typeof file.contentHash === "string" ? file.contentHash.trim() : "",
lineCount,
lastContentLine: Math.min(lineCount, lastContentLine),
};
}
}
const seenMessagesRaw = asRecord(record?.seenMessages);
const seenMessages: Record<string, string[]> = {};
if (seenMessagesRaw) {
for (const [scope, value] of Object.entries(seenMessagesRaw)) {
if (scope.trim().length === 0 || !Array.isArray(value)) {
continue;
}
const unique = normalizeStringEntries([
...new Set(value.filter((entry): entry is string => typeof entry === "string")),
]).slice(-SESSION_INGESTION_MAX_TRACKED_MESSAGES_PER_SESSION);
if (unique.length > 0) {
seenMessages[scope] = unique;
}
}
}
return { version: 3, files, seenMessages };
}
async function readSessionIngestionState(workspaceDir: string): Promise<SessionIngestionState> {
const [fileEntries, seenChunks] = await Promise.all([
readMemoryCoreWorkspaceEntries<SessionIngestionFileState>({
extensions/memory-core/src/migration/doctor-dreaming-state.ts
+14 -5
View File
@@ -6,11 +6,9 @@ import {
legacyStateFileExists,
} from "openclaw/plugin-sdk/runtime-doctor";
import {
DAILY_INGESTION_STATE_RELATIVE_PATH,
SESSION_INGESTION_STATE_RELATIVE_PATH,
normalizeDailyIngestionState,
normalizeSessionIngestionState,
} from "../dreaming-phases.js";
} from "../dreaming-ingestion-state.js";
import {
DREAMING_DAILY_INGESTION_NAMESPACE,
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
@@ -38,6 +36,17 @@ type LegacySource = {
filePath: string;
};
const LEGACY_DAILY_INGESTION_STATE_RELATIVE_PATH = path.join(
"memory",
".dreams",
"daily-ingestion.json",
);
const LEGACY_SESSION_INGESTION_STATE_RELATIVE_PATH = path.join(
"memory",
".dreams",
"session-ingestion.json",
);
async function readJsonFile(filePath: string): Promise<unknown> {
return JSON.parse(await fs.readFile(filePath, "utf8"));
}
@@ -49,8 +58,8 @@ async function collectLegacySources(
const sources: LegacySource[] = [];
for (const workspaceDir of resolveConfiguredWorkspaces(config, env)) {
const candidates = [
{ label: "daily ingestion", relativePath: DAILY_INGESTION_STATE_RELATIVE_PATH },
{ label: "session ingestion", relativePath: SESSION_INGESTION_STATE_RELATIVE_PATH },
{ label: "daily ingestion", relativePath: LEGACY_DAILY_INGESTION_STATE_RELATIVE_PATH },
{ label: "session ingestion", relativePath: LEGACY_SESSION_INGESTION_STATE_RELATIVE_PATH },
{ label: "short-term recall", relativePath: SHORT_TERM_STORE_RELATIVE_PATH },
{ label: "phase signals", relativePath: SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH },
];
extensions/memory-core/src/migration/dreaming-state-comparison.ts
+1 -1
View File
@@ -5,7 +5,7 @@ import { isDeepStrictEqual } from "node:util";
import {
normalizeDailyIngestionState,
normalizeSessionIngestionState,
} from "../dreaming-phases.js";
} from "../dreaming-ingestion-state.js";
import {
DREAMING_DAILY_INGESTION_NAMESPACE,
DREAMING_SESSION_INGESTION_FILES_NAMESPACE,
extensions/memory-core/src/test-helpers.ts
+4 -1
View File
@@ -5,7 +5,10 @@ import type { OpenKeyedStoreOptions } from "openclaw/plugin-sdk/plugin-state-run
import { createPluginStateKeyedStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
import { afterAll, beforeAll } from "vitest";
import { normalizeDailyIngestionState, normalizeSessionIngestionState } from "./dreaming-phases.js";
import {
normalizeDailyIngestionState,
normalizeSessionIngestionState,
} from "./dreaming-ingestion-state.js";
import {
configureMemoryCoreDreamingState,
DREAMING_DAILY_INGESTION_NAMESPACE,