From a7d2f0bd6852c689d4464e670eaf6950d78ba81f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 24 Jul 2026 00:02:05 -0700 Subject: [PATCH] refactor(memory-core): split short-term promotion (#113223) --- config/max-lines-baseline.txt | 1 - .../src/short-term-promotion-apply.ts | 365 +++ .../src/short-term-promotion-artifacts.ts | 337 ++ .../src/short-term-promotion-record.ts | 421 +++ .../src/short-term-promotion-rehydrate.ts | 339 ++ .../src/short-term-promotion-stats.ts | 387 +++ .../src/short-term-promotion-store.ts | 277 ++ .../src/short-term-promotion-types.ts | 245 ++ .../src/short-term-promotion-utils.ts | 528 ++++ .../memory-core/src/short-term-promotion.ts | 2814 +---------------- 10 files changed, 2955 insertions(+), 2759 deletions(-) create mode 100644 extensions/memory-core/src/short-term-promotion-apply.ts create mode 100644 extensions/memory-core/src/short-term-promotion-artifacts.ts create mode 100644 extensions/memory-core/src/short-term-promotion-record.ts create mode 100644 extensions/memory-core/src/short-term-promotion-rehydrate.ts create mode 100644 extensions/memory-core/src/short-term-promotion-stats.ts create mode 100644 extensions/memory-core/src/short-term-promotion-store.ts create mode 100644 extensions/memory-core/src/short-term-promotion-types.ts create mode 100644 extensions/memory-core/src/short-term-promotion-utils.ts diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 00976b917391..c03ca7bf6fd3 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -179,7 +179,6 @@ extensions/memory-core/src/memory/search-manager.test.ts extensions/memory-core/src/memory/search-manager.ts extensions/memory-core/src/rem-evidence.ts extensions/memory-core/src/short-term-promotion.test.ts -extensions/memory-core/src/short-term-promotion.ts extensions/memory-core/src/tools.test.ts extensions/memory-core/src/tools.ts extensions/memory-lancedb/index.test.ts diff --git a/extensions/memory-core/src/short-term-promotion-apply.ts b/extensions/memory-core/src/short-term-promotion-apply.ts new file mode 100644 index 000000000000..7ccb50905fa9 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-apply.ts @@ -0,0 +1,365 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { + DEFAULT_MEMORY_DEEP_DREAMING_MAX_PROMOTED_SNIPPET_TOKENS, + formatMemoryDreamingDay, +} from "openclaw/plugin-sdk/memory-core-host-status"; +import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events"; +import { replaceFileAtomic } from "openclaw/plugin-sdk/security-runtime"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { compactMemoryForBudget, DEFAULT_MEMORY_FILE_MAX_CHARS } from "./memory-budget.js"; +import { rehydratePromotionCandidate } from "./short-term-promotion-rehydrate.js"; +import { readStore, withShortTermLock, writeStore } from "./short-term-promotion-store.js"; +import { + DEFAULT_PROMOTION_MIN_RECALL_COUNT, + DEFAULT_PROMOTION_MIN_SCORE, + DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + type ApplyShortTermPromotionsOptions, + type ApplyShortTermPromotionsResult, + type PromotionCandidate, +} from "./short-term-promotion-types.js"; +import { + isContaminatedDreamingSnippet, + normalizeSnippet, + toFiniteNonNegativeInt, + toFiniteScore, +} from "./short-term-promotion-utils.js"; +import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js"; + +const PROMOTION_MARKER_PREFIX = "openclaw-memory-promotion:"; +const PROMOTED_SNIPPET_CHARS_PER_TOKEN_ESTIMATE = 4; + +function buildPromotionSection( + candidates: PromotionCandidate[], + nowMs: number, + timezone?: string, + maxPromotedSnippetTokens = DEFAULT_MEMORY_DEEP_DREAMING_MAX_PROMOTED_SNIPPET_TOKENS, +): string { + const sectionDate = formatMemoryDreamingDay(nowMs, timezone); + const lines = ["", `## Promoted From Short-Term Memory (${sectionDate})`, ""]; + + for (const candidate of candidates) { + const source = `${candidate.path}:${candidate.startLine}-${candidate.endLine}`; + const metadata = `[score=${candidate.score.toFixed(3)} signals=${candidate.signalCount} recalls=${candidate.recallCount} avg=${candidate.avgScore.toFixed(3)} source=${source}]`; + lines.push(``); + // Cap only the visible MEMORY.md text. The recall store keeps the full + // rehydrated snippet so ranking, provenance, and dream narratives remain + // tied to the source entry instead of this presentation budget. + lines.push( + `- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata}`, + ); + } + + lines.push(""); + return lines.join("\n"); +} + +function resolvePromotedSnippetCharLimit(maxTokens: number): number { + const tokenLimit = toFiniteNonNegativeInt( + maxTokens, + DEFAULT_MEMORY_DEEP_DREAMING_MAX_PROMOTED_SNIPPET_TOKENS, + ); + // This is an inexpensive display-size guard, not a tokenizer contract. + return tokenLimit * PROMOTED_SNIPPET_CHARS_PER_TOKEN_ESTIMATE; +} + +function truncatePromotedSnippet(snippet: string, maxTokens: number): string { + const limit = resolvePromotedSnippetCharLimit(maxTokens); + if (limit === 0 || snippet.length <= limit) { + return snippet; + } + const hardLimit = truncateUtf16Safe(snippet, limit); + const sentenceBoundary = Math.max( + hardLimit.lastIndexOf(". "), + hardLimit.lastIndexOf("! "), + hardLimit.lastIndexOf("? "), + ); + const wordBoundary = hardLimit.lastIndexOf(" "); + const cutAt = + sentenceBoundary >= Math.floor(limit * 0.55) + ? sentenceBoundary + 1 + : wordBoundary >= Math.floor(limit * 0.65) + ? wordBoundary + : limit; + return `${hardLimit.slice(0, cutAt).trimEnd()}...`; +} + +function formatPromotedSnippetForMemory(rawSnippet: string, maxTokens: number): string { + const normalized = normalizeSnippet(rawSnippet || "(no snippet captured)") + .replace(/^[-*+] +/, "") + .trim(); + return truncatePromotedSnippet(normalized || "(no snippet captured)", maxTokens); +} + +function withTrailingNewline(content: string): string { + if (!content) { + return ""; + } + return content.endsWith("\n") ? content : `${content}\n`; +} + +async function resolveMemoryWritePath(filePath: string): Promise { + try { + return await fs.realpath(filePath); + } catch (err) { + const hasTrailingSeparator = + filePath.endsWith(path.sep) || + (process.platform === "win32" && filePath.endsWith(path.posix.sep)); + if ((err as NodeJS.ErrnoException)?.code !== "ENOENT" || hasTrailingSeparator) { + throw err; + } + } + + // Canonicalize each parent before applying a relative link target. Lexical + // normalization would change `..` semantics when an earlier component is a symlink. + const parentPath = await fs.realpath(path.dirname(filePath)); + const canonicalPath = path.join(parentPath, path.basename(filePath)); + let linkTarget: string; + try { + linkTarget = await fs.readlink(canonicalPath); + } catch (err) { + const code = (err as NodeJS.ErrnoException)?.code; + if (code === "ENOENT" || code === "EINVAL") { + return canonicalPath; + } + throw err; + } + const isWindowsRootRelative = process.platform === "win32" && /^[\\/](?![\\/])/.test(linkTarget); + const targetPath = isWindowsRootRelative + ? `${path.parse(parentPath).root.replace(/[\\/]$/, "")}${linkTarget}` + : path.isAbsolute(linkTarget) + ? linkTarget + : `${parentPath}${parentPath.endsWith(path.sep) ? "" : path.sep}${linkTarget}`; + return await resolveMemoryWritePath(targetPath); +} + +function isAtomicReplacePermissionError(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException)?.code; + return code === "EACCES" || code === "EPERM" || code === "EEXIST" || code === "EROFS"; +} + +async function writeExistingMemoryInPlace(filePath: string, content: string): Promise { + let handle: Awaited>; + try { + handle = await fs.open(filePath, "r+"); + } catch { + return false; + } + try { + await handle.writeFile(content, { encoding: "utf-8" }); + await handle.truncate(Buffer.byteLength(content)); + await handle.sync(); + return true; + } finally { + await handle.close(); + } +} + +function extractPromotionMarkers(memoryText: string): Set { + const markers = new Set(); + // Marker keys include source paths, so spaces are valid. Capture until the + // comment close; otherwise a path like "memory/project alpha/..." is missed + // and the same candidate can be appended again. + const matches = memoryText.matchAll(//gi); + for (const match of matches) { + const key = match[1]?.trim(); + if (key) { + markers.add(key); + } + } + return markers; +} + +export async function applyShortTermPromotions( + options: ApplyShortTermPromotionsOptions, +): Promise { + const workspaceDir = options.workspaceDir.trim(); + const nowMs = resolveMemoryCoreNowMs(options.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + const limit = Number.isFinite(options.limit) + ? Math.max(0, Math.floor(options.limit as number)) + : options.candidates.length; + const minScore = toFiniteScore(options.minScore, DEFAULT_PROMOTION_MIN_SCORE); + const minRecallCount = toFiniteNonNegativeInt( + options.minRecallCount, + DEFAULT_PROMOTION_MIN_RECALL_COUNT, + ); + const minUniqueQueries = toFiniteNonNegativeInt( + options.minUniqueQueries, + DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + ); + const maxAgeDays = toFiniteNonNegativeInt(options.maxAgeDays, -1); + const memoryPath = path.join(workspaceDir, "MEMORY.md"); + + return await withShortTermLock(workspaceDir, async () => { + const store = await readStore(workspaceDir, nowIso); + const selected = options.candidates + .filter((candidate) => { + if (isContaminatedDreamingSnippet(candidate.snippet)) { + return false; + } + if (candidate.promotedAt) { + return false; + } + if (candidate.score < minScore) { + return false; + } + if (candidate.signalCount < minRecallCount) { + return false; + } + if (Math.max(candidate.uniqueQueries, candidate.recallDays.length) < minUniqueQueries) { + return false; + } + if (maxAgeDays >= 0 && candidate.ageDays > maxAgeDays) { + return false; + } + const latest = store.entries[candidate.key]; + if (latest?.promotedAt) { + return false; + } + return true; + }) + .slice(0, limit); + + const rehydratedSelected: PromotionCandidate[] = []; + for (const candidate of selected) { + const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate); + if (rehydrated && !isContaminatedDreamingSnippet(rehydrated.snippet)) { + rehydratedSelected.push(rehydrated); + } + } + + if (rehydratedSelected.length === 0) { + return { + memoryPath, + applied: 0, + appended: 0, + reconciledExisting: 0, + appliedCandidates: [], + compactedSections: 0, + compactedDates: [], + }; + } + + // Promotions historically follow user-managed MEMORY.md symlinks. Replace the + // final target atomically without severing the chain, matching the prior writeFile path. + const memoryWritePath = await resolveMemoryWritePath(memoryPath); + const existingMemory = await fs.readFile(memoryWritePath, "utf-8").catch((err: unknown) => { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + return ""; + } + throw err; + }); + const existingMarkers = extractPromotionMarkers(existingMemory); + const alreadyWritten = rehydratedSelected.filter((candidate) => + existingMarkers.has(candidate.key), + ); + const toAppend = rehydratedSelected.filter((candidate) => !existingMarkers.has(candidate.key)); + + let compactedDates: string[] = []; + if (toAppend.length > 0) { + const section = buildPromotionSection( + toAppend, + nowMs, + options.timezone, + options.maxPromotedSnippetTokens, + ); + const budgetChars = + typeof options.memoryFileMaxChars === "number" && + Number.isFinite(options.memoryFileMaxChars) + ? Math.max(0, Math.floor(options.memoryFileMaxChars)) + : DEFAULT_MEMORY_FILE_MAX_CHARS; + const compaction = compactMemoryForBudget({ + existingMemory, + newSection: section, + budgetChars, + }); + compactedDates = compaction.droppedDates; + const baseMemory = compaction.compacted; + const header = baseMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; + const content = `${header}${withTrailingNewline(baseMemory)}${section}`; + const memoryDirMode = (await fs.stat(path.dirname(memoryWritePath))).mode & 0o7777; + let atomicRenameCommitted = false; + const trackedRename: typeof fs.rename = async (source, destination) => { + await fs.rename(source, destination); + atomicRenameCommitted = true; + }; + try { + await replaceFileAtomic({ + filePath: memoryWritePath, + content, + dirMode: memoryDirMode, + mode: 0o600, + preserveExistingMode: true, + tempPrefix: `${path.basename(memoryPath)}.promotion`, + syncTempFile: true, + syncParentDir: true, + throwOnCleanupError: true, + // Stage proof prevents a future post-rename permission error from entering fallback. + fileSystem: { + promises: { + mkdir: fs.mkdir, + chmod: fs.chmod, + writeFile: fs.writeFile, + rename: trackedRename, + copyFile: fs.copyFile, + unlink: fs.unlink, + rm: fs.rm, + open: fs.open, + stat: fs.stat, + lstat: fs.lstat, + }, + }, + }); + } catch (error) { + // Released promotion writes could update an existing writable MEMORY.md even when + // directory ACLs blocked rename. Retain that in-place contract only after a real + // atomic permission failure and a successful writable-file open. + if ( + atomicRenameCommitted || + !isAtomicReplacePermissionError(error) || + !(await writeExistingMemoryInPlace(memoryWritePath, content)) + ) { + throw error; + } + } + } + + for (const candidate of rehydratedSelected) { + const entry = store.entries[candidate.key]; + if (!entry) { + continue; + } + entry.startLine = candidate.startLine; + entry.endLine = candidate.endLine; + entry.snippet = candidate.snippet; + entry.promotedAt = nowIso; + } + store.updatedAt = nowIso; + await writeStore(workspaceDir, store); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.promotion.applied", + timestamp: nowIso, + memoryPath, + applied: rehydratedSelected.length, + candidates: rehydratedSelected.map((candidate) => ({ + key: candidate.key, + path: candidate.path, + startLine: candidate.startLine, + endLine: candidate.endLine, + score: candidate.score, + recallCount: candidate.recallCount, + })), + }); + + return { + memoryPath, + applied: rehydratedSelected.length, + appended: toAppend.length, + reconciledExisting: alreadyWritten.length, + appliedCandidates: rehydratedSelected, + compactedSections: compactedDates.length, + compactedDates, + }; + }); +} diff --git a/extensions/memory-core/src/short-term-promotion-artifacts.ts b/extensions/memory-core/src/short-term-promotion-artifacts.ts new file mode 100644 index 000000000000..3c741976e447 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-artifacts.ts @@ -0,0 +1,337 @@ +import fs from "node:fs/promises"; +import { + deriveConceptTags, + summarizeConceptTagScriptCoverage, + type ConceptTagScriptCoverage, +} from "./concept-vocabulary.js"; +import { + SHORT_TERM_LOCK_MAX_ENTRIES, + SHORT_TERM_LOCK_NAMESPACE, + SHORT_TERM_RECALL_NAMESPACE, + memoryCoreWorkspaceStateKey, + openMemoryCoreStateStore, + readMemoryCoreWorkspaceEntries, +} from "./dreaming-state.js"; +import { + SHORT_TERM_LOCK_STALE_MS, + isProcessLikelyAlive, + parseLockOwnerPid, + readPhaseSignalStore, + readStore, + resolveLockPath, + resolveStorePath, + withShortTermLock, + writePhaseSignalStore, + writeStore, +} from "./short-term-promotion-store.js"; +import type { + RepairShortTermPromotionArtifactsResult, + ShortTermAuditIssue, + ShortTermAuditSummary, + ShortTermLockEntry, + ShortTermRecallEntry, + ShortTermRecallStore, +} from "./short-term-promotion-types.js"; +import { + MAX_QUERY_HASHES, + MAX_RECALL_DAYS, + SHORT_TERM_RECALL_MAX_ENTRIES, + enforceShortTermRecallStoreRetention, + mergeRecentDistinct, + normalizeIsoDay, + normalizeShortTermRecallStore, +} from "./short-term-promotion-utils.js"; + +export function resolveShortTermRecallStorePath(workspaceDir: string): string { + return resolveStorePath(workspaceDir); +} + +export function resolveShortTermRecallLockPath(workspaceDir: string): string { + return resolveLockPath(workspaceDir); +} + +export async function auditShortTermPromotionArtifacts(params: { + workspaceDir: string; + qmd?: { + dbPath?: string; + collections?: number; + }; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + const storePath = resolveStorePath(workspaceDir); + const lockPath = resolveLockPath(workspaceDir); + const issues: ShortTermAuditIssue[] = []; + let entryCount = 0; + let promotedCount = 0; + let spacedEntryCount = 0; + let conceptTaggedEntryCount = 0; + let conceptTagScripts: ConceptTagScriptCoverage | undefined; + let invalidEntryCount = 0; + let updatedAt: string | undefined; + + const nowIso = new Date().toISOString(); + const rawEntries = await readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_RECALL_NAMESPACE, + workspaceDir, + }); + const exists = rawEntries.length > 0; + if (exists) { + const parsed = { + version: 1, + updatedAt: nowIso, + entries: Object.fromEntries(rawEntries.map((entry) => [entry.key, entry.value])), + }; + const store = normalizeShortTermRecallStore(parsed, nowIso); + const normalizedEntryCount = Object.keys(store.entries).length; + updatedAt = store.updatedAt; + entryCount = normalizedEntryCount; + promotedCount = Object.values(store.entries).filter((entry) => + Boolean(entry.promotedAt), + ).length; + spacedEntryCount = Object.values(store.entries).filter( + (entry) => (entry.recallDays?.length ?? 0) > 1, + ).length; + conceptTaggedEntryCount = Object.values(store.entries).filter( + (entry) => (entry.conceptTags?.length ?? 0) > 0, + ).length; + conceptTagScripts = summarizeConceptTagScriptCoverage( + Object.values(store.entries) + .filter((entry) => (entry.conceptTags?.length ?? 0) > 0) + .map((entry) => entry.conceptTags ?? []), + ); + invalidEntryCount = rawEntries.length - entryCount; + if (invalidEntryCount > 0) { + issues.push({ + severity: "warn", + code: "recall-store-invalid", + message: `Short-term recall store contains ${invalidEntryCount} invalid entr${invalidEntryCount === 1 ? "y" : "ies"}.`, + fixable: true, + }); + } + if (normalizedEntryCount > SHORT_TERM_RECALL_MAX_ENTRIES) { + issues.push({ + severity: "warn", + code: "recall-store-over-limit", + message: `Short-term recall store contains ${normalizedEntryCount} entries; only the newest ${SHORT_TERM_RECALL_MAX_ENTRIES} are kept at runtime.`, + fixable: true, + }); + } + } + + const lockKey = memoryCoreWorkspaceStateKey(workspaceDir); + const lockStore = openMemoryCoreStateStore({ + namespace: SHORT_TERM_LOCK_NAMESPACE, + maxEntries: SHORT_TERM_LOCK_MAX_ENTRIES, + }); + const lockEntry = await lockStore.lookup(lockKey); + if (lockEntry) { + const ageMs = Date.now() - lockEntry.acquiredAt; + const ownerPid = parseLockOwnerPid(lockEntry.owner); + if ( + ageMs > SHORT_TERM_LOCK_STALE_MS && + (ownerPid === null || !isProcessLikelyAlive(ownerPid)) + ) { + issues.push({ + severity: "warn", + code: "recall-lock-stale", + message: "Short-term promotion lock appears stale.", + fixable: true, + }); + } + } + + let qmd: ShortTermAuditSummary["qmd"]; + if (params.qmd) { + qmd = { + dbPath: params.qmd.dbPath, + collections: params.qmd.collections, + }; + if (typeof params.qmd.collections === "number" && params.qmd.collections <= 0) { + issues.push({ + severity: "warn", + code: "qmd-collections-empty", + message: "QMD reports zero managed collections.", + fixable: false, + }); + } + const dbPath = params.qmd.dbPath?.trim(); + if (dbPath) { + try { + const stat = await fs.stat(dbPath); + qmd.dbBytes = stat.size; + if (!stat.isFile() || stat.size <= 0) { + issues.push({ + severity: "error", + code: "qmd-index-empty", + message: "QMD index file exists but is empty.", + fixable: false, + }); + } + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === "ENOENT") { + issues.push({ + severity: "error", + code: "qmd-index-missing", + message: "QMD index file is missing.", + fixable: false, + }); + } else { + throw err; + } + } + } + } + + return { + storePath, + lockPath, + updatedAt, + exists, + entryCount, + promotedCount, + spacedEntryCount, + conceptTaggedEntryCount, + ...(conceptTagScripts ? { conceptTagScripts } : {}), + invalidEntryCount, + issues, + ...(qmd ? { qmd } : {}), + }; +} + +export async function repairShortTermPromotionArtifacts(params: { + workspaceDir: string; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + const nowIso = new Date().toISOString(); + let rewroteStore = false; + let removedInvalidEntries = 0; + let removedOverflowEntries = 0; + let removedStaleLock = false; + + const lockKey = memoryCoreWorkspaceStateKey(workspaceDir); + const lockStore = openMemoryCoreStateStore({ + namespace: SHORT_TERM_LOCK_NAMESPACE, + maxEntries: SHORT_TERM_LOCK_MAX_ENTRIES, + }); + const lockEntry = await lockStore.lookup(lockKey); + if (lockEntry && Date.now() - lockEntry.acquiredAt > SHORT_TERM_LOCK_STALE_MS) { + const ownerPid = parseLockOwnerPid(lockEntry.owner); + if (ownerPid === null || !isProcessLikelyAlive(ownerPid)) { + removedStaleLock = await lockStore.delete(lockKey); + } + } + + await withShortTermLock(workspaceDir, async () => { + const rawEntries = await readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_RECALL_NAMESPACE, + workspaceDir, + }); + if (rawEntries.length > 0) { + const normalized = normalizeShortTermRecallStore( + { + version: 1, + updatedAt: nowIso, + entries: Object.fromEntries(rawEntries.map((entry) => [entry.key, entry.value])), + }, + nowIso, + ); + removedInvalidEntries = Math.max( + 0, + rawEntries.length - Object.keys(normalized.entries).length, + ); + const nextEntries = Object.fromEntries( + Object.entries(normalized.entries).map(([key, entry]) => { + const conceptTags = deriveConceptTags({ path: entry.path, snippet: entry.snippet }); + const fallbackDay = normalizeIsoDay(entry.lastRecalledAt) ?? nowIso.slice(0, 10); + return [ + key, + { + ...entry, + dailyCount: Math.max( + 0, + Math.floor((entry as { dailyCount?: number }).dailyCount ?? 0), + ), + groundedCount: Math.max( + 0, + Math.floor((entry as { groundedCount?: number }).groundedCount ?? 0), + ), + queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES), + recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS), + conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []), + } satisfies ShortTermRecallEntry, + ]; + }), + ); + const comparableStore: ShortTermRecallStore = { + version: 1, + updatedAt: normalized.updatedAt, + entries: nextEntries, + }; + removedOverflowEntries = enforceShortTermRecallStoreRetention(comparableStore); + const needsRewrite = + removedInvalidEntries > 0 || + removedOverflowEntries > 0 || + JSON.stringify(normalized.entries) !== JSON.stringify(comparableStore.entries); + if (needsRewrite) { + await writeStore(workspaceDir, { + ...comparableStore, + updatedAt: nowIso, + }); + rewroteStore = true; + } + } + }); + + return { + changed: rewroteStore || removedStaleLock, + removedInvalidEntries, + removedOverflowEntries, + rewroteStore, + removedStaleLock, + }; +} + +export async function removeGroundedShortTermCandidates(params: { + workspaceDir: string; +}): Promise<{ removed: number; storePath: string }> { + const workspaceDir = params.workspaceDir.trim(); + const storePath = resolveStorePath(workspaceDir); + const nowIso = new Date().toISOString(); + let removed = 0; + + await withShortTermLock(workspaceDir, async () => { + const [store, phaseSignals] = await Promise.all([ + readStore(workspaceDir, nowIso), + readPhaseSignalStore(workspaceDir, nowIso), + ]); + + for (const [key, entry] of Object.entries(store.entries)) { + if ( + Math.max(0, Math.floor(entry.groundedCount ?? 0)) > 0 && + Math.max(0, Math.floor(entry.recallCount ?? 0)) === 0 && + Math.max(0, Math.floor(entry.dailyCount ?? 0)) === 0 + ) { + delete store.entries[key]; + removed += 1; + } + } + + for (const key of Object.keys(phaseSignals.entries)) { + if (!Object.hasOwn(store.entries, key)) { + delete phaseSignals.entries[key]; + } + } + + if (removed > 0) { + store.updatedAt = nowIso; + phaseSignals.updatedAt = nowIso; + await Promise.all([ + writeStore(workspaceDir, store), + writePhaseSignalStore(workspaceDir, phaseSignals), + ]); + } + }); + + return { removed, storePath }; +} diff --git a/extensions/memory-core/src/short-term-promotion-record.ts b/extensions/memory-core/src/short-term-promotion-record.ts new file mode 100644 index 000000000000..8bda7318de5e --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-record.ts @@ -0,0 +1,421 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; +import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status"; +import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events"; +import pLimit from "p-limit"; +import { deriveConceptTags } from "./concept-vocabulary.js"; +import { readStore, withShortTermLock, writeStore } from "./short-term-promotion-store.js"; +import type { ShortTermRecallEntry } from "./short-term-promotion-types.js"; +import { + buildClaimHash, + buildEntryKey, + clampScore, + hashQuery, + isContaminatedDreamingSnippet, + isShortTermMemoryPath, + isShortTermSessionCorpusPath, + MAX_RECALL_DAYS, + mergeQueryHashes, + mergeRecentDistinct, + normalizeIsoDay, + normalizeMemoryPath, + normalizeSnippet, + truncateShortTermSnippet, +} from "./short-term-promotion-utils.js"; +import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js"; + +// One recall batch can inspect every retained entry; cap filesystem pressure. +const SHORT_TERM_SOURCE_FILE_CHECK_CONCURRENCY = 32; + +async function shortTermRecallSourceIsFile(sourcePath: string): Promise { + try { + const stat = await fs.stat(sourcePath); + return stat.isFile(); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw err; + } +} + +export async function filterLiveShortTermRecallEntries(params: { + workspaceDir: string; + entries: ShortTermRecallEntry[]; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + if (!workspaceDir) { + return []; + } + const sourceFileChecks = new Map>(); + const sourceFileLimit = pLimit(SHORT_TERM_SOURCE_FILE_CHECK_CONCURRENCY); + const checkSourceFile = (sourcePath: string): Promise => { + const existing = sourceFileChecks.get(sourcePath); + if (existing) { + return existing; + } + const check = sourceFileLimit(() => shortTermRecallSourceIsFile(sourcePath)); + sourceFileChecks.set(sourcePath, check); + return check; + }; + const results = await Promise.all( + params.entries.map(async (entry) => { + let exists = false; + for (const sourcePath of resolveShortTermSourcePathCandidates(workspaceDir, entry.path)) { + if (await checkSourceFile(sourcePath)) { + exists = true; + break; + } + } + return { entry, exists }; + }), + ); + return results.filter((result) => result.exists).map((result) => result.entry); +} + +function buildMemoryRecallSkippedEvent(params: { + timestamp: string; + query: string; + eligibleResultCount: number; + skipped: MemorySearchResult[]; +}) { + return { + type: "memory.recall.skipped" as const, + timestamp: params.timestamp, + query: params.query, + reason: "non-short-term-memory-path" as const, + eligibleResultCount: params.eligibleResultCount, + skippedResultCount: params.skipped.length, + results: params.skipped.map((result) => ({ + path: normalizeMemoryPath(result.path), + startLine: Math.max(1, Math.floor(result.startLine)), + endLine: Math.max(1, Math.floor(result.endLine)), + score: clampScore(result.score), + reason: "non-short-term-memory-path" as const, + })), + }; +} + +export async function recordShortTermRecalls(params: { + workspaceDir?: string; + query: string; + results: MemorySearchResult[]; + signalType?: "recall" | "daily"; + dedupeByQueryPerDay?: boolean; + dayBucket?: string; + nowMs?: number; + timezone?: string; +}): Promise { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return; + } + const query = params.query.trim(); + if (!query) { + return; + } + const memoryResults = params.results.filter((result) => result.source === "memory"); + const relevant = memoryResults.filter((result) => isShortTermMemoryPath(result.path)); + const skipped = memoryResults.filter((result) => !isShortTermMemoryPath(result.path)); + if (relevant.length === 0 && skipped.length === 0) { + return; + } + + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + if (relevant.length === 0) { + await appendMemoryHostEvent( + workspaceDir, + buildMemoryRecallSkippedEvent({ + timestamp: nowIso, + query, + eligibleResultCount: relevant.length, + skipped, + }), + ); + return; + } + const signalType = params.signalType ?? "recall"; + const queryHash = hashQuery(query); + const todayBucket = + normalizeIsoDay(params.dayBucket ?? "") ?? formatMemoryDreamingDay(nowMs, params.timezone); + await withShortTermLock(workspaceDir, async () => { + const store = await readStore(workspaceDir, nowIso); + + for (const result of relevant) { + const normalizedPath = normalizeMemoryPath(result.path); + const rawSnippet = normalizeSnippet(result.snippet); + const snippet = truncateShortTermSnippet(rawSnippet); + if ( + !rawSnippet || + isContaminatedDreamingSnippet(rawSnippet, { + allowTranscriptTurnSnippet: isShortTermSessionCorpusPath(normalizedPath), + }) + ) { + continue; + } + const claimHash = buildClaimHash(rawSnippet); + const groundedKey = claimHash + ? buildEntryKey({ + path: normalizedPath, + startLine: Math.max(1, Math.floor(result.startLine)), + endLine: Math.max(1, Math.floor(result.endLine)), + source: "memory", + claimHash, + }) + : null; + const baseKey = buildEntryKey(result); + const key = groundedKey && store.entries[groundedKey] ? groundedKey : baseKey; + const existing = store.entries[key]; + const score = clampScore(result.score); + const recallDaysBase = existing?.recallDays ?? []; + const queryHashesBase = existing?.queryHashes ?? []; + const dedupeSignal = + Boolean(params.dedupeByQueryPerDay) && + queryHashesBase.includes(queryHash) && + recallDaysBase.includes(todayBucket); + const recallCount = + signalType === "recall" + ? Math.max(0, Math.floor(existing?.recallCount ?? 0) + (dedupeSignal ? 0 : 1)) + : Math.max(0, Math.floor(existing?.recallCount ?? 0)); + const dailyCount = + signalType === "daily" + ? Math.max(0, Math.floor(existing?.dailyCount ?? 0) + (dedupeSignal ? 0 : 1)) + : Math.max(0, Math.floor(existing?.dailyCount ?? 0)); + const totalScore = Math.max(0, (existing?.totalScore ?? 0) + (dedupeSignal ? 0 : score)); + const maxScore = Math.max(existing?.maxScore ?? 0, dedupeSignal ? 0 : score); + const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash); + const recallDays = mergeRecentDistinct(recallDaysBase, todayBucket, MAX_RECALL_DAYS); + const conceptTags = deriveConceptTags({ path: normalizedPath, snippet }); + + const unchangedRepeatedSignal = + Boolean(params.dedupeByQueryPerDay) && + queryHashesBase.includes(queryHash) && + existing?.snippet === snippet; + const lastRecalledAt = unchangedRepeatedSignal + ? (existing?.lastRecalledAt ?? nowIso) + : nowIso; + + store.entries[key] = { + key, + path: normalizedPath, + startLine: Math.max(1, Math.floor(result.startLine)), + endLine: Math.max(1, Math.floor(result.endLine)), + source: "memory", + snippet: snippet || existing?.snippet || "", + recallCount, + dailyCount, + groundedCount: Math.max(0, Math.floor(existing?.groundedCount ?? 0)), + totalScore, + maxScore, + firstRecalledAt: existing?.firstRecalledAt ?? nowIso, + lastRecalledAt, + queryHashes, + recallDays, + conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []), + ...(existing?.claimHash ? { claimHash: existing.claimHash } : {}), + ...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}), + }; + } + + store.updatedAt = nowIso; + await writeStore(workspaceDir, store); + await appendMemoryHostEvent(workspaceDir, { + type: "memory.recall.recorded", + timestamp: nowIso, + query, + resultCount: relevant.length, + results: relevant.map((result) => ({ + path: normalizeMemoryPath(result.path), + startLine: Math.max(1, Math.floor(result.startLine)), + endLine: Math.max(1, Math.floor(result.endLine)), + score: clampScore(result.score), + })), + }); + if (skipped.length > 0) { + await appendMemoryHostEvent( + workspaceDir, + buildMemoryRecallSkippedEvent({ + timestamp: nowIso, + query, + eligibleResultCount: relevant.length, + skipped, + }), + ); + } + }); +} + +export async function recordGroundedShortTermCandidates(params: { + workspaceDir?: string; + query: string; + items: Array<{ + path: string; + startLine: number; + endLine: number; + snippet: string; + score: number; + query?: string; + signalCount?: number; + dayBucket?: string; + }>; + dedupeByQueryPerDay?: boolean; + dayBucket?: string; + nowMs?: number; + timezone?: string; +}): Promise { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return; + } + const query = params.query.trim(); + if (!query) { + return; + } + const relevant = params.items + .map((item) => { + const rawSnippet = normalizeSnippet(item.snippet); + const snippet = truncateShortTermSnippet(rawSnippet); + const normalizedPath = normalizeMemoryPath(item.path); + if ( + !rawSnippet || + isContaminatedDreamingSnippet(rawSnippet) || + !normalizedPath || + !isShortTermMemoryPath(normalizedPath) || + !Number.isFinite(item.startLine) || + !Number.isFinite(item.endLine) + ) { + return null; + } + return { + path: normalizedPath, + startLine: Math.max(1, Math.floor(item.startLine)), + endLine: Math.max(1, Math.floor(item.endLine)), + snippet, + identitySnippet: rawSnippet, + score: clampScore(item.score), + query: normalizeSnippet(item.query ?? query), + signalCount: Math.max(1, Math.floor(item.signalCount ?? 1)), + dayBucket: normalizeIsoDay(item.dayBucket ?? params.dayBucket ?? ""), + }; + }) + .filter((item): item is NonNullable => item !== null); + if (relevant.length === 0) { + return; + } + + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + const fallbackDayBucket = formatMemoryDreamingDay(nowMs, params.timezone); + await withShortTermLock(workspaceDir, async () => { + const store = await readStore(workspaceDir, nowIso); + + for (const item of relevant) { + const dayBucket = item.dayBucket ?? fallbackDayBucket; + const effectiveQuery = item.query || query; + if (!effectiveQuery) { + continue; + } + const queryHash = hashQuery(effectiveQuery); + const claimHash = buildClaimHash(item.identitySnippet); + const key = buildEntryKey({ + path: item.path, + startLine: item.startLine, + endLine: item.endLine, + source: "memory", + claimHash, + }); + const existing = store.entries[key]; + const recallDaysBase = existing?.recallDays ?? []; + const queryHashesBase = existing?.queryHashes ?? []; + const dedupeSignal = + Boolean(params.dedupeByQueryPerDay) && + queryHashesBase.includes(queryHash) && + recallDaysBase.includes(dayBucket); + const groundedCount = Math.max( + 0, + Math.floor(existing?.groundedCount ?? 0) + (dedupeSignal ? 0 : item.signalCount), + ); + const totalScore = Math.max( + 0, + (existing?.totalScore ?? 0) + (dedupeSignal ? 0 : item.score * item.signalCount), + ); + const maxScore = Math.max(existing?.maxScore ?? 0, dedupeSignal ? 0 : item.score); + const queryHashes = mergeQueryHashes(existing?.queryHashes ?? [], queryHash); + const recallDays = mergeRecentDistinct(recallDaysBase, dayBucket, MAX_RECALL_DAYS); + const conceptTags = deriveConceptTags({ path: item.path, snippet: item.snippet }); + + const unchangedRepeatedSignal = + Boolean(params.dedupeByQueryPerDay) && + queryHashesBase.includes(queryHash) && + existing?.snippet === item.snippet; + const lastRecalledAt = unchangedRepeatedSignal + ? (existing?.lastRecalledAt ?? nowIso) + : nowIso; + + store.entries[key] = { + key, + path: item.path, + startLine: item.startLine, + endLine: item.endLine, + source: "memory", + snippet: item.snippet, + recallCount: Math.max(0, Math.floor(existing?.recallCount ?? 0)), + dailyCount: Math.max(0, Math.floor(existing?.dailyCount ?? 0)), + groundedCount, + totalScore, + maxScore, + firstRecalledAt: existing?.firstRecalledAt ?? nowIso, + lastRecalledAt, + queryHashes, + recallDays, + conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []), + claimHash, + ...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}), + }; + } + + store.updatedAt = nowIso; + await writeStore(workspaceDir, store); + }); +} + +export async function readShortTermRecallEntries(params: { + workspaceDir: string; + nowMs?: number; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + if (!workspaceDir) { + return []; + } + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + const store = await readStore(workspaceDir, nowIso); + return Object.values(store.entries).filter( + (entry): entry is ShortTermRecallEntry => + Boolean(entry) && entry.source === "memory" && isShortTermMemoryPath(entry.path), + ); +} + +export function resolveShortTermSourcePathCandidates( + workspaceDir: string, + candidatePath: string, +): string[] { + const normalizedPath = normalizeMemoryPath(candidatePath); + const basenames = [normalizedPath]; + if (!normalizedPath.startsWith("memory/")) { + basenames.push(path.posix.join("memory", path.posix.basename(normalizedPath))); + } + const seen = new Set(); + const resolved: string[] = []; + for (const relativePath of basenames) { + const absolutePath = path.resolve(workspaceDir, relativePath); + if (seen.has(absolutePath)) { + continue; + } + seen.add(absolutePath); + resolved.push(absolutePath); + } + return resolved; +} diff --git a/extensions/memory-core/src/short-term-promotion-rehydrate.ts b/extensions/memory-core/src/short-term-promotion-rehydrate.ts new file mode 100644 index 000000000000..6b2e4360e514 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-rehydrate.ts @@ -0,0 +1,339 @@ +import fs from "node:fs/promises"; +import { resolveShortTermSourcePathCandidates } from "./short-term-promotion-record.js"; +import type { PromotionCandidate } from "./short-term-promotion-types.js"; +import { normalizeSnippet, SHORT_TERM_BASENAME_RE } from "./short-term-promotion-utils.js"; + +const GENERIC_DAY_HEADING_RE = + /^(?:(?:mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)(?:,\s+)?)?(?:(?:jan|january|feb|february|mar|march|apr|april|may|jun|june|jul|july|aug|august|sep|sept|september|oct|october|nov|november|dec|december)\s+\d{1,2}(?:st|nd|rd|th)?(?:,\s*\d{4})?|\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?|\d{4}[/-]\d{2}[/-]\d{2})$/i; +const PROMOTION_LIST_MARKER_RE = /^(?:\d+\.\s+|[-*+]\s+)/; +const MANAGED_DREAMING_HEADINGS = new Set(["light sleep", "rem sleep"]); + +function normalizeRangeSnippet(lines: string[], startLine: number, endLine: number): string { + const startIndex = Math.max(0, startLine - 1); + const endIndex = Math.min(lines.length, endLine); + if (startIndex >= endIndex) { + return ""; + } + return normalizeSnippet(lines.slice(startIndex, endIndex).join(" ")); +} + +function normalizeListMarkerFreeRangeSnippet( + lines: string[], + startLine: number, + endLine: number, +): string { + const startIndex = Math.max(0, startLine - 1); + const endIndex = Math.min(lines.length, endLine); + if (startIndex >= endIndex) { + return ""; + } + const strippedLines = lines.slice(startIndex, endIndex).map((line) => { + const trimmed = line.trim(); + const withoutMarker = trimmed.replace(PROMOTION_LIST_MARKER_RE, ""); + return { text: withoutMarker, hadListMarker: withoutMarker !== trimmed }; + }); + const joiner = + strippedLines.length > 1 && strippedLines.every((line) => line.hadListMarker) ? "; " : " "; + return normalizeSnippet(strippedLines.map((line) => line.text).join(joiner)); +} + +function normalizeDailyHeadingForPromotion(line: string): string | null { + const match = line.trim().match(/^#{1,6}\s+(.+)$/); + const heading = match?.[1]?.replace(PROMOTION_LIST_MARKER_RE, "").trim() ?? ""; + const normalized = normalizeSnippet(heading); + if ( + !normalized || + SHORT_TERM_BASENAME_RE.test(normalized) || + isGenericDailyHeadingForPromotion(normalized) + ) { + return null; + } + return normalized; +} + +function isGenericDailyHeadingForPromotion(heading: string): boolean { + const normalized = heading.trim().replace(/\s+/g, " "); + const lower = normalized.toLowerCase(); + if (MANAGED_DREAMING_HEADINGS.has(lower)) { + return true; + } + if (lower === "today" || lower === "yesterday" || lower === "tomorrow") { + return true; + } + if (lower === "morning" || lower === "afternoon" || lower === "evening" || lower === "night") { + return true; + } + return GENERIC_DAY_HEADING_RE.test(normalized); +} + +function buildRelocatedDailyHeadingLookup(lines: string[]): (string | null)[] { + const headings: (string | null)[] = Array.from({ length: lines.length + 1 }, () => null); + let currentHeading: string | null = null; + for (let index = 0; index < lines.length; index += 1) { + headings[index + 1] = currentHeading; + const line = lines[index] ?? ""; + if (DREAMING_FENCE_START_RE.test(line) || DREAMING_FENCE_END_RE.test(line)) { + currentHeading = null; + continue; + } + if (/^#{1,6}\s+.+$/.test(line.trim())) { + currentHeading = normalizeDailyHeadingForPromotion(line); + } + } + return headings; +} + +function buildListMarkerFreeMatchSnippet( + heading: string | null, + listMarkerFreeSnippet: string, +): string { + if (!listMarkerFreeSnippet) { + return listMarkerFreeSnippet; + } + return heading ? normalizeSnippet(`${heading}: ${listMarkerFreeSnippet}`) : listMarkerFreeSnippet; +} + +function targetSnippetHasHeadingContext(targetSnippet: string, bodySnippet: string): boolean { + if (!targetSnippet || !bodySnippet || targetSnippet === bodySnippet) { + return false; + } + const bodyIndex = targetSnippet.indexOf(bodySnippet); + if (bodyIndex <= 0) { + return false; + } + return targetSnippet.slice(0, bodyIndex).trimEnd().endsWith(":"); +} + +function extractTargetHeadingBodySnippet( + targetSnippet: string, + bodySnippet: string, +): string | null { + if (!targetSnippet || !bodySnippet || targetSnippet === bodySnippet) { + return null; + } + if (bodySnippet.startsWith(targetSnippet)) { + return null; + } + const normalizedBody = normalizeSnippet(bodySnippet); + for (let separatorIndex = targetSnippet.indexOf(": "); separatorIndex > 0;) { + const targetBody = normalizeSnippet(targetSnippet.slice(separatorIndex + 2)); + if (targetBody && normalizedBody.startsWith(targetBody)) { + return targetBody; + } + separatorIndex = targetSnippet.indexOf(": ", separatorIndex + 2); + } + return null; +} + +function compareCandidateWindow( + targetSnippet: string, + windowSnippet: string, +): { matched: boolean; quality: number } { + if (!targetSnippet || !windowSnippet) { + return { matched: false, quality: 0 }; + } + if (windowSnippet === targetSnippet) { + return { matched: true, quality: 3 }; + } + if (windowSnippet.includes(targetSnippet)) { + return { matched: true, quality: 2 }; + } + if (targetSnippet.includes(windowSnippet)) { + return { matched: true, quality: 1 }; + } + return { matched: false, quality: 0 }; +} + +function relocateCandidateRange( + lines: string[], + candidate: PromotionCandidate, +): { startLine: number; endLine: number; snippet: string } | null { + const targetSnippet = normalizeSnippet(candidate.snippet); + const preferredSpan = Math.max(1, candidate.endLine - candidate.startLine + 1); + if (targetSnippet.length === 0) { + const fallbackSnippet = normalizeRangeSnippet(lines, candidate.startLine, candidate.endLine); + if (!fallbackSnippet) { + return null; + } + return { + startLine: candidate.startLine, + endLine: candidate.endLine, + snippet: fallbackSnippet, + }; + } + + const exactSnippet = normalizeRangeSnippet(lines, candidate.startLine, candidate.endLine); + if (exactSnippet === targetSnippet) { + return { + startLine: candidate.startLine, + endLine: candidate.endLine, + snippet: exactSnippet, + }; + } + + const maxSpan = Math.min(lines.length, Math.max(preferredSpan + 3, 8)); + const headingLookup = buildRelocatedDailyHeadingLookup(lines); + let bestMatch: + | { startLine: number; endLine: number; snippet: string; quality: number; distance: number } + | undefined; + for (let startIndex = 0; startIndex < lines.length; startIndex += 1) { + for (let span = 1; span <= maxSpan && startIndex + span <= lines.length; span += 1) { + const startLine = startIndex + 1; + const endLine = startIndex + span; + const snippet = normalizeRangeSnippet(lines, startLine, endLine); + const comparison = compareCandidateWindow(targetSnippet, snippet); + const listMarkerFreeSnippet = normalizeListMarkerFreeRangeSnippet(lines, startLine, endLine); + const listMarkerFreeMatchSnippet = buildListMarkerFreeMatchSnippet( + headingLookup[startLine] ?? null, + listMarkerFreeSnippet, + ); + const listMarkerFreeComparison = + listMarkerFreeSnippet === snippet + ? { matched: false, quality: 0 } + : compareCandidateWindow(targetSnippet, listMarkerFreeSnippet); + const listMarkerFreeContextComparison = + listMarkerFreeMatchSnippet === listMarkerFreeSnippet + ? { matched: false, quality: 0 } + : compareCandidateWindow(targetSnippet, listMarkerFreeMatchSnippet); + const targetHeadingBodySnippet = extractTargetHeadingBodySnippet( + targetSnippet, + listMarkerFreeSnippet, + ); + const targetHeadingBodyComparison = + targetHeadingBodySnippet && listMarkerFreeMatchSnippet !== listMarkerFreeSnippet + ? compareCandidateWindow(targetHeadingBodySnippet, listMarkerFreeSnippet) + : { matched: false, quality: 0 }; + const useTargetHeadingBodyContext = + targetHeadingBodyComparison.matched && + targetHeadingBodyComparison.quality >= comparison.quality && + targetHeadingBodyComparison.quality >= listMarkerFreeComparison.quality; + const useListMarkerFreeContext = + !useTargetHeadingBodyContext && + listMarkerFreeContextComparison.quality > comparison.quality && + listMarkerFreeContextComparison.quality >= listMarkerFreeComparison.quality; + const useListMarkerFree = + !useListMarkerFreeContext && listMarkerFreeComparison.quality > comparison.quality; + const bestComparison = useTargetHeadingBodyContext + ? targetHeadingBodyComparison + : useListMarkerFreeContext + ? listMarkerFreeContextComparison + : useListMarkerFree + ? listMarkerFreeComparison + : comparison; + if (!bestComparison.matched) { + continue; + } + const matchedSnippet = + useTargetHeadingBodyContext || useListMarkerFreeContext + ? listMarkerFreeMatchSnippet + : useListMarkerFree + ? targetSnippetHasHeadingContext(targetSnippet, listMarkerFreeSnippet) + ? listMarkerFreeMatchSnippet + : listMarkerFreeSnippet + : snippet; + const distance = Math.abs(startLine - candidate.startLine); + if ( + !bestMatch || + bestComparison.quality > bestMatch.quality || + (bestComparison.quality === bestMatch.quality && distance < bestMatch.distance) || + (bestComparison.quality === bestMatch.quality && + distance === bestMatch.distance && + Math.abs(span - preferredSpan) < + Math.abs(bestMatch.endLine - bestMatch.startLine + 1 - preferredSpan)) + ) { + bestMatch = { + startLine, + endLine, + snippet: matchedSnippet, + quality: bestComparison.quality, + distance, + }; + } + } + } + + if (!bestMatch) { + return null; + } + return { + startLine: bestMatch.startLine, + endLine: bestMatch.endLine, + snippet: bestMatch.snippet, + }; +} + +const DREAMING_FENCE_START_RE = //i; +const DREAMING_FENCE_END_RE = //i; + +function lineRangeOverlapsDreamingFence( + lines: string[], + startLine: number, + endLine: number, +): boolean { + if (lines.length === 0) { + return false; + } + const safeStart = Math.max(1, Math.min(startLine, lines.length)); + const safeEnd = Math.max(safeStart, Math.min(endLine, lines.length)); + let insideFence = false; + for (let i = 0; i < safeEnd; i += 1) { + const line = lines[i] ?? ""; + const oneIndexed = i + 1; + const isStart = DREAMING_FENCE_START_RE.test(line); + const isEnd = DREAMING_FENCE_END_RE.test(line); + if (isStart || isEnd) { + // The marker line itself is managed-block content. A relocated range + // that includes a `` marker would + // build its snippet from raw lines that contain that marker text and + // leak it into MEMORY.md alongside any adjacent fenced content captured + // by the same window. (#80613) + if (oneIndexed >= safeStart && oneIndexed <= safeEnd) { + return true; + } + insideFence = isStart; + continue; + } + if (insideFence && oneIndexed >= safeStart && oneIndexed <= safeEnd) { + return true; + } + } + return false; +} + +export async function rehydratePromotionCandidate( + workspaceDir: string, + candidate: PromotionCandidate, +): Promise { + const sourcePaths = resolveShortTermSourcePathCandidates(workspaceDir, candidate.path); + for (const sourcePath of sourcePaths) { + let rawSource: string; + try { + rawSource = await fs.readFile(sourcePath, "utf-8"); + } catch (err) { + if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { + continue; + } + throw err; + } + + const lines = rawSource.split(/\r?\n/); + const relocated = relocateCandidateRange(lines, candidate); + if (!relocated) { + continue; + } + // Managed dreaming blocks in daily memory files are scratchwork, not durable + // content. If rehydration lands inside an openclaw:dreaming fence (for example + // because file edits shifted lines between ranking and apply), refuse the + // candidate so dream artifacts cannot be promoted into MEMORY.md. + if (lineRangeOverlapsDreamingFence(lines, relocated.startLine, relocated.endLine)) { + continue; + } + return { + ...candidate, + startLine: relocated.startLine, + endLine: relocated.endLine, + snippet: relocated.snippet, + }; + } + return null; +} diff --git a/extensions/memory-core/src/short-term-promotion-stats.ts b/extensions/memory-core/src/short-term-promotion-stats.ts new file mode 100644 index 000000000000..cfbef411adc8 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-stats.ts @@ -0,0 +1,387 @@ +import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; +import { isSameMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status"; +import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { formatErrorMessage } from "./dreaming-shared.js"; +import { + emptyPhaseSignalStore, + readPhaseSignalStore, + readStore, + resolvePhaseSignalPath, + resolveStorePath, + withShortTermLock, + writePhaseSignalStore, +} from "./short-term-promotion-store.js"; +import type { + ShortTermDreamingStats, + ShortTermDreamingStatsEntry, + ShortTermPhaseSignalStore, + ShortTermRecallEntry, +} from "./short-term-promotion-types.js"; +import { + isShortTermMemoryPath, + normalizeMemoryPathForWorkspace, + normalizeSnippet, + parseEntryRangeFromKey, + parseStoreTimestampMs, + toNonNegativeInt, +} from "./short-term-promotion-utils.js"; +import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js"; + +const DREAMING_ENTRY_LIST_LIMIT = 8; + +function compareDreamingStatsEntryByRecency( + a: ShortTermDreamingStatsEntry, + b: ShortTermDreamingStatsEntry, +): number { + const aMs = a.lastRecalledAt ? Date.parse(a.lastRecalledAt) : Number.NEGATIVE_INFINITY; + const bMs = b.lastRecalledAt ? Date.parse(b.lastRecalledAt) : Number.NEGATIVE_INFINITY; + if (Number.isFinite(aMs) || Number.isFinite(bMs)) { + if (bMs !== aMs) { + return bMs - aMs; + } + } + if (b.totalSignalCount !== a.totalSignalCount) { + return b.totalSignalCount - a.totalSignalCount; + } + return a.path.localeCompare(b.path); +} + +function compareDreamingStatsEntryBySignals( + a: ShortTermDreamingStatsEntry, + b: ShortTermDreamingStatsEntry, +): number { + if (b.totalSignalCount !== a.totalSignalCount) { + return b.totalSignalCount - a.totalSignalCount; + } + if (b.phaseHitCount !== a.phaseHitCount) { + return b.phaseHitCount - a.phaseHitCount; + } + return compareDreamingStatsEntryByRecency(a, b); +} + +function compareDreamingStatsEntryByPromotion( + a: ShortTermDreamingStatsEntry, + b: ShortTermDreamingStatsEntry, +): number { + const aMs = a.promotedAt ? Date.parse(a.promotedAt) : Number.NEGATIVE_INFINITY; + const bMs = b.promotedAt ? Date.parse(b.promotedAt) : Number.NEGATIVE_INFINITY; + if (Number.isFinite(aMs) || Number.isFinite(bMs)) { + if (bMs !== aMs) { + return bMs - aMs; + } + } + return compareDreamingStatsEntryBySignals(a, b); +} + +function trimDreamingStatsEntries( + entries: ShortTermDreamingStatsEntry[], + compare: (a: ShortTermDreamingStatsEntry, b: ShortTermDreamingStatsEntry) => number, +): ShortTermDreamingStatsEntry[] { + const selected: ShortTermDreamingStatsEntry[] = []; + for (const entry of entries) { + let insertAt = selected.length; + for (let index = 0; index < selected.length; index += 1) { + if (compare(entry, expectDefined(selected[index], "selected dreaming stats index")) < 0) { + insertAt = index; + break; + } + } + if (insertAt < DREAMING_ENTRY_LIST_LIMIT) { + selected.splice(insertAt, 0, entry); + if (selected.length > DREAMING_ENTRY_LIST_LIMIT) { + selected.pop(); + } + } else if (selected.length < DREAMING_ENTRY_LIST_LIMIT) { + selected.push(entry); + } + } + return selected; +} + +export async function loadShortTermPromotionDreamingStats(params: { + workspaceDir: string; + nowMs: number; + timezone?: string; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + const nowIso = new Date(params.nowMs).toISOString(); + const store = await readStore(workspaceDir, nowIso); + let phaseSignalError: string | undefined; + let phaseStore: ShortTermPhaseSignalStore; + try { + phaseStore = await readPhaseSignalStore(workspaceDir, nowIso); + } catch (err) { + phaseSignalError = formatErrorMessage(err); + phaseStore = emptyPhaseSignalStore(nowIso); + } + let shortTermCount = 0; + let recallSignalCount = 0; + let dailySignalCount = 0; + let groundedSignalCount = 0; + let totalSignalCount = 0; + let phaseSignalCount = 0; + let lightPhaseHitCount = 0; + let remPhaseHitCount = 0; + let promotedTotal = 0; + let promotedToday = 0; + let latestPromotedAtMs = Number.NEGATIVE_INFINITY; + let latestPromotedAt: string | undefined; + const activeKeys = new Set(); + const activeEntries = new Map(); + const shortTermEntries: ShortTermDreamingStatsEntry[] = []; + const promotedEntries: ShortTermDreamingStatsEntry[] = []; + + for (const [entryKey, entry] of Object.entries(store.entries)) { + if (entry.source !== "memory" || !entry.path || !isShortTermMemoryPath(entry.path)) { + continue; + } + const range = parseEntryRangeFromKey(entryKey, entry.startLine, entry.endLine); + const recallCount = toNonNegativeInt(entry.recallCount); + const dailyCount = toNonNegativeInt(entry.dailyCount); + const groundedCount = toNonNegativeInt(entry.groundedCount); + const totalEntrySignalCount = recallCount + dailyCount + groundedCount; + const normalizedEntryPath = normalizeMemoryPathForWorkspace(workspaceDir, entry.path); + const detail: ShortTermDreamingStatsEntry = { + key: entryKey, + path: normalizedEntryPath, + startLine: range.startLine, + endLine: Math.max(range.startLine, range.endLine), + snippet: normalizeSnippet(entry.snippet) || normalizedEntryPath, + recallCount, + dailyCount, + groundedCount, + totalSignalCount: totalEntrySignalCount, + lightHits: 0, + remHits: 0, + phaseHitCount: 0, + ...(entry.lastRecalledAt ? { lastRecalledAt: entry.lastRecalledAt } : {}), + }; + if (!entry.promotedAt) { + shortTermCount += 1; + activeKeys.add(entryKey); + recallSignalCount += recallCount; + dailySignalCount += dailyCount; + groundedSignalCount += groundedCount; + totalSignalCount += totalEntrySignalCount; + shortTermEntries.push(detail); + activeEntries.set(entryKey, detail); + continue; + } + promotedTotal += 1; + promotedEntries.push({ ...detail, promotedAt: entry.promotedAt }); + const promotedAtMs = Date.parse(entry.promotedAt); + if ( + Number.isFinite(promotedAtMs) && + isSameMemoryDreamingDay(promotedAtMs, params.nowMs, params.timezone) + ) { + promotedToday += 1; + } + if (Number.isFinite(promotedAtMs) && promotedAtMs > latestPromotedAtMs) { + latestPromotedAtMs = promotedAtMs; + latestPromotedAt = entry.promotedAt; + } + } + + for (const [key, phaseEntry] of Object.entries(phaseStore.entries)) { + if (!activeKeys.has(key)) { + continue; + } + const lightHits = toNonNegativeInt(phaseEntry.lightHits); + const remHits = toNonNegativeInt(phaseEntry.remHits); + lightPhaseHitCount += lightHits; + remPhaseHitCount += remHits; + phaseSignalCount += lightHits + remHits; + const detail = activeEntries.get(key); + if (detail) { + detail.lightHits = lightHits; + detail.remHits = remHits; + detail.phaseHitCount = lightHits + remHits; + } + } + + return { + shortTermCount, + recallSignalCount, + dailySignalCount, + groundedSignalCount, + totalSignalCount, + phaseSignalCount, + lightPhaseHitCount, + remPhaseHitCount, + promotedTotal, + promotedToday, + storePath: resolveStorePath(workspaceDir), + phaseSignalPath: resolvePhaseSignalPath(workspaceDir), + shortTermEntries: trimDreamingStatsEntries( + shortTermEntries, + compareDreamingStatsEntryByRecency, + ), + signalEntries: trimDreamingStatsEntries(shortTermEntries, compareDreamingStatsEntryBySignals), + promotedEntries: trimDreamingStatsEntries( + promotedEntries, + compareDreamingStatsEntryByPromotion, + ), + ...(phaseSignalError ? { phaseSignalError } : {}), + ...(latestPromotedAt ? { lastPromotedAt: latestPromotedAt } : {}), + }; +} + +export async function recordDreamingPhaseSignals(params: { + workspaceDir?: string; + phase: "light" | "rem"; + keys: string[]; + nowMs?: number; +}): Promise { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return; + } + const keys = uniqueStrings(normalizeStringEntries(params.keys)); + if (keys.length === 0) { + return; + } + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + + await withShortTermLock(workspaceDir, async () => { + const [store, phaseSignals] = await Promise.all([ + readStore(workspaceDir, nowIso), + readPhaseSignalStore(workspaceDir, nowIso), + ]); + const knownKeys = new Set(Object.keys(store.entries)); + + for (const key of keys) { + if (!knownKeys.has(key)) { + continue; + } + const entry = phaseSignals.entries[key] ?? { + key, + lightHits: 0, + remHits: 0, + }; + if (params.phase === "light") { + entry.lightHits = Math.min(9999, entry.lightHits + 1); + entry.lastLightAt = nowIso; + } else { + entry.remHits = Math.min(9999, entry.remHits + 1); + entry.lastRemAt = nowIso; + } + phaseSignals.entries[key] = entry; + } + + for (const [key, entry] of Object.entries(phaseSignals.entries)) { + if (!knownKeys.has(key) || (entry.lightHits <= 0 && entry.remHits <= 0)) { + delete phaseSignals.entries[key]; + } + } + + phaseSignals.updatedAt = nowIso; + await writePhaseSignalStore(workspaceDir, phaseSignals); + }); +} + +export async function recordRemConsideredPhaseSignals(params: { + workspaceDir?: string; + keys: string[]; + nowMs?: number; +}): Promise { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return; + } + const keys = uniqueStrings(normalizeStringEntries(params.keys)); + if (keys.length === 0) { + return; + } + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + + await withShortTermLock(workspaceDir, async () => { + const [store, phaseSignals] = await Promise.all([ + readStore(workspaceDir, nowIso), + readPhaseSignalStore(workspaceDir, nowIso), + ]); + const knownKeys = new Set(Object.keys(store.entries)); + + for (const key of keys) { + if (!knownKeys.has(key)) { + continue; + } + const entry = phaseSignals.entries[key] ?? { + key, + lightHits: 0, + remHits: 0, + }; + entry.lastRemConsideredAt = nowIso; + phaseSignals.entries[key] = entry; + } + + for (const [key, entry] of Object.entries(phaseSignals.entries)) { + if (!knownKeys.has(key) || (entry.lightHits <= 0 && entry.remHits <= 0)) { + delete phaseSignals.entries[key]; + } + } + + phaseSignals.updatedAt = nowIso; + await writePhaseSignalStore(workspaceDir, phaseSignals); + }); +} + +export async function readLightStagedKeys(params: { + workspaceDir: string; + nowMs?: number; +}): Promise> { + const workspaceDir = params.workspaceDir?.trim(); + if (!workspaceDir) { + return new Set(); + } + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + const store = await readPhaseSignalStore(workspaceDir, nowIso); + const keys = new Set(); + for (const [key, entry] of Object.entries(store.entries)) { + if (entry.lightHits <= 0) { + continue; + } + const lastLightMs = Date.parse(entry.lastLightAt ?? ""); + const lastRemMs = Date.parse(entry.lastRemAt ?? ""); + const lastRemConsideredMs = Date.parse(entry.lastRemConsideredAt ?? ""); + const lastConsumedMs = Math.max( + Number.isFinite(lastRemMs) ? lastRemMs : Number.NEGATIVE_INFINITY, + Number.isFinite(lastRemConsideredMs) ? lastRemConsideredMs : Number.NEGATIVE_INFINITY, + ); + const hasPendingLightSignal = Number.isFinite(lastLightMs) + ? lastLightMs > lastConsumedMs + : !entry.lastRemAt; + if (hasPendingLightSignal) { + keys.add(key); + } + } + return keys; +} + +export async function filterFreshLightDreamingEntries(params: { + workspaceDir: string; + entries: readonly ShortTermRecallEntry[]; + nowMs?: number; +}): Promise { + const workspaceDir = params.workspaceDir.trim(); + if (!workspaceDir || params.entries.length === 0) { + return []; + } + const nowMs = resolveMemoryCoreNowMs(params.nowMs); + const nowIso = resolveMemoryCoreTimestamp(nowMs); + const phaseSignals = await readPhaseSignalStore(workspaceDir, nowIso); + return params.entries.filter((entry) => { + const phaseSignal = phaseSignals.entries[entry.key]; + if (!phaseSignal || phaseSignal.lightHits <= 0) { + return true; + } + const lastLightMs = parseStoreTimestampMs(phaseSignal.lastLightAt); + if (!Number.isFinite(lastLightMs)) { + return true; + } + const lastRecalledAtMs = parseStoreTimestampMs(entry.lastRecalledAt); + return Number.isFinite(lastRecalledAtMs) && lastRecalledAtMs > lastLightMs; + }); +} diff --git a/extensions/memory-core/src/short-term-promotion-store.ts b/extensions/memory-core/src/short-term-promotion-store.ts new file mode 100644 index 000000000000..fb65484fac73 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-store.ts @@ -0,0 +1,277 @@ +import { KeyedAsyncQueue } from "openclaw/plugin-sdk/keyed-async-queue"; +import { sleep } from "openclaw/plugin-sdk/runtime-env"; +import { asRecord } from "./dreaming-shared.js"; +import { + SHORT_TERM_LOCK_MAX_ENTRIES, + SHORT_TERM_LOCK_NAMESPACE, + SHORT_TERM_META_NAMESPACE, + SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + SHORT_TERM_RECALL_NAMESPACE, + memoryCoreStateReference, + memoryCoreWorkspaceStateKey, + openMemoryCoreStateStore, + readMemoryCoreWorkspaceEntries, + writeMemoryCoreWorkspaceEntries, + writeMemoryCoreWorkspaceEntry, +} from "./dreaming-state.js"; +import type { + ShortTermLockEntry, + ShortTermPhaseSignalEntry, + ShortTermPhaseSignalStore, + ShortTermRecallEntry, + ShortTermRecallStore, + ShortTermStoreMeta, +} from "./short-term-promotion-types.js"; +import { + enforceShortTermRecallSnippetCap, + enforceShortTermRecallStoreRetention, + normalizeShortTermRecallStore, + toFiniteNonNegativeInt, +} from "./short-term-promotion-utils.js"; + +const SHORT_TERM_LOCK_WAIT_TIMEOUT_MS = 10_000; +export const SHORT_TERM_LOCK_STALE_MS = 60_000; +const SHORT_TERM_LOCK_RETRY_DELAY_MS = 40; +const inProcessShortTermLocks = new KeyedAsyncQueue(); + +export function resolveStorePath(workspaceDir: string): string { + return memoryCoreStateReference(SHORT_TERM_RECALL_NAMESPACE, workspaceDir); +} + +export function resolvePhaseSignalPath(workspaceDir: string): string { + return memoryCoreStateReference(SHORT_TERM_PHASE_SIGNAL_NAMESPACE, workspaceDir); +} + +export function resolveLockPath(workspaceDir: string): string { + return memoryCoreStateReference(SHORT_TERM_LOCK_NAMESPACE, workspaceDir); +} + +export function parseLockOwnerPid(raw: string): number | null { + const match = raw.trim().match(/^(\d+):/); + if (!match) { + return null; + } + const pid = Number.parseInt(match[1] ?? "", 10); + if (!Number.isInteger(pid) || pid <= 0) { + return null; + } + return pid; +} + +export function isProcessLikelyAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === "ESRCH") { + return false; + } + // EPERM and unknown errors are treated as alive to avoid stealing active locks. + return true; + } +} + +async function withInProcessShortTermLock(lockPath: string, task: () => Promise): Promise { + return await inProcessShortTermLocks.enqueue(lockPath, task); +} + +export async function withShortTermLock( + workspaceDir: string, + task: () => Promise, +): Promise { + const lockKey = memoryCoreWorkspaceStateKey(workspaceDir); + const lockRef = resolveLockPath(workspaceDir); + const lockStore = openMemoryCoreStateStore({ + namespace: SHORT_TERM_LOCK_NAMESPACE, + maxEntries: SHORT_TERM_LOCK_MAX_ENTRIES, + }); + return withInProcessShortTermLock(lockKey, async () => { + const startedAt = Date.now(); + + while (true) { + const owner = `${process.pid}:${Date.now()}`; + const acquired = await lockStore.registerIfAbsent(lockKey, { + owner, + acquiredAt: Date.now(), + }); + if (acquired) { + try { + return await task(); + } finally { + const current = await lockStore.lookup(lockKey).catch(() => undefined); + if (current?.owner === owner) { + await lockStore.delete(lockKey).catch(() => false); + } + } + } + + const existing = await lockStore.lookup(lockKey); + if (existing && Date.now() - existing.acquiredAt > SHORT_TERM_LOCK_STALE_MS) { + const ownerPid = parseLockOwnerPid(existing.owner); + if (ownerPid === null || !isProcessLikelyAlive(ownerPid)) { + await lockStore.delete(lockKey); + continue; + } + } + + if (Date.now() - startedAt >= SHORT_TERM_LOCK_WAIT_TIMEOUT_MS) { + throw new Error(`Timed out waiting for short-term promotion lock at ${lockRef}`); + } + + await sleep(SHORT_TERM_LOCK_RETRY_DELAY_MS); + } + }); +} + +export async function readStore( + workspaceDir: string, + nowIso: string, +): Promise { + const [entryRows, metaRows] = await Promise.all([ + readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_RECALL_NAMESPACE, + workspaceDir, + }), + readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_META_NAMESPACE, + workspaceDir, + }), + ]); + const meta = metaRows.find((entry) => entry.key === "recall")?.value; + const store = normalizeShortTermRecallStore( + { + version: 1, + updatedAt: meta?.updatedAt ?? nowIso, + entries: Object.fromEntries(entryRows.map((entry) => [entry.key, entry.value])), + }, + nowIso, + ); + enforceShortTermRecallStoreRetention(store); + return store; +} + +export function emptyPhaseSignalStore(nowIso: string): ShortTermPhaseSignalStore { + return { + version: 1, + updatedAt: nowIso, + entries: {}, + }; +} + +export function normalizeShortTermPhaseSignalStore( + raw: unknown, + nowIso: string, +): ShortTermPhaseSignalStore { + const record = asRecord(raw); + if (!record) { + return emptyPhaseSignalStore(nowIso); + } + const entriesRaw = asRecord(record?.entries); + if (!entriesRaw) { + return emptyPhaseSignalStore(nowIso); + } + const entries: Record = {}; + for (const [mapKey, value] of Object.entries(entriesRaw)) { + const entry = asRecord(value); + if (!entry) { + continue; + } + const key = typeof entry.key === "string" && entry.key.trim().length > 0 ? entry.key : mapKey; + const lightHits = toFiniteNonNegativeInt(entry.lightHits, 0); + const remHits = toFiniteNonNegativeInt(entry.remHits, 0); + if (lightHits === 0 && remHits === 0) { + continue; + } + const lastLightAt = + typeof entry.lastLightAt === "string" && entry.lastLightAt.trim().length > 0 + ? entry.lastLightAt + : undefined; + const lastRemAt = + typeof entry.lastRemAt === "string" && entry.lastRemAt.trim().length > 0 + ? entry.lastRemAt + : undefined; + const lastRemConsideredAt = + typeof entry.lastRemConsideredAt === "string" && entry.lastRemConsideredAt.trim().length > 0 + ? entry.lastRemConsideredAt + : undefined; + entries[key] = { + key, + lightHits, + remHits, + ...(lastLightAt ? { lastLightAt } : {}), + ...(lastRemAt ? { lastRemAt } : {}), + ...(lastRemConsideredAt ? { lastRemConsideredAt } : {}), + }; + } + return { + version: 1, + updatedAt: + typeof record.updatedAt === "string" && record.updatedAt.trim().length > 0 + ? record.updatedAt + : nowIso, + entries, + }; +} + +export async function readPhaseSignalStore( + workspaceDir: string, + nowIso: string, +): Promise { + const [entryRows, metaRows] = await Promise.all([ + readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + workspaceDir, + }), + readMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_META_NAMESPACE, + workspaceDir, + }), + ]); + const meta = metaRows.find((entry) => entry.key === "phase")?.value; + return normalizeShortTermPhaseSignalStore( + { + version: 1, + updatedAt: meta?.updatedAt ?? nowIso, + entries: Object.fromEntries(entryRows.map((entry) => [entry.key, entry.value])), + }, + nowIso, + ); +} + +export async function writePhaseSignalStore( + workspaceDir: string, + store: ShortTermPhaseSignalStore, +): Promise { + await Promise.all([ + writeMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + workspaceDir, + entries: Object.entries(store.entries).map(([key, value]) => ({ key, value })), + }), + writeMemoryCoreWorkspaceEntry({ + namespace: SHORT_TERM_META_NAMESPACE, + workspaceDir, + key: "phase", + value: { updatedAt: store.updatedAt }, + }), + ]); +} + +export async function writeStore(workspaceDir: string, store: ShortTermRecallStore): Promise { + enforceShortTermRecallSnippetCap(store); + enforceShortTermRecallStoreRetention(store); + await Promise.all([ + writeMemoryCoreWorkspaceEntries({ + namespace: SHORT_TERM_RECALL_NAMESPACE, + workspaceDir, + entries: Object.entries(store.entries).map(([key, value]) => ({ key, value })), + }), + writeMemoryCoreWorkspaceEntry({ + namespace: SHORT_TERM_META_NAMESPACE, + workspaceDir, + key: "recall", + value: { updatedAt: store.updatedAt }, + }), + ]); +} diff --git a/extensions/memory-core/src/short-term-promotion-types.ts b/extensions/memory-core/src/short-term-promotion-types.ts new file mode 100644 index 000000000000..e12a75d2c0d2 --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-types.ts @@ -0,0 +1,245 @@ +import path from "node:path"; +import type { ConceptTagScriptCoverage } from "./concept-vocabulary.js"; + +export const DEFAULT_PROMOTION_MIN_SCORE = 0.75; +export const DEFAULT_PROMOTION_MIN_RECALL_COUNT = 3; +export const DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES = 2; +export const SHORT_TERM_STORE_RELATIVE_PATH = path.join( + "memory", + ".dreams", + "short-term-recall.json", +); +export const SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH = path.join( + "memory", + ".dreams", + "phase-signals.json", +); + +export type PromotionWeights = { + frequency: number; + relevance: number; + diversity: number; + recency: number; + consolidation: number; + conceptual: number; +}; + +export type ShortTermRecallEntry = { + key: string; + path: string; + startLine: number; + endLine: number; + source: "memory"; + snippet: string; + recallCount: number; + dailyCount: number; + groundedCount: number; + totalScore: number; + maxScore: number; + firstRecalledAt: string; + lastRecalledAt: string; + queryHashes: string[]; + recallDays: string[]; + conceptTags: string[]; + claimHash?: string; + promotedAt?: string; +}; + +export type ShortTermRecallStore = { + version: 1; + updatedAt: string; + entries: Record; +}; + +export type ShortTermPhaseSignalEntry = { + key: string; + lightHits: number; + remHits: number; + lastLightAt?: string; + lastRemAt?: string; + lastRemConsideredAt?: string; +}; + +export type ShortTermPhaseSignalStore = { + version: 1; + updatedAt: string; + entries: Record; +}; + +export type ShortTermStoreMeta = { + updatedAt: string; +}; + +export type ShortTermLockEntry = { + owner: string; + acquiredAt: number; +}; + +type PromotionComponents = { + frequency: number; + relevance: number; + diversity: number; + recency: number; + consolidation: number; + conceptual: number; +}; + +export type PromotionCandidate = { + key: string; + path: string; + startLine: number; + endLine: number; + source: "memory"; + snippet: string; + recallCount: number; + dailyCount?: number; + groundedCount?: number; + signalCount: number; + avgScore: number; + maxScore: number; + uniqueQueries: number; + claimHash?: string; + promotedAt?: string; + firstRecalledAt: string; + lastRecalledAt: string; + ageDays: number; + score: number; + recallDays: string[]; + conceptTags: string[]; + components: PromotionComponents; +}; + +export type ShortTermAuditIssue = { + severity: "warn" | "error"; + code: + | "recall-store-unreadable" + | "recall-store-empty" + | "recall-store-invalid" + | "recall-store-over-limit" + | "recall-lock-stale" + | "recall-lock-unreadable" + | "qmd-index-missing" + | "qmd-index-empty" + | "qmd-collections-empty"; + message: string; + fixable: boolean; +}; + +export type ShortTermAuditSummary = { + storePath: string; + lockPath: string; + updatedAt?: string; + exists: boolean; + entryCount: number; + promotedCount: number; + spacedEntryCount: number; + conceptTaggedEntryCount: number; + conceptTagScripts?: ConceptTagScriptCoverage; + invalidEntryCount: number; + issues: ShortTermAuditIssue[]; + qmd?: + | { + dbPath?: string; + collections?: number; + dbBytes?: number; + } + | undefined; +}; + +export type RepairShortTermPromotionArtifactsResult = { + changed: boolean; + removedInvalidEntries: number; + removedOverflowEntries: number; + rewroteStore: boolean; + removedStaleLock: boolean; +}; + +export type RankShortTermPromotionOptions = { + workspaceDir: string; + limit?: number; + minScore?: number; + minRecallCount?: number; + minUniqueQueries?: number; + maxAgeDays?: number; + includePromoted?: boolean; + recencyHalfLifeDays?: number; + weights?: Partial; + nowMs?: number; +}; + +export type ApplyShortTermPromotionsOptions = { + workspaceDir: string; + candidates: PromotionCandidate[]; + limit?: number; + minScore?: number; + minRecallCount?: number; + minUniqueQueries?: number; + maxAgeDays?: number; + nowMs?: number; + timezone?: string; + /** + * Maximum size of MEMORY.md on disk after a promotion write, in + * characters. When the post-write size would exceed this budget, the + * oldest auto-promotion sections are compacted out before write so the + * file stays bounded and bootstrap injection keeps reaching new + * sessions. Pass `0` to disable compaction. Defaults to + * `DEFAULT_MEMORY_FILE_MAX_CHARS`. See #73691. + */ + memoryFileMaxChars?: number; + /** + * Maximum visible size of each promoted short-term snippet in MEMORY.md, in + * estimated tokens. This keeps daily journal ranges from being copied + * wholesale into long-term memory while preserving the candidate's provenance + * metadata. + */ + maxPromotedSnippetTokens?: number; +}; + +export type ApplyShortTermPromotionsResult = { + memoryPath: string; + applied: number; + appended: number; + reconciledExisting: number; + appliedCandidates: PromotionCandidate[]; + /** Number of older promotion sections compacted out to honor the budget. */ + compactedSections: number; + /** Dates of the compacted promotion sections, oldest first. */ + compactedDates: string[]; +}; + +export type ShortTermDreamingStatsEntry = { + key: string; + path: string; + startLine: number; + endLine: number; + snippet: string; + recallCount: number; + dailyCount: number; + groundedCount: number; + totalSignalCount: number; + lightHits: number; + remHits: number; + phaseHitCount: number; + promotedAt?: string; + lastRecalledAt?: string; +}; + +export type ShortTermDreamingStats = { + shortTermCount: number; + recallSignalCount: number; + dailySignalCount: number; + groundedSignalCount: number; + totalSignalCount: number; + phaseSignalCount: number; + lightPhaseHitCount: number; + remPhaseHitCount: number; + promotedTotal: number; + promotedToday: number; + storePath: string; + phaseSignalPath: string; + phaseSignalError?: string; + lastPromotedAt?: string; + shortTermEntries: ShortTermDreamingStatsEntry[]; + signalEntries: ShortTermDreamingStatsEntry[]; + promotedEntries: ShortTermDreamingStatsEntry[]; +}; diff --git a/extensions/memory-core/src/short-term-promotion-utils.ts b/extensions/memory-core/src/short-term-promotion-utils.ts new file mode 100644 index 000000000000..d2a61bad488c --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-utils.ts @@ -0,0 +1,528 @@ +import { createHash } from "node:crypto"; +import path from "node:path"; +import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; +import { deriveConceptTags, MAX_CONCEPT_TAGS } from "./concept-vocabulary.js"; +import type { + PromotionWeights, + ShortTermRecallEntry, + ShortTermRecallStore, +} from "./short-term-promotion-types.js"; + +const SHORT_TERM_PATH_RE = /(?:^|\/)memory\/(?:[^/]+\/)*(\d{4})-(\d{2})-(\d{2})(?:-[^/]+)?\.md$/; +const DREAMING_MEMORY_PATH_RE = /(?:^|\/)memory\/dreaming\//; +const SHORT_TERM_SESSION_CORPUS_RE = + /(?:^|\/)memory\/\.dreams\/session-corpus\/(\d{4})-(\d{2})-(\d{2})\.(?:md|txt)$/; +export const SHORT_TERM_BASENAME_RE = /^(\d{4})-(\d{2})-(\d{2})(?:-[^/]+)?\.md$/; +export const MAX_QUERY_HASHES = 32; +export const MAX_RECALL_DAYS = 16; +export const SHORT_TERM_RECALL_MAX_ENTRIES = 512; +const SHORT_TERM_RECALL_MAX_SNIPPET_CHARS = 800; +const DREAMING_TRANSCRIPT_PROMPT_LINE_RE = + /\[[^\]]*dreaming-narrative[^\]]*]\s*(?:User|Assistant):\s*Write a dream diary entry from these memory fragments:?/i; +const RAW_SESSION_METADATA_RE = + /\bSession Key\b.{0,260}\bSession ID\b|\bSession ID\b.{0,260}\bSession Key\b/i; +const RAW_CONVERSATION_SUMMARY_RE = /^(?:[-*+]\s*)?Conversation Summary:/i; +const RAW_TRANSCRIPT_TURN_RE = /^(?:[-*+]\s*)?(?:user|assistant):\s/i; +const MEMORY_FLUSH_PROMPT_RE = + /Save important context from this session to the daily memory file\.\s*STRICT RULES:/i; +const PROMOTION_SCORE_METADATA_RE = + /\[\s*score=\d+(?:\.\d+)?\s+(?:signals=\d+\s+)?recalls=\d+\s+avg=\d+(?:\.\d+)?\s+source=memory\//i; +const DREAMING_DIFF_PREFIX_RE = /@@\s*-\d+(?:,\d+)?\s+[-*+]\s+/iy; +const DEFAULT_PROMOTION_WEIGHTS: PromotionWeights = { + frequency: 0.24, + relevance: 0.3, + diversity: 0.15, + recency: 0.15, + consolidation: 0.1, + conceptual: 0.06, +}; + +export function clampScore(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + return Math.max(0, Math.min(1, value)); +} + +export function toFiniteScore(value: unknown, fallback: number): number { + const num = Number(value); + if (!Number.isFinite(num)) { + return fallback; + } + if (num < 0 || num > 1) { + return fallback; + } + return num; +} + +export function normalizeSnippet(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) { + return ""; + } + return trimmed.replace(/\s+/g, " "); +} + +export function truncateShortTermSnippet(snippet: string): string { + if (snippet.length <= SHORT_TERM_RECALL_MAX_SNIPPET_CHARS) { + return snippet; + } + return truncateUtf16Safe(snippet, SHORT_TERM_RECALL_MAX_SNIPPET_CHARS).trimEnd(); +} + +export function enforceShortTermRecallSnippetCap(store: ShortTermRecallStore): void { + for (const entry of Object.values(store.entries)) { + entry.snippet = truncateShortTermSnippet(entry.snippet); + } +} + +function consumeDreamingLeadPrefix(snippet: string): string { + let index = 0; + while (index < snippet.length) { + DREAMING_DIFF_PREFIX_RE.lastIndex = index; + const diffMatch = DREAMING_DIFF_PREFIX_RE.exec(snippet); + if (diffMatch) { + index = DREAMING_DIFF_PREFIX_RE.lastIndex; + continue; + } + const char = snippet[index]; + if (char === "[" || char === "(") { + index += 1; + while (snippet[index] === " ") { + index += 1; + } + continue; + } + if ( + (char === "-" || char === "*" || char === "+" || char === ">") && + snippet[index + 1] === " " + ) { + index += 2; + continue; + } + break; + } + return snippet.slice(index); +} + +function hasDreamingNarrativeLead(snippet: string): boolean { + const withoutPrefix = consumeDreamingLeadPrefix(snippet); + if (/^(?:Candidate|Reflections?):/i.test(withoutPrefix)) { + return true; + } + // Managed dreaming blocks occasionally serialize recall metadata (status:/confidence:/ + // evidence:/recalls:) inline before the Candidate or Reflections marker, so the + // start-of-string check misses shapes like "status: staged - Candidate: User: ...". + // The composite detector below still requires the full signal combination, so widening + // the lead check to anywhere in the first 200 chars closes the leak without creating + // false positives for ordinary durable notes that merely mention the word in prose. + const head = withoutPrefix.slice(0, 200); + return /\b(?:Candidate|Reflections?):/i.test(head); +} + +export function isContaminatedDreamingSnippet( + raw: string, + opts: { allowTranscriptTurnSnippet?: boolean } = {}, +): boolean { + const snippet = normalizeSnippet(raw); + if (!snippet) { + return false; + } + if ( + //i; -const DREAMING_FENCE_END_RE = //i; - -function lineRangeOverlapsDreamingFence( - lines: string[], - startLine: number, - endLine: number, -): boolean { - if (lines.length === 0) { - return false; - } - const safeStart = Math.max(1, Math.min(startLine, lines.length)); - const safeEnd = Math.max(safeStart, Math.min(endLine, lines.length)); - let insideFence = false; - for (let i = 0; i < safeEnd; i += 1) { - const line = lines[i] ?? ""; - const oneIndexed = i + 1; - const isStart = DREAMING_FENCE_START_RE.test(line); - const isEnd = DREAMING_FENCE_END_RE.test(line); - if (isStart || isEnd) { - // The marker line itself is managed-block content. A relocated range - // that includes a `` marker would - // build its snippet from raw lines that contain that marker text and - // leak it into MEMORY.md alongside any adjacent fenced content captured - // by the same window. (#80613) - if (oneIndexed >= safeStart && oneIndexed <= safeEnd) { - return true; - } - insideFence = isStart; - continue; - } - if (insideFence && oneIndexed >= safeStart && oneIndexed <= safeEnd) { - return true; - } - } - return false; -} - -async function rehydratePromotionCandidate( - workspaceDir: string, - candidate: PromotionCandidate, -): Promise { - const sourcePaths = resolveShortTermSourcePathCandidates(workspaceDir, candidate.path); - for (const sourcePath of sourcePaths) { - let rawSource: string; - try { - rawSource = await fs.readFile(sourcePath, "utf-8"); - } catch (err) { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - continue; - } - throw err; - } - - const lines = rawSource.split(/\r?\n/); - const relocated = relocateCandidateRange(lines, candidate); - if (!relocated) { - continue; - } - // Managed dreaming blocks in daily memory files are scratchwork, not durable - // content. If rehydration lands inside an openclaw:dreaming fence (for example - // because file edits shifted lines between ranking and apply), refuse the - // candidate so dream artifacts cannot be promoted into MEMORY.md. - if (lineRangeOverlapsDreamingFence(lines, relocated.startLine, relocated.endLine)) { - continue; - } - return { - ...candidate, - startLine: relocated.startLine, - endLine: relocated.endLine, - snippet: relocated.snippet, - }; - } - return null; -} - -function buildPromotionSection( - candidates: PromotionCandidate[], - nowMs: number, - timezone?: string, - maxPromotedSnippetTokens = DEFAULT_MEMORY_DEEP_DREAMING_MAX_PROMOTED_SNIPPET_TOKENS, -): string { - const sectionDate = formatMemoryDreamingDay(nowMs, timezone); - const lines = ["", `## Promoted From Short-Term Memory (${sectionDate})`, ""]; - - for (const candidate of candidates) { - const source = `${candidate.path}:${candidate.startLine}-${candidate.endLine}`; - const metadata = `[score=${candidate.score.toFixed(3)} signals=${candidate.signalCount} recalls=${candidate.recallCount} avg=${candidate.avgScore.toFixed(3)} source=${source}]`; - lines.push(``); - // Cap only the visible MEMORY.md text. The recall store keeps the full - // rehydrated snippet so ranking, provenance, and dream narratives remain - // tied to the source entry instead of this presentation budget. - lines.push( - `- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata}`, - ); - } - - lines.push(""); - return lines.join("\n"); -} - -function resolvePromotedSnippetCharLimit(maxTokens: number): number { - const tokenLimit = toFiniteNonNegativeInt( - maxTokens, - DEFAULT_MEMORY_DEEP_DREAMING_MAX_PROMOTED_SNIPPET_TOKENS, - ); - // This is an inexpensive display-size guard, not a tokenizer contract. - return tokenLimit * PROMOTED_SNIPPET_CHARS_PER_TOKEN_ESTIMATE; -} - -function truncatePromotedSnippet(snippet: string, maxTokens: number): string { - const limit = resolvePromotedSnippetCharLimit(maxTokens); - if (limit === 0 || snippet.length <= limit) { - return snippet; - } - const hardLimit = truncateUtf16Safe(snippet, limit); - const sentenceBoundary = Math.max( - hardLimit.lastIndexOf(". "), - hardLimit.lastIndexOf("! "), - hardLimit.lastIndexOf("? "), - ); - const wordBoundary = hardLimit.lastIndexOf(" "); - const cutAt = - sentenceBoundary >= Math.floor(limit * 0.55) - ? sentenceBoundary + 1 - : wordBoundary >= Math.floor(limit * 0.65) - ? wordBoundary - : limit; - return `${hardLimit.slice(0, cutAt).trimEnd()}...`; -} - -function formatPromotedSnippetForMemory(rawSnippet: string, maxTokens: number): string { - const normalized = normalizeSnippet(rawSnippet || "(no snippet captured)") - .replace(/^[-*+] +/, "") - .trim(); - return truncatePromotedSnippet(normalized || "(no snippet captured)", maxTokens); -} - -function withTrailingNewline(content: string): string { - if (!content) { - return ""; - } - return content.endsWith("\n") ? content : `${content}\n`; -} - -async function resolveMemoryWritePath(filePath: string): Promise { - try { - return await fs.realpath(filePath); - } catch (err) { - const hasTrailingSeparator = - filePath.endsWith(path.sep) || - (process.platform === "win32" && filePath.endsWith(path.posix.sep)); - if ((err as NodeJS.ErrnoException)?.code !== "ENOENT" || hasTrailingSeparator) { - throw err; - } - } - - // Canonicalize each parent before applying a relative link target. Lexical - // normalization would change `..` semantics when an earlier component is a symlink. - const parentPath = await fs.realpath(path.dirname(filePath)); - const canonicalPath = path.join(parentPath, path.basename(filePath)); - let linkTarget: string; - try { - linkTarget = await fs.readlink(canonicalPath); - } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code; - if (code === "ENOENT" || code === "EINVAL") { - return canonicalPath; - } - throw err; - } - const isWindowsRootRelative = process.platform === "win32" && /^[\\/](?![\\/])/.test(linkTarget); - const targetPath = isWindowsRootRelative - ? `${path.parse(parentPath).root.replace(/[\\/]$/, "")}${linkTarget}` - : path.isAbsolute(linkTarget) - ? linkTarget - : `${parentPath}${parentPath.endsWith(path.sep) ? "" : path.sep}${linkTarget}`; - return await resolveMemoryWritePath(targetPath); -} - -function isAtomicReplacePermissionError(error: unknown): boolean { - const code = (error as NodeJS.ErrnoException)?.code; - return code === "EACCES" || code === "EPERM" || code === "EEXIST" || code === "EROFS"; -} - -async function writeExistingMemoryInPlace(filePath: string, content: string): Promise { - let handle: Awaited>; - try { - handle = await fs.open(filePath, "r+"); - } catch { - return false; - } - try { - await handle.writeFile(content, { encoding: "utf-8" }); - await handle.truncate(Buffer.byteLength(content)); - await handle.sync(); - return true; - } finally { - await handle.close(); - } -} - -function extractPromotionMarkers(memoryText: string): Set { - const markers = new Set(); - // Marker keys include source paths, so spaces are valid. Capture until the - // comment close; otherwise a path like "memory/project alpha/..." is missed - // and the same candidate can be appended again. - const matches = memoryText.matchAll(//gi); - for (const match of matches) { - const key = match[1]?.trim(); - if (key) { - markers.add(key); - } - } - return markers; -} - -export async function applyShortTermPromotions( - options: ApplyShortTermPromotionsOptions, -): Promise { - const workspaceDir = options.workspaceDir.trim(); - const nowMs = resolveMemoryCoreNowMs(options.nowMs); - const nowIso = resolveMemoryCoreTimestamp(nowMs); - const limit = Number.isFinite(options.limit) - ? Math.max(0, Math.floor(options.limit as number)) - : options.candidates.length; - const minScore = toFiniteScore(options.minScore, DEFAULT_PROMOTION_MIN_SCORE); - const minRecallCount = toFiniteNonNegativeInt( - options.minRecallCount, - DEFAULT_PROMOTION_MIN_RECALL_COUNT, - ); - const minUniqueQueries = toFiniteNonNegativeInt( - options.minUniqueQueries, - DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, - ); - const maxAgeDays = toFiniteNonNegativeInt(options.maxAgeDays, -1); - const memoryPath = path.join(workspaceDir, "MEMORY.md"); - - return await withShortTermLock(workspaceDir, async () => { - const store = await readStore(workspaceDir, nowIso); - const selected = options.candidates - .filter((candidate) => { - if (isContaminatedDreamingSnippet(candidate.snippet)) { - return false; - } - if (candidate.promotedAt) { - return false; - } - if (candidate.score < minScore) { - return false; - } - if (candidate.signalCount < minRecallCount) { - return false; - } - if (Math.max(candidate.uniqueQueries, candidate.recallDays.length) < minUniqueQueries) { - return false; - } - if (maxAgeDays >= 0 && candidate.ageDays > maxAgeDays) { - return false; - } - const latest = store.entries[candidate.key]; - if (latest?.promotedAt) { - return false; - } - return true; - }) - .slice(0, limit); - - const rehydratedSelected: PromotionCandidate[] = []; - for (const candidate of selected) { - const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate); - if (rehydrated && !isContaminatedDreamingSnippet(rehydrated.snippet)) { - rehydratedSelected.push(rehydrated); - } - } - - if (rehydratedSelected.length === 0) { - return { - memoryPath, - applied: 0, - appended: 0, - reconciledExisting: 0, - appliedCandidates: [], - compactedSections: 0, - compactedDates: [], - }; - } - - // Promotions historically follow user-managed MEMORY.md symlinks. Replace the - // final target atomically without severing the chain, matching the prior writeFile path. - const memoryWritePath = await resolveMemoryWritePath(memoryPath); - const existingMemory = await fs.readFile(memoryWritePath, "utf-8").catch((err: unknown) => { - if ((err as NodeJS.ErrnoException)?.code === "ENOENT") { - return ""; - } - throw err; - }); - const existingMarkers = extractPromotionMarkers(existingMemory); - const alreadyWritten = rehydratedSelected.filter((candidate) => - existingMarkers.has(candidate.key), - ); - const toAppend = rehydratedSelected.filter((candidate) => !existingMarkers.has(candidate.key)); - - let compactedDates: string[] = []; - if (toAppend.length > 0) { - const section = buildPromotionSection( - toAppend, - nowMs, - options.timezone, - options.maxPromotedSnippetTokens, - ); - const budgetChars = - typeof options.memoryFileMaxChars === "number" && - Number.isFinite(options.memoryFileMaxChars) - ? Math.max(0, Math.floor(options.memoryFileMaxChars)) - : DEFAULT_MEMORY_FILE_MAX_CHARS; - const compaction = compactMemoryForBudget({ - existingMemory, - newSection: section, - budgetChars, - }); - compactedDates = compaction.droppedDates; - const baseMemory = compaction.compacted; - const header = baseMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; - const content = `${header}${withTrailingNewline(baseMemory)}${section}`; - const memoryDirMode = (await fs.stat(path.dirname(memoryWritePath))).mode & 0o7777; - let atomicRenameCommitted = false; - const trackedRename: typeof fs.rename = async (source, destination) => { - await fs.rename(source, destination); - atomicRenameCommitted = true; - }; - try { - await replaceFileAtomic({ - filePath: memoryWritePath, - content, - dirMode: memoryDirMode, - mode: 0o600, - preserveExistingMode: true, - tempPrefix: `${path.basename(memoryPath)}.promotion`, - syncTempFile: true, - syncParentDir: true, - throwOnCleanupError: true, - // Stage proof prevents a future post-rename permission error from entering fallback. - fileSystem: { - promises: { - mkdir: fs.mkdir, - chmod: fs.chmod, - writeFile: fs.writeFile, - rename: trackedRename, - copyFile: fs.copyFile, - unlink: fs.unlink, - rm: fs.rm, - open: fs.open, - stat: fs.stat, - lstat: fs.lstat, - }, - }, - }); - } catch (error) { - // Released promotion writes could update an existing writable MEMORY.md even when - // directory ACLs blocked rename. Retain that in-place contract only after a real - // atomic permission failure and a successful writable-file open. - if ( - atomicRenameCommitted || - !isAtomicReplacePermissionError(error) || - !(await writeExistingMemoryInPlace(memoryWritePath, content)) - ) { - throw error; - } - } - } - - for (const candidate of rehydratedSelected) { - const entry = store.entries[candidate.key]; - if (!entry) { - continue; - } - entry.startLine = candidate.startLine; - entry.endLine = candidate.endLine; - entry.snippet = candidate.snippet; - entry.promotedAt = nowIso; - } - store.updatedAt = nowIso; - await writeStore(workspaceDir, store); - await appendMemoryHostEvent(workspaceDir, { - type: "memory.promotion.applied", - timestamp: nowIso, - memoryPath, - applied: rehydratedSelected.length, - candidates: rehydratedSelected.map((candidate) => ({ - key: candidate.key, - path: candidate.path, - startLine: candidate.startLine, - endLine: candidate.endLine, - score: candidate.score, - recallCount: candidate.recallCount, - })), - }); - - return { - memoryPath, - applied: rehydratedSelected.length, - appended: toAppend.length, - reconciledExisting: alreadyWritten.length, - appliedCandidates: rehydratedSelected, - compactedSections: compactedDates.length, - compactedDates, - }; - }); -} - -export function resolveShortTermRecallStorePath(workspaceDir: string): string { - return resolveStorePath(workspaceDir); -} - -export function resolveShortTermRecallLockPath(workspaceDir: string): string { - return resolveLockPath(workspaceDir); -} - -export async function auditShortTermPromotionArtifacts(params: { - workspaceDir: string; - qmd?: { - dbPath?: string; - collections?: number; - }; -}): Promise { - const workspaceDir = params.workspaceDir.trim(); - const storePath = resolveStorePath(workspaceDir); - const lockPath = resolveLockPath(workspaceDir); - const issues: ShortTermAuditIssue[] = []; - let entryCount = 0; - let promotedCount = 0; - let spacedEntryCount = 0; - let conceptTaggedEntryCount = 0; - let conceptTagScripts: ConceptTagScriptCoverage | undefined; - let invalidEntryCount = 0; - let updatedAt: string | undefined; - - const nowIso = new Date().toISOString(); - const rawEntries = await readMemoryCoreWorkspaceEntries({ - namespace: SHORT_TERM_RECALL_NAMESPACE, - workspaceDir, - }); - const exists = rawEntries.length > 0; - if (exists) { - const parsed = { - version: 1, - updatedAt: nowIso, - entries: Object.fromEntries(rawEntries.map((entry) => [entry.key, entry.value])), - }; - const store = normalizeShortTermRecallStore(parsed, nowIso); - const normalizedEntryCount = Object.keys(store.entries).length; - updatedAt = store.updatedAt; - entryCount = normalizedEntryCount; - promotedCount = Object.values(store.entries).filter((entry) => - Boolean(entry.promotedAt), - ).length; - spacedEntryCount = Object.values(store.entries).filter( - (entry) => (entry.recallDays?.length ?? 0) > 1, - ).length; - conceptTaggedEntryCount = Object.values(store.entries).filter( - (entry) => (entry.conceptTags?.length ?? 0) > 0, - ).length; - conceptTagScripts = summarizeConceptTagScriptCoverage( - Object.values(store.entries) - .filter((entry) => (entry.conceptTags?.length ?? 0) > 0) - .map((entry) => entry.conceptTags ?? []), - ); - invalidEntryCount = rawEntries.length - entryCount; - if (invalidEntryCount > 0) { - issues.push({ - severity: "warn", - code: "recall-store-invalid", - message: `Short-term recall store contains ${invalidEntryCount} invalid entr${invalidEntryCount === 1 ? "y" : "ies"}.`, - fixable: true, - }); - } - if (normalizedEntryCount > SHORT_TERM_RECALL_MAX_ENTRIES) { - issues.push({ - severity: "warn", - code: "recall-store-over-limit", - message: `Short-term recall store contains ${normalizedEntryCount} entries; only the newest ${SHORT_TERM_RECALL_MAX_ENTRIES} are kept at runtime.`, - fixable: true, - }); - } - } - - const lockKey = memoryCoreWorkspaceStateKey(workspaceDir); - const lockStore = openMemoryCoreStateStore({ - namespace: SHORT_TERM_LOCK_NAMESPACE, - maxEntries: SHORT_TERM_LOCK_MAX_ENTRIES, - }); - const lockEntry = await lockStore.lookup(lockKey); - if (lockEntry) { - const ageMs = Date.now() - lockEntry.acquiredAt; - const ownerPid = parseLockOwnerPid(lockEntry.owner); - if ( - ageMs > SHORT_TERM_LOCK_STALE_MS && - (ownerPid === null || !isProcessLikelyAlive(ownerPid)) - ) { - issues.push({ - severity: "warn", - code: "recall-lock-stale", - message: "Short-term promotion lock appears stale.", - fixable: true, - }); - } - } - - let qmd: ShortTermAuditSummary["qmd"]; - if (params.qmd) { - qmd = { - dbPath: params.qmd.dbPath, - collections: params.qmd.collections, - }; - if (typeof params.qmd.collections === "number" && params.qmd.collections <= 0) { - issues.push({ - severity: "warn", - code: "qmd-collections-empty", - message: "QMD reports zero managed collections.", - fixable: false, - }); - } - const dbPath = params.qmd.dbPath?.trim(); - if (dbPath) { - try { - const stat = await fs.stat(dbPath); - qmd.dbBytes = stat.size; - if (!stat.isFile() || stat.size <= 0) { - issues.push({ - severity: "error", - code: "qmd-index-empty", - message: "QMD index file exists but is empty.", - fixable: false, - }); - } - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code === "ENOENT") { - issues.push({ - severity: "error", - code: "qmd-index-missing", - message: "QMD index file is missing.", - fixable: false, - }); - } else { - throw err; - } - } - } - } - - return { - storePath, - lockPath, - updatedAt, - exists, - entryCount, - promotedCount, - spacedEntryCount, - conceptTaggedEntryCount, - ...(conceptTagScripts ? { conceptTagScripts } : {}), - invalidEntryCount, - issues, - ...(qmd ? { qmd } : {}), - }; -} - -export async function repairShortTermPromotionArtifacts(params: { - workspaceDir: string; -}): Promise { - const workspaceDir = params.workspaceDir.trim(); - const nowIso = new Date().toISOString(); - let rewroteStore = false; - let removedInvalidEntries = 0; - let removedOverflowEntries = 0; - let removedStaleLock = false; - - const lockKey = memoryCoreWorkspaceStateKey(workspaceDir); - const lockStore = openMemoryCoreStateStore({ - namespace: SHORT_TERM_LOCK_NAMESPACE, - maxEntries: SHORT_TERM_LOCK_MAX_ENTRIES, - }); - const lockEntry = await lockStore.lookup(lockKey); - if (lockEntry && Date.now() - lockEntry.acquiredAt > SHORT_TERM_LOCK_STALE_MS) { - const ownerPid = parseLockOwnerPid(lockEntry.owner); - if (ownerPid === null || !isProcessLikelyAlive(ownerPid)) { - removedStaleLock = await lockStore.delete(lockKey); - } - } - - await withShortTermLock(workspaceDir, async () => { - const rawEntries = await readMemoryCoreWorkspaceEntries({ - namespace: SHORT_TERM_RECALL_NAMESPACE, - workspaceDir, - }); - if (rawEntries.length > 0) { - const normalized = normalizeShortTermRecallStore( - { - version: 1, - updatedAt: nowIso, - entries: Object.fromEntries(rawEntries.map((entry) => [entry.key, entry.value])), - }, - nowIso, - ); - removedInvalidEntries = Math.max( - 0, - rawEntries.length - Object.keys(normalized.entries).length, - ); - const nextEntries = Object.fromEntries( - Object.entries(normalized.entries).map(([key, entry]) => { - const conceptTags = deriveConceptTags({ path: entry.path, snippet: entry.snippet }); - const fallbackDay = normalizeIsoDay(entry.lastRecalledAt) ?? nowIso.slice(0, 10); - return [ - key, - { - ...entry, - dailyCount: Math.max( - 0, - Math.floor((entry as { dailyCount?: number }).dailyCount ?? 0), - ), - groundedCount: Math.max( - 0, - Math.floor((entry as { groundedCount?: number }).groundedCount ?? 0), - ), - queryHashes: (entry.queryHashes ?? []).slice(-MAX_QUERY_HASHES), - recallDays: mergeRecentDistinct(entry.recallDays ?? [], fallbackDay, MAX_RECALL_DAYS), - conceptTags: conceptTags.length > 0 ? conceptTags : (entry.conceptTags ?? []), - } satisfies ShortTermRecallEntry, - ]; - }), - ); - const comparableStore: ShortTermRecallStore = { - version: 1, - updatedAt: normalized.updatedAt, - entries: nextEntries, - }; - removedOverflowEntries = enforceShortTermRecallStoreRetention(comparableStore); - const needsRewrite = - removedInvalidEntries > 0 || - removedOverflowEntries > 0 || - JSON.stringify(normalized.entries) !== JSON.stringify(comparableStore.entries); - if (needsRewrite) { - await writeStore(workspaceDir, { - ...comparableStore, - updatedAt: nowIso, - }); - rewroteStore = true; - } - } - }); - - return { - changed: rewroteStore || removedStaleLock, - removedInvalidEntries, - removedOverflowEntries, - rewroteStore, - removedStaleLock, - }; -} - -export async function removeGroundedShortTermCandidates(params: { - workspaceDir: string; -}): Promise<{ removed: number; storePath: string }> { - const workspaceDir = params.workspaceDir.trim(); - const storePath = resolveStorePath(workspaceDir); - const nowIso = new Date().toISOString(); - let removed = 0; - - await withShortTermLock(workspaceDir, async () => { - const [store, phaseSignals] = await Promise.all([ - readStore(workspaceDir, nowIso), - readPhaseSignalStore(workspaceDir, nowIso), - ]); - - for (const [key, entry] of Object.entries(store.entries)) { - if ( - Math.max(0, Math.floor(entry.groundedCount ?? 0)) > 0 && - Math.max(0, Math.floor(entry.recallCount ?? 0)) === 0 && - Math.max(0, Math.floor(entry.dailyCount ?? 0)) === 0 - ) { - delete store.entries[key]; - removed += 1; - } - } - - for (const key of Object.keys(phaseSignals.entries)) { - if (!Object.hasOwn(store.entries, key)) { - delete phaseSignals.entries[key]; - } - } - - if (removed > 0) { - store.updatedAt = nowIso; - phaseSignals.updatedAt = nowIso; - await Promise.all([ - writeStore(workspaceDir, store), - writePhaseSignalStore(workspaceDir, phaseSignals), - ]); - } - }); - - return { removed, storePath }; -} -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ +export { + DEFAULT_PROMOTION_MIN_RECALL_COUNT, + DEFAULT_PROMOTION_MIN_SCORE, + DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH, + SHORT_TERM_STORE_RELATIVE_PATH, + type PromotionCandidate, + type RepairShortTermPromotionArtifactsResult, + type ShortTermAuditSummary, + type ShortTermDreamingStats, + type ShortTermDreamingStatsEntry, + type ShortTermRecallEntry, +} from "./short-term-promotion-types.js"; +export { normalizeShortTermPhaseSignalStore } from "./short-term-promotion-store.js"; +export { normalizeShortTermRecallStore } from "./short-term-promotion-utils.js"; +export { + filterFreshLightDreamingEntries, + loadShortTermPromotionDreamingStats, + readLightStagedKeys, + recordDreamingPhaseSignals, + recordRemConsideredPhaseSignals, +} from "./short-term-promotion-stats.js"; +export { + filterLiveShortTermRecallEntries, + readShortTermRecallEntries, + recordGroundedShortTermCandidates, + recordShortTermRecalls, +} from "./short-term-promotion-record.js"; +export { applyShortTermPromotions } from "./short-term-promotion-apply.js"; +export { + auditShortTermPromotionArtifacts, + removeGroundedShortTermCandidates, + repairShortTermPromotionArtifacts, + resolveShortTermRecallLockPath, + resolveShortTermRecallStorePath, +} from "./short-term-promotion-artifacts.js";