From b2cc5f50420b5d273f71a01a4474b2f23038a2bb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Jul 2026 03:51:59 -0400 Subject: [PATCH] fix(memory): promote durable facts after repeated recall (#115715) * fix(memory): calibrate promotion gate defaults * docs: refresh generated docs map --- docs/.generated/config-baseline.sha256 | 4 +- docs/cli/memory.md | 16 +- extensions/memory-core/openclaw.plugin.json | 9 +- extensions/memory-core/src/config.test.ts | 22 ++ .../src/short-term-promotion-scoring.test.ts | 239 ++++++++++++++++++ .../src/short-term-promotion-types.ts | 11 +- .../src/short-term-promotion.test.ts | 14 +- src/memory-host-sdk/dreaming.test.ts | 2 +- src/memory-host-sdk/dreaming.ts | 4 +- 9 files changed, 300 insertions(+), 21 deletions(-) create mode 100644 extensions/memory-core/src/short-term-promotion-scoring.test.ts diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 0a5b358ba74f..008a9afbd82d 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -1702c4a729d952844880aa6f45dfbab1ff4a8408bc8338d5c41354a043869f7f config-baseline.json +4dc7458ef793bc3c483e57be2af7e5118f3de824483a4b8e6ef2afcdf7a38e0a config-baseline.json 82dbf6abbd417071aa545c3d398afbab297591b117627ff5766c1fad689b8f40 config-baseline.core.json b3d7fdc2662b49b2e6567573852545a36a8eab13bb2111011510aa96a80a3b8b config-baseline.channel.json -c7c25cc8e151d9db96fb6497e9f2e69870b4106f88e6c0f6cd481100217a556d config-baseline.plugin.json +99e8401cbeff42d79d84e08cdadb5b1b285e2b6734e41ab77ed2b42438aea991 config-baseline.plugin.json diff --git a/docs/cli/memory.md b/docs/cli/memory.md index 3848eb14f0cb..bf888a1fe3fc 100644 --- a/docs/cli/memory.md +++ b/docs/cli/memory.md @@ -79,14 +79,13 @@ openclaw memory promote [--agent ] [--limit ] [--min-score ] \ | `--limit ` | | Max candidates to return/apply. | | `--min-score ` | `0.75` | Minimum weighted promotion score. | | `--min-recall-count ` | `3` | Minimum recall count required. | -| `--min-unique-queries ` | `2` | Minimum distinct query count required. | +| `--min-unique-queries ` | `3` | Minimum distinct query count required. | | `--apply` | preview only | Append selected candidates to `MEMORY.md` and mark them promoted. | | `--include-promoted` | | Include candidates already promoted in previous cycles. | | `--json` | | Print JSON. | -These CLI defaults differ from the scheduled dreaming sweep's deep-phase -thresholds (see [Dreaming](#dreaming) below); pass explicit flags to match -sweep behavior for a one-off manual run. +The CLI and scheduled dreaming sweep share the deep-phase defaults below. +Explicit CLI flags override them for a one-off manual run. Ranking signals: recall frequency, retrieval relevance, query diversity, temporal recency, cross-day consolidation, and derived concept richness, drawn @@ -191,7 +190,7 @@ material), **REM** (reflect and surface themes), **deep** (promote durable facts into `MEMORY.md`). Only deep writes to `MEMORY.md`. - Enable with `plugins.entries.memory-core.config.dreaming.enabled: true` - (default `false`); `memory-core` auto-manages the sweep cron job, no manual + (default `true`); `memory-core` auto-manages the sweep cron job, no manual `openclaw cron add` required. - Toggle from chat with `/dreaming on|off`; inspect with `/dreaming status` (or `/dreaming`/`/dreaming help`). `on`/`off` requires channel owner status @@ -202,9 +201,8 @@ facts into `MEMORY.md`). Only deep writes to `MEMORY.md`. standalone report to `memory/dreaming//YYYY-MM-DD.md`; set `mode: "inline"` to fold reports into the daily memory file instead, or `"both"` for both. -- Scheduled and manual `memory promote` runs share the same deep-phase - ranking signals; only the default thresholds differ (see table above vs. - scheduled defaults below). +- Scheduled and manual `memory promote` runs share the same deep-phase ranking + signals and default thresholds; explicit CLI flags remain one-run overrides. - Scheduled runs fan out across every configured agent's memory workspace. Scheduled defaults (`plugins.entries.memory-core.config.dreaming`): @@ -212,7 +210,7 @@ Scheduled defaults (`plugins.entries.memory-core.config.dreaming`): | Key | Default | | -------------------------------------- | ----------- | | `frequency` | `0 3 * * *` | -| `phases.deep.minScore` | `0.8` | +| `phases.deep.minScore` | `0.75` | | `phases.deep.minRecallCount` | `3` | | `phases.deep.minUniqueQueries` | `3` | | `phases.deep.recencyHalfLifeDays` | `14` | diff --git a/extensions/memory-core/openclaw.plugin.json b/extensions/memory-core/openclaw.plugin.json index 6b772208a65e..a5e033f97e70 100644 --- a/extensions/memory-core/openclaw.plugin.json +++ b/extensions/memory-core/openclaw.plugin.json @@ -140,15 +140,18 @@ "minScore": { "type": "number", "minimum": 0, - "maximum": 1 + "maximum": 1, + "default": 0.75 }, "minRecallCount": { "type": "integer", - "minimum": 0 + "minimum": 0, + "default": 3 }, "minUniqueQueries": { "type": "integer", - "minimum": 0 + "minimum": 0, + "default": 3 }, "recencyHalfLifeDays": { "type": "integer", diff --git a/extensions/memory-core/src/config.test.ts b/extensions/memory-core/src/config.test.ts index c37775e84a33..d9a01f7459a4 100644 --- a/extensions/memory-core/src/config.test.ts +++ b/extensions/memory-core/src/config.test.ts @@ -11,6 +11,28 @@ const manifest = JSON.parse( ) as { configSchema: JsonSchemaObject }; describe("memory-core manifest config schema", () => { + it("publishes the canonical promotion gate defaults", () => { + expect(manifest.configSchema).toMatchObject({ + properties: { + dreaming: { + properties: { + phases: { + properties: { + deep: { + properties: { + minScore: { default: 0.75 }, + minRecallCount: { default: 3 }, + minUniqueQueries: { default: 3 }, + }, + }, + }, + }, + }, + }, + }, + }); + }); + it("accepts dreaming phase thresholds used by QA and runtime", () => { const result = validateJsonSchemaValue({ schema: manifest.configSchema, diff --git a/extensions/memory-core/src/short-term-promotion-scoring.test.ts b/extensions/memory-core/src/short-term-promotion-scoring.test.ts new file mode 100644 index 000000000000..3d2f86d7b55b --- /dev/null +++ b/extensions/memory-core/src/short-term-promotion-scoring.test.ts @@ -0,0 +1,239 @@ +// Memory Core tests calibrate deterministic short-term promotion scoring. +import { + DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT, + DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE, + DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES, +} from "openclaw/plugin-sdk/memory-core-host-status"; +import { describe, expect, it } from "vitest"; +import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js"; +import type { PromotionWeights } from "./short-term-promotion-types.js"; +import { + DEFAULT_PROMOTION_MIN_RECALL_COUNT, + DEFAULT_PROMOTION_MIN_SCORE, + DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + rankShortTermPromotionCandidates, + type ShortTermRecallEntry, +} from "./short-term-promotion.js"; +import { createMemoryCoreTestHarness, shortTermTestState } from "./test-helpers.js"; + +const NOW_ISO = "2026-04-03T10:01:00.000Z"; +const NOW_MS = Date.parse(NOW_ISO); +const RECALL_DAYS = ["2026-04-01", "2026-04-02", "2026-04-03"]; +const THREE_QUERY_HASHES = ["query-a", "query-b", "query-c"]; + +const { createTempWorkspace } = createMemoryCoreTestHarness(); + +type CalibrationClass = "genuine" | "filler" | "oneoff"; + +function createRecallEntry(params: { + key: string; + snippet?: string; + signalCount: number; + avgScore: number; + queryHashes: string[]; + recallDays: string[]; + conceptTags: string[]; +}): ShortTermRecallEntry { + return { + key: params.key, + path: "memory/2026-04-01.md", + startLine: 1, + endLine: 1, + source: "memory", + snippet: params.snippet ?? `${params.key} durable note`, + recallCount: 0, + dailyCount: params.signalCount, + groundedCount: 0, + totalScore: params.avgScore * params.signalCount, + maxScore: params.avgScore, + firstRecalledAt: "2026-04-01T10:00:00.000Z", + lastRecalledAt: NOW_ISO, + queryHashes: params.queryHashes, + recallDays: params.recallDays, + conceptTags: params.conceptTags, + }; +} + +async function writeCalibrationStore(workspaceDir: string): Promise { + const entries = [ + createRecallEntry({ + key: "genuine-a", + signalCount: 3, + avgScore: 0.58, + queryHashes: THREE_QUERY_HASHES, + recallDays: RECALL_DAYS, + conceptTags: ["backup", "backups", "glacier", "s3"], + }), + createRecallEntry({ + key: "genuine-b", + signalCount: 3, + avgScore: 0.6, + queryHashes: THREE_QUERY_HASHES, + recallDays: RECALL_DAYS, + conceptTags: ["backup", "backups", "glacier", "s3"], + }), + createRecallEntry({ + key: "genuine-c", + signalCount: 3, + avgScore: 0.62, + queryHashes: THREE_QUERY_HASHES, + recallDays: RECALL_DAYS, + conceptTags: ["backup", "glacier", "s3"], + }), + ...[0.2, 0.3, 0.4].map((avgScore, index) => + createRecallEntry({ + key: `filler-${index}`, + snippet: "Routine heartbeat completed successfully.", + signalCount: 3, + avgScore, + queryHashes: THREE_QUERY_HASHES, + recallDays: RECALL_DAYS, + conceptTags: [], + }), + ), + ...[0.8, 0.9, 0.99].map((avgScore, index) => + createRecallEntry({ + key: `oneoff-${index}`, + signalCount: 1, + avgScore, + queryHashes: ["query-a"], + recallDays: ["2026-04-03"], + conceptTags: ["novel", "signal", "single", "mention"].slice(0, index + 2), + }), + ), + ]; + await shortTermTestState.writeRawRecallStore(workspaceDir, { + version: 1, + updatedAt: NOW_ISO, + entries: Object.fromEntries(entries.map((entry) => [entry.key, entry])), + }); + await shortTermTestState.writeRawPhaseSignalStore(workspaceDir, { + version: 1, + updatedAt: NOW_ISO, + entries: Object.fromEntries( + entries + .filter((entry) => entry.key.startsWith("genuine-")) + .map((entry) => [ + entry.key, + { + key: entry.key, + lightHits: 3, + remHits: 3, + lastLightAt: NOW_ISO, + lastRemAt: NOW_ISO, + }, + ]), + ), + }); +} + +function scoresForClass( + candidates: Awaited>, + calibrationClass: CalibrationClass, +): number[] { + return candidates + .filter((candidate) => candidate.key.startsWith(`${calibrationClass}-`)) + .toSorted((left, right) => left.key.localeCompare(right.key)) + .map((candidate) => Number(candidate.score.toFixed(6))); +} + +describe("short-term promotion score calibration", () => { + it("separates repeated durable facts from filler and high-signal one-offs", async () => { + const workspaceDir = await createTempWorkspace("promotion-score-distribution-"); + await writeCalibrationStore(workspaceDir); + + const measured = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + nowMs: NOW_MS, + }); + const distribution = { + genuine: scoresForClass(measured, "genuine"), + filler: scoresForClass(measured, "filler"), + oneoff: scoresForClass(measured, "oneoff"), + }; + + expect(distribution).toEqual({ + genuine: [0.750014, 0.756014, 0.752014], + filler: [0.489152, 0.519152, 0.549152], + oneoff: [0.529376, 0.569376, 0.606376], + }); + + const promoted = await rankShortTermPromotionCandidates({ workspaceDir, nowMs: NOW_MS }); + expect(promoted.map((candidate) => candidate.key).toSorted()).toEqual([ + "genuine-a", + "genuine-b", + "genuine-c", + ]); + }); + + it("keeps sweep and direct promotion fallback defaults aligned", () => { + const sweep = resolveShortTermPromotionDreamingConfig({ pluginConfig: {} }); + expect({ + minScore: sweep.minScore, + minRecallCount: sweep.minRecallCount, + minUniqueQueries: sweep.minUniqueQueries, + }).toEqual({ + minScore: DEFAULT_PROMOTION_MIN_SCORE, + minRecallCount: DEFAULT_PROMOTION_MIN_RECALL_COUNT, + minUniqueQueries: DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + }); + expect({ + minScore: DEFAULT_PROMOTION_MIN_SCORE, + minRecallCount: DEFAULT_PROMOTION_MIN_RECALL_COUNT, + minUniqueQueries: DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES, + }).toEqual({ + minScore: DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE, + minRecallCount: DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT, + minUniqueQueries: DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES, + }); + }); + + it("keeps the minimum score boundary inclusive", async () => { + const workspaceDir = await createTempWorkspace("promotion-score-boundary-"); + const boundary = createRecallEntry({ + key: "boundary", + signalCount: 1, + avgScore: DEFAULT_PROMOTION_MIN_SCORE, + queryHashes: ["query-a"], + recallDays: ["2026-04-03"], + conceptTags: [], + }); + await shortTermTestState.writeRawRecallStore(workspaceDir, { + version: 1, + updatedAt: NOW_ISO, + entries: { boundary }, + }); + const relevanceOnly: PromotionWeights = { + frequency: 0, + relevance: 1, + diversity: 0, + recency: 0, + consolidation: 0, + conceptual: 0, + }; + + await expect( + rankShortTermPromotionCandidates({ + workspaceDir, + minScore: DEFAULT_PROMOTION_MIN_SCORE, + minRecallCount: 0, + minUniqueQueries: 0, + weights: relevanceOnly, + nowMs: NOW_MS, + }), + ).resolves.toHaveLength(1); + await expect( + rankShortTermPromotionCandidates({ + workspaceDir, + minScore: DEFAULT_PROMOTION_MIN_SCORE + 0.000001, + minRecallCount: 0, + minUniqueQueries: 0, + weights: relevanceOnly, + nowMs: NOW_MS, + }), + ).resolves.toHaveLength(0); + }); +}); diff --git a/extensions/memory-core/src/short-term-promotion-types.ts b/extensions/memory-core/src/short-term-promotion-types.ts index cc17a091652a..116378c6507e 100644 --- a/extensions/memory-core/src/short-term-promotion-types.ts +++ b/extensions/memory-core/src/short-term-promotion-types.ts @@ -1,10 +1,15 @@ import path from "node:path"; import type { MemoryEntryProvenance } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; +import { + DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT, + DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE, + DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES, +} from "openclaw/plugin-sdk/memory-core-host-status"; import type { ConceptTagScriptCoverage } from "./concept-vocabulary.js"; -export const DEFAULT_PROMOTION_MIN_SCORE = 0.75; -export const DEFAULT_PROMOTION_MIN_RECALL_COUNT = 3; -export const DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES = 2; +export const DEFAULT_PROMOTION_MIN_SCORE = DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE; +export const DEFAULT_PROMOTION_MIN_RECALL_COUNT = DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT; +export const DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES = DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES; export const SHORT_TERM_STORE_RELATIVE_PATH = path.join( "memory", ".dreams", diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index f61d844a1bbf..f58654f6acbc 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -999,7 +999,7 @@ describe("short-term promotion", () => { snippet: 'Always use "Happy Together" calendar for flights and reservations.', score: 0.92, query: "__dreaming_grounded_backfill__:lasting-update", - signalCount: 2, + signalCount: 1, dayBucket: "2026-04-03", }, { @@ -1012,6 +1012,16 @@ describe("short-term promotion", () => { signalCount: 1, dayBucket: "2026-04-03", }, + { + path: "memory/2026-04-03.md", + startLine: 1, + endLine: 1, + snippet: 'Always use "Happy Together" calendar for flights and reservations.', + score: 0.86, + query: "__dreaming_grounded_backfill__:durable-fact", + signalCount: 1, + dayBucket: "2026-04-03", + }, ], dedupeByQueryPerDay: true, nowMs: Date.parse("2026-04-03T10:00:00.000Z"), @@ -1024,7 +1034,7 @@ describe("short-term promotion", () => { expect(ranked).toHaveLength(1); expect(ranked[0]?.groundedCount).toBe(3); - expect(ranked[0]?.uniqueQueries).toBe(2); + expect(ranked[0]?.uniqueQueries).toBe(3); expect(ranked[0]?.avgScore).toBeGreaterThan(0.85); const applied = await applyShortTermPromotions({ diff --git a/src/memory-host-sdk/dreaming.test.ts b/src/memory-host-sdk/dreaming.test.ts index 26285a68838a..b2365fd027aa 100644 --- a/src/memory-host-sdk/dreaming.test.ts +++ b/src/memory-host-sdk/dreaming.test.ts @@ -162,7 +162,7 @@ describe("memory dreaming host helpers", () => { expect(resolved.timezone).toBe("America/Los_Angeles"); expect(resolved.phases.deep.cron).toBe("0 3 * * *"); expect(resolved.phases.deep.limit).toBe(10); - expect(resolved.phases.deep.minScore).toBe(0.8); + expect(resolved.phases.deep.minScore).toBe(0.75); expect(resolved.phases.deep.recencyHalfLifeDays).toBe(14); expect(resolved.phases.deep.maxAgeDays).toBe(30); }); diff --git a/src/memory-host-sdk/dreaming.ts b/src/memory-host-sdk/dreaming.ts index 61af154da710..f970825eb0f4 100644 --- a/src/memory-host-sdk/dreaming.ts +++ b/src/memory-host-sdk/dreaming.ts @@ -40,7 +40,9 @@ const DEFAULT_MEMORY_LIGHT_DREAMING_LOOKBACK_DAYS = 2; const DEFAULT_MEMORY_LIGHT_DREAMING_LIMIT = 100; const DEFAULT_MEMORY_LIGHT_DREAMING_DEDUPE_SIMILARITY = 0.9; export const DEFAULT_MEMORY_DEEP_DREAMING_LIMIT = 10; -export const DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE = 0.8; +// Deterministic calibration scores 3-day/3-query durable facts at 0.750-0.756, +// versus repeated filler at 0.489-0.549 and high-relevance one-offs at 0.529-0.606. +export const DEFAULT_MEMORY_DEEP_DREAMING_MIN_SCORE = 0.75; export const DEFAULT_MEMORY_DEEP_DREAMING_MIN_RECALL_COUNT = 3; export const DEFAULT_MEMORY_DEEP_DREAMING_MIN_UNIQUE_QUERIES = 3; export const DEFAULT_MEMORY_DEEP_DREAMING_RECENCY_HALF_LIFE_DAYS = 14;