fix(memory): preserve session recall when upgrading from QMD (#130016)

* fix(memory): preserve QMD session search across builtin migration

* fix(memory): migrate agent defaults to canonical memory owner
This commit is contained in:
Peter Steinberger
2026-08-26 03:18:42 -07:00
committed by GitHub
parent 01ac8db8ce
commit 49d916e8e9
9 changed files with 396 additions and 38 deletions
+6 -1
View File
@@ -126,7 +126,12 @@ Doctor removes the retired `memory.backend`, `memory.qmd`, and
`memory.search.qmd` settings, including agent-scoped `memory.search.qmd`
forms. It preserves QMD paths and extra collections as the corresponding
`memory.search.extraPaths` entries, including `{ path, pattern }` globs. When
Memory Core finds a retired per-agent QMD workspace under
QMD session indexing was enabled, Doctor also enables builtin session indexing
and adds `sessions` to `memory.search.sources` without enabling broader
cross-conversation recall. Retained session-reset transcripts remain in the
agent's sessions directory and are indexed from those original artifacts.
When Memory Core finds a retired per-agent QMD workspace under
`~/.openclaw/agents/<agentId>/qmd/`, Doctor also offers to remove its derived
indexes, model downloads, collection metadata, and session exports.
+4
View File
@@ -501,6 +501,10 @@ allows it).
separate runtime-only authorization limited to same-agent private
transcripts during the bounded Active Memory pass.
An explicit `memory_search` request for the `sessions` corpus requires session
search to be enabled for that agent. If it is unavailable, OpenClaw explains
how to enable session indexing instead of silently searching memory files.
The examples below place these settings under top-level `memory.search`. You can also
apply equivalent settings in a per-agent `memory.search` override when only one
agent should index and search session transcripts.
@@ -4,6 +4,7 @@ import os from "node:os";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { buildSessionEntry } from "openclaw/plugin-sdk/memory-core-host-engine-sessions";
import {
ensureMemoryIndexSchema,
loadSqliteVecExtension,
@@ -2577,6 +2578,13 @@ describe("memory-core doctor dreaming migration", () => {
"agent",
"openclaw-agent.sqlite",
);
const retainedResetTranscript = path.join(
stateDir,
"agents",
"main",
"sessions",
"session-1.jsonl.reset.2026-08-23T07-10-59.000Z",
);
const invalidAgentQmdHome = path.join(stateDir, "agents", "main!", "qmd");
const externalModels = path.join(rootDir, "shared-qmd-models");
const symlinkHomeTarget = path.join(rootDir, "symlink-qmd-home-target");
@@ -2586,6 +2594,7 @@ describe("memory-core doctor dreaming migration", () => {
path.join(qmdHome, "xdg-config", "qmd", "index.yml"),
path.join(qmdHome, "sessions", "session.md"),
canonicalAgentFile,
retainedResetTranscript,
path.join(invalidAgentQmdHome, "index.sqlite"),
path.join(externalModels, "model.bin"),
path.join(symlinkHomeTarget, "index.sqlite"),
@@ -2593,6 +2602,14 @@ describe("memory-core doctor dreaming migration", () => {
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, "derived", "utf8");
}
await fs.writeFile(
retainedResetTranscript,
JSON.stringify({
type: "message",
message: { role: "user", content: "Retained reset transcript recall fact" },
}),
"utf8",
);
await fs.symlink(externalModels, path.join(qmdHome, "xdg-cache", "qmd", "models"));
await fs.mkdir(path.dirname(symlinkHome), { recursive: true });
await fs.symlink(symlinkHomeTarget, symlinkHome);
@@ -2611,6 +2628,12 @@ describe("memory-core doctor dreaming migration", () => {
await expect(fs.access(qmdHome)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.access(canonicalAgentFile)).resolves.toBeUndefined();
await expect(fs.readFile(retainedResetTranscript, "utf8")).resolves.toContain(
"Retained reset transcript recall fact",
);
expect((await buildSessionEntry(retainedResetTranscript))?.content).toBe(
"User: Retained reset transcript recall fact",
);
await expect(fs.access(invalidAgentQmdHome)).resolves.toBeUndefined();
await expect(fs.access(path.join(externalModels, "model.bin"))).resolves.toBeUndefined();
expect((await fs.lstat(symlinkHome)).isSymbolicLink()).toBe(true);
+116 -29
View File
@@ -871,43 +871,73 @@ describe("memory_search corpus labels", () => {
expect(details.results[2]?.score).toBeCloseTo(0.765);
});
it.each(["sessions", "all"] as const)(
"does not let ordinary corpus=%s broaden implicitly indexed recall transcripts",
async (corpus) => {
let seenSources: readonly string[] | undefined;
setMemorySearchImpl(async (opts) => {
seenSources = opts?.sources;
return [
{
path: "sessions/private-group.jsonl",
startLine: 1,
endLine: 2,
score: 0.95,
snippet: "private transcript",
source: "sessions" as const,
},
];
});
it("does not let corpus=all broaden implicitly indexed recall transcripts", async () => {
let seenSources: readonly string[] | undefined;
setMemorySearchImpl(async (opts) => {
seenSources = opts?.sources;
return [
{
path: "sessions/private-group.jsonl",
startLine: 1,
endLine: 2,
score: 0.95,
snippet: "private transcript",
source: "sessions" as const,
},
];
});
const tool = createMemorySearchToolOrThrow({
config: {
agents: {
defaults: {},
list: [{ id: "main", default: true }],
},
memory: {
citations: "off",
search: { rememberAcrossConversations: true },
},
tools: { sessions: { visibility: "all" } },
},
agentSessionKey: "agent:main:main",
});
const result = await tool.execute("ordinary-search", {
query: "favorite food",
corpus: "all",
});
const details = result.details as { results: Array<{ source: string }> };
expect(seenSources).toEqual(["memory"]);
expect(details.results).toEqual([]);
});
it.each([
{ name: "recall-only session indexing", rememberAcrossConversations: true },
{ name: "disabled session indexing", rememberAcrossConversations: false },
])(
"reports unavailable for corpus=sessions with $name instead of searching memory files",
async ({ rememberAcrossConversations }) => {
const tool = createMemorySearchToolOrThrow({
config: {
agents: {
defaults: {},
list: [{ id: "main", default: true }],
},
memory: {
citations: "off",
search: { rememberAcrossConversations: true },
},
agents: { list: [{ id: "main", default: true }] },
memory: { search: { rememberAcrossConversations } },
tools: { sessions: { visibility: "all" } },
},
agentSessionKey: "agent:main:main",
});
const result = await tool.execute("ordinary-search", { query: "favorite food", corpus });
const details = result.details as { results: Array<{ source: string }> };
const result = await tool.execute("sessions-unavailable", {
query: "favorite food",
corpus: "sessions",
});
expect(seenSources).toEqual(["memory"]);
expect(details.results).toEqual([]);
expectUnavailableMemorySearchDetails(result.details, {
error: "Session transcript search is not enabled.",
warning: "Session transcript search is unavailable for this agent.",
action:
'Enable memory.search.experimental.sessionMemory and add "sessions" to memory.search.sources, then retry memory_search.',
});
expect(getMemorySearchManagerMockCalls()).toBe(0);
},
);
@@ -943,6 +973,63 @@ describe("memory_search corpus labels", () => {
},
);
it.each([
{ visibility: "agent" as const, visible: true },
{ visibility: "self" as const, visible: false },
])(
"keeps migrated isolated-DM reset recall within visibility=$visibility",
async ({ visibility, visible }) => {
let seenSources: readonly string[] | undefined;
setMemorySearchImpl(async (opts) => {
seenSources = opts?.sources;
return [
{
path: "sessions/main/past-thread.jsonl.reset.2026-08-23T07-10-59.000Z",
startLine: 1,
endLine: 2,
score: 0.9,
snippet: "Retained pre-reset conversation fact",
source: "sessions" as const,
},
];
});
const tool = createMemorySearchToolOrThrow({
config: {
agents: { list: [{ id: "main", default: true }] },
session: { dmScope: "per-channel-peer" },
memory: {
citations: "off",
search: {
rememberAcrossConversations: false,
experimental: { sessionMemory: true },
sources: ["memory", "sessions"],
},
},
tools: { sessions: { visibility } },
},
agentSessionKey: "agent:main:main",
});
const result = await tool.execute("isolated-session-search", {
query: "pre-reset conversation",
corpus: "sessions",
});
const details = result.details as { results: Array<{ corpus: string; snippet: string }> };
expect(seenSources).toEqual(["sessions"]);
expect(details.results).toEqual(
visible
? [
expect.objectContaining({
corpus: "sessions",
snippet: "Retained pre-reset conversation fact",
}),
]
: [],
);
},
);
it("forces trusted conversation recall onto its authorized transcript corpus", async () => {
let seenSources: readonly string[] | undefined;
setMemorySearchImpl(async (opts) => {
+13
View File
@@ -292,6 +292,19 @@ export function createMemorySearchTool(options: MemoryToolOptions) {
// The trusted runtime chooses the recall corpus; model-authored arguments cannot broaden it.
const requestedCorpus =
options.conversationRecall?.corpus === "sessions" ? "sessions" : modelRequestedCorpus;
if (
requestedCorpus === "sessions" &&
!options.conversationRecall &&
!resolveMemorySearchConfig(cfg, agentId)?.searchSources.includes("sessions")
) {
return jsonResult(
buildMemorySearchUnavailableResult("Session transcript search is not enabled.", {
warning: "Session transcript search is unavailable for this agent.",
action:
'Enable memory.search.experimental.sessionMemory and add "sessions" to memory.search.sources, then retry memory_search.',
}),
);
}
const cooldownKey = resolveMemorySearchToolCooldownKey({
agentId,
agentSessionKey: options.agentSessionKey,
@@ -307,6 +307,17 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => {
message: { role: "user", content: "Archived JSONL transcript text" },
}),
);
const resetArchivePath = path.join(
sessionsDir,
`${sessionId}.jsonl.reset.2026-06-25T12-02-00.000Z`,
);
fsSync.writeFileSync(
resetArchivePath,
JSON.stringify({
type: "message",
message: { role: "user", content: "Retained pre-reset conversation fact" },
}),
);
expect(fsSync.existsSync(path.join(sessionsDir, `${sessionId}.jsonl`))).toBe(false);
const entries = await listSessionTranscriptCorpusEntriesForAgent("main");
@@ -330,6 +341,10 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => {
sessionFile: archivePath,
sessionId,
}),
expect.objectContaining({
artifactKind: "archive-artifact",
sessionFile: resetArchivePath,
}),
]),
);
@@ -350,6 +365,7 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => {
updatedAtMs: updatedAt,
});
const archiveEntry = requireSessionEntry(await buildSessionEntry(archivePath));
const resetArchiveEntry = requireSessionEntry(await buildSessionEntry(resetArchivePath));
expect(liveEntry.path).toBe("sessions/main/sqlite-live.jsonl");
expect(liveEntry.content).toBe("User: Live SQLite transcript text");
@@ -363,6 +379,10 @@ describe("listSessionTranscriptCorpusEntriesForAgent", () => {
"sessions/main/sqlite-live.jsonl.deleted.2026-06-25T12-01-00.000Z",
);
expect(archiveEntry.content).toBe("User: Archived JSONL transcript text");
expect(resetArchiveEntry.path).toBe(
"sessions/main/sqlite-live.jsonl.reset.2026-06-25T12-02-00.000Z",
);
expect(resetArchiveEntry.content).toBe("User: Retained pre-reset conversation fact");
});
it("exposes content revisions that change with SQLite appends and file replacement", async () => {
@@ -1,4 +1,5 @@
import { describe, expect, it } from "vitest";
import { resolveMemorySearchConfig } from "../../../agents/memory-search.js";
import { validateConfigObjectRaw } from "../../../config/validation.js";
import { applyLegacyDoctorMigrations } from "./legacy-config-compat.js";
import { migrateLegacyConfig } from "./legacy-config-migrate.js";
@@ -42,6 +43,80 @@ describe("legacy config migration end to end", () => {
expect(validateConfigObjectRaw({ agents: { defaults: { tts: {} } } }).ok).toBe(false);
});
it.each([
{
name: "defaults-only QMD session indexing",
canonical: undefined,
defaults: {
provider: "none",
rememberAcrossConversations: false,
extraPaths: ["/defaults-existing"],
},
expectedSources: ["memory", "sessions"],
expectedPaths: ["/defaults-existing", "/defaults-qmd"],
},
{
name: "explicit canonical privacy and indexing policy",
canonical: {
provider: "none",
rememberAcrossConversations: false,
experimental: { sessionMemory: false },
sources: ["memory"],
extraPaths: ["/canonical"],
},
defaults: {
provider: "openai",
rememberAcrossConversations: true,
experimental: { sessionMemory: true },
extraPaths: ["/defaults-existing"],
},
expectedSources: ["memory"],
expectedPaths: ["/canonical", "/defaults-qmd"],
},
])(
"migrates $name into validated effective memory settings",
({ canonical, defaults, expectedSources, expectedPaths }) => {
const result = migrateLegacyConfig({
...(canonical ? { memory: { search: canonical } } : {}),
session: { dmScope: "per-peer" },
agents: {
entries: { main: {} },
defaults: {
memory: {
search: {
...defaults,
qmd: {
sessions: { enabled: true },
extraCollections: [{ path: "/defaults-qmd" }],
},
},
},
},
},
});
expect(result.partiallyValid).toBeUndefined();
expect(result.config).not.toHaveProperty("agents.defaults.memory");
const validation = validateConfigObjectRaw(result.config);
expect(validation.ok, validation.ok ? undefined : JSON.stringify(validation.issues)).toBe(
true,
);
if (!validation.ok) {
return;
}
const resolved = resolveMemorySearchConfig(validation.config, "main");
expect(resolved).toMatchObject({
provider: "none",
rememberAcrossConversations: false,
sources: expectedSources,
searchSources: expectedSources,
extraPaths: expectedPaths,
});
expect(validation.config.memory?.search?.experimental?.sessionMemory).toBe(!canonical);
expect(migrateLegacyConfig(validation.config)).toEqual({ config: null, changes: [] });
},
);
it("canonicalizes a multi-family legacy config and is idempotent", () => {
const result = migrateLegacyConfig({
env: { shellEnv: { enabled: true }, API_ORIGIN: "https://example.test" },
@@ -2,6 +2,7 @@ import {
defineLegacyConfigMigration,
ensureRecord,
getRecord,
mergeMissing,
type LegacyConfigMigrationSpec,
type LegacyConfigRule,
} from "../../../config/legacy.shared.js";
@@ -91,11 +92,50 @@ function migrateRetiredQmdExternalPaths(params: {
}
}
function migrateRetiredQmdSessionIndexing(
qmd: Record<string, unknown> | null,
scope: Record<string, unknown>,
sourcePath: string,
changes: string[],
targetPath = sourcePath === "memory.qmd" ? "memory.search" : sourcePath.slice(0, -4),
): void {
if (getRecord(qmd?.sessions)?.enabled !== true) {
return;
}
const search = ensureRecord(ensureRecord(scope, "memory"), "search");
const experimental = getRecord(search.experimental);
let changed = false;
if (
(experimental || search.experimental === undefined) &&
experimental?.sessionMemory === undefined
) {
ensureRecord(search, "experimental").sessionMemory = true;
changed = true;
}
if (
search.sources === undefined ||
(Array.isArray(search.sources) && search.sources.length === 0)
) {
search.sources = ["memory", "sessions"];
changed = true;
} else if (Array.isArray(search.sources) && !search.sources.includes("sessions")) {
search.sources.push("sessions");
changed = true;
}
if (changed) {
changes.push(
`Migrated ${sourcePath}.sessions.enabled → ${targetPath}.experimental.sessionMemory and ${targetPath}.sources.`,
);
}
}
function migrateRetiredMemoryQmd(raw: Record<string, unknown>, changes: string[]): void {
const memory = getRecord(raw.memory);
const search = getRecord(memory?.search);
const qmd = getRecord(memory?.qmd);
const searchQmd = getRecord(search?.qmd);
migrateRetiredQmdSessionIndexing(qmd, raw, "memory.qmd", changes);
migrateRetiredQmdSessionIndexing(searchQmd, raw, "memory.search.qmd", changes);
migrateRetiredQmdExternalPaths({
changes,
entries: [
@@ -117,14 +157,32 @@ function migrateRetiredMemoryQmd(raw: Record<string, unknown>, changes: string[]
visitAgentConfigScopes(raw, (scope, scopePath) => {
const agentSearch = getRecord(getRecord(scope.memory)?.search);
const agentSearchQmd = getRecord(agentSearch?.qmd);
const isAgentDefaults = scopePath === "agents.defaults" && agentSearch?.qmd !== undefined;
const targetScope = isAgentDefaults ? raw : scope;
const targetPath = isAgentDefaults ? "memory.search" : `${scopePath}.memory.search`;
if (isAgentDefaults && agentSearch) {
// Agent defaults have no memory owner; global memory.search owns their policy.
removed = deleteRetiredPath(scope, ["memory", "search", "qmd"]) || removed;
mergeMissing(ensureRecord(ensureRecord(raw, "memory"), "search"), agentSearch);
delete scope.memory;
}
migrateRetiredQmdSessionIndexing(
agentSearchQmd,
targetScope,
`${scopePath}.memory.search.qmd`,
changes,
targetPath,
);
migrateRetiredQmdExternalPaths({
changes,
entries: readRetiredQmdExternalPaths(agentSearchQmd?.extraCollections),
scope,
scope: targetScope,
sourcePath: `${scopePath}.memory.search.qmd.extraCollections`,
targetPath: `${scopePath}.memory.search.extraPaths`,
targetPath: `${targetPath}.extraPaths`,
});
removed = deleteRetiredPath(scope, ["memory", "search", "qmd"]) || removed;
if (!isAgentDefaults) {
removed = deleteRetiredPath(scope, ["memory", "search", "qmd"]) || removed;
}
});
if (removed) {
changes.push(
@@ -152,7 +210,7 @@ export const LEGACY_CONFIG_MIGRATION_RUNTIME_MEMORY_QMD: LegacyConfigMigrationSp
),
rule(
["agents", "defaults", "memory", "search", "qmd"],
"agents.defaults.memory.search.qmd is retired because the QMD memory backend was removed; configured external collections migrate to agents.defaults.memory.search.extraPaths.",
"agents.defaults.memory.search.qmd is retired because the QMD memory backend was removed; configured external collections migrate to memory.search.extraPaths.",
),
rule(
["agents", "entries"],
@@ -94,11 +94,13 @@ describe("retired QMD memory config migration", () => {
expect(result.raw).not.toHaveProperty("memory.backend");
expect(result.raw).not.toHaveProperty("memory.qmd");
expect(result.raw).not.toHaveProperty("memory.search.qmd");
expect(result.raw).not.toHaveProperty("agents.defaults.memory.search.qmd");
expect(result.raw).not.toHaveProperty("agents.defaults.memory");
expect(result.raw).not.toHaveProperty("agents.entries.research.memory.search.qmd");
expect(result.raw).not.toHaveProperty("agents.list.0.memory.search.qmd");
expect(result.raw).toHaveProperty("memory.citations", "on");
expect(result.raw).toHaveProperty("memory.search.provider", "openai");
expect(result.raw).toHaveProperty("memory.search.experimental.sessionMemory", true);
expect(result.raw).toHaveProperty("memory.search.sources", ["memory", "sessions"]);
expect(result.raw).toHaveProperty("memory.search.extraPaths", [
"notes",
"/tmp/shared",
@@ -106,9 +108,6 @@ describe("retired QMD memory config migration", () => {
{ path: "/tmp/patterned", pattern: "notes/*.md" },
{ path: "/tmp/shared", pattern: "**/*.md" },
"/tmp/search",
]);
expect(result.raw).toHaveProperty("agents.defaults.memory.search.extraPaths", [
"notes",
"/tmp/defaults",
]);
expect(result.raw).toHaveProperty("agents.entries.research.memory.search.enabled", false);
@@ -120,9 +119,83 @@ describe("retired QMD memory config migration", () => {
expect(result.changes).toContain(
"Migrated 4 external QMD paths from memory.qmd.paths and memory.search.qmd.extraCollections → memory.search.extraPaths.",
);
expect(result.changes).toContain(
"Migrated memory.qmd.sessions.enabled → memory.search.experimental.sessionMemory and memory.search.sources.",
);
expect(result.changes).toContain(
"Removed retired QMD memory configuration; builtin memory is now the only memory engine.",
);
expect(applyRetired(result.raw).changes).toEqual([]);
});
it.each([
{
name: "global search",
raw: { memory: { search: { qmd: { sessions: { enabled: true } } } } },
target: "memory.search",
},
{
name: "agent defaults",
raw: {
agents: { defaults: { memory: { search: { qmd: { sessions: { enabled: true } } } } } },
},
target: "memory.search",
},
{
name: "named agent",
raw: {
agents: {
entries: {
research: { memory: { search: { qmd: { sessions: { enabled: true } } } } },
},
},
},
target: "agents.entries.research.memory.search",
},
{
name: "listed agent",
raw: {
agents: {
list: [{ id: "research", memory: { search: { qmd: { sessions: { enabled: true } } } } }],
},
},
target: "agents.list.0.memory.search",
},
])("preserves explicitly enabled QMD transcript indexing for $name", ({ raw, target }) => {
const result = applyRetired(raw);
expect(result.raw).toHaveProperty(`${target}.experimental.sessionMemory`, true);
expect(result.raw).toHaveProperty(`${target}.sources`, ["memory", "sessions"]);
expect(result.raw).not.toHaveProperty(`${target}.rememberAcrossConversations`);
expect(applyRetired(result.raw).changes).toEqual([]);
});
it.each([{ sources: [] }, { sources: ["memory"] }])(
"preserves explicit builtin privacy and indexing settings while merging sources=$sources",
({ sources }) => {
const result = applyRetired({
memory: {
qmd: { sessions: { enabled: true } },
search: {
rememberAcrossConversations: false,
experimental: { sessionMemory: false },
sources,
},
},
});
expect(result.raw).toHaveProperty("memory.search.rememberAcrossConversations", false);
expect(result.raw).toHaveProperty("memory.search.experimental.sessionMemory", false);
expect(result.raw).toHaveProperty("memory.search.sources", ["memory", "sessions"]);
},
);
it.each([false, undefined])("does not enable builtin sessions for QMD enabled=%s", (enabled) => {
const result = applyRetired({
memory: { qmd: { sessions: enabled === undefined ? {} : { enabled } } },
});
expect(result.raw).not.toHaveProperty("memory.search.experimental.sessionMemory");
expect(result.raw).not.toHaveProperty("memory.search.sources");
});
});