mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-24 11:25:50 -06:00
Peter Steinberger
702eb6eb57
* fix(qa): prove session memory outranks stale durable facts * fix(qa): require the canonical memory search result window * fix(qa): honor the optional canonical memory search limit
268 lines
13 KiB
YAML
268 lines
13 KiB
YAML
title: Session memory ranking
|
|
|
|
scenario:
|
|
id: session-memory-ranking
|
|
surface: session-memory
|
|
coverage:
|
|
primary:
|
|
- session-memory.embedding-search-ranking
|
|
secondary:
|
|
- session-memory.embedding-search-recall
|
|
objective: Verify session-transcript memory can outrank stale durable notes and drive the final answer toward the newer fact.
|
|
successCriteria:
|
|
- Session memory indexing is enabled for the scenario.
|
|
- Search ranks the newer transcript-backed fact ahead of the stale durable note.
|
|
- The agent uses memory tools and answers with the current fact, not the stale one.
|
|
docsRefs:
|
|
- docs/concepts/memory-search.md
|
|
- docs/reference/memory-config.md
|
|
codeRefs:
|
|
- extensions/memory-core/src/tools.ts
|
|
- extensions/memory-core/src/memory/manager.ts
|
|
- extensions/qa-lab/src/suite.ts
|
|
execution:
|
|
kind: flow
|
|
summary: Verify session-transcript memory can outrank stale durable notes and drive the final answer toward the newer fact.
|
|
channel: qa-channel
|
|
config:
|
|
staleFact: ORBIT-9
|
|
currentFact: ORBIT-10
|
|
transcriptId: qa-session-memory-ranking
|
|
transcriptQuestion: "What is the current Project Nebula codename?"
|
|
transcriptAnswer: "The current Project Nebula codename is ORBIT-10."
|
|
prompt: "Session memory ranking check: what is the current Project Nebula codename? Run memory_search without a corpus filter using maxResults=6 and the answer-free query 'current Project Nebula codename' so durable notes and indexed session transcripts compete in the same ranked result. Answer with the codename from the newer indexed session transcript. If durable notes conflict with newer indexed session transcripts, prefer the newer current fact."
|
|
|
|
flow:
|
|
steps:
|
|
- name: prefers the newer transcript-backed fact over the stale durable note
|
|
actions:
|
|
- set: staleFact
|
|
value:
|
|
expr: config.staleFact
|
|
- set: currentFact
|
|
value:
|
|
expr: config.currentFact
|
|
- call: readConfigSnapshot
|
|
saveAs: original
|
|
args:
|
|
- ref: env
|
|
- set: originalMemorySearch
|
|
value:
|
|
expr: "original.config.memory && typeof original.config.memory === 'object' ? structuredClone(original.config.memory.search) : undefined"
|
|
- set: originalToolsSessions
|
|
value:
|
|
expr: "original.config.tools && typeof original.config.tools === 'object' && typeof original.config.tools.sessions === 'object' ? structuredClone(original.config.tools.sessions) : undefined"
|
|
- call: patchConfig
|
|
args:
|
|
- env:
|
|
ref: env
|
|
patch:
|
|
tools:
|
|
sessions:
|
|
visibility: all
|
|
memory:
|
|
search:
|
|
sources:
|
|
- memory
|
|
- sessions
|
|
experimental:
|
|
sessionMemory: true
|
|
query:
|
|
minScore: 0
|
|
- call: waitForGatewayHealthy
|
|
args:
|
|
- ref: env
|
|
- call: waitForQaChannelReady
|
|
args:
|
|
- ref: env
|
|
- 60000
|
|
- try:
|
|
actions:
|
|
- set: memoryDir
|
|
value:
|
|
expr: "path.join(env.gateway.workspaceDir, 'memory')"
|
|
- call: fs.mkdir
|
|
args:
|
|
- ref: memoryDir
|
|
- recursive: true
|
|
- set: now
|
|
value:
|
|
expr: "Date.now()"
|
|
- set: staleAt
|
|
value:
|
|
expr: "new Date(now - 3 * 86_400_000)"
|
|
- set: staleMemoryPath
|
|
value:
|
|
expr: "path.join(memoryDir, `${staleAt.toISOString().slice(0, 10)}.md`)"
|
|
- call: fs.writeFile
|
|
args:
|
|
- ref: staleMemoryPath
|
|
- expr: "`${'Project Nebula current codename: '}${staleFact}.\\n`"
|
|
- utf8
|
|
- call: fs.utimes
|
|
args:
|
|
- ref: staleMemoryPath
|
|
- ref: staleAt
|
|
- ref: staleAt
|
|
- call: seedQaSessionTranscript
|
|
args:
|
|
- ref: env
|
|
- sessionId:
|
|
expr: config.transcriptId
|
|
sessionKey: agent:qa:seed-session-memory-ranking
|
|
updatedAt:
|
|
ref: now
|
|
label: QA seeded session memory ranking transcript
|
|
messages:
|
|
- role: user
|
|
text:
|
|
expr: config.transcriptQuestion
|
|
timestamp:
|
|
expr: now - 90000
|
|
- role: assistant
|
|
text:
|
|
expr: config.transcriptAnswer
|
|
timestamp:
|
|
expr: now - 60000
|
|
- call: forceMemoryIndex
|
|
args:
|
|
- env:
|
|
ref: env
|
|
query:
|
|
expr: "'current Project Nebula codename'"
|
|
expectedNeedle:
|
|
ref: currentFact
|
|
- call: reset
|
|
- call: runAgentPrompt
|
|
args:
|
|
- ref: env
|
|
- sessionKey: agent:qa:session-memory-ranking
|
|
message:
|
|
expr: config.prompt
|
|
timeoutMs:
|
|
expr: liveTurnTimeoutMs(env, 45000)
|
|
transcriptToolName: memory_search
|
|
requireSuccessfulTranscriptToolResult: true
|
|
- call: waitForOutboundMessage
|
|
saveAs: outbound
|
|
args:
|
|
- ref: state
|
|
- lambda:
|
|
params: [candidate]
|
|
expr: "candidate.conversation.id === 'qa-operator' && (candidate.text.includes(currentFact) || candidate.text.includes(staleFact) || /no hits|unknown|not available/i.test(candidate.text))"
|
|
- expr: liveTurnTimeoutMs(env, 45000)
|
|
- assert:
|
|
expr: "outbound.text.includes(currentFact)"
|
|
message:
|
|
expr: "`expected current transcript-backed fact ${currentFact}, got: ${outbound.text}`"
|
|
- set: staleLeak
|
|
value:
|
|
expr: "outbound.text.includes(staleFact) && !/(stale|durable|conflict|older|previous)/i.test(outbound.text)"
|
|
- assert:
|
|
expr: "!staleLeak"
|
|
message:
|
|
expr: "`stale durable fact leaked through: ${outbound.text}`"
|
|
- set: rankingHistory
|
|
value:
|
|
expr: "await env.gateway.call('chat.history', { sessionKey: 'agent:qa:session-memory-ranking', limit: 100, maxChars: 131072 }, { timeoutMs: liveTurnTimeoutMs(env, 15000) })"
|
|
- set: searchCalls
|
|
value:
|
|
expr: |-
|
|
(rankingHistory.messages ?? []).flatMap((message, messageIndex) =>
|
|
message?.role === 'assistant' && Array.isArray(message.content)
|
|
? message.content.flatMap((item) =>
|
|
['toolCall', 'toolUse', 'tool_use'].includes(item?.type) &&
|
|
(item.name ?? item.toolName) === 'memory_search' &&
|
|
typeof item.id === 'string' && item.id.length > 0
|
|
? [{ messageIndex, callId: item.id, args: item.arguments ?? item.input ?? {} }]
|
|
: [])
|
|
: [])
|
|
- assert:
|
|
expr: searchCalls.length > 0
|
|
message: expected a persisted planned memory_search call
|
|
- assert:
|
|
expr: "searchCalls.every((call) => !String(call.args?.query ?? '').includes(currentFact) && !String(call.args?.query ?? '').includes(staleFact))"
|
|
message: memory_search query revealed an expected answer
|
|
- assert:
|
|
expr: "searchCalls.every((call) => !Object.hasOwn(call.args ?? {}, 'corpus'))"
|
|
message: memory_search corpus filter prevented durable and session memory from competing
|
|
- set: correlatedSearchResults
|
|
value:
|
|
expr: |-
|
|
searchCalls.flatMap((call) => {
|
|
const result = (rankingHistory.messages ?? []).find((message, messageIndex) =>
|
|
messageIndex > call.messageIndex &&
|
|
message?.role === 'toolResult' &&
|
|
message.toolName === 'memory_search' &&
|
|
message.toolCallId === call.callId &&
|
|
message.isError === false);
|
|
if (!result) return [];
|
|
const content = typeof result.content === 'string'
|
|
? result.content
|
|
: Array.isArray(result.content)
|
|
? result.content
|
|
.filter((item) => ['text', 'input_text', 'output_text'].includes(item?.type))
|
|
.map((item) => item.text ?? '')
|
|
.join('\n')
|
|
: '';
|
|
try {
|
|
const parsed = JSON.parse(content);
|
|
return Array.isArray(parsed?.results) ? [{ ...call, results: parsed.results }] : [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
})
|
|
- assert:
|
|
expr: correlatedSearchResults.length > 0
|
|
message: expected a correlated successful memory_search result
|
|
- set: rankedSearchResult
|
|
value:
|
|
expr: |-
|
|
correlatedSearchResults.find(({ results }) =>
|
|
results.some((result) =>
|
|
(result.source === 'sessions' || result.corpus === 'sessions' ||
|
|
String(result.path ?? '').startsWith('sessions/')) &&
|
|
String(result.snippet ?? result.text ?? '').includes(currentFact)) &&
|
|
results.some((result) =>
|
|
(result.source === 'memory' || result.corpus === 'memory' ||
|
|
String(result.path ?? '').startsWith('memory/') ||
|
|
String(result.path ?? '') === 'MEMORY.md') &&
|
|
String(result.snippet ?? result.text ?? '').includes(staleFact)))
|
|
- assert:
|
|
expr: Boolean(rankedSearchResult)
|
|
message:
|
|
expr: "`expected current session and stale durable memory to compete in the same search result: ${JSON.stringify(correlatedSearchResults.map(({ results }) => results.map(({ path, source, corpus, score, snippet, text }) => ({ path, source, corpus, score, snippet: String(snippet ?? text ?? '').slice(0, 140) }))))}`"
|
|
- assert:
|
|
expr: "(rankedSearchResult.args?.maxResults ?? 6) === 6"
|
|
message: expected the correlated memory_search call to use maxResults=6
|
|
- set: currentFactRank
|
|
value:
|
|
expr: "rankedSearchResult.results.findIndex((result) => (result.source === 'sessions' || result.corpus === 'sessions' || String(result.path ?? '').startsWith('sessions/')) && String(result.snippet ?? result.text ?? '').includes(currentFact))"
|
|
- set: staleFactRank
|
|
value:
|
|
expr: "rankedSearchResult.results.findIndex((result) => (result.source === 'memory' || result.corpus === 'memory' || String(result.path ?? '').startsWith('memory/') || String(result.path ?? '') === 'MEMORY.md') && String(result.snippet ?? result.text ?? '').includes(staleFact))"
|
|
- assert:
|
|
expr: "currentFactRank >= 0 && staleFactRank >= 0 && currentFactRank < staleFactRank"
|
|
message:
|
|
expr: "`stale durable memory outranked the current session fact: ${JSON.stringify(rankedSearchResult.results)}`"
|
|
finally:
|
|
- call: patchConfig
|
|
args:
|
|
- env:
|
|
ref: env
|
|
patch:
|
|
tools:
|
|
sessions:
|
|
expr: "originalToolsSessions === undefined ? null : structuredClone(originalToolsSessions)"
|
|
memory:
|
|
search:
|
|
expr: "originalMemorySearch === undefined ? null : structuredClone(originalMemorySearch)"
|
|
- call: waitForGatewayHealthy
|
|
args:
|
|
- ref: env
|
|
- call: waitForQaChannelReady
|
|
args:
|
|
- ref: env
|
|
- 60000
|
|
detailsExpr: outbound.text
|