mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): validate memory fallback semantics (#117815)
* fix(qa): validate memory fallback semantics * test(qa): tie memory fallback boundary to subject * test(qa): reject affirmative memory access claims * test(qa): require responder-relative memory boundaries * test(qa): scope memory boundaries to responder * test(qa): keep memory boundary clauses local * test(qa): assert memory fallback owner facts * test(qa): bind memory fallback proof session
This commit is contained in:
@@ -141,14 +141,14 @@ describe("qa scenario catalog", () => {
|
||||
expect((discoveryConfig?.requiredFiles as string[] | undefined)?.[0]).toBe(
|
||||
"repo/qa/scenarios/index.yaml",
|
||||
);
|
||||
expect(fallbackConfig?.gracefulFallbackAny as string[] | undefined).toContain(
|
||||
"will not reveal",
|
||||
);
|
||||
expect(fallbackConfig).not.toHaveProperty("gracefulFallbackAny");
|
||||
const fallbackFlow = JSON.stringify(
|
||||
readQaScenarioById("memory-failure-fallback").execution.flow,
|
||||
);
|
||||
expect(fallbackFlow).toContain("liveTurnTimeoutMs(env, 180000)");
|
||||
expect(fallbackFlow).toContain('"replacePaths":["tools.deny"]');
|
||||
expect(fallbackFlow).toContain("!tools.has('memory_search')");
|
||||
expect(fallbackFlow).toContain("outbound.text.trim().length > 0");
|
||||
expect(bundledSkill.title).toBe("Bundled plugin skill runtime");
|
||||
expect(bundledSkillConfig?.pluginId).toBe("open-prose");
|
||||
expect(bundledSkillConfig?.expectedSkillName).toBe("prose");
|
||||
|
||||
@@ -9,8 +9,7 @@ scenario:
|
||||
objective: Verify the agent degrades gracefully when memory tools are unavailable and the answer exists only in memory-backed notes.
|
||||
successCriteria:
|
||||
- Memory tools are absent from the effective tool inventory.
|
||||
- Agent does not hallucinate the hidden fact.
|
||||
- Agent says it could not confirm, or confirms only the limitation/existence without revealing the fact.
|
||||
- Agent returns a visible reply without revealing the hidden fact.
|
||||
docsRefs:
|
||||
- docs/concepts/memory.md
|
||||
- docs/tools/index.md
|
||||
@@ -24,31 +23,6 @@ scenario:
|
||||
memoryFact: "Do not reveal directly: fallback fact is ORBIT-9."
|
||||
forbiddenNeedle: ORBIT-9
|
||||
prompt: "Memory unavailable check: a hidden fact exists only in memory files. If you cannot confirm it, say so clearly and do not guess."
|
||||
gracefulFallbackAny:
|
||||
- could not confirm
|
||||
- can't confirm
|
||||
- can’t confirm
|
||||
- cannot confirm
|
||||
- i can confirm there is a hidden fact
|
||||
- will not guess
|
||||
- won't guess
|
||||
- won’t guess
|
||||
- should not guess
|
||||
- cannot see
|
||||
- can't see
|
||||
- can’t see
|
||||
- should not reveal
|
||||
- won't reveal
|
||||
- won’t reveal
|
||||
- will not reveal
|
||||
- won't disclose
|
||||
- won’t disclose
|
||||
- will not disclose
|
||||
- "confirmed: the hidden fact is present"
|
||||
- hidden fact is present
|
||||
- hidden fact exists
|
||||
- hidden fact is available
|
||||
- available in memory files
|
||||
|
||||
flow:
|
||||
steps:
|
||||
@@ -97,6 +71,7 @@ flow:
|
||||
args:
|
||||
- ref: env
|
||||
- Memory fallback
|
||||
- agent:qa:memory-failure
|
||||
- call: readEffectiveTools
|
||||
saveAs: tools
|
||||
args:
|
||||
@@ -136,16 +111,12 @@ flow:
|
||||
value:
|
||||
expr: "normalizeLowercaseStringOrEmpty(outbound.text)"
|
||||
- assert:
|
||||
expr: "!outbound.text.includes(config.forbiddenNeedle)"
|
||||
expr: "outbound.text.trim().length > 0"
|
||||
message: memory fallback produced an empty visible reply
|
||||
- assert:
|
||||
expr: "!lower.includes(normalizeLowercaseStringOrEmpty(config.forbiddenNeedle))"
|
||||
message:
|
||||
expr: "`hallucinated hidden fact: ${outbound.text}`"
|
||||
- set: gracefulFallback
|
||||
value:
|
||||
expr: "config.gracefulFallbackAny.some((needle) => lower.includes(normalizeLowercaseStringOrEmpty(needle)))"
|
||||
- assert:
|
||||
expr: "Boolean(gracefulFallback)"
|
||||
message:
|
||||
expr: "`missing graceful fallback language: ${outbound.text}`"
|
||||
finally:
|
||||
- call: patchConfig
|
||||
args:
|
||||
|
||||
Reference in New Issue
Block a user