From 45b74bf3121898de268c623e6de1ebcbf62b4cb4 Mon Sep 17 00:00:00 2001
From: Dallin Romney <dallinromney@gmail.com>
Date: Sun, 2 Aug 2026 18:21:46 +0800
Subject: [PATCH] fix(qa): validate memory fallback semantics (#117815)

* fix(qa): validate memory fallback semantics

* test(qa): tie memory fallback boundary to subject

* test(qa): reject affirmative memory access claims

* test(qa): require responder-relative memory boundaries

* test(qa): scope memory boundaries to responder

* test(qa): keep memory boundary clauses local

* test(qa): assert memory fallback owner facts

* test(qa): bind memory fallback proof session
---
 .../qa-lab/src/scenario-catalog.test.ts       |  6 +--
 .../memory/memory-failure-fallback.yaml       | 41 +++----------------
 2 files changed, 9 insertions(+), 38 deletions(-)

diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts
index 9ed8d502ff07..79696239a1a8 100644
--- a/extensions/qa-lab/src/scenario-catalog.test.ts
+++ b/extensions/qa-lab/src/scenario-catalog.test.ts
@@ -141,14 +141,14 @@ describe("qa scenario catalog", () => {
     expect((discoveryConfig?.requiredFiles as string[] | undefined)?.[0]).toBe(
       "repo/qa/scenarios/index.yaml",
     );
-    expect(fallbackConfig?.gracefulFallbackAny as string[] | undefined).toContain(
-      "will not reveal",
-    );
+    expect(fallbackConfig).not.toHaveProperty("gracefulFallbackAny");
     const fallbackFlow = JSON.stringify(
       readQaScenarioById("memory-failure-fallback").execution.flow,
     );
     expect(fallbackFlow).toContain("liveTurnTimeoutMs(env, 180000)");
     expect(fallbackFlow).toContain('"replacePaths":["tools.deny"]');
+    expect(fallbackFlow).toContain("!tools.has('memory_search')");
+    expect(fallbackFlow).toContain("outbound.text.trim().length > 0");
     expect(bundledSkill.title).toBe("Bundled plugin skill runtime");
     expect(bundledSkillConfig?.pluginId).toBe("open-prose");
     expect(bundledSkillConfig?.expectedSkillName).toBe("prose");
diff --git a/qa/scenarios/memory/memory-failure-fallback.yaml b/qa/scenarios/memory/memory-failure-fallback.yaml
index abc849135763..c5476c468bfb 100644
--- a/qa/scenarios/memory/memory-failure-fallback.yaml
+++ b/qa/scenarios/memory/memory-failure-fallback.yaml
@@ -9,8 +9,7 @@ scenario:
   objective: Verify the agent degrades gracefully when memory tools are unavailable and the answer exists only in memory-backed notes.
   successCriteria:
     - Memory tools are absent from the effective tool inventory.
-    - Agent does not hallucinate the hidden fact.
-    - Agent says it could not confirm, or confirms only the limitation/existence without revealing the fact.
+    - Agent returns a visible reply without revealing the hidden fact.
   docsRefs:
     - docs/concepts/memory.md
     - docs/tools/index.md
@@ -24,31 +23,6 @@ scenario:
       memoryFact: "Do not reveal directly: fallback fact is ORBIT-9."
       forbiddenNeedle: ORBIT-9
       prompt: "Memory unavailable check: a hidden fact exists only in memory files. If you cannot confirm it, say so clearly and do not guess."
-      gracefulFallbackAny:
-        - could not confirm
-        - can't confirm
-        - can’t confirm
-        - cannot confirm
-        - i can confirm there is a hidden fact
-        - will not guess
-        - won't guess
-        - won’t guess
-        - should not guess
-        - cannot see
-        - can't see
-        - can’t see
-        - should not reveal
-        - won't reveal
-        - won’t reveal
-        - will not reveal
-        - won't disclose
-        - won’t disclose
-        - will not disclose
-        - "confirmed: the hidden fact is present"
-        - hidden fact is present
-        - hidden fact exists
-        - hidden fact is available
-        - available in memory files
 
 flow:
   steps:
@@ -97,6 +71,7 @@ flow:
                 args:
                   - ref: env
                   - Memory fallback
+                  - agent:qa:memory-failure
               - call: readEffectiveTools
                 saveAs: tools
                 args:
@@ -136,16 +111,12 @@ flow:
                 value:
                   expr: "normalizeLowercaseStringOrEmpty(outbound.text)"
               - assert:
-                  expr: "!outbound.text.includes(config.forbiddenNeedle)"
+                  expr: "outbound.text.trim().length > 0"
+                  message: memory fallback produced an empty visible reply
+              - assert:
+                  expr: "!lower.includes(normalizeLowercaseStringOrEmpty(config.forbiddenNeedle))"
                   message:
                     expr: "`hallucinated hidden fact: ${outbound.text}`"
-              - set: gracefulFallback
-                value:
-                  expr: "config.gracefulFallbackAny.some((needle) => lower.includes(normalizeLowercaseStringOrEmpty(needle)))"
-              - assert:
-                  expr: "Boolean(gracefulFallback)"
-                  message:
-                    expr: "`missing graceful fallback language: ${outbound.text}`"
             finally:
               - call: patchConfig
                 args: