diff --git a/extensions/qa-lab/src/scenario-catalog-causality.test.ts b/extensions/qa-lab/src/scenario-catalog-causality.test.ts index 3df445cf9b04..51af7045bf13 100644 --- a/extensions/qa-lab/src/scenario-catalog-causality.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-causality.test.ts @@ -333,4 +333,62 @@ describe("qa scenario catalog causality", () => { }), ).resolves.toMatchObject({ status: "pass" }); }); + + it("isolates Active Memory request traces from interleaved heartbeats", async () => { + const scenario = requireFlowScenario(readQaScenarioById("active-memory-preprompt-recall")); + const actions = scenario.execution.flow?.steps[0]?.actions ?? []; + const baselineTrace = actions.find( + (action) => (action as { set?: string }).set === "baselineMockRequests", + ); + const activeTrace = actions.find( + (action) => (action as { set?: string }).set === "activeRequests", + ); + expect(baselineTrace).toBeDefined(); + expect(activeTrace).toBeDefined(); + if (!baselineTrace || !activeTrace) { + throw new Error("active-memory-preprompt-recall request trace actions are missing"); + } + + const marker = String(scenario.execution.config?.turnMarker); + const heartbeat = { allInputText: "[OpenClaw heartbeat poll]" }; + const scenarioRequest = (suffix: string) => ({ allInputText: `${marker} ${suffix}` }); + const traces = new Map([ + ["10", [heartbeat, scenarioRequest("baseline main")]], + [ + "20", + [ + heartbeat, + scenarioRequest("You are a memory search agent. search plan"), + scenarioRequest("You are a memory search agent. search result"), + scenarioRequest("You are a memory search agent. memory get result"), + scenarioRequest("active main"), + ], + ], + ]); + + await expect( + runLoadedScenarioFlow("active-memory-preprompt-recall", { + flow: { + steps: [ + { + name: "filters provider-global traces before exact counts", + actions: [ + { set: "requestCursorBeforeBaseline", value: { expr: "10" } }, + baselineTrace, + { assert: "baselineMockRequests.length === 1" }, + { set: "requestCursorBeforeActive", value: { expr: "20" } }, + activeTrace, + { assert: "activeRequests.length === 4" }, + ], + }, + ], + }, + api: { + env: { mock: { baseUrl: "http://mock.invalid" } }, + fetchJson: async (url: string) => + traces.get(new URL(url).searchParams.get("after") ?? "") ?? [], + }, + }), + ).resolves.toMatchObject({ status: "pass" }); + }); }); diff --git a/qa/scenarios/memory/active-memory-preprompt-recall.yaml b/qa/scenarios/memory/active-memory-preprompt-recall.yaml index 304828440418..a71ff1efab03 100644 --- a/qa/scenarios/memory/active-memory-preprompt-recall.yaml +++ b/qa/scenarios/memory/active-memory-preprompt-recall.yaml @@ -174,7 +174,7 @@ flow: expr: "normalizeLowercaseStringOrEmpty(baselineReply.text)" - set: baselineMockRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeBaseline}`))" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeBaseline}`)).filter((request) => String(request.allInputText ?? '').includes(config.turnMarker))" - assert: expr: baselineMockRequests.length === 1 message: @@ -243,7 +243,7 @@ flow: message: active memory recall left a transient SQLite session behind - set: activeRequests value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeActive}`))" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeActive}`)).filter((request) => String(request.allInputText ?? '').includes(config.turnMarker))" - assert: expr: activeRequests.length === 4 message: