From c578d324ef1990e0cdab326ee993d8436dd5fcf7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 26 Aug 2026 19:54:39 -0700 Subject: [PATCH] fix(qa): preserve truthful debug snapshots for large AIMock requests (#130605) --- .../src/providers/aimock/server.test.ts | 323 +++++++++++++++++- .../qa-lab/src/providers/aimock/server.ts | 256 +++++++++----- .../memory/remember-across-conversations.yaml | 30 +- 3 files changed, 510 insertions(+), 99 deletions(-) diff --git a/extensions/qa-lab/src/providers/aimock/server.test.ts b/extensions/qa-lab/src/providers/aimock/server.test.ts index 99da0be17718..a04fec3ffc40 100644 --- a/extensions/qa-lab/src/providers/aimock/server.test.ts +++ b/extensions/qa-lab/src/providers/aimock/server.test.ts @@ -15,6 +15,323 @@ function makeResponsesInput(text: string) { } describe("qa aimock server", () => { + it("keeps complete large input when the upstream body is still retained", async () => { + const server = await startQaAimockServer(); + const prompt = "u".repeat(40_000); + try { + const response = await fetch(`${server.baseUrl}/v1/responses`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "aimock/gpt-5.6-luna", + stream: false, + input: [makeResponsesInput(prompt)], + }), + }); + expect(response.status).toBe(200); + await response.json(); + const debug = await fetch(`${server.baseUrl}/debug/last-request`); + expect(debug.status).toBe(200); + const snapshot = await debug.json(); + expect(snapshot).toMatchObject({ prompt, allInputText: prompt }); + expect(snapshot.body).not.toHaveProperty("__aimock_truncated"); + } finally { + await server.stop(); + } + }); + + it.each(["chat", "responses"])( + "retains exact %s request facts when tool schemas exceed the upstream journal cap", + async (dialect) => { + const server = await startQaAimockServer(); + const model = "aimock/gpt-5.6-luna"; + const prompt = "current user marker"; + const tool = { name: "echo", description: "schema".repeat(14_000), parameters: {} }; + try { + const response = await fetch( + `${server.baseUrl}/v1/${dialect === "chat" ? "chat/completions" : "responses"}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: false, + ...(dialect === "chat" + ? { + messages: [ + { role: "user", content: prompt }, + { role: "tool", content: "observed tool output", tool_call_id: "call-qa" }, + ], + tools: [{ type: "function", function: tool }], + } + : { + input: [ + makeResponsesInput(prompt), + { + type: "function_call_output", + output: "observed tool output", + call_id: "call-qa", + }, + ], + tools: [{ type: "function", ...tool }], + }), + }), + }, + ); + expect(response.status).toBe(200); + await response.json(); + const debug = await fetch(`${server.baseUrl}/debug/last-request`); + expect(debug.status).toBe(200); + const snapshot = await debug.json(); + expect(snapshot).toMatchObject({ + model, + prompt, + allInputText: `${prompt}\nobserved tool output`, + toolOutput: "observed tool output", + toolOutputCallId: "call-qa", + providerVariant: "openai", + body: { __aimock_truncated: true }, + }); + expect(JSON.parse(snapshot.raw)).toEqual(snapshot.body); + expect(snapshot.body.originalByteSize).toBeGreaterThan(64 * 1024); + } finally { + await server.stop(); + } + }, + ); + + it.each([ + { role: "system", content: "s".repeat(70_000), omittedFields: ["allInputText"] }, + { role: "user", content: "😀".repeat(20_000), omittedFields: ["prompt", "allInputText"] }, + ])("reports incomplete $role text without poisoning later queries or metadata", async (input) => { + const server = await startQaAimockServer(); + const post = async (messages: Array<{ role: string; content: string }>) => { + const response = await fetch(`${server.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "aimock/gpt-5.6-luna", stream: false, messages }), + }); + expect(response.status).toBe(200); + await response.json(); + }; + try { + await post([ + ...(input.role === "system" ? [{ role: "user", content: "exact current user" }] : []), + { role: input.role, content: input.content }, + ]); + for (const endpoint of ["last-request", "requests", "requests?after=0"]) { + const debug = await fetch(`${server.baseUrl}/debug/${endpoint}`); + expect(debug.status).toBe(413); + const incomplete = await debug.json(); + expect(incomplete).toMatchObject({ + code: "QA_DEBUG_SNAPSHOT_INCOMPLETE", + maxBytes: 64 * 1024, + requests: [{ cursor: 1, omittedFields: input.omittedFields }], + }); + expect(incomplete.error).toContain("/debug/request-cursor"); + expect(incomplete.requests[0].facts.model).toBe("aimock/gpt-5.6-luna"); + for (const field of input.omittedFields) { + expect(incomplete.requests[0].facts).not.toHaveProperty(field); + } + if (input.role === "system") { + expect(incomplete.requests[0].facts.prompt).toBe("exact current user"); + } + expect(Buffer.byteLength(JSON.stringify(incomplete.requests[0]))).toBeLessThan(64 * 1024); + } + expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({ + cursor: 1, + }); + const images = await fetch(`${server.baseUrl}/debug/image-generations`); + expect(images.status).toBe(200); + expect(await images.json()).toEqual([]); + expect((await fetch(`${server.baseUrl}/debug/requests?after=1.5`)).status).toBe(400); + expect((await fetch(`${server.baseUrl}/debug/requests?after=2`)).status).toBe(409); + + await post([{ role: "user", content: "later complete request" }]); + const latest = await fetch(`${server.baseUrl}/debug/last-request`); + expect(latest.status).toBe(200); + expect(await latest.json()).toMatchObject({ prompt: "later complete request" }); + const after = await fetch(`${server.baseUrl}/debug/requests?after=1`); + expect(after.status).toBe(200); + expect(await after.json()).toMatchObject([{ prompt: "later complete request" }]); + expect((await fetch(`${server.baseUrl}/debug/requests`)).status).toBe(413); + + const reset = await fetch(`${server.baseUrl}/__aimock/reset/journal`, { method: "POST" }); + expect(reset.status).toBe(200); + expect(await fetch(`${server.baseUrl}/debug/requests`).then((r) => r.json())).toEqual([]); + expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({ + cursor: 2, + }); + await post([{ role: "user", content: "after reset" }]); + expect( + await fetch(`${server.baseUrl}/debug/requests?after=2`).then((r) => r.json()), + ).toMatchObject([{ prompt: "after reset" }]); + expect((await fetch(`${server.baseUrl}/__aimock/reset`, { method: "POST" })).status).toBe( + 200, + ); + expect(await fetch(`${server.baseUrl}/debug/requests`).then((r) => r.json())).toEqual([]); + expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({ + cursor: 3, + }); + } finally { + await server.stop(); + } + }); + + it("preserves image evidence and tool-call pairing across capped journal bodies", async () => { + const server = await startQaAimockServer(); + const prompt = "inspect and use echo"; + const user = { + role: "user", + content: [ + { type: "text", text: prompt }, + { type: "image_url", image_url: { url: "data:image/png;base64,cWE=" } }, + ], + }; + const post = async (messages: unknown[]) => { + const response = await fetch(`${server.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "aimock/gpt-5.6-luna", + stream: false, + messages, + tools: [ + { + type: "function", + function: { name: "echo", description: "t".repeat(70_000), parameters: {} }, + }, + ], + }), + }); + expect(response.status).toBe(200); + return response.json(); + }; + try { + expect( + (await fetch(`${server.baseUrl}/__aimock/fixtures`, { method: "DELETE" })).status, + ).toBe(200); + const configured = await fetch(`${server.baseUrl}/__aimock/fixtures`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + fixtures: [ + { + match: { sequenceIndex: 0 }, + response: { toolCalls: [{ name: "echo", arguments: "{}" }] }, + }, + { match: { sequenceIndex: 1 }, response: { content: "tool observed" } }, + ], + }), + }); + expect(configured.status).toBe(200); + const planned = await post([user]); + const assistant = planned.choices[0].message; + const toolCallId = assistant.tool_calls[0].id; + await post([ + user, + assistant, + { + role: "tool", + content: "independent tool evidence", + tool_call_id: toolCallId, + isError: true, + }, + { + role: "user", + content: + "<<>>\nprivate runtime context\n<<>>", + }, + ]); + const requests = await fetch(`${server.baseUrl}/debug/requests`); + expect(requests.status).toBe(200); + expect(await requests.json()).toMatchObject([ + { + prompt, + imageInputCount: 1, + plannedToolName: "echo", + plannedToolCallId: toolCallId, + body: { __aimock_truncated: true }, + }, + { + prompt, + imageInputCount: 1, + toolOutput: "independent tool evidence", + toolOutputCallId: toolCallId, + toolOutputStructuredError: true, + body: { __aimock_truncated: true }, + }, + ]); + } finally { + await server.stop(); + } + }); + + it.each([ + { label: "complete", olderOverflow: false, olderPrompt: "first conversation" }, + { label: "overflow", olderOverflow: true, olderPrompt: "first conversation" }, + { label: "near-budget prompt", olderOverflow: true, olderPrompt: "u".repeat(65_280) }, + ])( + "keeps tool-result pairing before the selected cursor window ($label)", + async ({ olderOverflow, olderPrompt }) => { + const server = await startQaAimockServer(); + const post = async (messages: unknown[]) => { + const response = await fetch(`${server.baseUrl}/v1/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "aimock/gpt-5.6-luna", stream: false, messages }), + }); + expect(response.status).toBe(200); + return response.json(); + }; + try { + await fetch(`${server.baseUrl}/__aimock/fixtures`, { method: "DELETE" }); + const fixtures = [0, 1].map((sequenceIndex) => ({ + match: { sequenceIndex }, + response: { toolCalls: [{ name: "echo", arguments: "{}" }] }, + })); + const configured = await fetch(`${server.baseUrl}/__aimock/fixtures`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + fixtures: [...fixtures, { match: { sequenceIndex: 2 }, response: { content: "done" } }], + }), + }); + expect(configured.status).toBe(200); + const first = await post([ + ...(olderOverflow ? [{ role: "system", content: "s".repeat(70_000) }] : []), + { role: "user", content: olderPrompt }, + ]); + await post([{ role: "user", content: "second conversation" }]); + const firstToolCallId = first.choices[0].message.tool_calls[0].id; + await post([ + { role: "user", content: "first conversation" }, + first.choices[0].message, + { role: "tool", content: "first result", tool_call_id: firstToolCallId }, + ]); + const after = await fetch(`${server.baseUrl}/debug/requests?after=1`); + expect(after.status).toBe(200); + const selected = await after.json(); + expect(selected).toHaveLength(2); + expect(selected[0]).toMatchObject({ + prompt: "second conversation", + plannedToolName: "echo", + }); + expect(selected[0]).not.toHaveProperty("plannedToolCallId"); + expect(selected[1]).toMatchObject({ toolOutputCallId: firstToolCallId }); + if (olderOverflow) { + const all = await fetch(`${server.baseUrl}/debug/requests`); + expect(all.status).toBe(413); + const incomplete = await all.json(); + expect(incomplete.requests[0].facts.plannedToolCallId).toBe(firstToolCallId); + expect(incomplete.requests[0].omittedFields).not.toContain("plannedToolCallId"); + } + } finally { + await server.stop(); + } + }, + ); + it("serves OpenAI Responses text replies and debug request snapshots", async () => { const server = await startQaAimockServer({ host: "127.0.0.1", @@ -107,7 +424,7 @@ describe("qa aimock server", () => { host: "127.0.0.1", port: 0, }); - const post = async (text: string) => { + const post = async (text: string, instructions?: string) => { const response = await fetch(`${server.baseUrl}/v1/responses`, { method: "POST", headers: { "content-type": "application/json" }, @@ -115,6 +432,7 @@ describe("qa aimock server", () => { model: "aimock/gpt-5.6-luna", stream: false, input: [makeResponsesInput(text)], + instructions, }), }); expect(response.status).toBe(200); @@ -125,12 +443,13 @@ describe("qa aimock server", () => { ).toEqual({ cursor: 0 }); const debugRequestLimit = 1_000; for (let index = 0; index < debugRequestLimit; index += 1) { - await post(`aimock cursor ${index}`); + await post(`aimock cursor ${index}`, index === 0 ? "s".repeat(70_000) : undefined); } const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) => response.json(), ); expect(cursor).toEqual({ cursor: debugRequestLimit }); + expect((await fetch(`${server.baseUrl}/debug/requests`)).status).toBe(413); await post("aimock cursor overflow"); const retained = (await fetch(`${server.baseUrl}/debug/requests`).then((response) => diff --git a/extensions/qa-lab/src/providers/aimock/server.ts b/extensions/qa-lab/src/providers/aimock/server.ts index bdc0365ccf51..a047818f40b6 100644 --- a/extensions/qa-lab/src/providers/aimock/server.ts +++ b/extensions/qa-lab/src/providers/aimock/server.ts @@ -27,6 +27,23 @@ type AimockRequestSnapshot = { }; const AIMOCK_DEBUG_REQUEST_LIMIT = 1_000; +const AIMOCK_DEBUG_FACTS_MAX_BYTES = 64 * 1024; + +type AimockRequestFacts = Omit; +type AimockRequestProjection = + | { complete: true; facts: AimockRequestFacts } + | { + complete: false; + facts: Partial; + omittedFields: Array; + }; +type AimockToolFacts = Pick< + AimockRequestFacts, + "plannedToolName" | "plannedToolCallId" | "toolOutputCallId" +>; +type AimockRequestObservation = + | { kind: "retained-body"; tools: AimockToolFacts } + | { kind: "projected"; projection: AimockRequestProjection }; // Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts. // This mock mirrors the wire shape so delimiter drift fails through QA timeouts. @@ -139,7 +156,7 @@ function resolveProviderVariant(model: string): AimockRequestSnapshot["providerV return "unknown"; } -function extractPlannedToolName(entry: JournalEntry) { +function extractPlannedToolName(entry: Pick) { const response = entry.response.fixture?.response as | { toolCalls?: Array<{ name?: unknown }> } | undefined; @@ -147,7 +164,7 @@ function extractPlannedToolName(entry: JournalEntry) { return typeof name === "string" && name.length > 0 ? name : undefined; } -function extractPlannedToolCallId(entry: JournalEntry) { +function extractPlannedToolCallId(entry: Pick) { const response = entry.response.fixture?.response as | { toolCalls?: Array<{ id?: unknown; callId?: unknown; toolCallId?: unknown }> } | undefined; @@ -158,53 +175,73 @@ function extractPlannedToolCallId(entry: JournalEntry) { return typeof candidate === "string" && candidate.length > 0 ? candidate : undefined; } -function toRequestSnapshot(entry: JournalEntry): AimockRequestSnapshot { - const body = entry.body ?? null; +function extractRequestFacts( + body: JournalEntry["body"], + tools: AimockToolFacts, +): AimockRequestFacts { const model = typeof body?.model === "string" ? body.model : ""; return { - raw: JSON.stringify(body ?? {}), - body: (body ?? {}) as Record, - prompt: extractLastUserText(body), - allInputText: extractAllInputText(body), - toolOutput: extractToolOutput(body), + ...tools, model, + prompt: extractLastUserText(body), providerVariant: resolveProviderVariant(model), imageInputCount: countImageInputs(requestMessages(body)), - plannedToolCallId: extractPlannedToolCallId(entry), - plannedToolName: extractPlannedToolName(entry), - toolOutputCallId: extractToolOutputCallId(body) || undefined, ...(extractToolOutputStructuredError(body) ? { toolOutputStructuredError: true } : {}), + toolOutput: extractToolOutput(body), + allInputText: extractAllInputText(body), }; } -function toRequestSnapshots(entries: JournalEntry[]): AimockRequestSnapshot[] { - const snapshots = entries.map((entry) => toRequestSnapshot(entry)); +function boundRequestFacts(projection: AimockRequestProjection): AimockRequestProjection { + if (Buffer.byteLength(JSON.stringify(projection)) <= AIMOCK_DEBUG_FACTS_MAX_BYTES) { + return projection; + } + const { facts } = projection; + const retained: Partial = {}; + const fields = Object.keys(facts) as Array; + const omittedFields = projection.complete ? [] : [...projection.omittedFields]; + const reservedOmissions = [...omittedFields, ...fields]; + // Reserve correlation facts before text: an older overflowing prompt must not + // redirect its tool result to a newer plan. Omission names share the byte budget. + for (const field of fields) { + const candidate = { ...retained, [field]: facts[field] }; + const diagnostic = { + complete: false, + cursor: Number.MAX_SAFE_INTEGER, + facts: candidate, + omittedFields: reservedOmissions, + }; + if (Buffer.byteLength(JSON.stringify(diagnostic)) <= AIMOCK_DEBUG_FACTS_MAX_BYTES) { + Object.assign(retained, { [field]: facts[field] }); + } else { + omittedFields.push(field); + } + } + return { complete: false, facts: retained, omittedFields }; +} + +function resolvePlannedToolCallIds(snapshots: AimockToolFacts[]): Map { + const callIds = new Map(); const pendingPlannedIndexes: number[] = []; for (const [index, snapshot] of snapshots.entries()) { if (snapshot.toolOutputCallId && pendingPlannedIndexes.length > 0) { const plannedIndex = pendingPlannedIndexes.shift(); if (plannedIndex !== undefined) { - const plannedSnapshot = snapshots[plannedIndex]; - if (!plannedSnapshot) { - continue; - } - snapshots[plannedIndex] = { - ...plannedSnapshot, - plannedToolCallId: snapshot.toolOutputCallId, - }; + callIds.set(plannedIndex, snapshot.toolOutputCallId); } } if (snapshot.plannedToolName && !snapshot.plannedToolCallId) { pendingPlannedIndexes.push(index); } } - return snapshots; + return callIds; } function createDebugMount(): Mountable { let journal: Journal | undefined; let nextRequestCursor = 1; const requestCursors = new Map(); + const observations = new WeakMap(); return { setJournal(nextJournal) { @@ -219,7 +256,26 @@ function createDebugMount(): Mountable { // AIMock evicts its request journal FIFO. Assign cursors at insertion time // so the debug boundary remains monotonic after retained entries rotate. journal.add = (entry) => { + const tools: AimockToolFacts = { + plannedToolName: extractPlannedToolName(entry), + plannedToolCallId: extractPlannedToolCallId(entry), + toolOutputCallId: extractToolOutputCallId(entry.body) || undefined, + }; const recorded = addJournalEntry(entry); + // Upstream keeps <=64 KiB bodies intact; only discarded bodies need an + // extra bounded projection. Weak entry ownership follows eviction/reset. + observations.set( + recorded, + recorded.body === entry.body + ? { kind: "retained-body", tools } + : { + kind: "projected", + projection: boundRequestFacts({ + complete: true, + facts: extractRequestFacts(entry.body, tools), + }), + }, + ); requestCursors.set(recorded.id, nextRequestCursor++); if (requestCursors.size > AIMOCK_DEBUG_REQUEST_LIMIT) { const oldestRequestId = requestCursors.keys().next().value; @@ -231,68 +287,11 @@ function createDebugMount(): Mountable { }; }, async handleRequest(req: IncomingMessage, res: ServerResponse, pathname: string) { - const entries = journal?.getAll() ?? []; - const snapshots = toRequestSnapshots(entries); - const cursorSnapshots = entries.map((entry, index) => { - const cursor = requestCursors.get(entry.id); - if (cursor === undefined) { - throw new Error(`AIMock debug request cursor missing for ${entry.id}`); - } - const snapshot = snapshots[index]; - if (!snapshot) { - throw new Error(`AIMock debug request snapshot missing for ${entry.id}`); - } - return { cursor, snapshot }; - }); - const url = new URL(req.url ?? "/", "http://127.0.0.1"); - if (pathname === "/last-request") { - const lastSnapshot = snapshots.at(-1); - writeJson(res, 200, lastSnapshot ?? { ok: false, error: "no request recorded" }); - return true; - } if (pathname === "/request-cursor") { writeJson(res, 200, { cursor: nextRequestCursor - 1 }); return true; } - if (pathname === "/requests") { - const afterText = url.searchParams.get("after"); - if (afterText === null) { - writeJson(res, 200, snapshots); - return true; - } - const after = parseQaDebugRequestCursor(afterText); - if (after === null) { - writeJson(res, 400, { error: "after must be a non-negative safe integer" }); - return true; - } - const latestCursor = nextRequestCursor - 1; - const oldestCursor = cursorSnapshots[0]?.cursor ?? nextRequestCursor; - if (after > latestCursor) { - writeJson(res, 409, { - error: "request cursor is ahead of the latest recorded request", - after, - latestCursor, - }); - return true; - } - if (after < oldestCursor - 1) { - writeJson(res, 409, { - error: "request cursor expired", - after, - oldestCursor, - latestCursor, - }); - return true; - } - writeJson( - res, - 200, - cursorSnapshots - .filter((request) => request.cursor > after) - .map((request) => request.snapshot), - ); - return true; - } + const entries = journal?.getAll() ?? []; if (pathname === "/image-generations") { writeJson( res, @@ -303,7 +302,100 @@ function createDebugMount(): Mountable { ); return true; } - return false; + if (pathname !== "/last-request" && pathname !== "/requests") { + return false; + } + let selected = entries.map((entry, index) => { + const cursor = requestCursors.get(entry.id); + const observation = observations.get(entry); + if (cursor === undefined || observation === undefined) { + throw new Error(`AIMock debug request observation missing for ${entry.id}`); + } + return { cursor, entry, observation, index }; + }); + // Pair against retained tool facts before selecting a window: a result + // inside the window may belong to a plan before its cursor. + const plannedToolCallIds = resolvePlannedToolCallIds( + selected.map(({ observation }) => + observation.kind === "retained-body" ? observation.tools : observation.projection.facts, + ), + ); + if (pathname === "/requests") { + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + const afterText = url.searchParams.get("after"); + if (afterText !== null) { + const after = parseQaDebugRequestCursor(afterText); + if (after === null) { + writeJson(res, 400, { error: "after must be a non-negative safe integer" }); + return true; + } + const latestCursor = nextRequestCursor - 1; + const oldestCursor = selected[0]?.cursor ?? nextRequestCursor; + if (after > latestCursor) { + writeJson(res, 409, { + error: "request cursor is ahead of the latest recorded request", + after, + latestCursor, + }); + return true; + } + if (after < oldestCursor - 1) { + writeJson(res, 409, { + error: "request cursor expired", + after, + oldestCursor, + latestCursor, + }); + return true; + } + selected = selected.filter((request) => request.cursor > after); + } + } else { + selected = selected.slice(-1); + } + const snapshots: AimockRequestSnapshot[] = []; + const incomplete: Array< + { cursor: number } & Extract + > = []; + for (const { cursor, entry, observation, index } of selected) { + const plannedToolCallId = plannedToolCallIds.get(index); + let projection: AimockRequestProjection = + observation.kind === "retained-body" + ? { complete: true, facts: extractRequestFacts(entry.body, observation.tools) } + : observation.projection; + if (plannedToolCallId) { + projection = projection.complete + ? { complete: true, facts: { ...projection.facts, plannedToolCallId } } + : { ...projection, facts: { ...projection.facts, plannedToolCallId } }; + if (observation.kind === "projected") { + projection = boundRequestFacts(projection); + } + } + if (!projection.complete) { + incomplete.push({ cursor, ...projection }); + continue; + } + const body = entry.body ?? {}; + snapshots.push({ raw: JSON.stringify(body), body, ...projection.facts }); + } + if (incomplete.length > 0) { + writeJson(res, 413, { + code: "QA_DEBUG_SNAPSHOT_INCOMPLETE", + error: + "Semantic facts exceeded the retained byte limit; omitted fields cannot prove presence or absence. Use /debug/request-cursor for request count deltas, or /debug/requests?after= for a later window.", + maxBytes: AIMOCK_DEBUG_FACTS_MAX_BYTES, + requests: incomplete, + }); + return true; + } + writeJson( + res, + 200, + pathname === "/requests" + ? snapshots + : (snapshots[0] ?? { ok: false, error: "no request recorded" }), + ); + return true; }, }; } diff --git a/qa/scenarios/memory/remember-across-conversations.yaml b/qa/scenarios/memory/remember-across-conversations.yaml index 9edb97748684..a30a747b4e8a 100644 --- a/qa/scenarios/memory/remember-across-conversations.yaml +++ b/qa/scenarios/memory/remember-across-conversations.yaml @@ -245,9 +245,9 @@ flow: - ref: transcriptRoot - recursive: true force: true - - set: requestCountBeforeRecall + - set: requestCursorBeforeRecall value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0" + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - set: targetStartIndex value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" @@ -286,7 +286,7 @@ flow: - utf8 - set: recallRequests value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeRecall) : []" + expr: "env.mock ? await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeRecall}`) : []" - set: recallRequestDebug value: expr: "recallRequests.map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: String(request.toolOutput ?? '').slice(0, 1200), finalText: String(request.finalText ?? '').slice(0, 300), allInputText: String(request.allInputText ?? '').slice(-500) }))" @@ -329,9 +329,9 @@ flow: - set: helperCountBeforeGroupDestination value: expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length" - - set: groupRequestCountBefore + - set: groupRequestCursorBefore value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - sendInbound: conversation: id: @@ -347,7 +347,7 @@ flow: args: - lambda: async: true - expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > groupRequestCountBefore ? requests.slice(groupRequestCountBefore) : undefined; })()" + expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${groupRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()" - expr: liveTurnTimeoutMs(env, 60000) - 100 - set: helperCountAfterGroupDestination @@ -391,9 +391,9 @@ flow: - set: helperCountBeforePaused value: expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length" - - set: pausedRequestCountBefore + - set: pausedRequestCursorBefore value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - sendInbound: conversation: id: @@ -409,7 +409,7 @@ flow: args: - lambda: async: true - expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > pausedRequestCountBefore ? requests.slice(pausedRequestCountBefore) : undefined; })()" + expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${pausedRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()" - expr: liveTurnTimeoutMs(env, 60000) - 100 - set: helperCountAfterPaused @@ -422,9 +422,9 @@ flow: expr: "pausedRequests.every((request) => !String(request.allInputText ?? '').includes(config.privateFact) && !String(request.allInputText ?? '').includes(''))" message: expr: "`paused conversation received private recall context: ${JSON.stringify(pausedRequests)}`" - - set: freshRequestCountBefore + - set: freshRequestCursorBefore value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - sendInbound: conversation: id: @@ -440,7 +440,7 @@ flow: args: - lambda: async: true - expr: "await (async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(freshRequestCountBefore); return requests.some((request) => String(request.allInputText ?? '').includes('')) ? requests : undefined; })()" + expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${freshRequestCursorBefore}`); return requests.some((request) => String(request.allInputText ?? '').includes('')) ? requests : undefined; })()" - expr: liveTurnTimeoutMs(env, 60000) - 100 - set: helperCountAfterFresh @@ -472,9 +472,9 @@ flow: - set: helperCountBeforeDisabled value: expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length" - - set: disabledRequestCountBefore + - set: disabledRequestCursorBefore value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length" + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" - sendInbound: conversation: id: @@ -490,7 +490,7 @@ flow: args: - lambda: async: true - expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > disabledRequestCountBefore ? requests.slice(disabledRequestCountBefore) : undefined; })()" + expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${disabledRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()" - expr: liveTurnTimeoutMs(env, 60000) - 100 - set: helperCountAfterDisabled