mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-27 12:56:01 -06:00
fix(qa): preserve truthful debug snapshots for large AIMock requests (#130605)
This commit is contained in:
committed by
GitHub
parent
e3c78c3487
commit
c578d324ef
@@ -15,6 +15,323 @@ function makeResponsesInput(text: string) {
|
||||
}
|
||||
|
||||
describe("qa aimock server", () => {
|
||||
it("keeps complete large input when the upstream body is still retained", async () => {
|
||||
const server = await startQaAimockServer();
|
||||
const prompt = "u".repeat(40_000);
|
||||
try {
|
||||
const response = await fetch(`${server.baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "aimock/gpt-5.6-luna",
|
||||
stream: false,
|
||||
input: [makeResponsesInput(prompt)],
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
await response.json();
|
||||
const debug = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debug.status).toBe(200);
|
||||
const snapshot = await debug.json();
|
||||
expect(snapshot).toMatchObject({ prompt, allInputText: prompt });
|
||||
expect(snapshot.body).not.toHaveProperty("__aimock_truncated");
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["chat", "responses"])(
|
||||
"retains exact %s request facts when tool schemas exceed the upstream journal cap",
|
||||
async (dialect) => {
|
||||
const server = await startQaAimockServer();
|
||||
const model = "aimock/gpt-5.6-luna";
|
||||
const prompt = "current user marker";
|
||||
const tool = { name: "echo", description: "schema".repeat(14_000), parameters: {} };
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${server.baseUrl}/v1/${dialect === "chat" ? "chat/completions" : "responses"}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
stream: false,
|
||||
...(dialect === "chat"
|
||||
? {
|
||||
messages: [
|
||||
{ role: "user", content: prompt },
|
||||
{ role: "tool", content: "observed tool output", tool_call_id: "call-qa" },
|
||||
],
|
||||
tools: [{ type: "function", function: tool }],
|
||||
}
|
||||
: {
|
||||
input: [
|
||||
makeResponsesInput(prompt),
|
||||
{
|
||||
type: "function_call_output",
|
||||
output: "observed tool output",
|
||||
call_id: "call-qa",
|
||||
},
|
||||
],
|
||||
tools: [{ type: "function", ...tool }],
|
||||
}),
|
||||
}),
|
||||
},
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
await response.json();
|
||||
const debug = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debug.status).toBe(200);
|
||||
const snapshot = await debug.json();
|
||||
expect(snapshot).toMatchObject({
|
||||
model,
|
||||
prompt,
|
||||
allInputText: `${prompt}\nobserved tool output`,
|
||||
toolOutput: "observed tool output",
|
||||
toolOutputCallId: "call-qa",
|
||||
providerVariant: "openai",
|
||||
body: { __aimock_truncated: true },
|
||||
});
|
||||
expect(JSON.parse(snapshot.raw)).toEqual(snapshot.body);
|
||||
expect(snapshot.body.originalByteSize).toBeGreaterThan(64 * 1024);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ role: "system", content: "s".repeat(70_000), omittedFields: ["allInputText"] },
|
||||
{ role: "user", content: "😀".repeat(20_000), omittedFields: ["prompt", "allInputText"] },
|
||||
])("reports incomplete $role text without poisoning later queries or metadata", async (input) => {
|
||||
const server = await startQaAimockServer();
|
||||
const post = async (messages: Array<{ role: string; content: string }>) => {
|
||||
const response = await fetch(`${server.baseUrl}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ model: "aimock/gpt-5.6-luna", stream: false, messages }),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
await response.json();
|
||||
};
|
||||
try {
|
||||
await post([
|
||||
...(input.role === "system" ? [{ role: "user", content: "exact current user" }] : []),
|
||||
{ role: input.role, content: input.content },
|
||||
]);
|
||||
for (const endpoint of ["last-request", "requests", "requests?after=0"]) {
|
||||
const debug = await fetch(`${server.baseUrl}/debug/${endpoint}`);
|
||||
expect(debug.status).toBe(413);
|
||||
const incomplete = await debug.json();
|
||||
expect(incomplete).toMatchObject({
|
||||
code: "QA_DEBUG_SNAPSHOT_INCOMPLETE",
|
||||
maxBytes: 64 * 1024,
|
||||
requests: [{ cursor: 1, omittedFields: input.omittedFields }],
|
||||
});
|
||||
expect(incomplete.error).toContain("/debug/request-cursor");
|
||||
expect(incomplete.requests[0].facts.model).toBe("aimock/gpt-5.6-luna");
|
||||
for (const field of input.omittedFields) {
|
||||
expect(incomplete.requests[0].facts).not.toHaveProperty(field);
|
||||
}
|
||||
if (input.role === "system") {
|
||||
expect(incomplete.requests[0].facts.prompt).toBe("exact current user");
|
||||
}
|
||||
expect(Buffer.byteLength(JSON.stringify(incomplete.requests[0]))).toBeLessThan(64 * 1024);
|
||||
}
|
||||
expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({
|
||||
cursor: 1,
|
||||
});
|
||||
const images = await fetch(`${server.baseUrl}/debug/image-generations`);
|
||||
expect(images.status).toBe(200);
|
||||
expect(await images.json()).toEqual([]);
|
||||
expect((await fetch(`${server.baseUrl}/debug/requests?after=1.5`)).status).toBe(400);
|
||||
expect((await fetch(`${server.baseUrl}/debug/requests?after=2`)).status).toBe(409);
|
||||
|
||||
await post([{ role: "user", content: "later complete request" }]);
|
||||
const latest = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(latest.status).toBe(200);
|
||||
expect(await latest.json()).toMatchObject({ prompt: "later complete request" });
|
||||
const after = await fetch(`${server.baseUrl}/debug/requests?after=1`);
|
||||
expect(after.status).toBe(200);
|
||||
expect(await after.json()).toMatchObject([{ prompt: "later complete request" }]);
|
||||
expect((await fetch(`${server.baseUrl}/debug/requests`)).status).toBe(413);
|
||||
|
||||
const reset = await fetch(`${server.baseUrl}/__aimock/reset/journal`, { method: "POST" });
|
||||
expect(reset.status).toBe(200);
|
||||
expect(await fetch(`${server.baseUrl}/debug/requests`).then((r) => r.json())).toEqual([]);
|
||||
expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({
|
||||
cursor: 2,
|
||||
});
|
||||
await post([{ role: "user", content: "after reset" }]);
|
||||
expect(
|
||||
await fetch(`${server.baseUrl}/debug/requests?after=2`).then((r) => r.json()),
|
||||
).toMatchObject([{ prompt: "after reset" }]);
|
||||
expect((await fetch(`${server.baseUrl}/__aimock/reset`, { method: "POST" })).status).toBe(
|
||||
200,
|
||||
);
|
||||
expect(await fetch(`${server.baseUrl}/debug/requests`).then((r) => r.json())).toEqual([]);
|
||||
expect(await fetch(`${server.baseUrl}/debug/request-cursor`).then((r) => r.json())).toEqual({
|
||||
cursor: 3,
|
||||
});
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves image evidence and tool-call pairing across capped journal bodies", async () => {
|
||||
const server = await startQaAimockServer();
|
||||
const prompt = "inspect and use echo";
|
||||
const user = {
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: prompt },
|
||||
{ type: "image_url", image_url: { url: "data:image/png;base64,cWE=" } },
|
||||
],
|
||||
};
|
||||
const post = async (messages: unknown[]) => {
|
||||
const response = await fetch(`${server.baseUrl}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "aimock/gpt-5.6-luna",
|
||||
stream: false,
|
||||
messages,
|
||||
tools: [
|
||||
{
|
||||
type: "function",
|
||||
function: { name: "echo", description: "t".repeat(70_000), parameters: {} },
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
return response.json();
|
||||
};
|
||||
try {
|
||||
expect(
|
||||
(await fetch(`${server.baseUrl}/__aimock/fixtures`, { method: "DELETE" })).status,
|
||||
).toBe(200);
|
||||
const configured = await fetch(`${server.baseUrl}/__aimock/fixtures`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
fixtures: [
|
||||
{
|
||||
match: { sequenceIndex: 0 },
|
||||
response: { toolCalls: [{ name: "echo", arguments: "{}" }] },
|
||||
},
|
||||
{ match: { sequenceIndex: 1 }, response: { content: "tool observed" } },
|
||||
],
|
||||
}),
|
||||
});
|
||||
expect(configured.status).toBe(200);
|
||||
const planned = await post([user]);
|
||||
const assistant = planned.choices[0].message;
|
||||
const toolCallId = assistant.tool_calls[0].id;
|
||||
await post([
|
||||
user,
|
||||
assistant,
|
||||
{
|
||||
role: "tool",
|
||||
content: "independent tool evidence",
|
||||
tool_call_id: toolCallId,
|
||||
isError: true,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"<<<BEGIN_OPENCLAW_INTERNAL_CONTEXT>>>\nprivate runtime context\n<<<END_OPENCLAW_INTERNAL_CONTEXT>>>",
|
||||
},
|
||||
]);
|
||||
const requests = await fetch(`${server.baseUrl}/debug/requests`);
|
||||
expect(requests.status).toBe(200);
|
||||
expect(await requests.json()).toMatchObject([
|
||||
{
|
||||
prompt,
|
||||
imageInputCount: 1,
|
||||
plannedToolName: "echo",
|
||||
plannedToolCallId: toolCallId,
|
||||
body: { __aimock_truncated: true },
|
||||
},
|
||||
{
|
||||
prompt,
|
||||
imageInputCount: 1,
|
||||
toolOutput: "independent tool evidence",
|
||||
toolOutputCallId: toolCallId,
|
||||
toolOutputStructuredError: true,
|
||||
body: { __aimock_truncated: true },
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "complete", olderOverflow: false, olderPrompt: "first conversation" },
|
||||
{ label: "overflow", olderOverflow: true, olderPrompt: "first conversation" },
|
||||
{ label: "near-budget prompt", olderOverflow: true, olderPrompt: "u".repeat(65_280) },
|
||||
])(
|
||||
"keeps tool-result pairing before the selected cursor window ($label)",
|
||||
async ({ olderOverflow, olderPrompt }) => {
|
||||
const server = await startQaAimockServer();
|
||||
const post = async (messages: unknown[]) => {
|
||||
const response = await fetch(`${server.baseUrl}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ model: "aimock/gpt-5.6-luna", stream: false, messages }),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
return response.json();
|
||||
};
|
||||
try {
|
||||
await fetch(`${server.baseUrl}/__aimock/fixtures`, { method: "DELETE" });
|
||||
const fixtures = [0, 1].map((sequenceIndex) => ({
|
||||
match: { sequenceIndex },
|
||||
response: { toolCalls: [{ name: "echo", arguments: "{}" }] },
|
||||
}));
|
||||
const configured = await fetch(`${server.baseUrl}/__aimock/fixtures`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
fixtures: [...fixtures, { match: { sequenceIndex: 2 }, response: { content: "done" } }],
|
||||
}),
|
||||
});
|
||||
expect(configured.status).toBe(200);
|
||||
const first = await post([
|
||||
...(olderOverflow ? [{ role: "system", content: "s".repeat(70_000) }] : []),
|
||||
{ role: "user", content: olderPrompt },
|
||||
]);
|
||||
await post([{ role: "user", content: "second conversation" }]);
|
||||
const firstToolCallId = first.choices[0].message.tool_calls[0].id;
|
||||
await post([
|
||||
{ role: "user", content: "first conversation" },
|
||||
first.choices[0].message,
|
||||
{ role: "tool", content: "first result", tool_call_id: firstToolCallId },
|
||||
]);
|
||||
const after = await fetch(`${server.baseUrl}/debug/requests?after=1`);
|
||||
expect(after.status).toBe(200);
|
||||
const selected = await after.json();
|
||||
expect(selected).toHaveLength(2);
|
||||
expect(selected[0]).toMatchObject({
|
||||
prompt: "second conversation",
|
||||
plannedToolName: "echo",
|
||||
});
|
||||
expect(selected[0]).not.toHaveProperty("plannedToolCallId");
|
||||
expect(selected[1]).toMatchObject({ toolOutputCallId: firstToolCallId });
|
||||
if (olderOverflow) {
|
||||
const all = await fetch(`${server.baseUrl}/debug/requests`);
|
||||
expect(all.status).toBe(413);
|
||||
const incomplete = await all.json();
|
||||
expect(incomplete.requests[0].facts.plannedToolCallId).toBe(firstToolCallId);
|
||||
expect(incomplete.requests[0].omittedFields).not.toContain("plannedToolCallId");
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("serves OpenAI Responses text replies and debug request snapshots", async () => {
|
||||
const server = await startQaAimockServer({
|
||||
host: "127.0.0.1",
|
||||
@@ -107,7 +424,7 @@ describe("qa aimock server", () => {
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
});
|
||||
const post = async (text: string) => {
|
||||
const post = async (text: string, instructions?: string) => {
|
||||
const response = await fetch(`${server.baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
@@ -115,6 +432,7 @@ describe("qa aimock server", () => {
|
||||
model: "aimock/gpt-5.6-luna",
|
||||
stream: false,
|
||||
input: [makeResponsesInput(text)],
|
||||
instructions,
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
@@ -125,12 +443,13 @@ describe("qa aimock server", () => {
|
||||
).toEqual({ cursor: 0 });
|
||||
const debugRequestLimit = 1_000;
|
||||
for (let index = 0; index < debugRequestLimit; index += 1) {
|
||||
await post(`aimock cursor ${index}`);
|
||||
await post(`aimock cursor ${index}`, index === 0 ? "s".repeat(70_000) : undefined);
|
||||
}
|
||||
const cursor = await fetch(`${server.baseUrl}/debug/request-cursor`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
expect(cursor).toEqual({ cursor: debugRequestLimit });
|
||||
expect((await fetch(`${server.baseUrl}/debug/requests`)).status).toBe(413);
|
||||
await post("aimock cursor overflow");
|
||||
|
||||
const retained = (await fetch(`${server.baseUrl}/debug/requests`).then((response) =>
|
||||
|
||||
@@ -27,6 +27,23 @@ type AimockRequestSnapshot = {
|
||||
};
|
||||
|
||||
const AIMOCK_DEBUG_REQUEST_LIMIT = 1_000;
|
||||
const AIMOCK_DEBUG_FACTS_MAX_BYTES = 64 * 1024;
|
||||
|
||||
type AimockRequestFacts = Omit<AimockRequestSnapshot, "raw" | "body">;
|
||||
type AimockRequestProjection =
|
||||
| { complete: true; facts: AimockRequestFacts }
|
||||
| {
|
||||
complete: false;
|
||||
facts: Partial<AimockRequestFacts>;
|
||||
omittedFields: Array<keyof AimockRequestFacts>;
|
||||
};
|
||||
type AimockToolFacts = Pick<
|
||||
AimockRequestFacts,
|
||||
"plannedToolName" | "plannedToolCallId" | "toolOutputCallId"
|
||||
>;
|
||||
type AimockRequestObservation =
|
||||
| { kind: "retained-body"; tools: AimockToolFacts }
|
||||
| { kind: "projected"; projection: AimockRequestProjection };
|
||||
|
||||
// Runtime-context delimiters are owned by src/agents/internal-runtime-context.ts.
|
||||
// This mock mirrors the wire shape so delimiter drift fails through QA timeouts.
|
||||
@@ -139,7 +156,7 @@ function resolveProviderVariant(model: string): AimockRequestSnapshot["providerV
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function extractPlannedToolName(entry: JournalEntry) {
|
||||
function extractPlannedToolName(entry: Pick<JournalEntry, "response">) {
|
||||
const response = entry.response.fixture?.response as
|
||||
| { toolCalls?: Array<{ name?: unknown }> }
|
||||
| undefined;
|
||||
@@ -147,7 +164,7 @@ function extractPlannedToolName(entry: JournalEntry) {
|
||||
return typeof name === "string" && name.length > 0 ? name : undefined;
|
||||
}
|
||||
|
||||
function extractPlannedToolCallId(entry: JournalEntry) {
|
||||
function extractPlannedToolCallId(entry: Pick<JournalEntry, "response">) {
|
||||
const response = entry.response.fixture?.response as
|
||||
| { toolCalls?: Array<{ id?: unknown; callId?: unknown; toolCallId?: unknown }> }
|
||||
| undefined;
|
||||
@@ -158,53 +175,73 @@ function extractPlannedToolCallId(entry: JournalEntry) {
|
||||
return typeof candidate === "string" && candidate.length > 0 ? candidate : undefined;
|
||||
}
|
||||
|
||||
function toRequestSnapshot(entry: JournalEntry): AimockRequestSnapshot {
|
||||
const body = entry.body ?? null;
|
||||
function extractRequestFacts(
|
||||
body: JournalEntry["body"],
|
||||
tools: AimockToolFacts,
|
||||
): AimockRequestFacts {
|
||||
const model = typeof body?.model === "string" ? body.model : "";
|
||||
return {
|
||||
raw: JSON.stringify(body ?? {}),
|
||||
body: (body ?? {}) as Record<string, unknown>,
|
||||
prompt: extractLastUserText(body),
|
||||
allInputText: extractAllInputText(body),
|
||||
toolOutput: extractToolOutput(body),
|
||||
...tools,
|
||||
model,
|
||||
prompt: extractLastUserText(body),
|
||||
providerVariant: resolveProviderVariant(model),
|
||||
imageInputCount: countImageInputs(requestMessages(body)),
|
||||
plannedToolCallId: extractPlannedToolCallId(entry),
|
||||
plannedToolName: extractPlannedToolName(entry),
|
||||
toolOutputCallId: extractToolOutputCallId(body) || undefined,
|
||||
...(extractToolOutputStructuredError(body) ? { toolOutputStructuredError: true } : {}),
|
||||
toolOutput: extractToolOutput(body),
|
||||
allInputText: extractAllInputText(body),
|
||||
};
|
||||
}
|
||||
|
||||
function toRequestSnapshots(entries: JournalEntry[]): AimockRequestSnapshot[] {
|
||||
const snapshots = entries.map((entry) => toRequestSnapshot(entry));
|
||||
function boundRequestFacts(projection: AimockRequestProjection): AimockRequestProjection {
|
||||
if (Buffer.byteLength(JSON.stringify(projection)) <= AIMOCK_DEBUG_FACTS_MAX_BYTES) {
|
||||
return projection;
|
||||
}
|
||||
const { facts } = projection;
|
||||
const retained: Partial<AimockRequestFacts> = {};
|
||||
const fields = Object.keys(facts) as Array<keyof AimockRequestFacts>;
|
||||
const omittedFields = projection.complete ? [] : [...projection.omittedFields];
|
||||
const reservedOmissions = [...omittedFields, ...fields];
|
||||
// Reserve correlation facts before text: an older overflowing prompt must not
|
||||
// redirect its tool result to a newer plan. Omission names share the byte budget.
|
||||
for (const field of fields) {
|
||||
const candidate = { ...retained, [field]: facts[field] };
|
||||
const diagnostic = {
|
||||
complete: false,
|
||||
cursor: Number.MAX_SAFE_INTEGER,
|
||||
facts: candidate,
|
||||
omittedFields: reservedOmissions,
|
||||
};
|
||||
if (Buffer.byteLength(JSON.stringify(diagnostic)) <= AIMOCK_DEBUG_FACTS_MAX_BYTES) {
|
||||
Object.assign(retained, { [field]: facts[field] });
|
||||
} else {
|
||||
omittedFields.push(field);
|
||||
}
|
||||
}
|
||||
return { complete: false, facts: retained, omittedFields };
|
||||
}
|
||||
|
||||
function resolvePlannedToolCallIds(snapshots: AimockToolFacts[]): Map<number, string> {
|
||||
const callIds = new Map<number, string>();
|
||||
const pendingPlannedIndexes: number[] = [];
|
||||
for (const [index, snapshot] of snapshots.entries()) {
|
||||
if (snapshot.toolOutputCallId && pendingPlannedIndexes.length > 0) {
|
||||
const plannedIndex = pendingPlannedIndexes.shift();
|
||||
if (plannedIndex !== undefined) {
|
||||
const plannedSnapshot = snapshots[plannedIndex];
|
||||
if (!plannedSnapshot) {
|
||||
continue;
|
||||
}
|
||||
snapshots[plannedIndex] = {
|
||||
...plannedSnapshot,
|
||||
plannedToolCallId: snapshot.toolOutputCallId,
|
||||
};
|
||||
callIds.set(plannedIndex, snapshot.toolOutputCallId);
|
||||
}
|
||||
}
|
||||
if (snapshot.plannedToolName && !snapshot.plannedToolCallId) {
|
||||
pendingPlannedIndexes.push(index);
|
||||
}
|
||||
}
|
||||
return snapshots;
|
||||
return callIds;
|
||||
}
|
||||
|
||||
function createDebugMount(): Mountable {
|
||||
let journal: Journal | undefined;
|
||||
let nextRequestCursor = 1;
|
||||
const requestCursors = new Map<string, number>();
|
||||
const observations = new WeakMap<JournalEntry, AimockRequestObservation>();
|
||||
|
||||
return {
|
||||
setJournal(nextJournal) {
|
||||
@@ -219,7 +256,26 @@ function createDebugMount(): Mountable {
|
||||
// AIMock evicts its request journal FIFO. Assign cursors at insertion time
|
||||
// so the debug boundary remains monotonic after retained entries rotate.
|
||||
journal.add = (entry) => {
|
||||
const tools: AimockToolFacts = {
|
||||
plannedToolName: extractPlannedToolName(entry),
|
||||
plannedToolCallId: extractPlannedToolCallId(entry),
|
||||
toolOutputCallId: extractToolOutputCallId(entry.body) || undefined,
|
||||
};
|
||||
const recorded = addJournalEntry(entry);
|
||||
// Upstream keeps <=64 KiB bodies intact; only discarded bodies need an
|
||||
// extra bounded projection. Weak entry ownership follows eviction/reset.
|
||||
observations.set(
|
||||
recorded,
|
||||
recorded.body === entry.body
|
||||
? { kind: "retained-body", tools }
|
||||
: {
|
||||
kind: "projected",
|
||||
projection: boundRequestFacts({
|
||||
complete: true,
|
||||
facts: extractRequestFacts(entry.body, tools),
|
||||
}),
|
||||
},
|
||||
);
|
||||
requestCursors.set(recorded.id, nextRequestCursor++);
|
||||
if (requestCursors.size > AIMOCK_DEBUG_REQUEST_LIMIT) {
|
||||
const oldestRequestId = requestCursors.keys().next().value;
|
||||
@@ -231,68 +287,11 @@ function createDebugMount(): Mountable {
|
||||
};
|
||||
},
|
||||
async handleRequest(req: IncomingMessage, res: ServerResponse, pathname: string) {
|
||||
const entries = journal?.getAll() ?? [];
|
||||
const snapshots = toRequestSnapshots(entries);
|
||||
const cursorSnapshots = entries.map((entry, index) => {
|
||||
const cursor = requestCursors.get(entry.id);
|
||||
if (cursor === undefined) {
|
||||
throw new Error(`AIMock debug request cursor missing for ${entry.id}`);
|
||||
}
|
||||
const snapshot = snapshots[index];
|
||||
if (!snapshot) {
|
||||
throw new Error(`AIMock debug request snapshot missing for ${entry.id}`);
|
||||
}
|
||||
return { cursor, snapshot };
|
||||
});
|
||||
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
||||
if (pathname === "/last-request") {
|
||||
const lastSnapshot = snapshots.at(-1);
|
||||
writeJson(res, 200, lastSnapshot ?? { ok: false, error: "no request recorded" });
|
||||
return true;
|
||||
}
|
||||
if (pathname === "/request-cursor") {
|
||||
writeJson(res, 200, { cursor: nextRequestCursor - 1 });
|
||||
return true;
|
||||
}
|
||||
if (pathname === "/requests") {
|
||||
const afterText = url.searchParams.get("after");
|
||||
if (afterText === null) {
|
||||
writeJson(res, 200, snapshots);
|
||||
return true;
|
||||
}
|
||||
const after = parseQaDebugRequestCursor(afterText);
|
||||
if (after === null) {
|
||||
writeJson(res, 400, { error: "after must be a non-negative safe integer" });
|
||||
return true;
|
||||
}
|
||||
const latestCursor = nextRequestCursor - 1;
|
||||
const oldestCursor = cursorSnapshots[0]?.cursor ?? nextRequestCursor;
|
||||
if (after > latestCursor) {
|
||||
writeJson(res, 409, {
|
||||
error: "request cursor is ahead of the latest recorded request",
|
||||
after,
|
||||
latestCursor,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (after < oldestCursor - 1) {
|
||||
writeJson(res, 409, {
|
||||
error: "request cursor expired",
|
||||
after,
|
||||
oldestCursor,
|
||||
latestCursor,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
writeJson(
|
||||
res,
|
||||
200,
|
||||
cursorSnapshots
|
||||
.filter((request) => request.cursor > after)
|
||||
.map((request) => request.snapshot),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
const entries = journal?.getAll() ?? [];
|
||||
if (pathname === "/image-generations") {
|
||||
writeJson(
|
||||
res,
|
||||
@@ -303,7 +302,100 @@ function createDebugMount(): Mountable {
|
||||
);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
if (pathname !== "/last-request" && pathname !== "/requests") {
|
||||
return false;
|
||||
}
|
||||
let selected = entries.map((entry, index) => {
|
||||
const cursor = requestCursors.get(entry.id);
|
||||
const observation = observations.get(entry);
|
||||
if (cursor === undefined || observation === undefined) {
|
||||
throw new Error(`AIMock debug request observation missing for ${entry.id}`);
|
||||
}
|
||||
return { cursor, entry, observation, index };
|
||||
});
|
||||
// Pair against retained tool facts before selecting a window: a result
|
||||
// inside the window may belong to a plan before its cursor.
|
||||
const plannedToolCallIds = resolvePlannedToolCallIds(
|
||||
selected.map(({ observation }) =>
|
||||
observation.kind === "retained-body" ? observation.tools : observation.projection.facts,
|
||||
),
|
||||
);
|
||||
if (pathname === "/requests") {
|
||||
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
||||
const afterText = url.searchParams.get("after");
|
||||
if (afterText !== null) {
|
||||
const after = parseQaDebugRequestCursor(afterText);
|
||||
if (after === null) {
|
||||
writeJson(res, 400, { error: "after must be a non-negative safe integer" });
|
||||
return true;
|
||||
}
|
||||
const latestCursor = nextRequestCursor - 1;
|
||||
const oldestCursor = selected[0]?.cursor ?? nextRequestCursor;
|
||||
if (after > latestCursor) {
|
||||
writeJson(res, 409, {
|
||||
error: "request cursor is ahead of the latest recorded request",
|
||||
after,
|
||||
latestCursor,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (after < oldestCursor - 1) {
|
||||
writeJson(res, 409, {
|
||||
error: "request cursor expired",
|
||||
after,
|
||||
oldestCursor,
|
||||
latestCursor,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
selected = selected.filter((request) => request.cursor > after);
|
||||
}
|
||||
} else {
|
||||
selected = selected.slice(-1);
|
||||
}
|
||||
const snapshots: AimockRequestSnapshot[] = [];
|
||||
const incomplete: Array<
|
||||
{ cursor: number } & Extract<AimockRequestProjection, { complete: false }>
|
||||
> = [];
|
||||
for (const { cursor, entry, observation, index } of selected) {
|
||||
const plannedToolCallId = plannedToolCallIds.get(index);
|
||||
let projection: AimockRequestProjection =
|
||||
observation.kind === "retained-body"
|
||||
? { complete: true, facts: extractRequestFacts(entry.body, observation.tools) }
|
||||
: observation.projection;
|
||||
if (plannedToolCallId) {
|
||||
projection = projection.complete
|
||||
? { complete: true, facts: { ...projection.facts, plannedToolCallId } }
|
||||
: { ...projection, facts: { ...projection.facts, plannedToolCallId } };
|
||||
if (observation.kind === "projected") {
|
||||
projection = boundRequestFacts(projection);
|
||||
}
|
||||
}
|
||||
if (!projection.complete) {
|
||||
incomplete.push({ cursor, ...projection });
|
||||
continue;
|
||||
}
|
||||
const body = entry.body ?? {};
|
||||
snapshots.push({ raw: JSON.stringify(body), body, ...projection.facts });
|
||||
}
|
||||
if (incomplete.length > 0) {
|
||||
writeJson(res, 413, {
|
||||
code: "QA_DEBUG_SNAPSHOT_INCOMPLETE",
|
||||
error:
|
||||
"Semantic facts exceeded the retained byte limit; omitted fields cannot prove presence or absence. Use /debug/request-cursor for request count deltas, or /debug/requests?after=<cursor> for a later window.",
|
||||
maxBytes: AIMOCK_DEBUG_FACTS_MAX_BYTES,
|
||||
requests: incomplete,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
writeJson(
|
||||
res,
|
||||
200,
|
||||
pathname === "/requests"
|
||||
? snapshots
|
||||
: (snapshots[0] ?? { ok: false, error: "no request recorded" }),
|
||||
);
|
||||
return true;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -245,9 +245,9 @@ flow:
|
||||
- ref: transcriptRoot
|
||||
- recursive: true
|
||||
force: true
|
||||
- set: requestCountBeforeRecall
|
||||
- set: requestCursorBeforeRecall
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length : 0"
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
|
||||
- set: targetStartIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
@@ -286,7 +286,7 @@ flow:
|
||||
- utf8
|
||||
- set: recallRequests
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(requestCountBeforeRecall) : []"
|
||||
expr: "env.mock ? await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeRecall}`) : []"
|
||||
- set: recallRequestDebug
|
||||
value:
|
||||
expr: "recallRequests.map((request) => ({ plannedToolName: request.plannedToolName ?? null, plannedToolArgs: request.plannedToolArgs ?? null, toolOutput: String(request.toolOutput ?? '').slice(0, 1200), finalText: String(request.finalText ?? '').slice(0, 300), allInputText: String(request.allInputText ?? '').slice(-500) }))"
|
||||
@@ -329,9 +329,9 @@ flow:
|
||||
- set: helperCountBeforeGroupDestination
|
||||
value:
|
||||
expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length"
|
||||
- set: groupRequestCountBefore
|
||||
- set: groupRequestCursorBefore
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- sendInbound:
|
||||
conversation:
|
||||
id:
|
||||
@@ -347,7 +347,7 @@ flow:
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > groupRequestCountBefore ? requests.slice(groupRequestCountBefore) : undefined; })()"
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${groupRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- 100
|
||||
- set: helperCountAfterGroupDestination
|
||||
@@ -391,9 +391,9 @@ flow:
|
||||
- set: helperCountBeforePaused
|
||||
value:
|
||||
expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length"
|
||||
- set: pausedRequestCountBefore
|
||||
- set: pausedRequestCursorBefore
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- sendInbound:
|
||||
conversation:
|
||||
id:
|
||||
@@ -409,7 +409,7 @@ flow:
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > pausedRequestCountBefore ? requests.slice(pausedRequestCountBefore) : undefined; })()"
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${pausedRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- 100
|
||||
- set: helperCountAfterPaused
|
||||
@@ -422,9 +422,9 @@ flow:
|
||||
expr: "pausedRequests.every((request) => !String(request.allInputText ?? '').includes(config.privateFact) && !String(request.allInputText ?? '').includes('<active_memory_plugin>'))"
|
||||
message:
|
||||
expr: "`paused conversation received private recall context: ${JSON.stringify(pausedRequests)}`"
|
||||
- set: freshRequestCountBefore
|
||||
- set: freshRequestCursorBefore
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- sendInbound:
|
||||
conversation:
|
||||
id:
|
||||
@@ -440,7 +440,7 @@ flow:
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).slice(freshRequestCountBefore); return requests.some((request) => String(request.allInputText ?? '').includes('<active_memory_plugin>')) ? requests : undefined; })()"
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${freshRequestCursorBefore}`); return requests.some((request) => String(request.allInputText ?? '').includes('<active_memory_plugin>')) ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- 100
|
||||
- set: helperCountAfterFresh
|
||||
@@ -472,9 +472,9 @@ flow:
|
||||
- set: helperCountBeforeDisabled
|
||||
value:
|
||||
expr: "(await fs.readdir(transcriptRoot).catch(() => [])).filter((entry) => entry.endsWith('.jsonl')).length"
|
||||
- set: disabledRequestCountBefore
|
||||
- set: disabledRequestCursorBefore
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests`)).length"
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- sendInbound:
|
||||
conversation:
|
||||
id:
|
||||
@@ -490,7 +490,7 @@ flow:
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests`); return requests.length > disabledRequestCountBefore ? requests.slice(disabledRequestCountBefore) : undefined; })()"
|
||||
expr: "await (async () => { const requests = await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${disabledRequestCursorBefore}`); return requests.length > 0 ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 60000)
|
||||
- 100
|
||||
- set: helperCountAfterDisabled
|
||||
|
||||
Reference in New Issue
Block a user