diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index 545d6f772661..77473ed6ca39 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -8,6 +8,10 @@ import { writeJson } from "../shared/http-json.js"; export type ResponsesInputItem = Record; export type MockOpenAiRequestKind = "agent-initial" | "compaction-summary" | "tool-continuation"; +export type MockCompactionSummaryFaultMode = + | "none" + | "empty-output-once" + | "reasoning-only-output-once"; type MockOpenAiRequestOutcome = "success" | "error"; @@ -127,6 +131,7 @@ export type MockOpenAiRequestSnapshot = { providerVariant: MockOpenAiProviderVariant; imageInputCount: number; requestKind: MockOpenAiRequestKind; + compactionSummaryFaultMode: MockCompactionSummaryFaultMode; outcome: MockOpenAiRequestOutcome; errorCode?: string; rawByteLength: number; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts index 0ff83045c477..150b28e387c4 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.test.ts @@ -6,6 +6,8 @@ import { startQaMockOpenAiServer } from "./server.js"; const cleanups: Array<() => Promise> = []; const QA_COMPACTION_RETRY_PROMPT = "Compaction retry mutating tool check. Current durable context marker: QA-COMPACTION-DURABLE-MARKER. Create compaction-retry-summary.txt."; +const QA_COMPACTION_SUMMARY_INSTRUCTIONS = + "You are a context summarization assistant. Produce a structured summary. Do not continue."; afterEach(async () => { while (cleanups.length > 0) { @@ -409,6 +411,49 @@ describe("QA mock OpenAI Responses WebSocket", () => { ]); }); + it.each([ + { + faultMode: "empty-output-once", + marker: "QA-COMPACTION-EMPTY-OUTPUT-ONCE-websocket", + recoveredMarker: "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY", + }, + { + faultMode: "reasoning-only-output-once", + marker: "QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE-websocket", + recoveredMarker: "QA-COMPACTION-REASONING-RECOVERED-SUMMARY", + }, + ] as const)("preserves $faultMode compaction recovery on WebSocket", async (testCase) => { + const server = await startServer(); + const socket = await connectResponsesWebSocket(server.baseUrl); + const request = { + type: "response.create", + model: "gpt-5.6-sol", + stream: true, + instructions: QA_COMPACTION_SUMMARY_INSTRUCTIONS, + input: [ + { + role: "user", + content: [ + { type: "input_text", text: `${testCase.marker}` }, + ], + }, + ], + }; + + const first = readCompletedResponse(await collectResponseEvents(socket, request)); + expect(JSON.stringify(first.output)).not.toContain(testCase.recoveredMarker); + const recovered = readCompletedResponse(await collectResponseEvents(socket, request)); + expect(JSON.stringify(recovered.output)).toContain(testCase.recoveredMarker); + + const requests = (await fetch(`${server.baseUrl}/debug/requests`).then((response) => + response.json(), + )) as Array>; + expect(requests.map((entry) => entry.compactionSummaryFaultMode)).toEqual([ + testCase.faultMode, + "none", + ]); + }); + it("rejects a response delta whose previous response belongs to another connection", async () => { const server = await startServer(); const socket = await connectResponsesWebSocket(server.baseUrl); diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index ddac11c8a093..a8630f816361 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -62,6 +62,8 @@ const QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT = { const QA_COMPACTION_RETRY_PROMPT = "Compaction retry mutating tool check. Current durable context marker: QA-COMPACTION-DURABLE-MARKER. Create compaction-retry-summary.txt."; const QA_COMPACTION_RETRY_OVERFLOW_PADDING = "x".repeat(300_000); +const QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY"; +const QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-REASONING-RECOVERED-SUMMARY"; const QA_COMPACTION_SUMMARY_INSTRUCTIONS = `You are a context summarization assistant. Your task is to read a conversation between a user and an AI assistant, then produce a structured summary following the exact format specified. Do NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.`; @@ -2350,6 +2352,150 @@ describe("qa mock openai server", () => { expect(summaryRequest.allInputText).toContain("[Chunk 1 - oldest messages]"); }); + it.each([ + { + faultMode: "empty-output-once", + markerPrefix: "QA-COMPACTION-EMPTY-OUTPUT-ONCE", + recoveredMarker: QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER, + reasoningOnly: false, + }, + { + faultMode: "reasoning-only-output-once", + markerPrefix: "QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE", + recoveredMarker: QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER, + reasoningOnly: true, + }, + ] as const)( + "injects one coded overflow for $faultMode scenario markers", + async ({ markerPrefix }) => { + const server = await startMockServer(); + const body = { + model: "gpt-5.6-luna", + instructions: `Runtime: embedded | sessionId=compaction-output-${markerPrefix}`, + input: [makeUserInput(`${markerPrefix}-http\n${"x".repeat(100_000)}`)], + }; + + const first = await postNonStreamingResponses(server, body); + expect(first.status).toBe(400); + expect(await first.json()).toMatchObject({ + error: { code: "context_length_exceeded" }, + }); + expect((await postNonStreamingResponses(server, body)).status).toBe(200); + + const requests = requireArray( + await getJson(server, "/debug/requests"), + "compaction output overflow requests", + ).map((request) => requireRecord(request, "compaction output overflow request")); + expect(requests).toHaveLength(2); + expect(Number(requests[0]?.rawByteLength)).toBeLessThan(256 * 1024); + expect(requests[0]).toMatchObject({ + requestKind: "agent-initial", + compactionSummaryFaultMode: "none", + outcome: "error", + errorCode: "context_length_exceeded", + }); + expect(requests[1]).toMatchObject({ + requestKind: "agent-initial", + compactionSummaryFaultMode: "none", + outcome: "success", + }); + }, + ); + + it.each([ + { + faultMode: "empty-output-once", + markerPrefix: "QA-COMPACTION-EMPTY-OUTPUT-ONCE", + recoveredMarker: QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER, + reasoningOnly: false, + }, + { + faultMode: "reasoning-only-output-once", + markerPrefix: "QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE", + recoveredMarker: QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER, + reasoningOnly: true, + }, + ] as const)( + "scopes $faultMode compaction faults to one scenario session", + async ({ faultMode, markerPrefix, recoveredMarker, reasoningOnly }) => { + const server = await startMockServer(); + const requestFor = (session: string) => ({ + model: "gpt-5.6-luna", + instructions: QA_COMPACTION_SUMMARY_INSTRUCTIONS, + input: `\n${markerPrefix}-${session}\nretain current work\n\n\nCreate a structured summary.`, + }); + + const first = await expectOpenAiNonStreamingResponsesJson(server, requestFor("session-a")); + if (reasoningOnly) { + expect(JSON.stringify(first)).toContain("reasoning_compaction_summary_fault"); + expect(JSON.stringify(first)).not.toContain(recoveredMarker); + } else { + expect(outputText(first)).toBe(""); + } + const recovered = await expectOpenAiNonStreamingResponsesJson( + server, + requestFor("session-a"), + ); + const recoveredText = outputText(recovered); + expect(recoveredText).toContain(recoveredMarker); + expect(recoveredText).toContain(`${markerPrefix}-session-a`); + expect(recoveredText).toContain("## Decisions"); + expect(recoveredText).toContain("## Open TODOs"); + expect(recoveredText).toContain("## Constraints/Rules"); + expect(recoveredText).toContain("## Pending user asks"); + expect(recoveredText).toContain("## Exact identifiers"); + const independent = await expectOpenAiNonStreamingResponsesJson( + server, + requestFor("session-b"), + ); + if (reasoningOnly) { + expect(JSON.stringify(independent)).toContain("reasoning_compaction_summary_fault"); + expect(JSON.stringify(independent)).not.toContain(recoveredMarker); + } else { + expect(outputText(independent)).toBe(""); + } + + const requests = requireArray( + await getJson(server, "/debug/requests"), + "compaction output fault requests", + ).map((request) => requireRecord(request, "compaction output fault request")); + expect(requests.map((request) => request.compactionSummaryFaultMode)).toEqual([ + faultMode, + "none", + faultMode, + ]); + expect(requests.every((request) => request.requestKind === "compaction-summary")).toBe(true); + }, + ); + + it("classifies developer-role compaction instructions before applying a typed fault", async () => { + const server = await startMockServer(); + const response = await expectOpenAiNonStreamingResponsesJson(server, { + model: "gpt-5.6-luna", + input: [ + { + role: "developer", + content: [{ type: "input_text", text: QA_COMPACTION_SUMMARY_INSTRUCTIONS }], + }, + makeUserInput( + "QA-COMPACTION-EMPTY-OUTPUT-ONCE-developer-wire", + ), + ], + }); + + expect(outputText(response)).toBe(""); + const requests = requireArray( + await getJson(server, "/debug/requests"), + "developer-role compaction requests", + ).map((request) => requireRecord(request, "developer-role compaction request")); + expect(requests).toEqual([ + expect.objectContaining({ + requestKind: "compaction-summary", + compactionSummaryFaultMode: "empty-output-once", + }), + ]); + }); + it("handles staged scalar compaction summaries and promotes the durable merge without state leakage", async () => { const server = await startMockServer(); const genericChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index bfcd306163db..ee34564eb1bf 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -22,6 +22,7 @@ import { type MockOpenAiRequestSnapshot, type MockOpenAiRequestSnapshotInput, type MockOpenAiRequestKind, + type MockCompactionSummaryFaultMode, type AnthropicMessagesRequest, TINY_PNG_BASE64, QA_REASONING_ONLY_RECOVERY_PROMPT_RE, @@ -177,8 +178,15 @@ const QA_COMPACTION_RETRY_PROMPT_RE = /compaction retry mutating tool check/i; const QA_COMPACTION_SUMMARY_INSTRUCTIONS_RE = /context summarization assistant[\s\S]*structured summary[\s\S]*do not continue/i; const QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES = 256 * 1024; +const QA_COMPACTION_OUTPUT_RECOVERY_OVERFLOW_THRESHOLD_BYTES = 96 * 1024; const QA_COMPACTION_RETRY_DURABLE_MARKER = "QA-COMPACTION-DURABLE-MARKER"; const QA_COMPACTION_RETRY_BULKY_MARKER = "QA-COMPACTION-BULKY-HISTORICAL-MARKER"; +const QA_COMPACTION_EMPTY_OUTPUT_ONCE_MARKER_RE = + /\bQA-COMPACTION-EMPTY-OUTPUT-ONCE-[A-Za-z0-9_-]+\b/u; +const QA_COMPACTION_REASONING_ONLY_OUTPUT_ONCE_MARKER_RE = + /\bQA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE-[A-Za-z0-9_-]+\b/u; +const QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY"; +const QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER = "QA-COMPACTION-REASONING-RECOVERED-SUMMARY"; const QA_COMPACTION_RETRY_SUMMARY = `## Goal Complete the compaction retry mutating tool check. @@ -228,6 +236,40 @@ Preserve the active conversation context. ## Critical Context - Refer to the retained recent turns for current task details.`; +const QA_COMPACTION_OUTPUT_RECOVERY_SUMMARY = `## Decisions +- Retry the typed compaction-summary fault at the compaction owner. + +## Open TODOs +- Continue the active task after compaction. + +## Constraints/Rules +- Preserve the historical recovery user block and current continuation. + +## Pending user asks +- Retain the historical recovery user block context. + +## Exact identifiers`; + +function resolveCompactionRecoverySummary(allInputText: string) { + const faultMarker = + QA_COMPACTION_EMPTY_OUTPUT_ONCE_MARKER_RE.exec(allInputText)?.[0] ?? + QA_COMPACTION_REASONING_ONLY_OUTPUT_ONCE_MARKER_RE.exec(allInputText)?.[0]; + const recoveryMarker = faultMarker?.startsWith("QA-COMPACTION-EMPTY-") + ? QA_COMPACTION_EMPTY_RECOVERY_SUMMARY_MARKER + : faultMarker + ? QA_COMPACTION_REASONING_RECOVERY_SUMMARY_MARKER + : undefined; + return recoveryMarker && faultMarker + ? `${QA_COMPACTION_OUTPUT_RECOVERY_SUMMARY}\n- ${recoveryMarker}\n- ${faultMarker}` + : QA_GENERIC_COMPACTION_SUMMARY; +} + +function hasCompactionOutputRecoveryMarker(allInputText: string) { + return ( + QA_COMPACTION_EMPTY_OUTPUT_ONCE_MARKER_RE.test(allInputText) || + QA_COMPACTION_REASONING_ONLY_OUTPUT_ONCE_MARKER_RE.test(allInputText) + ); +} const QA_STREAMING_TOOL_PROGRESS_FAMILY_PROMPT_RE = /(?:partial|quiet) streaming qa check|final-only marker streaming qa check|block streaming qa check|tool progress(?: error)? qa check/i; @@ -602,18 +644,53 @@ function classifyMockOpenAiRequest( input: ResponsesInputItem[], body: Record, ): MockOpenAiRequestKind { - if (QA_COMPACTION_SUMMARY_INSTRUCTIONS_RE.test(extractInstructionsText(body))) { + const instructionText = extractAllRequestTexts( + input.filter((item) => item.role === "developer" || item.role === "system"), + body, + ); + if (QA_COMPACTION_SUMMARY_INSTRUCTIONS_RE.test(instructionText)) { return "compaction-summary"; } return hasToolOutput(input) ? "tool-continuation" : "agent-initial"; } +function resolveCompactionSummaryFaultMode(params: { + allInputText: string; + requestKind: MockOpenAiRequestKind; + servedFaultMarkers: Set; +}): MockCompactionSummaryFaultMode { + if (params.requestKind !== "compaction-summary") { + return "none"; + } + const emptyMarker = QA_COMPACTION_EMPTY_OUTPUT_ONCE_MARKER_RE.exec(params.allInputText)?.[0]; + const reasoningMarker = QA_COMPACTION_REASONING_ONLY_OUTPUT_ONCE_MARKER_RE.exec( + params.allInputText, + )?.[0]; + const selected = emptyMarker + ? { + key: emptyMarker, + mode: "empty-output-once" as const, + } + : reasoningMarker + ? { + key: reasoningMarker, + mode: "reasoning-only-output-once" as const, + } + : undefined; + if (!selected?.key || params.servedFaultMarkers.has(selected.key)) { + return "none"; + } + params.servedFaultMarkers.add(selected.key); + return selected.mode; +} + async function buildResponsesPayload( body: Record, scenarioState: MockScenarioState, options: { waitForTerminalRequesterSettled?: (caseName: string, childSessionKey: string) => Promise; requestKind?: MockOpenAiRequestKind; + compactionSummaryFaultMode?: MockCompactionSummaryFaultMode; } = {}, ) { const providerVariant = resolveProviderVariant( @@ -651,10 +728,19 @@ async function buildResponsesPayload( allInputText.includes(QA_COMPACTION_RETRY_BULKY_MARKER); const requestKind = options.requestKind ?? classifyMockOpenAiRequest(input, body); if (requestKind === "compaction-summary") { + if (options.compactionSummaryFaultMode === "empty-output-once") { + return buildAssistantEvents(""); + } + if (options.compactionSummaryFaultMode === "reasoning-only-output-once") { + return buildReasoningOnlyEvents( + "Compaction summary reasoning completed without final summary text.", + "reasoning_compaction_summary_fault", + ); + } return buildAssistantEvents( hasCompactionRetryDurableContext ? QA_COMPACTION_RETRY_SUMMARY - : QA_GENERIC_COMPACTION_SUMMARY, + : resolveCompactionRecoverySummary(allInputText), ); } if ( @@ -2097,6 +2183,7 @@ export async function startQaMockOpenAiServer(params?: { const finalOnlyMarkerPauseMs = params?.finalOnlyMarkerPauseMs ?? 1_500; const terminalRequesterSettleGate = createTerminalRequesterSettleGate(); const scenarioStates = new Map(); + const servedCompactionSummaryFaultMarkers = new Set(); const scenarioStateFor = (body: Record): MockScenarioState => { const input = typeof body.input === "string" || Array.isArray(body.input) @@ -2149,10 +2236,18 @@ export async function startQaMockOpenAiServer(params?: { const allInputText = extractAllRequestTexts(input, request.body); const scenarioState = scenarioStateFor(request.body); const requestKind = classifyMockOpenAiRequest(input, request.body); + const compactionSummaryFaultMode = resolveCompactionSummaryFaultMode({ + allInputText, + requestKind, + servedFaultMarkers: servedCompactionSummaryFaultMarkers, + }); if (requestKind !== "compaction-summary" && QA_COMPACTION_RETRY_PROMPT_RE.test(allInputText)) { scenarioState.compactionRetryActive = true; } const rawByteLength = Buffer.byteLength(request.raw); + const compactionOverflowThresholdBytes = hasCompactionOutputRecoveryMarker(allInputText) + ? QA_COMPACTION_OUTPUT_RECOVERY_OVERFLOW_THRESHOLD_BYTES + : QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES; const resolvedModel = typeof request.body.model === "string" ? request.body.model : ""; const requestSnapshotBase = { raw: request.raw, @@ -2165,6 +2260,7 @@ export async function startQaMockOpenAiServer(params?: { providerVariant: resolveProviderVariant(resolvedModel), imageInputCount: countImageInputs(input), requestKind, + compactionSummaryFaultMode, rawByteLength, } satisfies Omit< MockOpenAiRequestSnapshotInput, @@ -2179,8 +2275,9 @@ export async function startQaMockOpenAiServer(params?: { >; if ( requestKind === "agent-initial" && - QA_COMPACTION_RETRY_PROMPT_RE.test(allInputText) && - rawByteLength > QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES && + (QA_COMPACTION_RETRY_PROMPT_RE.test(allInputText) || + hasCompactionOutputRecoveryMarker(allInputText)) && + rawByteLength > compactionOverflowThresholdBytes && !scenarioState.compactionOverflowInjected ) { scenarioState.compactionOverflowInjected = true; @@ -2206,6 +2303,7 @@ export async function startQaMockOpenAiServer(params?: { events = await buildResponsesPayload(request.body, scenarioState, { waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled, requestKind, + compactionSummaryFaultMode, }); } finally { inflightRequests.delete(inflightRequestId); @@ -2472,6 +2570,7 @@ export async function startQaMockOpenAiServer(params?: { providerVariant: resolveProviderVariant(normalizedModel), imageInputCount: countImageInputs(input), requestKind: classifyMockOpenAiRequest(input, body as Record), + compactionSummaryFaultMode: "none", outcome: "success", rawByteLength: Buffer.byteLength(raw), plannedToolCallId: extractPlannedToolCallId(events), diff --git a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts index b9a8bd74ec80..5a36b3bb4ef9 100644 --- a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts @@ -3,6 +3,55 @@ import { readQaScenarioById } from "./scenario-catalog.js"; import { requireFlowScenario } from "./scenario-catalog.test-utils.js"; describe("qa compaction scenario catalog", () => { + it.each([ + { + id: "compaction-empty-response-recovery", + coverage: "session-memory.compaction-empty-response-recovery", + faultMode: "empty-output-once", + summaryMarker: "QA-COMPACTION-EMPTY-RECOVERED-SUMMARY", + }, + { + id: "compaction-reasoning-only-recovery", + coverage: "session-memory.compaction-reasoning-only-recovery", + faultMode: "reasoning-only-output-once", + summaryMarker: "QA-COMPACTION-REASONING-RECOVERED-SUMMARY", + }, + ])("keeps $id on the OpenClaw compaction owner", ({ id, coverage, faultMode, summaryMarker }) => { + const scenario = requireFlowScenario(readQaScenarioById(id)); + const flow = JSON.stringify(scenario.execution.flow); + const serializedScenario = JSON.stringify(scenario); + + expect(scenario.runtimePairLane).toBeUndefined(); + expect(scenario.coverage?.primary).toEqual([coverage]); + expect(scenario.coverage?.secondary ?? []).toEqual([]); + expect(scenario.gatewayConfigPatch).toMatchObject({ + agents: { defaults: { compaction: { mode: "default" } } }, + }); + expect(flow).toContain("OPENCLAW_QA_FORCE_RUNTIME === 'openclaw'"); + expect(flow).toContain("initialRequests[0].errorCode === 'context_length_exceeded'"); + expect(flow).toContain("initialRequests.length === 2"); + expect(flow).toContain("compactionSummaryRequests.length === 2"); + expect(flow).toContain( + `compactionSummaryRequests[0].compactionSummaryFaultMode === config.faultMode`, + ); + expect(flow).toContain("compactionSummaryRequests[1].compactionSummaryFaultMode === 'none'"); + expect(flow).toContain( + "compactionSummaryRequests[0].cursor < compactionSummaryRequests[1].cursor", + ); + expect(flow).toContain( + "scenarioRequests.every((request) => request.model === scenarioRequests[0].model)", + ); + expect(flow).toContain("transcript.compactionSummaries.length === 1"); + expect(flow).toContain("transcript.compactionSummaries[0].includes(config.summaryMarker)"); + expect(flow).toContain("String(transcript.finalText ?? '').trim() === config.finalMarker"); + expect(flow).toContain("sessionEntry?.compactionCount === 1"); + expect(flow).toContain("request.requestKind === 'tool-continuation'"); + expect(flow).toContain("finalOutbound.length === 1"); + expect(serializedScenario).toContain(faultMode); + expect(serializedScenario).toContain(summaryMarker); + expect(serializedScenario).not.toContain("codex"); + }); + it("assigns compaction retry and pruning to OpenClaw with an early Codex gap", () => { const scenario = requireFlowScenario(readQaScenarioById("compaction-retry-mutating-tool")); const flow = JSON.stringify(scenario.execution.flow); diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts index a09d108557af..174d1bc38db7 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.test.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.test.ts @@ -6,6 +6,7 @@ import { upsertSessionEntry, } from "openclaw/plugin-sdk/session-store-runtime"; import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { appendSqliteSessionTranscriptEventForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createSession, @@ -315,6 +316,49 @@ describe("qa suite runtime agent session helpers", () => { ).rejects.toMatchObject({ code: "ENOENT" }); }); + it("reports bounded persisted compaction summaries", async () => { + const tempRoot = await makeTempDir("qa-session-compaction-summaries-"); + const sessionId = "compaction-summary"; + const sessionKey = "agent:qa:compaction-summary"; + const summaries = Array.from({ length: 18 }, (_, index) => `summary-${index}`); + await seedQaSession({ tempRoot, sessionId, sessionKey }); + + let parentId: string | null = null; + for (const [index, summary] of summaries.entries()) { + const id = `compaction-${index}`; + await appendSqliteSessionTranscriptEventForTest({ + agentId: "qa", + env: qaSessionEnv(tempRoot), + sessionId, + sessionKey, + event: { + type: "compaction", + id, + parentId, + timestamp: new Date(index).toISOString(), + summary, + firstKeptEntryId: id, + tokensBefore: 100, + }, + }); + parentId = id; + } + await appendQaTranscriptMessage({ + tempRoot, + sessionId, + sessionKey, + message: { role: "assistant", content: "done" }, + }); + + const result = await readSessionTranscriptSummary( + { gateway: { tempRoot } } as never, + sessionKey, + ); + + expect(result.compactionSummaries).toEqual(summaries.slice(-16)); + expect(result.finalText).toBe("done"); + }); + it("rejects an empty QA session transcript seed", async () => { const tempRoot = await makeTempDir("qa-session-seed-empty-"); @@ -363,6 +407,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + compactionSummaries: [], completedToolCallCounts: {}, eventCursor: 2, userMessageCount: 0, @@ -391,6 +436,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + compactionSummaries: [], completedToolCallCounts: {}, eventCursor: 3, userMessageCount: 0, @@ -447,6 +493,7 @@ describe("qa suite runtime agent session helpers", () => { ), ).resolves.toEqual({ assistantToolCallCounts: { message: 1 }, + compactionSummaries: [], completedToolCallCounts: {}, eventCursor: 4, userMessageCount: 1, @@ -734,6 +781,7 @@ describe("qa suite runtime agent session helpers", () => { }), ).resolves.toEqual({ assistantToolCallCounts: {}, + compactionSummaries: [], completedToolCallCounts: {}, eventCursor: 0, userMessageCount: 0, diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index 575c52527d22..5501e2c51cbc 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -43,11 +43,13 @@ type QaSessionTranscriptSeedParams = { const SESSION_STORE_LOCK_RETRY_DELAYS_MS = [1_000, 3_000, 5_000] as const; const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const; +const MAX_COMPACTION_SUMMARIES = 16; const MAX_SUCCESSFUL_TOOL_CALL_EVENTS = 64; type QaSessionTranscriptSummary = { assistantMirrors?: Array<{ identity: string; text: string }>; assistantToolCallCounts: Record; + compactionSummaries: string[]; completedToolCallCounts: Record; eventCursor: number; userMessageCount: number; @@ -121,6 +123,7 @@ function summarizeSessionTranscriptEvents( const assistantMirrors: Array<{ identity: string; text: string }> = []; const assistantToolCallCounts: Record = {}; const completedToolCallCounts: Record = {}; + const compactionSummaries: string[] = []; const successfulToolCallCounts: Record = {}; const successfulToolCallEvents: NonNullable< QaSessionTranscriptSummary["successfulToolCallEvents"] @@ -137,6 +140,16 @@ function summarizeSessionTranscriptEvents( let userMessageCount = 0; for (const event of events) { + if (isRecord(event) && event.type === "compaction") { + const summary = readNonEmptyString(event.summary); + if (summary) { + if (compactionSummaries.length === MAX_COMPACTION_SUMMARIES) { + compactionSummaries.shift(); + } + compactionSummaries.push(summary); + } + continue; + } const message = readSessionTranscriptEventMessage(event); if (!message) { continue; @@ -219,6 +232,7 @@ function summarizeSessionTranscriptEvents( return { ...(assistantMirrors.length > 0 ? { assistantMirrors } : {}), assistantToolCallCounts, + compactionSummaries, completedToolCallCounts, eventCursor, userMessageCount, @@ -237,6 +251,7 @@ function summarizeSessionTranscriptEvents( function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscriptSummary { return { assistantToolCallCounts: {}, + compactionSummaries: [], completedToolCallCounts: {}, eventCursor, userMessageCount: 0, diff --git a/packages/agent-core/src/harness/compaction/branch-summarization.test.ts b/packages/agent-core/src/harness/compaction/branch-summarization.test.ts index 5ee3f1b0cf07..163b7b67e6a1 100644 --- a/packages/agent-core/src/harness/compaction/branch-summarization.test.ts +++ b/packages/agent-core/src/harness/compaction/branch-summarization.test.ts @@ -30,10 +30,13 @@ function createMessageEntry(message: AgentMessage, index: number): SessionTreeEn }; } -function createResponse(model: Model): AssistantMessage { +function createResponse( + model: Model, + content: AssistantMessage["content"] = [{ type: "text", text: "Branch summary" }], +): AssistantMessage { return { role: "assistant", - content: [{ type: "text", text: "Branch summary" }], + content, api: model.api, provider: model.provider, model: model.id, @@ -50,6 +53,13 @@ function createResponse(model: Model): AssistantMessage { }; } +function createResponseStream(model: Model, content?: AssistantMessage["content"]) { + const stream = createAssistantMessageEventStream(); + stream.push({ type: "done", reason: "stop", message: createResponse(model, content) }); + stream.end(); + return stream; +} + function createCapturingStream(model: Model) { let prompt = ""; let systemPrompt = ""; @@ -65,10 +75,7 @@ function createCapturingStream(model: Model) { : userMessage.content.map((block) => (block.type === "text" ? block.text : "")).join(""); systemPrompt = context.systemPrompt ?? ""; maxOutputTokens = options?.maxTokens; - const stream = createAssistantMessageEventStream(); - stream.push({ type: "done", reason: "stop", message: createResponse(model) }); - stream.end(); - return stream; + return createResponseStream(model); }); return { streamFn, @@ -86,6 +93,89 @@ function createLongBranchEntries(count: number): SessionTreeEntry[] { } describe("branch summarization", () => { + it.each([ + ["empty", []], + ["whitespace-only", [{ type: "text" as const, text: " \n\t " }]], + ["reasoning-only", [{ type: "thinking" as const, thinking: "internal reasoning" }]], + ])("rejects %s model output before creating a summary", async (_name, content) => { + const model = createModel(128_000); + const streamFn = vi.fn(() => createResponseStream(model, content)); + const entries = [ + createMessageEntry({ role: "user", content: "summarize this branch", timestamp: 1 }, 0), + ]; + + const result = await generateBranchSummary(entries, { + model, + apiKey: "test-key", + signal: new AbortController().signal, + streamFn, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected invalid branch summary output to fail"); + } + expect(result.error).toMatchObject({ + name: "BranchSummaryError", + code: "summarization_failed", + message: "Branch summary failed: model returned no summary text", + }); + }); + + it("preserves valid summary whitespace, preamble, and file metadata", async () => { + const model = createModel(128_000); + const summaryText = " Branch summary body \ncontinues "; + const streamFn = vi.fn(() => + createResponseStream(model, [ + { type: "text", text: " Branch summary body " }, + { type: "text", text: "continues " }, + ]), + ); + const entries: SessionTreeEntry[] = [ + createMessageEntry({ role: "user", content: "inspect files", timestamp: 1 }, 0), + createMessageEntry( + createResponse(model, [ + { type: "toolCall", id: "read-1", name: "read", arguments: { path: "src/read.ts" } }, + { + type: "toolCall", + id: "write-1", + name: "write", + arguments: { path: "src/write.ts" }, + }, + ]), + 1, + ), + ]; + + const result = await generateBranchSummary(entries, { + model, + apiKey: "test-key", + signal: new AbortController().signal, + streamFn, + }); + + expect(result.ok).toBe(true); + if (!result.ok) { + throw result.error; + } + expect(result.value).toEqual({ + summary: `The user explored a different conversation branch before returning here. +Summary of that exploration: + +${summaryText} + + +src/read.ts + + + +src/write.ts +`, + readFiles: ["src/read.ts"], + modifiedFiles: ["src/write.ts"], + }); + }); + it("retains failed tool results when preparing a branch", () => { const entries: SessionTreeEntry[] = [ createMessageEntry({ role: "user", content: "run deployment", timestamp: 1 }, 0), diff --git a/packages/agent-core/src/harness/compaction/branch-summarization.ts b/packages/agent-core/src/harness/compaction/branch-summarization.ts index 76e2fd9db905..bbe7ac5dd933 100644 --- a/packages/agent-core/src/harness/compaction/branch-summarization.ts +++ b/packages/agent-core/src/harness/compaction/branch-summarization.ts @@ -19,6 +19,7 @@ import { computeFileLists, createFileOps, extractFileOpsFromMessage, + extractSummaryText, type FileOperations, formatFileOperations, serializeConversation, @@ -292,16 +293,22 @@ export async function generateBranchSummary( ); } - let summary = response.content - .filter((c): c is { type: "text"; text: string } => c.type === "text") - .map((c) => c.text) - .join("\n"); - summary = BRANCH_SUMMARY_PREAMBLE + summary; + const summaryText = extractSummaryText(response); + if (summaryText === undefined) { + return err( + new BranchSummaryError( + "summarization_failed", + "Branch summary failed: model returned no summary text", + ), + ); + } + + let summary = BRANCH_SUMMARY_PREAMBLE + summaryText; const { readFiles, modifiedFiles } = computeFileLists(fileOps); summary += formatFileOperations(readFiles, modifiedFiles); return ok({ - summary: summary || "No summary generated", + summary, readFiles, modifiedFiles, }); diff --git a/packages/agent-core/src/harness/compaction/compaction.test.ts b/packages/agent-core/src/harness/compaction/compaction.test.ts index 8ffa0bd3a74f..2e0b0c04a7a1 100644 --- a/packages/agent-core/src/harness/compaction/compaction.test.ts +++ b/packages/agent-core/src/harness/compaction/compaction.test.ts @@ -499,6 +499,67 @@ describe("generateSummary thinking options", () => { expect(result).toEqual({ ok: true, value: "summary" }); expect(streamFn).toHaveBeenCalledOnce(); }); + + it.each([ + ["empty", []], + ["whitespace-only", [{ type: "text" as const, text: " \n\t " }]], + ["reasoning-only", [{ type: "thinking" as const, thinking: "internal summary reasoning" }]], + ])("rejects %s compaction output", async (_name, content) => { + const model: Model = { + id: "summary-model", + name: "Summary Model", + api: "test-api", + provider: "test-provider", + baseUrl: "https://example.test", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 100_000, + maxTokens: 8_000, + }; + const streamFn = vi.fn(() => { + const stream = createAssistantMessageEventStream(); + stream.push({ + type: "done", + reason: "stop", + message: { + role: "assistant", + content, + api: model.api, + provider: model.provider, + model: model.id, + usage: createUsage(1), + stopReason: "stop", + timestamp: 1, + }, + }); + stream.end(); + return stream; + }); + + const result = await generateSummary( + [{ role: "user", content: "hello", timestamp: 1 }], + model, + 1_000, + undefined, + undefined, + undefined, + undefined, + undefined, + "low", + streamFn, + ); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected empty compaction output to fail"); + } + expect(result.error).toMatchObject({ + name: "CompactionError", + code: "summarization_failed", + message: "Summarization failed: model returned no summary text", + }); + }); }); describe("split-turn compaction", () => { diff --git a/packages/agent-core/src/harness/compaction/compaction.ts b/packages/agent-core/src/harness/compaction/compaction.ts index 3e474a50ca94..f792d65d203a 100644 --- a/packages/agent-core/src/harness/compaction/compaction.ts +++ b/packages/agent-core/src/harness/compaction/compaction.ts @@ -31,6 +31,7 @@ import { type CompactionEntry, CompactionError, err, + InvalidSummaryOutputError, ok, type Result, type SessionTreeEntry, @@ -39,6 +40,7 @@ import { computeFileLists, createFileOps, extractFileOpsFromMessage, + extractSummaryText, type FileOperations, formatFileOperations, getCompactionContentBlockText, @@ -646,12 +648,13 @@ async function runSummarizationCompletion(params: { ); } - return ok( - response.content - .filter((c): c is { type: "text"; text: string } => c.type === "text") - .map((c) => c.text) - .join("\n"), - ); + const summary = extractSummaryText(response); + if (summary === undefined) { + return err( + new InvalidSummaryOutputError(`${params.errorLabel} failed: model returned no summary text`), + ); + } + return ok(summary); } /** Generate or update a conversation summary for compaction. */ diff --git a/packages/agent-core/src/harness/compaction/utils.ts b/packages/agent-core/src/harness/compaction/utils.ts index 1edc36504401..5bda4caa824b 100644 --- a/packages/agent-core/src/harness/compaction/utils.ts +++ b/packages/agent-core/src/harness/compaction/utils.ts @@ -1,4 +1,4 @@ -import type { Message } from "@openclaw/llm-core"; +import type { AssistantMessage, Message } from "@openclaw/llm-core"; // Agent Core helper module supports utils behavior. import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { AgentMessage } from "../../types.js"; @@ -85,6 +85,15 @@ export function formatFileOperations(readFiles: string[], modifiedFiles: string[ return `\n\n${sections.join("\n\n")}`; } +/** Extract visible summary text without normalizing valid model output. */ +export function extractSummaryText(response: AssistantMessage): string | undefined { + const summary = response.content + .filter((block): block is { type: "text"; text: string } => block.type === "text") + .map((block) => block.text) + .join("\n"); + return summary.trim() ? summary : undefined; +} + const TOOL_RESULT_MAX_CHARS = 2000; const IMPORTANT_TOOL_RESULT_TAIL = /(error|exception|failed|fatal|traceback|panic|stack trace|errno|exit code)/i; diff --git a/packages/agent-core/src/harness/types.ts b/packages/agent-core/src/harness/types.ts index a3b550dfa3dd..6ad21fde641a 100644 --- a/packages/agent-core/src/harness/types.ts +++ b/packages/agent-core/src/harness/types.ts @@ -16,6 +16,13 @@ export class CompactionError extends Error { } } +/** Internal typed signal for a completed summary response with no usable text. */ +export class InvalidSummaryOutputError extends CompactionError { + constructor(message: string) { + super("summarization_failed", message); + } +} + type BranchSummaryErrorCode = "aborted" | "summarization_failed" | "invalid_session"; export class BranchSummaryError extends Error { diff --git a/qa/scenarios/runtime/compaction-empty-response-recovery.yaml b/qa/scenarios/runtime/compaction-empty-response-recovery.yaml new file mode 100644 index 000000000000..4e05a3e35a56 --- /dev/null +++ b/qa/scenarios/runtime/compaction-empty-response-recovery.yaml @@ -0,0 +1,155 @@ +title: Compaction empty-response recovery + +scenario: + id: compaction-empty-response-recovery + surface: runtime + coverage: + primary: + - session-memory.compaction-empty-response-recovery + objective: Verify an empty overflow-compaction summary is rejected, retried by the compaction owner, and persisted only after a non-empty summary is returned. + successCriteria: + - The OpenClaw compaction owner issues exactly two ordered summary requests. + - The first summary response is the typed empty-output fault and the second returns a durable non-empty summary. + - Every provider request stays on one model with explicit default-mode compaction; no model fallback or safeguard retry owns recovery. + - Exactly one compaction and one final visible answer are persisted without generic incomplete-turn recovery. + gatewayConfigPatch: + agents: + defaults: + compaction: + mode: default + docsRefs: + - docs/concepts/compaction.md + codeRefs: + - packages/agent-core/src/harness/compaction/compaction.ts + - src/agents/sessions/agent-session-compaction.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + execution: + kind: flow + summary: Reject one empty overflow-compaction summary, retry it, and verify the recovered summary persists. + config: + faultMode: empty-output-once + faultMarkerPrefix: QA-COMPACTION-EMPTY-OUTPUT-ONCE + summaryMarker: QA-COMPACTION-EMPTY-RECOVERED-SUMMARY + finalMarker: QA-COMPACTION-EMPTY-FINAL-OK + emptyRetryNeedle: The previous attempt did not produce a user-visible answer. + reasoningRetryNeedle: The previous assistant turn recorded reasoning but did not produce a user-visible answer. + +flow: + steps: + - name: retries an empty compaction summary at the compaction owner + actions: + - assert: + expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)" + message: compaction output recovery requires the mock-openai OpenClaw runtime + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: reset + - set: sessionId + value: + expr: "`qa-compaction-empty-${randomUUID()}`" + - set: sessionKey + value: + expr: "`agent:qa:compaction-empty:${randomUUID().slice(0, 8)}`" + - set: faultMarker + value: + expr: "`${config.faultMarkerPrefix}-${sessionId}`" + - set: now + value: + expr: "Date.now()" + - call: seedQaSessionTranscript + args: + - ref: env + - sessionId: + ref: sessionId + sessionKey: + ref: sessionKey + updatedAt: + ref: now + label: QA empty compaction output recovery + messages: + expr: "[...Array.from({ length: 10 }, (_, index) => [{ role: 'user', text: `${index === 0 ? faultMarker + ' ' : ''}historical recovery user block ${String(index).padStart(2, '0')} ${'u'.repeat(8192)}`, timestamp: now - (50000 - index * 2000) }, { role: 'assistant', text: `historical recovery assistant block ${String(index).padStart(2, '0')} ${'r'.repeat(8192)}`, timestamp: now - (49000 - index * 2000) }]).flat(), { role: 'user', text: 'Retain the current recovery continuation.', timestamp: now - 1000 }]" + - set: requestCursorBefore + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - try: + actions: + - call: runAgentPrompt + args: + - ref: env + - sessionKey: + ref: sessionKey + message: + expr: "`Reply with only this exact marker: ${config.finalMarker}`" + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + catchAs: agentRunError + catch: + - set: failedRequests + value: + expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)" + - throw: + expr: "`compaction recovery agent run failed: ${agentRunError?.message ?? agentRunError}; requests=${JSON.stringify(failedRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, mode: request.compactionSummaryFaultMode, outcome: request.outcome, code: request.errorCode ?? null, marker: String(request.allInputText ?? '').match(/QA-COMPACTION-EMPTY-OUTPUT-ONCE-[A-Za-z0-9_-]+/)?.[0] ?? null })))}`" + - call: waitForCondition + saveAs: outbound + args: + - lambda: + expr: "state.getSnapshot().messages.findLast((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text === config.finalMarker)" + - expr: liveTurnTimeoutMs(env, 30000) + - 100 + - set: scenarioRequests + value: + expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)" + - set: compactionSummaryRequests + value: + expr: "scenarioRequests.filter((request) => request.requestKind === 'compaction-summary')" + - set: initialRequests + value: + expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial')" + - assert: + expr: "initialRequests.length === 2 && initialRequests[0].outcome === 'error' && initialRequests[0].errorCode === 'context_length_exceeded' && initialRequests[1].outcome === 'success' && initialRequests[0].cursor < initialRequests[1].cursor" + message: + expr: "`expected one overflow and one compacted final request: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null })))}`" + - assert: + expr: "compactionSummaryRequests.length === 2 && compactionSummaryRequests[0].cursor < compactionSummaryRequests[1].cursor" + message: + expr: "`expected two ordered compaction summary requests: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, mode: request.compactionSummaryFaultMode })))}`" + - assert: + expr: "compactionSummaryRequests[0].compactionSummaryFaultMode === config.faultMode && compactionSummaryRequests[1].compactionSummaryFaultMode === 'none'" + message: + expr: "`unexpected compaction response modes: ${JSON.stringify(compactionSummaryRequests)}`" + - assert: + expr: "scenarioRequests.length > 0 && scenarioRequests.every((request) => request.model === scenarioRequests[0].model)" + message: + expr: "`compaction recovery must stay on one model without fallback: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, model: request.model })))}`" + - assert: + expr: "!scenarioRequests.some((request) => request.requestKind === 'tool-continuation')" + message: compaction output recovery must not enter tool continuation + - assert: + expr: "!scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.emptyRetryNeedle) || String(request.allInputText ?? '').includes(config.reasoningRetryNeedle))" + message: compaction output recovery must not use generic incomplete-turn instructions + - call: readSessionTranscriptSummary + saveAs: transcript + args: + - ref: env + - ref: sessionKey + - call: readRawQaSessionStore + saveAs: store + args: + - ref: env + - set: sessionEntry + value: + expr: "store[sessionKey]" + - assert: + expr: "sessionEntry?.compactionCount === 1 && transcript.compactionSummaries.length === 1 && transcript.compactionSummaries[0].includes(config.summaryMarker) && String(transcript.finalText ?? '').trim() === config.finalMarker" + message: + expr: "`recovered compaction output was not persisted: ${JSON.stringify({ compactionCount: sessionEntry?.compactionCount, summaries: transcript.compactionSummaries, finalText: transcript.finalText })}`" + - set: finalOutbound + value: + expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text === config.finalMarker)" + - assert: + expr: "outbound.text === config.finalMarker && finalOutbound.length === 1" + message: + expr: "`expected one final visible output: ${JSON.stringify(finalOutbound.map((candidate) => candidate.text))}`" + detailsExpr: "`${outbound.text}\\ninitialRequests=${String(initialRequests.length)} compactionRequests=${String(compactionSummaryRequests.length)} compactions=${String(sessionEntry.compactionCount)}`" diff --git a/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml b/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml new file mode 100644 index 000000000000..16d38baf8b1f --- /dev/null +++ b/qa/scenarios/runtime/compaction-reasoning-only-recovery.yaml @@ -0,0 +1,155 @@ +title: Compaction reasoning-only recovery + +scenario: + id: compaction-reasoning-only-recovery + surface: runtime + coverage: + primary: + - session-memory.compaction-reasoning-only-recovery + objective: Verify a reasoning-only overflow-compaction summary is rejected, retried by the compaction owner, and persisted only after a non-empty summary is returned. + successCriteria: + - The OpenClaw compaction owner issues exactly two ordered summary requests. + - The first summary response is the typed reasoning-only fault and the second returns a durable non-empty summary. + - Every provider request stays on one model with explicit default-mode compaction; no model fallback or safeguard retry owns recovery. + - Exactly one compaction and one final visible answer are persisted without generic incomplete-turn recovery. + gatewayConfigPatch: + agents: + defaults: + compaction: + mode: default + docsRefs: + - docs/concepts/compaction.md + codeRefs: + - packages/agent-core/src/harness/compaction/compaction.ts + - src/agents/sessions/agent-session-compaction.ts + - extensions/qa-lab/src/providers/mock-openai/server.ts + execution: + kind: flow + summary: Reject one reasoning-only overflow-compaction summary, retry it, and verify the recovered summary persists. + config: + faultMode: reasoning-only-output-once + faultMarkerPrefix: QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE + summaryMarker: QA-COMPACTION-REASONING-RECOVERED-SUMMARY + finalMarker: QA-COMPACTION-REASONING-FINAL-OK + emptyRetryNeedle: The previous attempt did not produce a user-visible answer. + reasoningRetryNeedle: The previous assistant turn recorded reasoning but did not produce a user-visible answer. + +flow: + steps: + - name: retries a reasoning-only compaction summary at the compaction owner + actions: + - assert: + expr: "env.providerMode === 'mock-openai' && env.gateway.runtimeEnv.OPENCLAW_QA_FORCE_RUNTIME === 'openclaw' && Boolean(env.mock)" + message: compaction output recovery requires the mock-openai OpenClaw runtime + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: reset + - set: sessionId + value: + expr: "`qa-compaction-reasoning-${randomUUID()}`" + - set: sessionKey + value: + expr: "`agent:qa:compaction-reasoning:${randomUUID().slice(0, 8)}`" + - set: faultMarker + value: + expr: "`${config.faultMarkerPrefix}-${sessionId}`" + - set: now + value: + expr: "Date.now()" + - call: seedQaSessionTranscript + args: + - ref: env + - sessionId: + ref: sessionId + sessionKey: + ref: sessionKey + updatedAt: + ref: now + label: QA reasoning-only compaction output recovery + messages: + expr: "[...Array.from({ length: 10 }, (_, index) => [{ role: 'user', text: `${index === 0 ? faultMarker + ' ' : ''}historical recovery user block ${String(index).padStart(2, '0')} ${'u'.repeat(8192)}`, timestamp: now - (50000 - index * 2000) }, { role: 'assistant', text: `historical recovery assistant block ${String(index).padStart(2, '0')} ${'r'.repeat(8192)}`, timestamp: now - (49000 - index * 2000) }]).flat(), { role: 'user', text: 'Retain the current recovery continuation.', timestamp: now - 1000 }]" + - set: requestCursorBefore + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - try: + actions: + - call: runAgentPrompt + args: + - ref: env + - sessionKey: + ref: sessionKey + message: + expr: "`Reply with only this exact marker: ${config.finalMarker}`" + timeoutMs: + expr: liveTurnTimeoutMs(env, 60000) + catchAs: agentRunError + catch: + - set: failedRequests + value: + expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)" + - throw: + expr: "`compaction recovery agent run failed: ${agentRunError?.message ?? agentRunError}; requests=${JSON.stringify(failedRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, mode: request.compactionSummaryFaultMode, outcome: request.outcome, code: request.errorCode ?? null, marker: String(request.allInputText ?? '').match(/QA-COMPACTION-REASONING-ONLY-OUTPUT-ONCE-[A-Za-z0-9_-]+/)?.[0] ?? null })))}`" + - call: waitForCondition + saveAs: outbound + args: + - lambda: + expr: "state.getSnapshot().messages.findLast((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text === config.finalMarker)" + - expr: liveTurnTimeoutMs(env, 30000) + - 100 + - set: scenarioRequests + value: + expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)" + - set: compactionSummaryRequests + value: + expr: "scenarioRequests.filter((request) => request.requestKind === 'compaction-summary')" + - set: initialRequests + value: + expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial')" + - assert: + expr: "initialRequests.length === 2 && initialRequests[0].outcome === 'error' && initialRequests[0].errorCode === 'context_length_exceeded' && initialRequests[1].outcome === 'success' && initialRequests[0].cursor < initialRequests[1].cursor" + message: + expr: "`expected one overflow and one compacted final request: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null })))}`" + - assert: + expr: "compactionSummaryRequests.length === 2 && compactionSummaryRequests[0].cursor < compactionSummaryRequests[1].cursor" + message: + expr: "`expected two ordered compaction summary requests: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, mode: request.compactionSummaryFaultMode })))}`" + - assert: + expr: "compactionSummaryRequests[0].compactionSummaryFaultMode === config.faultMode && compactionSummaryRequests[1].compactionSummaryFaultMode === 'none'" + message: + expr: "`unexpected compaction response modes: ${JSON.stringify(compactionSummaryRequests)}`" + - assert: + expr: "scenarioRequests.length > 0 && scenarioRequests.every((request) => request.model === scenarioRequests[0].model)" + message: + expr: "`compaction recovery must stay on one model without fallback: ${JSON.stringify(scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, model: request.model })))}`" + - assert: + expr: "!scenarioRequests.some((request) => request.requestKind === 'tool-continuation')" + message: compaction output recovery must not enter tool continuation + - assert: + expr: "!scenarioRequests.some((request) => String(request.allInputText ?? '').includes(config.emptyRetryNeedle) || String(request.allInputText ?? '').includes(config.reasoningRetryNeedle))" + message: compaction output recovery must not use generic incomplete-turn instructions + - call: readSessionTranscriptSummary + saveAs: transcript + args: + - ref: env + - ref: sessionKey + - call: readRawQaSessionStore + saveAs: store + args: + - ref: env + - set: sessionEntry + value: + expr: "store[sessionKey]" + - assert: + expr: "sessionEntry?.compactionCount === 1 && transcript.compactionSummaries.length === 1 && transcript.compactionSummaries[0].includes(config.summaryMarker) && String(transcript.finalText ?? '').trim() === config.finalMarker" + message: + expr: "`recovered compaction output was not persisted: ${JSON.stringify({ compactionCount: sessionEntry?.compactionCount, summaries: transcript.compactionSummaries, finalText: transcript.finalText })}`" + - set: finalOutbound + value: + expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text === config.finalMarker)" + - assert: + expr: "outbound.text === config.finalMarker && finalOutbound.length === 1" + message: + expr: "`expected one final visible output: ${JSON.stringify(finalOutbound.map((candidate) => candidate.text))}`" + detailsExpr: "`${outbound.text}\\ninitialRequests=${String(initialRequests.length)} compactionRequests=${String(compactionSummaryRequests.length)} compactions=${String(sessionEntry.compactionCount)}`" diff --git a/src/agents/compaction.summarize-fallback.test.ts b/src/agents/compaction.summarize-fallback.test.ts index 54841c9aa72e..7b09fc17f226 100644 --- a/src/agents/compaction.summarize-fallback.test.ts +++ b/src/agents/compaction.summarize-fallback.test.ts @@ -3,6 +3,7 @@ import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import type { ExtensionContext } from "openclaw/plugin-sdk/agent-sessions"; import type { UserMessage } from "openclaw/plugin-sdk/llm"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { CompactionError } from "../../packages/agent-core/src/harness/types.js"; import { summarizeWithFallback } from "./compaction.test-support.js"; const agentSessionMocks = vi.hoisted(() => ({ @@ -106,6 +107,36 @@ describe("summarizeWithFallback", () => { expect(agentSessionMocks.generateSummary).toHaveBeenCalledTimes(2); }); + it("retries a summarization_failed result and persists the recovered summary", async () => { + agentSessionMocks.generateSummary + .mockRejectedValueOnce( + new CompactionError( + "summarization_failed", + "Summarization failed: model returned no summary text", + ), + ) + .mockResolvedValueOnce("recovered non-empty summary"); + + await expect( + summarizeWithFallback({ + messages: [ + { + role: "user", + content: "hello", + timestamp: 1, + } satisfies UserMessage, + ], + model: testModel, + apiKey: "test-key", // pragma: allowlist secret + signal: new AbortController().signal, + reserveTokens: 1000, + maxChunkTokens: 50_000, + contextWindow: 200_000, + }), + ).resolves.toBe("recovered non-empty summary"); + expect(agentSessionMocks.generateSummary).toHaveBeenCalledTimes(2); + }); + it("does not retry and propagates AbortError immediately when caller signal is already aborted", async () => { const controller = new AbortController(); controller.abort(); diff --git a/src/agents/embedded-agent-runner/compact.hooks.harness.ts b/src/agents/embedded-agent-runner/compact.hooks.harness.ts index ae7efddec768..5a1274ccf09d 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.harness.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.harness.ts @@ -8,6 +8,7 @@ import { clearAgentHarnesses } from "../harness/registry.js"; import type { AgentHarness } from "../harness/types.js"; import type { ModelAuthMode } from "../model-auth.js"; import type { AgentRuntimePlan, BuildAgentRuntimePlanParams } from "../runtime-plan/types.js"; +import { agentSessionAutomaticCompaction } from "../sessions/agent-session-compaction.js"; type MockResolvedModel = { model: { @@ -77,6 +78,8 @@ export const sessionCompactImpl = vi.fn(async () => ({ tokensBefore: 120, details: { ok: true }, })); +export const sessionManualCompactionMock = vi.fn(); +export const sessionAutomaticCompactionMock = vi.fn(); export const triggerInternalHook: Mock<(event?: unknown) => void> = vi.fn(); const sanitizeSessionHistoryMock = vi.fn( async (params: { messages: unknown[] }) => params.messages, @@ -158,6 +161,12 @@ function createMockCompactionSession() { }, }, compact: vi.fn(async () => { + sessionManualCompactionMock(); + session.messages.splice(1); + return await sessionCompactImpl(); + }), + [agentSessionAutomaticCompaction]: vi.fn(async () => { + sessionAutomaticCompactionMock(); session.messages.splice(1); return await sessionCompactImpl(); }), @@ -187,7 +196,8 @@ function createMockToolDefinitions(tools: unknown[] = []) { }); } export const createOpenClawCodingToolsMock = vi.fn(() => []); -const buildEmbeddedExtensionFactoriesMock = vi.fn(() => []); +export const buildEmbeddedExtensionFactoriesMock = vi.fn(() => []); +export const resolveEffectiveCompactionModeMock = vi.fn(() => "default"); export const guardSessionManagerMock = vi.fn(() => ({ flushPendingToolResults: vi.fn(), })); @@ -474,6 +484,10 @@ export function resetCompactSessionStateMocks(): void { estimateTokensMock.mockReturnValue(10); sessionMessages.splice(0, sessionMessages.length, ...createDefaultSessionMessages()); sessionAbortCompactionMock.mockReset(); + sessionManualCompactionMock.mockReset(); + sessionAutomaticCompactionMock.mockReset(); + resolveEffectiveCompactionModeMock.mockReset(); + resolveEffectiveCompactionModeMock.mockReturnValue("default"); createAgentSessionMock.mockReset(); createAgentSessionMock.mockImplementation(async () => ({ session: createMockCompactionSession(), @@ -741,6 +755,7 @@ export async function loadCompactHooksHarness(): Promise<{ applyAgentAutoCompactionGuard: vi.fn(() => ({ supported: true, disabled: false })), applyAgentCompactionSettingsFromConfig: applyAgentCompactionSettingsFromConfigMock, isSilentOverflowProneModel: vi.fn(() => false), + resolveEffectiveCompactionMode: resolveEffectiveCompactionModeMock, })); vi.doMock("../models-config.js", () => ({ diff --git a/src/agents/embedded-agent-runner/compact.hooks.test.ts b/src/agents/embedded-agent-runner/compact.hooks.test.ts index b48fee2262ca..c3a7a3112804 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.test.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.test.ts @@ -14,6 +14,7 @@ import { acquireAgentRunPreparedModelRuntimeMock, applyExtraParamsToAgentMock, applyAgentCompactionSettingsFromConfigMock, + buildEmbeddedExtensionFactoriesMock, buildAgentRuntimePlanMock, buildEmbeddedSystemPromptMock, contextEngineCompactMock, @@ -36,6 +37,7 @@ import { resolveProviderEntryApiKeyProfileReferenceMock, resolveContextWindowInfoMock, resolveContextEngineMock, + resolveEffectiveCompactionModeMock, resolveEmbeddedAgentStreamFnMock, resolveMemorySearchConfigMock, resolveModelAsyncMock, @@ -50,8 +52,10 @@ import { resetCompactHooksHarnessMocks, resetCompactSessionStateMocks, sessionAbortCompactionMock, + sessionAutomaticCompactionMock, sessionMessages, sessionCompactImpl, + sessionManualCompactionMock, triggerInternalHook, } from "./compact.hooks.harness.js"; import { @@ -2195,6 +2199,28 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { expect(sessionCompactImpl).toHaveBeenCalledOnce(); }); + it.each(["overflow", "budget", "timeout_recovery"] as const)( + "uses caller-owned automatic recovery once for default-mode %s compaction", + async (trigger) => { + hookRunner.hasHooks.mockReturnValue(true); + resolveEffectiveCompactionModeMock.mockReturnValue("default"); + + const result = await compactEmbeddedAgentSessionDirect( + wrappedCompactionArgs({ + trigger, + config: { agents: { defaults: { compaction: { mode: "default" } } } }, + }), + ); + + expect(result).toMatchObject({ ok: true, compacted: true }); + expect(sessionAutomaticCompactionMock).toHaveBeenCalledOnce(); + expect(sessionManualCompactionMock).not.toHaveBeenCalled(); + expect(buildEmbeddedExtensionFactoriesMock).toHaveBeenCalledOnce(); + expect(hookRunner.runBeforeCompaction).toHaveBeenCalledOnce(); + expect(hookRunner.runAfterCompaction).toHaveBeenCalledOnce(); + }, + ); + it("skips compaction when the transcript only contains boilerplate replies and tool output", () => { const messages = [ { role: "user", content: "HEARTBEAT_OK", timestamp: 1 }, diff --git a/src/agents/embedded-agent-runner/compaction-session-execution.ts b/src/agents/embedded-agent-runner/compaction-session-execution.ts index 441196629111..8bff05bb2efd 100644 --- a/src/agents/embedded-agent-runner/compaction-session-execution.ts +++ b/src/agents/embedded-agent-runner/compaction-session-execution.ts @@ -18,6 +18,7 @@ import { applyAgentAutoCompactionGuard, applyAgentCompactionSettingsFromConfig, isSilentOverflowProneModel, + resolveEffectiveCompactionMode, } from "../agent-settings.js"; import { pickFallbackThinkingLevel } from "../embedded-agent-helpers.js"; import { resolveAgentRunSessionTarget } from "../run-session-target.js"; @@ -29,6 +30,7 @@ import { resolveSessionWriteLockTargetKey, resolveSessionWriteLockOptions, } from "../session-write-lock.js"; +import { agentSessionAutomaticCompaction } from "../sessions/agent-session-compaction.js"; import { createAgentSession, estimateTokens, SessionManager } from "../sessions/index.js"; import { getModelRegistryRuntime } from "../sessions/model-registry-runtime.js"; import { resolveCompactionFailureReason } from "./compact-reasons.js"; @@ -424,7 +426,10 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti const result = await compactWithSafetyTimeout( () => { setCompactionSafeguardCancelReason(compactionSessionManager, undefined); - return activeSession.compact(params.customInstructions); + return resolveEffectiveCompactionMode(params.config) === "default" && + trigger !== "manual" + ? activeSession[agentSessionAutomaticCompaction](params.customInstructions) + : activeSession.compact(params.customInstructions); }, compactionTimeoutMs, { diff --git a/src/agents/sessions/agent-session-compaction.ts b/src/agents/sessions/agent-session-compaction.ts index a28e1c25f048..7c5feee6f349 100644 --- a/src/agents/sessions/agent-session-compaction.ts +++ b/src/agents/sessions/agent-session-compaction.ts @@ -1,4 +1,5 @@ import { isContextOverflow } from "@openclaw/ai/internal/runtime"; +import { InvalidSummaryOutputError } from "../../../packages/agent-core/src/harness/types.js"; import type { AssistantMessage, Model } from "../../llm/types.js"; import { calculateContextTokens, @@ -18,11 +19,17 @@ import { getLatestCompactionEntry, type CompactionEntry } from "./session-manage import type { SettingsManager } from "./settings-manager.js"; type CompactionReason = "manual" | "threshold" | "overflow"; +type SummaryOutputPolicy = "none" | "retry-invalid-once"; type CompactionWorkOutcome = | { status: "compacted"; result: CompactionResult } | { status: "aborted" } | { status: "skipped" }; +/** @internal */ +export const agentSessionAutomaticCompaction: unique symbol = Symbol.for( + "openclaw.agent-session.automatic-compaction", +); + export abstract class AgentSessionCompaction extends AgentSessionInspection { // ========================================================================= // Compaction @@ -35,12 +42,19 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { */ async compact(customInstructions?: string): Promise { return await this.runWithSessionWriteLock( - async () => await this.compactWithSessionWriteLock(customInstructions), + async () => await this.compactWithSessionWriteLock(customInstructions, "none"), + ); + } + + async [agentSessionAutomaticCompaction](customInstructions?: string): Promise { + return await this.runWithSessionWriteLock( + async () => await this.compactWithSessionWriteLock(customInstructions, "retry-invalid-once"), ); } private async compactWithSessionWriteLock( customInstructions?: string, + summaryOutputPolicy: SummaryOutputPolicy = "none", ): Promise { this.disconnectFromAgent(); await this.abort(); @@ -52,6 +66,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { const outcome = await this.runCompactionWork({ customInstructions, mode: "manual", + summaryOutputPolicy, settings, signal: this.compactionAbortController.signal, }); @@ -129,6 +144,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { signal: AbortSignal; customInstructions?: string; mode: "manual" | "auto"; + summaryOutputPolicy: SummaryOutputPolicy; }): Promise { const isManual = options.mode === "manual"; if (!this.model) { @@ -137,10 +153,11 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { } return { status: "skipped" }; } + const model = this.model; const auth = isManual - ? await this.getCompactionRequestAuth(this.model) - : await this.getAutoCompactionRequestAuth(this.model); + ? await this.getCompactionRequestAuth(model) + : await this.getAutoCompactionRequestAuth(model); if (!auth) { return { status: "skipped" }; } @@ -185,18 +202,36 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { } } - compactionResult ??= unwrapCoreResult( - await compact( - preparation, - this.model, - auth.apiKey, - auth.headers, - options.customInstructions, - options.signal, - this.thinkingLevel, - this.agent.streamFn, - ), - ); + if (!compactionResult) { + const runCoreCompaction = () => + compact( + preparation, + model, + auth.apiKey, + auth.headers, + options.customInstructions, + options.signal, + this.thinkingLevel, + this.agent.streamFn, + ); + let result = await runCoreCompaction(); + // Automatic core compaction owns one retry for invalid summary output. + // Manual, provider-error, and extension-owned paths keep their own policy. + if (options.signal.aborted) { + return { status: "aborted" }; + } + if ( + options.summaryOutputPolicy === "retry-invalid-once" && + !result.ok && + result.error instanceof InvalidSummaryOutputError + ) { + result = await runCoreCompaction(); + if (options.signal.aborted) { + return { status: "aborted" }; + } + } + compactionResult = unwrapCoreResult(result); + } if (options.signal.aborted) { return { status: "aborted" }; @@ -359,6 +394,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { try { const outcome = await this.runCompactionWork({ mode: "auto", + summaryOutputPolicy: "retry-invalid-once", settings, signal: this.autoCompactionAbortController.signal, }); diff --git a/src/agents/sessions/agent-session-loop-correctness.test.ts b/src/agents/sessions/agent-session-loop-correctness.test.ts index d44abec04294..7c1044ad7318 100644 --- a/src/agents/sessions/agent-session-loop-correctness.test.ts +++ b/src/agents/sessions/agent-session-loop-correctness.test.ts @@ -13,6 +13,7 @@ const streamMocks = vi.hoisted(() => ({ })); import type { AgentTool } from "../runtime/index.js"; +import { agentSessionAutomaticCompaction } from "./agent-session-compaction.js"; import type { AgentSessionEvent } from "./agent-session-types.js"; import { AgentSession } from "./agent-session.js"; import { AuthStorage } from "./auth-storage.js"; @@ -87,6 +88,32 @@ function createAssistantResultStream(message: AssistantMessage) { return stream; } +const createOverflowAssistant = (activeModel: Model) => ({ + ...createAssistant(activeModel, [{ type: "text", text: "truncated answer" }], "length", 100), + usage: { ...createUsage(100), output: 0 }, +}); + +const createAutoCompactionSettings = () => + SettingsManager.inMemory({ + compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, + retry: { enabled: false }, + }); + +function mockInvalidThenTextSummary(recoveredText: string) { + let requests = 0; + streamMocks.streamSimple.mockImplementation((activeModel: Model) => { + return createAssistantResultStream( + createAssistant( + activeModel, + ++requests === 1 + ? [{ type: "thinking", thinking: "internal summary reasoning" }] + : [{ type: "text", text: recoveredText }], + ), + ); + }); + return () => requests; +} + function createResourceLoader( handlers: Map Promise>> = new Map(), ): ResourceLoader { @@ -305,10 +332,7 @@ describe("AgentSession loop correctness", () => { }); it("keeps a successful high-usage response and performs threshold maintenance without retry", async () => { - const settingsManager = SettingsManager.inMemory({ - compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, - retry: { enabled: false }, - }); + const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; streamMocks.streamSimple.mockImplementation((activeModel: Model) => createAssistantResultStream( @@ -378,10 +402,7 @@ describe("AgentSession loop correctness", () => { terminate: true, }), }; - const settingsManager = SettingsManager.inMemory({ - compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, - retry: { enabled: false }, - }); + const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; streamMocks.streamSimple.mockImplementation((activeModel: Model) => createAssistantResultStream( @@ -413,25 +434,14 @@ describe("AgentSession loop correctness", () => { }); it("compacts and retries a high-usage length-truncated response", async () => { - const settingsManager = SettingsManager.inMemory({ - compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, - retry: { enabled: false }, - }); + const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; let requestCount = 0; streamMocks.streamSimple.mockImplementation((activeModel: Model) => { requestCount += 1; return createAssistantResultStream( requestCount === 1 - ? { - ...createAssistant( - activeModel, - [{ type: "text", text: "truncated answer" }], - "length", - 100, - ), - usage: { ...createUsage(100), output: 0 }, - } + ? createOverflowAssistant(activeModel) : createAssistant(activeModel, [{ type: "text", text: "complete retry" }]), ); }); @@ -454,11 +464,242 @@ describe("AgentSession loop correctness", () => { expect(session.getLastAssistantText()).toBe("complete retry"); }); - it("leaves reactive overflow recovery to the caller when configured", async () => { - const settingsManager = SettingsManager.inMemory({ - compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, - retry: { enabled: false }, + it("retries a reasoning-only summary once during default auto-compaction", async () => { + const settingsManager = createAutoCompactionSettings(); + const compactionEvents: AgentSessionEvent[] = []; + let agentRequests = 0; + let summaryRequests = 0; + streamMocks.streamSimple.mockImplementation((activeModel: Model, context: Context) => { + const isSummary = context.systemPrompt?.includes("context summarization assistant") === true; + if (isSummary) { + summaryRequests += 1; + return createAssistantResultStream( + createAssistant( + activeModel, + summaryRequests === 1 + ? [{ type: "thinking", thinking: "internal summary reasoning" }] + : [{ type: "text", text: "recovered default summary" }], + ), + ); + } + agentRequests += 1; + return createAssistantResultStream( + agentRequests === 1 + ? createOverflowAssistant(activeModel) + : createAssistant(activeModel, [{ type: "text", text: "complete retry" }]), + ); }); + const { session, sessionManager } = await createTestSession({ + settingsManager, + resourceLoader: createResourceLoader(), + }); + session.subscribe((event) => { + if (event.type === "compaction_end") { + compactionEvents.push(event); + } + }); + + await session.prompt("long request"); + + expect({ agentRequests, summaryRequests }).toEqual({ agentRequests: 2, summaryRequests: 2 }); + expect(compactionEvents).toContainEqual( + expect.objectContaining({ type: "compaction_end", reason: "overflow", willRetry: true }), + ); + const compactionEntry = sessionManager.getBranch().find((entry) => entry.type === "compaction"); + expect(compactionEntry).toMatchObject({ type: "compaction", fromHook: false }); + expect(compactionEntry?.summary).toContain("recovered default summary"); + expect(session.getLastAssistantText()).toBe("complete retry"); + }); + + it("shares invalid-summary recovery with caller-owned automatic compaction", async () => { + const sessionManager = SessionManager.inMemory(); + appendHistory( + sessionManager, + createAssistant(testModel, [{ type: "text", text: "historical answer to summarize" }]), + ); + const settingsManager = createAutoCompactionSettings(); + const getSummaryRequests = mockInvalidThenTextSummary("recovered caller-owned summary"); + const { session } = await createTestSession({ + sessionManager, + settingsManager, + resourceLoader: createResourceLoader(), + }); + + const result = await session[agentSessionAutomaticCompaction](); + + expect(getSummaryRequests()).toBe(2); + expect(result.summary).toContain("recovered caller-owned summary"); + const compactions = sessionManager.getBranch().filter((entry) => entry.type === "compaction"); + expect(compactions).toHaveLength(1); + }); + + it("keeps public manual compaction one-shot for invalid summary output", async () => { + const sessionManager = SessionManager.inMemory(); + appendHistory( + sessionManager, + createAssistant(testModel, [{ type: "text", text: "historical answer to summarize" }]), + ); + const settingsManager = createAutoCompactionSettings(); + const getSummaryRequests = mockInvalidThenTextSummary("must not be requested"); + const { session } = await createTestSession({ + sessionManager, + settingsManager, + resourceLoader: createResourceLoader(), + }); + + await expect(session.compact()).rejects.toThrow( + "Turn prefix summarization failed: model returned no summary text", + ); + + expect(getSummaryRequests()).toBe(1); + expect(sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe(false); + }); + + it("stops default auto-compaction after two invalid summaries", async () => { + const settingsManager = createAutoCompactionSettings(); + const compactionEvents: AgentSessionEvent[] = []; + let agentRequests = 0; + let summaryRequests = 0; + streamMocks.streamSimple.mockImplementation((activeModel: Model, context: Context) => { + if (context.systemPrompt?.includes("context summarization assistant")) { + summaryRequests += 1; + return createAssistantResultStream( + createAssistant(activeModel, [ + { type: "thinking", thinking: `internal summary reasoning ${summaryRequests}` }, + ]), + ); + } + agentRequests += 1; + return createAssistantResultStream(createOverflowAssistant(activeModel)); + }); + const { session, sessionManager } = await createTestSession({ + settingsManager, + resourceLoader: createResourceLoader(), + }); + session.subscribe((event) => { + if (event.type === "compaction_end") { + compactionEvents.push(event); + } + }); + + await session.prompt("long request"); + + expect({ agentRequests, summaryRequests }).toEqual({ agentRequests: 1, summaryRequests: 2 }); + expect(compactionEvents).toContainEqual( + expect.objectContaining({ + type: "compaction_end", + reason: "overflow", + willRetry: false, + errorMessage: + "Context overflow recovery failed: Turn prefix summarization failed: model returned no summary text", + }), + ); + expect(sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe(false); + }); + + it.each([1, 2])( + "preserves cancellation when aborting during summary attempt %i", + async (abortAttempt) => { + const settingsManager = createAutoCompactionSettings(); + const compactionEvents: Array> = []; + let agentRequests = 0; + let summaryRequests = 0; + const created = await createTestSession({ + settingsManager, + resourceLoader: createResourceLoader(), + }); + const { session } = created; + streamMocks.streamSimple.mockImplementation((activeModel: Model, context: Context) => { + if (context.systemPrompt?.includes("context summarization assistant")) { + const summaryAttempt = ++summaryRequests; + const stream = createAssistantMessageEventStream(); + queueMicrotask(() => { + if (summaryAttempt === abortAttempt) { + session?.abortCompaction(); + } + stream.push({ + type: "done", + reason: "stop", + message: createAssistant(activeModel, [ + { type: "thinking", thinking: `internal summary reasoning ${summaryAttempt}` }, + ]), + }); + stream.end(); + }); + return stream; + } + agentRequests += 1; + return createAssistantResultStream(createOverflowAssistant(activeModel)); + }); + session.subscribe((event) => { + if (event.type === "compaction_end") { + compactionEvents.push(event); + } + }); + + await session.prompt("long request"); + + expect({ agentRequests, summaryRequests }).toEqual({ + agentRequests: 1, + summaryRequests: abortAttempt, + }); + expect(compactionEvents).toHaveLength(1); + expect(compactionEvents[0]).toMatchObject({ + type: "compaction_end", + reason: "overflow", + aborted: true, + willRetry: false, + }); + expect(compactionEvents[0]?.errorMessage).toBeUndefined(); + expect(created.sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe( + false, + ); + }, + ); + + it("does not retry provider errors during default auto-compaction", async () => { + const settingsManager = createAutoCompactionSettings(); + const compactionEvents: AgentSessionEvent[] = []; + let agentRequests = 0; + let summaryRequests = 0; + streamMocks.streamSimple.mockImplementation((activeModel: Model, context: Context) => { + if (context.systemPrompt?.includes("context summarization assistant")) { + summaryRequests += 1; + return createAssistantResultStream({ + ...createAssistant(activeModel, [], "error"), + errorMessage: "provider unavailable", + }); + } + agentRequests += 1; + return createAssistantResultStream(createOverflowAssistant(activeModel)); + }); + const { session, sessionManager } = await createTestSession({ + settingsManager, + resourceLoader: createResourceLoader(), + }); + session.subscribe((event) => { + if (event.type === "compaction_end") { + compactionEvents.push(event); + } + }); + + await session.prompt("long request"); + + expect({ agentRequests, summaryRequests }).toEqual({ agentRequests: 1, summaryRequests: 1 }); + expect(compactionEvents).toContainEqual( + expect.objectContaining({ + type: "compaction_end", + reason: "overflow", + willRetry: false, + errorMessage: + "Context overflow recovery failed: Turn prefix summarization failed: provider unavailable", + }), + ); + expect(sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe(false); + }); + + it("leaves reactive overflow recovery to the caller when configured", async () => { + const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; streamMocks.streamSimple.mockImplementation((activeModel: Model) => createAssistantResultStream({ @@ -489,10 +730,7 @@ describe("AgentSession loop correctness", () => { }); it("keeps threshold maintenance session-owned when the caller owns overflow recovery", async () => { - const settingsManager = SettingsManager.inMemory({ - compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 1 }, - retry: { enabled: false }, - }); + const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; streamMocks.streamSimple.mockImplementation((activeModel: Model) => createAssistantResultStream( diff --git a/src/agents/sessions/sdk.test.ts b/src/agents/sessions/sdk.test.ts index 186c9925e6c4..990cbca11ac4 100644 --- a/src/agents/sessions/sdk.test.ts +++ b/src/agents/sessions/sdk.test.ts @@ -313,6 +313,85 @@ describe("AgentSession getLastAssistantText", () => { }); }); +describe("AgentSession tree navigation", () => { + it("leaves the tree unchanged when branch summarization returns reasoning only", async () => { + const authStorage = AuthStorage.inMemory(); + authStorage.setRuntimeApiKey(testModel.provider, "test-api-key"); + const sessionManager = SessionManager.inMemory(); + const rootId = sessionManager.appendMessage({ + role: "user", + content: "shared root", + timestamp: 1, + }); + const abandonedLeafId = sessionManager.appendMessage({ + role: "user", + content: "abandoned branch", + timestamp: 2, + }); + sessionManager.branch(rootId); + const targetId = sessionManager.appendMessage({ + role: "assistant", + content: [{ type: "text", text: "target branch" }], + api: testModel.api, + provider: testModel.provider, + model: testModel.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 3, + }); + sessionManager.branch(abandonedLeafId); + streamMocks.streamSimple.mockReset(); + streamMocks.streamSimple.mockImplementation(() => + createAssistantResultStream({ + role: "assistant", + content: [{ type: "thinking", thinking: "internal summary reasoning" }], + api: testModel.api, + provider: testModel.provider, + model: testModel.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 4, + }), + ); + const { session } = await createAgentSession({ + authStorage, + model: testModel, + resourceLoader: createEmptyResourceLoader(), + sessionManager, + settingsManager: SettingsManager.inMemory(), + modelRegistry: createTestModelRegistry(authStorage), + }); + const entriesBefore = sessionManager.getEntries(); + const leafBefore = sessionManager.getLeafId(); + + await expect(session.navigateTree(targetId, { summarize: true })).rejects.toThrow( + "Branch summary failed: model returned no summary text", + ); + + expect(streamMocks.streamSimple).toHaveBeenCalledOnce(); + expect(sessionManager.getEntries()).toEqual(entriesBefore); + expect(sessionManager.getLeafId()).toBe(leafBefore); + expect(sessionManager.getEntries().some((entry) => entry.type === "branch_summary")).toBe( + false, + ); + session.dispose(); + }); +}); + describe("AgentSession queued user turns", () => { it("carries prepared transcript context on the exact steered message", async () => { const session = await createSessionFromManager(SessionManager.inMemory());