From 6a8e11e63ceed04aed5476180b2a9a4dccea44a0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Wed, 5 Aug 2026 09:56:06 +0800 Subject: [PATCH] fix(qa): repair release validation scenarios (#119150) * fix(qa): repair parity and matrix media setup Punchcard-Session: silver-valley-valley-dt * test(qa): accept token-budgeted compaction suffix Punchcard-Session: silver-valley-valley-dt * test(qa): scope request diagnostics Punchcard-Session: silver-valley-valley-dt * test(qa): prove thread-memory causality Punchcard-Session: silver-valley-valley-dt * test(qa): configure Matrix image generation and fail fast Punchcard-Session: silver-valley-valley-dt * fix(qa): require memory get before thread recall Punchcard-Session: silver-valley-valley-dt * fix(qa): unify mock provider dispatch Punchcard-Session: silver-valley-valley-dt * fix(qa): repair Anthropic IDs and compaction causality Punchcard-Session: silver-valley-valley-dt * fix(qa): authenticate compaction wire causality Punchcard-Session: silver-valley-valley-dt * fix(qa): preserve Responses tool item identity Punchcard-Session: silver-valley-valley-dt * fix(qa): scope restart heartbeat proof Punchcard-Session: silver-valley-valley-dt * test(qa): assert causal traces before delivery Punchcard-Session: silver-valley-valley-dt * fix(qa): validate code mode completion evidence Punchcard-Session: silver-valley-valley-dt * test(qa): split causal catalog checks Punchcard-Session: silver-valley-valley-dt * style(qa): format matrix scenario tests Punchcard-Session: silver-valley-valley-dt --- .../matrix/matrix-scenario-flows.test.ts | 43 +- .../scenarios/scenario-runtime-media.test.ts | 132 +++- .../scenarios/scenario-runtime-media.ts | 63 +- .../matrix/substrate/config.test.ts | 29 + .../mock-openai/mock-anthropic-messages.ts | 155 ++-- .../mock-openai/mock-anthropic-wire.ts | 70 +- .../mock-openai/mock-openai-contracts.ts | 23 + .../mock-openai/mock-openai-events.ts | 14 +- .../mock-openai/mock-openai-input.ts | 5 +- .../mock-openai-responses-websocket.ts | 16 +- .../src/providers/mock-openai/server.test.ts | 733 ++++++++++++++++-- .../src/providers/mock-openai/server.ts | 406 ++++++---- .../qa-lab/src/qa-gateway-config.test.ts | 2 +- extensions/qa-lab/src/qa-gateway-config.ts | 2 +- .../src/scenario-catalog-causality.test.ts | 215 +++++ .../src/scenario-catalog-compaction.test.ts | 182 ++++- .../qa-lab/src/scenario-catalog.test-utils.ts | 14 + .../qa-lab/src/scenario-catalog.test.ts | 66 -- ...struction-followthrough-repo-contract.yaml | 5 +- qa/scenarios/agents/subagent-handoff.yaml | 5 +- .../matrix-room-generated-image-delivery.yaml | 19 +- .../memory/memory-tools-channel-context.yaml | 25 +- .../memory/thread-memory-isolation.yaml | 48 +- .../runtime/agent-tool-consumption.yaml | 20 +- .../compaction-retry-mutating-tool.yaml | 87 ++- .../runtime/gateway-restart-inflight-run.yaml | 52 +- 26 files changed, 1903 insertions(+), 528 deletions(-) create mode 100644 extensions/qa-lab/src/scenario-catalog-causality.test.ts diff --git a/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts b/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts index fe142c12294b..ab27d012e8d4 100644 --- a/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/matrix-scenario-flows.test.ts @@ -5,6 +5,7 @@ import { readQaScenarioExecutionConfig, } from "../../scenario-catalog.js"; import { requireFlowScenario } from "../../scenario-catalog.test-utils.js"; +import { collectQaSuitePluginIds } from "../../suite-planning.js"; const MATRIX_MENTION_GATE_PRIMARY_SCENARIOS = [ "matrix-allowbots-default-block", @@ -221,14 +222,17 @@ describe("Matrix QA Lab scenario flows", () => { }); it("loads the voice preflight provider and media overrides", () => { - expect(readQaScenarioById("matrix-voice-preflight-mention").plugins).toEqual(["openai"]); - expect(readQaScenarioById("matrix-voice-preflight-mention").execution).toMatchObject({ + const scenario = readQaScenarioById("matrix-voice-preflight-mention"); + + expect(scenario.execution).toMatchObject({ kind: "flow", providerMode: "mock-openai", retryCount: 0, timeoutMs: 90_000, }); - expect(readQaScenarioById("matrix-voice-preflight-mention").gatewayConfigPatch).toMatchObject({ + expect(scenario.plugins).toEqual(["openai"]); + expect(collectQaSuitePluginIds([scenario])).toEqual(["openai"]); + expect(scenario.gatewayConfigPatch).toMatchObject({ tools: { media: { models: [{ capabilities: ["audio"], model: "gpt-4o-transcribe", provider: "openai" }], @@ -274,4 +278,37 @@ describe("Matrix QA Lab scenario flows", () => { }, }); }); + + it("configures image generation before the single generated-image flow call", () => { + const scenario = requireFlowScenario( + readQaScenarioById("matrix-room-generated-image-delivery"), + ); + const actions = scenario.execution.flow?.steps[0]?.actions ?? []; + + expect(scenario.execution).toMatchObject({ + channel: "matrix", + retryCount: 0, + timeoutMs: 180_000, + config: { + requiredChannelDriver: "live", + }, + }); + expect(actions).toEqual([ + { + call: "ensureImageGenerationConfigured", + args: [{ ref: "env" }], + }, + { + set: "scenarioModule", + value: { + expr: "await qaImport('./live-transports/matrix/scenarios/scenario-runtime-media.js')", + }, + }, + { + call: "scenarioModule.runGeneratedImageDeliveryScenario", + args: [{ expr: "scenarioContext" }], + saveAs: "result", + }, + ]); + }); }); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.test.ts b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.test.ts index e178a6b0b1f7..b602abd9b5e1 100644 --- a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.test.ts @@ -1,5 +1,16 @@ -import { describe, expect, it } from "vitest"; -import { testing } from "./scenario-runtime-media.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const advanceMatrixQaActorCursor = vi.hoisted(() => vi.fn()); +const primeMatrixQaActorCursor = vi.hoisted(() => vi.fn()); + +vi.mock("./scenario-runtime-shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + advanceMatrixQaActorCursor, + primeMatrixQaActorCursor, +})); + +import { runGeneratedImageDeliveryScenario, testing } from "./scenario-runtime-media.js"; +import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js"; describe("Matrix voice preflight reply matching", () => { it("accepts punctuation differences in the transcribed marker", () => { @@ -10,3 +21,120 @@ describe("Matrix voice preflight reply matching", () => { ).toBe(true); }); }); + +const roomId = "!media:matrix-qa.test"; + +function createGeneratedImageContext( + observedEvents: MatrixQaScenarioContext["observedEvents"] = [], +): MatrixQaScenarioContext { + return { + baseUrl: "http://127.0.0.1:28008", + driverAccessToken: "driver-token", + driverUserId: "@driver:matrix-qa.test", + observedEvents, + observerAccessToken: "observer-token", + observerUserId: "@observer:matrix-qa.test", + roomId, + sutAccessToken: "sut-token", + sutUserId: "@sut:matrix-qa.test", + syncState: {}, + timeoutMs: 180_000, + topology: { + defaultRoomId: roomId, + defaultRoomKey: "media", + rooms: [ + { + encrypted: false, + key: "media", + kind: "group", + memberRoles: ["driver", "observer", "sut"], + memberUserIds: [ + "@driver:matrix-qa.test", + "@observer:matrix-qa.test", + "@sut:matrix-qa.test", + ], + name: "Matrix QA Media Room", + requireMention: true, + roomId, + }, + ], + }, + }; +} + +describe("Matrix generated image delivery", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("sends one trigger and accepts one fresh top-level image", async () => { + const sendTextMessage = vi.fn(async () => "$driver-trigger"); + const waitForOptionalRoomEvent = vi.fn(async ({ predicate }) => { + const event = { + attachment: { filename: "generated.png", kind: "image" as const }, + eventId: "$generated-image", + kind: "message" as const, + msgtype: "m.image", + originServerTs: Date.now() + 1, + roomId, + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }; + expect(predicate(event)).toBe(true); + return { event, matched: true as const, since: "next" }; + }); + primeMatrixQaActorCursor.mockResolvedValue({ + client: { sendTextMessage, waitForOptionalRoomEvent }, + startSince: "start", + }); + + const result = await runGeneratedImageDeliveryScenario(createGeneratedImageContext()); + + expect(sendTextMessage).toHaveBeenCalledTimes(1); + expect(waitForOptionalRoomEvent).toHaveBeenCalledTimes(1); + expect(result.artifacts?.driverEventId).toBe("$driver-trigger"); + expect(result.artifacts?.attachmentEventId).toBe("$generated-image"); + }); + + it("bounds timeout diagnostics to the last eight room events", async () => { + const observedEvents = Array.from({ length: 10 }, (_, index) => ({ + body: `event body ${index + 1}`, + eventId: `$event-${index + 1}`, + kind: "message" as const, + roomId, + sender: "@sut:matrix-qa.test", + type: "m.room.message", + })); + observedEvents.push({ + body: "other room", + eventId: "$other-room", + kind: "message", + roomId: "!other:matrix-qa.test", + sender: "@sut:matrix-qa.test", + type: "m.room.message", + }); + const sendTextMessage = vi.fn(async () => "$driver-trigger"); + primeMatrixQaActorCursor.mockResolvedValue({ + client: { + sendTextMessage, + waitForOptionalRoomEvent: vi.fn(async () => ({ + matched: false as const, + since: "next", + })), + }, + startSince: "start", + }); + + const error = await runGeneratedImageDeliveryScenario( + createGeneratedImageContext(observedEvents), + ).catch((cause: unknown) => cause); + const message = error instanceof Error ? error.message : String(error); + + expect(sendTextMessage).toHaveBeenCalledTimes(1); + expect(message).toContain("$event-3"); + expect(message).toContain("$event-10"); + expect(message).not.toContain('$event-1"'); + expect(message).not.toContain('$event-2"'); + expect(message).not.toContain("$other-room"); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.ts b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.ts index ac495c9fd0f9..2d5d7832c2a4 100644 --- a/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.ts +++ b/extensions/qa-lab/src/live-transports/matrix/scenarios/scenario-runtime-media.ts @@ -425,41 +425,43 @@ export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenari const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY); const { client, startSince } = await primeMatrixQaDriverMediaClient(context); const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId); - const driverEventIds: string[] = []; + const triggerSentAt = Date.now(); + const driverEventId = await client.sendTextMessage({ + body: triggerBody, + mentionUserIds: [context.sutUserId], + roomId, + }); const isGeneratedImageEvent = (event: MatrixQaObservedEvent) => event.roomId === roomId && event.sender === context.sutUserId && event.type === "m.room.message" && event.relatesTo === undefined && event.msgtype === "m.image" && - event.attachment?.kind === "image"; - let matched: Awaited> | undefined; - for (let attempt = 1; attempt <= 2; attempt += 1) { - const triggerSentAt = Date.now(); - const driverEventId = await client.sendTextMessage({ - body: triggerBody, - mentionUserIds: [context.sutUserId], - roomId, - }); - driverEventIds.push(driverEventId); - matched = await client.waitForOptionalRoomEvent({ - observedEvents: context.observedEvents, - // The start cursor can still receive delayed images from an earlier run. - predicate: (event) => - isGeneratedImageEvent(event) && - typeof event.originServerTs === "number" && - event.originServerTs >= triggerSentAt, - roomId, - since: matched?.since ?? startSince, - timeoutMs: context.timeoutMs, - }); - if (matched.matched) { - break; - } - } - if (!matched?.matched) { + event.attachment?.kind === "image" && + typeof event.originServerTs === "number" && + event.originServerTs >= triggerSentAt; + const matched = await client.waitForOptionalRoomEvent({ + observedEvents: context.observedEvents, + // The start cursor can still receive delayed images from an earlier run. + predicate: isGeneratedImageEvent, + roomId, + since: startSince, + timeoutMs: context.timeoutMs, + }); + if (!matched.matched) { + const recentRoomEvents = context.observedEvents + .filter((event) => event.roomId === roomId) + .slice(-8) + .map((event) => ({ + body: truncateMatrixQaPreview(event.body), + eventId: event.eventId, + kind: event.kind, + msgtype: event.msgtype, + sender: event.sender, + type: event.type, + })); throw new Error( - `timed out after ${context.timeoutMs}ms waiting for Matrix generated image after ${driverEventIds.length} attempt(s)`, + `timed out after ${context.timeoutMs}ms waiting for Matrix generated image; recent room events: ${JSON.stringify(recentRoomEvents)}`, ); } const matchedEvent = matched.event; @@ -480,14 +482,13 @@ export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenari attachmentFilename: attachment.filename, attachmentKind: attachment.kind, attachmentMsgtype: matchedEvent.msgtype, - driverEventId: driverEventIds[0], - driverEventIds, + driverEventId, roomId, triggerBody, }, details: [ `room id: ${roomId}`, - `driver events: ${driverEventIds.join(", ")}`, + `driver event: ${driverEventId}`, ...buildMatrixQaAttachmentDetailLines({ attachmentEvent: matchedEvent, label: "generated image", diff --git a/extensions/qa-lab/src/live-transports/matrix/substrate/config.test.ts b/extensions/qa-lab/src/live-transports/matrix/substrate/config.test.ts index d823de464282..8b306b06e21d 100644 --- a/extensions/qa-lab/src/live-transports/matrix/substrate/config.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/substrate/config.test.ts @@ -83,6 +83,35 @@ describe("matrix qa config", () => { expect(next.messages?.groupChat?.visibleReplies).toBe("automatic"); }); + it("preserves the scenario provider plugin without enabling unrelated plugins", () => { + const next = buildMatrixQaConfig( + { + plugins: { + allow: ["acpx", "memory-core", "qa-lab", "openai"], + entries: { + openai: { enabled: true }, + unrelated: { enabled: true }, + }, + }, + } as OpenClawConfig, + { + driverUserId: "@driver:matrix-qa.test", + homeserver: "http://127.0.0.1:28008/", + observerUserId: "@observer:matrix-qa.test", + sutAccessToken: "sut-token", + sutAccountId: "sut", + sutUserId: "@sut:matrix-qa.test", + topology, + }, + ); + + expect(next.plugins?.allow).toEqual(["acpx", "memory-core", "qa-lab", "openai", "matrix"]); + expect(next.plugins?.allow).not.toContain("unrelated"); + expect(next.plugins?.allow).not.toContain("anthropic"); + expect(next.plugins?.entries?.matrix).toEqual({ enabled: true }); + expect(next.plugins?.entries?.openai).toEqual({ enabled: true }); + }); + it("honors an explicit DM disable with a provisioned DM room", () => { const next = buildMatrixQaConfig({} as OpenClawConfig, { driverUserId: "@driver:matrix-qa.test", diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts index 88cb7485ab2d..64289203d5ce 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-messages.ts @@ -1,109 +1,70 @@ +// QA Lab Anthropic Messages wire adapter. import { - convertAnthropicMessagesToResponsesInput, - type ExtractedAssistantOutput, - extractFinalAssistantOutputFromEvents, + buildAnthropicFailureResponse, buildAnthropicMessageResponse, + buildAnthropicMessageStreamEvents, buildAnthropicThinkingErrorResponse, buildAnthropicThinkingErrorStreamEvents, - buildAnthropicMessageStreamEvents, + convertAnthropicMessagesToResponsesInput, + extractFinalAssistantOutputFromEvents, + normalizeAnthropicSystemToString, } from "./mock-anthropic-wire.js"; -// QA Lab Anthropic Messages request dispatcher. -import { - type ResponsesInputItem, - type StreamEvent, - type AnthropicMessagesRequest, - QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE, - type MockScenarioState, - type AnthropicStreamEvent, +import type { + AnthropicMessagesRequest, + AnthropicStreamEvent, + QaMockProviderDispatchResult, + ResponsesInputItem, } from "./mock-openai-contracts.js"; -import { buildAssistantEvents } from "./mock-openai-events.js"; -import { - extractAllRequestTexts, - extractLastUserText, - extractToolOutput, - extractToolOutputCallId, -} from "./mock-openai-input.js"; -import { buildToolCallEventsWithArgs } from "./mock-openai-tooling.js"; -export async function buildMessagesPayload( - body: AnthropicMessagesRequest, - scenarioState: MockScenarioState, - dispatchResponses: ( - body: Record, - scenarioState: MockScenarioState, - ) => Promise, -): Promise<{ - events: StreamEvent[]; +export function normalizeAnthropicMessagesRequest(body: AnthropicMessagesRequest): { + body: Record; input: ResponsesInputItem[]; - extracted: ExtractedAssistantOutput; + model: string; +} { + const model = + typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8"; + const input = convertAnthropicMessagesToResponsesInput({ + messages: Array.isArray(body.messages) ? body.messages : [], + }); + const instructions = normalizeAnthropicSystemToString(body.system); + return { + body: { + input, + model, + stream: false, + ...(instructions ? { instructions } : {}), + ...(Array.isArray(body.tools) ? { tools: body.tools } : {}), + }, + input, + model, + }; +} + +export function buildMessagesPayload(dispatched: QaMockProviderDispatchResult): { responseBody: Record; streamEvents: AnthropicStreamEvent[]; - model: string; -}> { - const messages = Array.isArray(body.messages) ? body.messages : []; - const input = convertAnthropicMessagesToResponsesInput({ - system: body.system, - messages, - }); - // Treat empty-string model the same as absent. A bare typeof check lets - // `""` leak through to `responseBody.model` and `lastRequest.model`, - // which then confuses parity consumers that assume the mock always - // echoes the real provider label. Normalize once and reuse everywhere. - const normalizedModel = - typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8"; - // Dispatch through the same scenario logic the /v1/responses route uses. - // Preserve declared tools so route-specific adapters mirror what the - // real provider request made available to the model. - const dispatchBody: Record = { - input, - model: normalizedModel, - stream: false, - ...(Array.isArray(body.tools) ? { tools: body.tools } : {}), - }; - const allInputText = extractAllRequestTexts(input, dispatchBody); - if (QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText)) { - const toolOutput = extractToolOutput(input); - const toolOutputCallId = extractToolOutputCallId(input); - const scenarioKey = `${normalizedModel}\n${extractLastUserText(input)}`; - const shouldEmitThinkingError = - toolOutput.length > 0 && - toolOutputCallId.length > 0 && - !scenarioState.anthropicThinkingErrorScenarioKeys.has(scenarioKey); - // Safe retries generate fresh read call IDs. The original user prompt stays - // stable, so fail once per model and nonce-bearing logical scenario instead. - if (shouldEmitThinkingError) { - scenarioState.anthropicThinkingErrorScenarioKeys.add(scenarioKey); - } - const events = - toolOutput.length === 0 - ? buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" }) - : shouldEmitThinkingError - ? buildAssistantEvents("") - : buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK"); - const extracted = extractFinalAssistantOutputFromEvents(events); - const responseBody = shouldEmitThinkingError - ? buildAnthropicThinkingErrorResponse({ model: normalizedModel }) - : buildAnthropicMessageResponse({ - model: normalizedModel, - extracted, - }); - const streamEvents = shouldEmitThinkingError - ? buildAnthropicThinkingErrorStreamEvents({ model: normalizedModel }) - : buildAnthropicMessageStreamEvents({ - model: normalizedModel, - extracted, - }); - return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; +} { + if (dispatched.failure?.presentation === "anthropic-thinking") { + return { + responseBody: buildAnthropicThinkingErrorResponse({ model: dispatched.model }), + streamEvents: buildAnthropicThinkingErrorStreamEvents({ model: dispatched.model }), + }; } - const events = await dispatchResponses(dispatchBody, scenarioState); - const extracted = extractFinalAssistantOutputFromEvents(events); - const responseBody = buildAnthropicMessageResponse({ - model: normalizedModel, - extracted, - }); - const streamEvents = buildAnthropicMessageStreamEvents({ - model: normalizedModel, - extracted, - }); - return { events, input, extracted, responseBody, streamEvents, model: normalizedModel }; + if (dispatched.failure) { + return { + responseBody: buildAnthropicFailureResponse(dispatched.failure), + streamEvents: [], + }; + } + const extracted = extractFinalAssistantOutputFromEvents(dispatched.events); + return { + responseBody: buildAnthropicMessageResponse({ + model: dispatched.model, + extracted, + }), + streamEvents: buildAnthropicMessageStreamEvents({ + model: dispatched.model, + extracted, + }), + }; } diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts index 6738702ffe42..bc8b893e2fd1 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-anthropic-wire.ts @@ -1,4 +1,5 @@ // QA Lab Anthropic Messages wire conversion and response events. +import { createHash } from "node:crypto"; import { type ResponsesInputItem, type StreamEvent, @@ -6,13 +7,14 @@ import { type AnthropicMessage, type AnthropicMessagesRequest, type AnthropicStreamEvent, + type QaMockProviderFailure, countApproxTokens, } from "./mock-openai-contracts.js"; // Anthropic Messages conversion preserves role and tool ordering while reusing // the shared Responses scenario dispatcher for provider parity. -function normalizeAnthropicSystemToString( +export function normalizeAnthropicSystemToString( system: AnthropicMessagesRequest["system"], ): string | undefined { if (typeof system === "string") { @@ -145,11 +147,62 @@ export function convertAnthropicMessagesToResponsesInput(params: { return items; } -export type ExtractedAssistantOutput = { +type ExtractedAssistantOutput = { text: string; toolCalls: Array<{ id: string; name: string; input: Record }>; }; +const NATIVE_ANTHROPIC_TOOL_USE_ID_RE = /^toolu_[A-Za-z0-9_]+$/; +const ANTHROPIC_TOOL_USE_ID_MAX_LENGTH = 64; + +function isNativeAnthropicToolUseId(id: string): boolean { + return id.length <= ANTHROPIC_TOOL_USE_ID_MAX_LENGTH && NATIVE_ANTHROPIC_TOOL_USE_ID_RE.test(id); +} + +export function adaptAnthropicToolCallIds(events: StreamEvent[]): StreamEvent[] { + const adaptedIds = new Map(); + const adaptId = (id: string) => { + if (isNativeAnthropicToolUseId(id)) { + return id; + } + const existing = adaptedIds.get(id); + if (existing) { + return existing; + } + const adapted = `toolu_${createHash("sha256").update(id).digest("hex").slice(0, 48)}`; + adaptedIds.set(id, adapted); + return adapted; + }; + const adaptItem = (item: Record) => { + if ( + (item.type === "function_call" || item.type === "custom_tool_call") && + typeof item.call_id === "string" + ) { + return { ...item, call_id: adaptId(item.call_id) }; + } + return item; + }; + + return events.map((event) => { + if (event.type === "response.output_item.added" || event.type === "response.output_item.done") { + return { ...event, item: adaptItem(event.item) }; + } + if (event.type === "response.custom_tool_call_input.delta") { + return { ...event, call_id: adaptId(event.call_id) }; + } + if (event.type === "response.completed") { + return { + ...event, + response: { + ...event.response, + output: event.response.output.map(adaptItem), + }, + }; + } + return event; + }); +} + export function extractFinalAssistantOutputFromEvents( events: StreamEvent[], ): ExtractedAssistantOutput { @@ -237,6 +290,19 @@ export function buildAnthropicMessageResponse(params: { }; } +export function buildAnthropicFailureResponse( + failure: QaMockProviderFailure, +): Record { + return { + type: "error", + error: { + type: failure.type, + ...(failure.code ? { code: failure.code } : {}), + message: failure.message, + }, + }; +} + const QA_ANTHROPIC_THINKING_ERROR_TEXT = "QA replay-safe read completed, but the provider stream failed after signed thinking."; const QA_ANTHROPIC_THINKING_ERROR_SIGNATURE = "qa_signed_thinking_block_91953"; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts index 77473ed6ca39..ad202368d6d5 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-contracts.ts @@ -15,6 +15,28 @@ export type MockCompactionSummaryFaultMode = type MockOpenAiRequestOutcome = "success" | "error"; +export type QaMockProviderDispatchRequest = { + route: "responses" | "anthropic-messages"; + body: Record; + raw: string; +}; + +export type QaMockProviderFailure = { + status: number; + type: string; + code?: string; + message: string; + presentation?: "anthropic-thinking"; +}; + +export type QaMockProviderDispatchResult = { + events: StreamEvent[]; + model: string; + failure?: QaMockProviderFailure; + onResponseSent?: () => void; + previewPauseMs?: number; +}; + export type StreamEvent = | { type: "response.created"; response: { id: string } } | { @@ -136,6 +158,7 @@ export type MockOpenAiRequestSnapshot = { errorCode?: string; rawByteLength: number; plannedToolCallId?: string; + plannedToolItemId?: string; plannedToolName?: string; plannedWireToolName?: string; plannedToolArgs?: Record; diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts index dcdd6fe9b959..af92b2965de1 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-events.ts @@ -108,20 +108,26 @@ export function extractPlannedToolName(events: StreamEvent[]) { return undefined; } -export function extractPlannedToolCallId(events: StreamEvent[]) { +export function extractPlannedToolIdentity(events: StreamEvent[]): { + callId?: string; + itemId?: string; +} { for (const event of events) { if (event.type !== "response.output_item.done") { continue; } - const item = event.item as { type?: unknown; call_id?: unknown }; + const item = event.item as { type?: unknown; id?: unknown; call_id?: unknown }; if ( (item.type === "function_call" || item.type === "custom_tool_call") && typeof item.call_id === "string" ) { - return item.call_id; + return { + callId: item.call_id, + itemId: typeof item.id === "string" ? item.id : undefined, + }; } } - return undefined; + return {}; } export function extractPlannedToolArgs(events: StreamEvent[]) { diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts index 3278c034fba6..6c60a99f6030 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-input.ts @@ -158,6 +158,9 @@ export function extractToolOutput(input: ResponsesInputItem[]) { return item ? stringifyFunctionCallOutput(item.output) : ""; } +export const extractToolOutputValue = (input: ResponsesInputItem[]) => + findCurrentToolOutput(input)?.output; + export function extractToolOutputStructuredError(input: ResponsesInputItem[]) { const item = findCurrentToolOutput(input); return item?.is_error === true || item?.isError === true; @@ -264,7 +267,7 @@ export function extractSlackMpimRetainedBotNonce( return undefined; } -export function extractAllInputTexts(input: ResponsesInputItem[]) { +function extractAllInputTexts(input: ResponsesInputItem[]) { const texts: string[] = []; for (const item of input) { if (typeof item.output === "string" && item.output.trim()) { diff --git a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts index 78a6c00d8111..4619079daa41 100644 --- a/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts +++ b/extensions/qa-lab/src/providers/mock-openai/mock-openai-responses-websocket.ts @@ -1,24 +1,12 @@ import type { Server } from "node:http"; import { setTimeout as sleep } from "node:timers/promises"; import { WebSocket, WebSocketServer, type RawData } from "ws"; -import type { ResponsesInputItem, StreamEvent } from "./mock-openai-contracts.js"; - -export type QaMockResponsesDispatchResult = { - events: StreamEvent[]; - failure?: { - status: number; - type: string; - code?: string; - message: string; - }; - onResponseSent?: () => void; - previewPauseMs?: number; -}; +import type { QaMockProviderDispatchResult, ResponsesInputItem } from "./mock-openai-contracts.js"; type QaMockResponsesWebSocketDispatch = (params: { body: Record; raw: string; -}) => Promise; +}) => Promise; type QaMockResponsesWebSocketHistory = { id: string; diff --git a/extensions/qa-lab/src/providers/mock-openai/server.test.ts b/extensions/qa-lab/src/providers/mock-openai/server.test.ts index a8630f816361..88c57ef550c7 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.test.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.test.ts @@ -3,6 +3,8 @@ import { once } from "node:events"; import { afterEach, describe, expect, it } from "vitest"; import { WebSocket } from "ws"; import { readQaMockRequestCursor } from "../shared/debug-request-cursor.js"; +import { adaptAnthropicToolCallIds } from "./mock-anthropic-wire.js"; +import type { StreamEvent } from "./mock-openai-contracts.js"; import { readTargetFromPrompt } from "./mock-openai-tooling.js"; import { startQaMockOpenAiServer } from "./server.js"; @@ -2315,6 +2317,65 @@ describe("qa mock openai server", () => { expect(requests.every((request) => request.errorCode === "context_length_exceeded")).toBe(true); }); + it("injects one Anthropic overflow per session before planning the logical write", async () => { + const server = await startMockServer(); + const bodyFor = (sessionId: string) => ({ + system: `Runtime: embedded | sessionId=${sessionId}`, + tools: [ + { + name: "exec", + input_schema: { + type: "object", + properties: { + language: { type: "string" }, + code: { type: "string" }, + }, + required: ["code"], + }, + }, + { + name: "wait", + input_schema: { + type: "object", + properties: { runId: { type: "string" } }, + required: ["runId"], + }, + }, + ], + messages: [ + makeAnthropicUserText( + `${QA_COMPACTION_RETRY_PROMPT}\n${QA_COMPACTION_RETRY_OVERFLOW_PADDING}`, + ), + ], + }); + + const first = await postAnthropicMessages(server, bodyFor("anthropic-overflow-a")); + expect(first.status).toBe(400); + expect(await first.json()).toEqual({ + type: "error", + error: { + type: "invalid_request_error", + code: "context_length_exceeded", + message: "This model's maximum context length was exceeded.", + }, + }); + + const second = await postAnthropicMessages(server, bodyFor("anthropic-overflow-a")); + expect(second.status).toBe(200); + const content = requireArray( + requireRecord(await second.json(), "Anthropic response").content, + "content", + ); + expect(content).toContainEqual(expect.objectContaining({ type: "tool_use", name: "exec" })); + expect(await getJson(server, "/debug/last-request")).toMatchObject({ + plannedToolName: "write", + plannedWireToolName: "exec", + }); + + const independent = await postAnthropicMessages(server, bodyFor("anthropic-overflow-b")); + expect(independent.status).toBe(400); + }); + it("excludes compaction summary requests from overflow injection", async () => { const server = await startMockServer(); const initial = await postNonStreamingResponses(server, { @@ -2496,6 +2557,28 @@ describe("qa mock openai server", () => { ]); }); + it("excludes Anthropic compaction summary requests from overflow injection", async () => { + const server = await startMockServer(); + const response = await postAnthropicMessages(server, { + system: QA_COMPACTION_SUMMARY_INSTRUCTIONS, + messages: [ + makeAnthropicUserText( + `\n${QA_COMPACTION_RETRY_OVERFLOW_PADDING}\n`, + ), + ], + }); + + expect(response.status).toBe(200); + const body = requireRecord(await response.json(), "Anthropic summary response"); + expect(requireArray(body.content, "content")).toContainEqual( + expect.objectContaining({ type: "text", text: expect.stringContaining("## Goal") }), + ); + expect(await getJson(server, "/debug/last-request")).toMatchObject({ + requestKind: "compaction-summary", + outcome: "success", + }); + }); + it("handles staged scalar compaction summaries and promotes the durable merge without state leakage", async () => { const server = await startMockServer(); const genericChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, { @@ -3549,6 +3632,115 @@ Update and merge these partial structured summaries.`, expect(imageRequest.size).toBe("1024x1024"); }); + it("requires memory_get before answering thread recall in Code Mode", async () => { + const server = await startMockServer(); + const prompt = + "@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread."; + const codeModeTools = [ + { + type: "function", + name: "exec", + parameters: { + type: "object", + properties: { + language: { type: "string" }, + code: { type: "string" }, + }, + required: ["code"], + }, + }, + { + type: "function", + name: "wait", + parameters: { + type: "object", + properties: { runId: { type: "string" } }, + required: ["runId"], + }, + }, + ]; + const initialInput: Array> = [ + { + type: "additional_tools", + role: "developer", + tools: codeModeTools, + }, + makeUserInput(prompt), + ]; + + const searchPlan = await expectOpenAiNonStreamingResponsesJson(server, { + input: initialInput, + }); + const searchCall = outputToolCall(searchPlan, "exec"); + const searchCallId = outputToolCallId(searchCall, "call_mock_memory_search"); + const continuationInput: Array> = [ + makeUserInput(prompt), + searchCall, + makeToolOutputWithCallId( + searchCallId, + JSON.stringify({ + status: "completed", + value: { + results: [ + { + path: "MEMORY.md", + startLine: 1, + endLine: 1, + snippet: "Thread-hidden codename: ORBIT-21.", + }, + ], + }, + }), + ), + ]; + + const getPlan = await expectOpenAiNonStreamingResponsesJson(server, { + input: continuationInput, + }); + const getCall = outputToolCall(getPlan, "memory_get"); + const getCallId = outputToolCallId(getCall, "call_mock_memory_get"); + expect(getCallId).not.toBe(searchCallId); + expect(outputItems(getPlan).some((item) => item.type === "message")).toBe(false); + expect(JSON.stringify(getPlan)).not.toContain("hidden thread codename is ORBIT-21"); + continuationInput.push( + getCall, + makeToolOutputWithCallId( + getCallId, + JSON.stringify({ + status: "completed", + value: { text: "Thread-hidden codename: ORBIT-22." }, + }), + ), + ); + + const final = await expectOpenAiNonStreamingResponsesJson(server, { + input: continuationInput, + }); + expect(outputText(final)).toContain("ORBIT-22"); + expect(outputText(final)).not.toContain("ORBIT-21"); + expect(outputItems(final).some((item) => item.type === "function_call")).toBe(false); + + const requests = requireArray(await getJson(server, "/debug/requests"), "debug requests").map( + (request, index) => requireRecord(request, `debug request ${index}`), + ); + expect(requests).toHaveLength(3); + expect(requests[0]).toMatchObject({ + plannedToolName: "memory_search", + plannedWireToolName: "exec", + plannedToolCallId: searchCallId, + }); + expect(requests[1]).toMatchObject({ + toolOutputCallId: searchCallId, + plannedToolName: "memory_get", + plannedToolCallId: getCallId, + }); + expect(requests[1]).not.toHaveProperty("plannedWireToolName"); + expect(requests[2]).toMatchObject({ + toolOutputCallId: getCallId, + }); + expect(requests[2]).not.toHaveProperty("plannedToolName"); + }); + it("supports advanced QA memory and subagent recovery prompts", async () => { const server = await startMockServer(); @@ -3570,6 +3762,29 @@ Update and merge these partial structured summaries.`, expect(threadMemorySearchText).toContain('"name":"memory_search"'); expect(threadMemorySearchText).toContain("ORBIT-22"); + const threadMemoryGetText = await expectStreamingResponsesText(server, { + instructions: + "@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.", + input: [ + makeToolOutput( + JSON.stringify({ + results: [ + { + path: "MEMORY.md", + startLine: 1, + endLine: 1, + snippet: "Thread-hidden codename: ORBIT-22.", + }, + ], + }), + ), + makeUserInput("Protocol note: acknowledged. Continue with the QA scenario plan."), + ], + }); + expect(threadMemoryGetText).toContain('"name":"memory_get"'); + expect(threadMemoryGetText).toContain('\\"path\\":\\"MEMORY.md\\"'); + expect(threadMemoryGetText).not.toContain("hidden thread codename is ORBIT-22"); + const threadMemorySummary = await expectNonStreamingResponses(server, { instructions: "@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.", @@ -3584,17 +3799,17 @@ Update and merge these partial structured summaries.`, }); expect(JSON.stringify(await threadMemorySummary.json())).toContain("ORBIT-22"); - const structuredThreadMemorySummary = await expectNonStreamingResponses(server, { + const rawThreadMemorySummary = await expectNonStreamingResponses(server, { instructions: "@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.", input: [ - makeToolOutput({ - text: "Thread-hidden codename: ORBIT-22.", - }), + makeToolOutput("Thread-hidden codename: ORBIT-23."), makeUserInput("Protocol note: acknowledged. Continue with the QA scenario plan."), ], }); - expect(JSON.stringify(await structuredThreadMemorySummary.json())).toContain("ORBIT-22"); + const rawThreadMemoryText = JSON.stringify(await rawThreadMemorySummary.json()); + expect(rawThreadMemoryText).toContain("NONE"); + expect(rawThreadMemoryText).not.toContain("ORBIT-23"); const unavailableThreadMemorySummary = await expectNonStreamingResponses(server, { input: [ @@ -5213,7 +5428,8 @@ Update and merge these partial structured summaries.`, }); const payload = await response.json(); - expect(outputItem(payload)).toMatchObject({ type: "function_call", name: "apply_patch" }); + const item = outputItem(payload); + expect(item).toMatchObject({ type: "function_call", name: "apply_patch" }); const args = outputToolArgs(payload); expect(args).not.toHaveProperty("__qaFailureMode"); expect(args.input).toBeTypeOf("string"); @@ -5223,6 +5439,9 @@ Update and merge these partial structured summaries.`, expect(args.input).toContain("\n@@\n-runtime-tool-fixture-denied-original\n"); } expect(args.input).toContain("\n*** End Patch\n"); + const debug = requireRecord(await getJson(server, "/debug/last-request"), "function plan"); + expect(debug.plannedToolCallId).toBe(item.call_id); + expect(debug.plannedToolItemId).toBe(item.id); }); it.each([ @@ -5265,6 +5484,7 @@ Update and merge these partial structured summaries.`, ); expect(debug.plannedToolName).toBe("apply_patch"); expect(debug.plannedToolCallId).toBe(item.call_id); + expect(debug.plannedToolItemId).toBe(item.id); expect(debug.plannedToolArgs).toEqual({ input: item.input }); }); @@ -6081,6 +6301,16 @@ Update and merge these partial structured summaries.`, const server = await startMockServer(); const body = (await expectAnthropicMessagesJson(server, { + tools: [ + { + name: "read", + input_schema: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + }, + ], messages: [ makeAnthropicUserText( "Read the seeded docs and report worked, failed, blocked, and follow-up items.", @@ -6098,8 +6328,10 @@ Update and merge these partial structured summaries.`, expect(body.model).toBe("claude-opus-4-8"); expect(body.stop_reason).toBe("tool_use"); const toolUseBlock = body.content.find((block) => block.type === "tool_use") as - | { name: string; input: Record } + | { id: string; name: string; input: Record } | undefined; + expect(toolUseBlock?.id).toMatch(/^toolu_[A-Za-z0-9_]+$/); + expect(toolUseBlock?.id.length).toBeLessThanOrEqual(64); expect(toolUseBlock?.name).toBe("read"); expect(toolUseBlock?.input).toEqual({ path: "repo/docs/help/testing.md" }); @@ -6108,9 +6340,85 @@ Update and merge these partial structured summaries.`, "debug request", ); expect(debugPayload.model).toBe("claude-opus-4-8"); + expect(debugPayload.plannedToolCallId).toBe(toolUseBlock?.id); + expect(debugPayload).not.toHaveProperty("plannedToolItemId"); expect(debugPayload.plannedToolName).toBe("read"); }); + it("preserves already-native Anthropic tool IDs while adapting shared generated IDs", () => { + const nativeId = "toolu_native_123"; + const generatedId = "call_mock_read_generated_1"; + const events: StreamEvent[] = [ + { + type: "response.output_item.added", + item: { type: "function_call", name: "read", call_id: nativeId, arguments: "{}" }, + }, + { + type: "response.output_item.done", + item: { type: "function_call", name: "read", call_id: nativeId, arguments: "{}" }, + }, + { + type: "response.output_item.added", + item: { type: "function_call", name: "read", call_id: generatedId, arguments: "{}" }, + }, + { + type: "response.output_item.done", + item: { type: "function_call", name: "read", call_id: generatedId, arguments: "{}" }, + }, + { + type: "response.completed", + response: { + id: "response_mock", + status: "completed", + output: [ + { type: "function_call", name: "read", call_id: nativeId, arguments: "{}" }, + { type: "function_call", name: "read", call_id: generatedId, arguments: "{}" }, + ], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + }, + ]; + + const adapted = adaptAnthropicToolCallIds(events); + const callIds = adapted.flatMap((event) => { + if ( + event.type === "response.output_item.added" || + event.type === "response.output_item.done" + ) { + return typeof event.item.call_id === "string" ? [event.item.call_id] : []; + } + if (event.type === "response.completed") { + return event.response.output.flatMap((item) => + typeof item.call_id === "string" ? [item.call_id] : [], + ); + } + return []; + }); + const adaptedGeneratedIds = callIds.filter((id) => id !== nativeId); + const repeatedGeneratedIds = adaptAnthropicToolCallIds(events).flatMap((event) => { + if ( + event.type === "response.output_item.added" || + event.type === "response.output_item.done" + ) { + return event.item.call_id === nativeId || typeof event.item.call_id !== "string" + ? [] + : [event.item.call_id]; + } + if (event.type === "response.completed") { + return event.response.output.flatMap((item) => + item.call_id === nativeId || typeof item.call_id !== "string" ? [] : [item.call_id], + ); + } + return []; + }); + + expect(callIds.filter((id) => id === nativeId)).toHaveLength(3); + expect(new Set(adaptedGeneratedIds).size).toBe(1); + expect(repeatedGeneratedIds).toEqual(adaptedGeneratedIds); + expect(adaptedGeneratedIds[0]).toMatch(/^toolu_[A-Za-z0-9_]+$/); + expect(adaptedGeneratedIds[0]?.length).toBeLessThanOrEqual(64); + }); + it("routes Anthropic hidden tools through Code Mode and preserves scenario evidence", async () => { const server = await startMockServer(); const prompt = @@ -6137,16 +6445,24 @@ Update and merge these partial structured summaries.`, }, ]; const messages: Array> = [makeAnthropicUserText(prompt)]; + const emittedToolUseIds: string[] = []; - const request = async () => { + const request = async (expectedToolResultId?: string) => { const response = await expectAnthropicMessages(server, { tools, messages, }); - return (await response.json()) as { + const body = (await response.json()) as { stop_reason: string; content: Array>; }; + const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request"); + if (expectedToolResultId) { + expect(debug.toolOutputCallId).toBe(expectedToolResultId); + } else { + expect(debug).not.toHaveProperty("toolOutputCallId"); + } + return body; }; const readToolUse = (body: { stop_reason: string; @@ -6157,6 +6473,9 @@ Update and merge these partial structured summaries.`, if (!toolUse || typeof toolUse.id !== "string" || typeof toolUse.name !== "string") { throw new Error("Expected Anthropic tool_use block"); } + expect(toolUse.id).toMatch(/^toolu_[A-Za-z0-9_]+$/); + expect(toolUse.id.length).toBeLessThanOrEqual(64); + emittedToolUseIds.push(toolUse.id); return toolUse; }; const appendToolResult = ( @@ -6168,8 +6487,14 @@ Update and merge these partial structured summaries.`, makeAnthropicToolResult(toolUse.id, JSON.stringify(result)), ); }; - const expectPlan = async (name: string, args: Record, wireName = "exec") => { + const expectPlan = async ( + name: string, + args: Record, + callId: string, + wireName = "exec", + ) => { const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request"); + expect(debug.plannedToolCallId).toBe(callId); expect(debug.plannedToolName).toBe(name); expect(debug.plannedWireToolName).toBe(wireName); expect(debug.plannedToolArgs).toEqual(args); @@ -6180,15 +6505,16 @@ Update and merge these partial structured summaries.`, const readAgentCode = String(requireRecord(readAgent.input, "exec input").code); expect(readAgentCode).toContain("tools.callValue(target.id, targetArgs)"); expect(readAgentCode).toContain("value.content.slice(0, 2048)"); - await expectPlan("read", { path: "AGENT.md" }); + await expectPlan("read", { path: "AGENT.md" }, String(readAgent.id)); appendToolResult(readAgent, { status: "waiting", runId: "qa-code-mode-read-agent" }); - const waitForAgent = readToolUse(await request()); + const waitForAgent = readToolUse(await request(String(readAgent.id))); expect(waitForAgent.name).toBe("wait"); const waitDebug = requireRecord( await fetch(`${server.baseUrl}/debug/last-request`).then((response) => response.json()), "wait debug request", ); + expect(waitDebug.plannedToolCallId).toBe(waitForAgent.id); expect(waitDebug.plannedToolName).toBe("wait"); expect(waitDebug).not.toHaveProperty("plannedWireToolName"); expect(waitDebug.plannedToolArgs).toEqual({ runId: "qa-code-mode-read-agent" }); @@ -6197,17 +6523,17 @@ Update and merge these partial structured summaries.`, status: "completed", value: { kind: "text", content: "# Repo contract\nDo not stop after planning." }, }); - const readSoul = readToolUse(await request()); + const readSoul = readToolUse(await request(String(waitForAgent.id))); expect(readSoul.name).toBe("exec"); - await expectPlan("read", { path: "SOUL.md" }); + await expectPlan("read", { path: "SOUL.md" }, String(readSoul.id)); appendToolResult(readSoul, { status: "completed", value: { kind: "text", content: "# Execution style\nStay action-first." }, }); - const readInput = readToolUse(await request()); + const readInput = readToolUse(await request(String(readSoul.id))); expect(readInput.name).toBe("exec"); - await expectPlan("read", { path: "FOLLOWTHROUGH_INPUT.md" }); + await expectPlan("read", { path: "FOLLOWTHROUGH_INPUT.md" }, String(readInput.id)); appendToolResult(readInput, { status: "completed", @@ -6217,24 +6543,215 @@ Update and merge these partial structured summaries.`, "Mission: prove you followed the repo contract.\nEvidence path: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md -> repo-contract-summary.txt", }, }); - const writeSummary = readToolUse(await request()); + const writeSummary = readToolUse(await request(String(readInput.id))); expect(writeSummary.name).toBe("exec"); - await expectPlan("write", { - path: "repo-contract-summary.txt", - content: - "Mission: prove you followed the repo contract.\nEvidence: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md\nStatus: complete", - }); + await expectPlan( + "write", + { + path: "repo-contract-summary.txt", + content: + "Mission: prove you followed the repo contract.\nEvidence: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md\nStatus: complete", + }, + String(writeSummary.id), + ); appendToolResult(writeSummary, { status: "completed", value: "Successfully wrote 146 bytes to repo-contract-summary.txt.", }); - const final = await request(); + const final = await request(String(writeSummary.id)); expect(final.stop_reason).toBe("end_turn"); const text = final.content.find((block) => block.type === "text")?.text; expect(text).toBe( "Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md\nWrote: repo-contract-summary.txt\nStatus: complete", ); + expect(new Set(emittedToolUseIds).size).toBe(emittedToolUseIds.length); + }); + + it("uses native Codex custom exec, output arrays, and cell_id waits", async () => { + const server = await startMockServer(); + const tools = [ + { type: "custom", name: "exec", format: { type: "grammar", syntax: "lark", definition: "" } }, + { + type: "function", + name: "wait", + parameters: { + type: "object", + properties: { cell_id: { type: "string" } }, + required: ["cell_id"], + }, + }, + ]; + const prompt = QA_COMPACTION_RETRY_PROMPT; + const execPayload = await expectOpenAiNonStreamingResponsesJson(server, { + tools, + input: [makeUserInput(prompt)], + }); + const execCall = outputItem(execPayload); + expect(execCall).toMatchObject({ type: "custom_tool_call", name: "exec" }); + const source = String(execCall.input); + expect(source).toContain("tools[target.name](targetArgs)"); + expect(source).toContain("text(JSON.stringify(value));"); + expect(source).not.toContain("tools.callValue"); + expect(source).not.toContain("target.id"); + expect(source).not.toMatch(/ALL_TOOLS[^\n]*\.id/); + expect(source).not.toContain("return value"); + + const execCallId = outputToolCallId(execCall, "native-exec"); + const waitPayload = await expectOpenAiNonStreamingResponsesJson(server, { + tools, + input: [ + makeUserInput(prompt), + execCall, + { + type: "custom_tool_call_output", + call_id: execCallId, + output: [ + { + type: "input_text", + text: "Script running with cell ID cell-write-1\nLive output:\n", + }, + ], + }, + ], + }); + const waitCall = outputToolCall(waitPayload, "wait"); + expect(outputToolArgsFromItem(waitCall)).toEqual({ cell_id: "cell-write-1" }); + + const finalPayload = await expectOpenAiNonStreamingResponsesJson(server, { + tools, + input: [ + makeUserInput(prompt), + execCall, + { + type: "custom_tool_call_output", + call_id: execCallId, + output: [ + { + type: "input_text", + text: "Script running with cell ID cell-write-1\nLive output:\n", + }, + ], + }, + waitCall, + { + type: "function_call_output", + call_id: outputToolCallId(waitCall, "native-wait"), + output: [ + { type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" }, + { + type: "input_text", + text: JSON.stringify({ status: "completed", value: { changed: false } }), + }, + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + ], + }, + ], + }); + expect(outputText(finalPayload)).toBe("Protocol note: replay unsafe after write."); + }); + + it.each([ + { + label: "failed header with canonical JSON", + output: [ + { type: "input_text", text: "Script failed\nWall time: 0.1 seconds\nOutput:\n" }, + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + ], + }, + { + label: "terminated header with canonical JSON", + output: [ + { type: "input_text", text: "Script terminated\nWall time: 0.1 seconds\nOutput:\n" }, + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + ], + }, + { + label: "unknown header with canonical JSON", + output: [ + { type: "input_text", text: "Script paused\nWall time: 0.1 seconds\nOutput:\n" }, + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + ], + }, + { + label: "failed header followed by a running marker", + output: [ + { type: "input_text", text: "Script failed\nWall time: 0.1 seconds\nOutput:\n" }, + { + type: "input_text", + text: "Script running with cell ID cell-write-late\nLive output:\n", + }, + ], + }, + { + label: "missing header with canonical JSON", + output: [ + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + ], + }, + { + label: "reordered completed header and canonical JSON", + output: [ + { + type: "input_text", + text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT), + }, + { type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" }, + ], + }, + { + label: "completed header without JSON", + output: [{ type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" }], + }, + ])("rejects native Code Mode compaction evidence with $label", async ({ output }) => { + const server = await startMockServer(); + const tools = [ + { type: "custom", name: "exec", format: { type: "grammar", syntax: "lark", definition: "" } }, + { + type: "function", + name: "wait", + parameters: { + type: "object", + properties: { cell_id: { type: "string" } }, + required: ["cell_id"], + }, + }, + ]; + const execPayload = await expectOpenAiNonStreamingResponsesJson(server, { + tools, + input: [makeUserInput(QA_COMPACTION_RETRY_PROMPT)], + }); + const execCall = outputItem(execPayload); + const payload = await expectOpenAiNonStreamingResponsesJson(server, { + tools, + input: [ + makeUserInput(QA_COMPACTION_RETRY_PROMPT), + execCall, + { + type: "custom_tool_call_output", + call_id: outputToolCallId(execCall, "native-exec"), + output, + }, + ], + }); + + expect(outputItems(payload).some((item) => item.type === "function_call")).toBe(false); + expect(outputText(payload)).not.toBe("Protocol note: replay unsafe after write."); }); it("routes Anthropic image generation through Code Mode when only exec and wait are visible", async () => { @@ -6336,70 +6853,83 @@ Update and merge these partial structured summaries.`, if (!readToolUse || typeof readToolUse.id !== "string") { throw new Error("Expected Anthropic read tool_use block"); } - messages.push( - { role: "assistant", content: [readToolUse] }, - makeAnthropicToolResult( - readToolUse.id, - JSON.stringify({ status: "waiting", runId: "ordinary-read" }), - ), - ); - - const body = (await expectAnthropicMessagesJson(server, { - tools, - messages, - })) as { - stop_reason: string; - content: Array>; - }; - expect(body.stop_reason).toBe("end_turn"); - expect(body.content.some((block) => block.type === "tool_use")).toBe(false); + for (const result of [ + { status: "waiting", runId: "ordinary-read" }, + { + status: "completed", + value: { status: "waiting", runId: "ordinary-read-completed-value" }, + }, + ]) { + const body = (await expectAnthropicMessagesJson(server, { + tools, + messages: [ + ...messages, + { role: "assistant", content: [readToolUse] }, + makeAnthropicToolResult(readToolUse.id, JSON.stringify(result)), + ], + })) as { + stop_reason: string; + content: Array>; + }; + expect(body.stop_reason).toBe("end_turn"); + expect(body.content.some((block) => block.type === "tool_use")).toBe(false); + } }); it("does not interpret unmarked direct exec results as Code Mode control envelopes", async () => { const server = await startMockServer(); - const body = (await expectAnthropicMessagesJson(server, { - tools: [ - { - name: "exec", - input_schema: { - type: "object", - properties: { code: { type: "string" } }, - required: ["code"], + const tools = [ + { + name: "exec", + input_schema: { + type: "object", + properties: { code: { type: "string" } }, + required: ["code"], + }, + }, + { + name: "wait", + input_schema: { + type: "object", + properties: { runId: { type: "string" } }, + required: ["runId"], + }, + }, + ]; + const messages = [ + makeAnthropicUserText("Direct exec envelope isolation check."), + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_direct_exec", + name: "exec", + input: { language: "javascript", code: "return 1;" }, }, - }, - { - name: "wait", - input_schema: { - type: "object", - properties: { runId: { type: "string" } }, - required: ["runId"], - }, - }, - ], - messages: [ - makeAnthropicUserText("Direct exec envelope isolation check."), - { - role: "assistant", - content: [ - { - type: "tool_use", - id: "toolu_direct_exec", - name: "exec", - input: { language: "javascript", code: "return 1;" }, - }, - ], - }, - makeAnthropicToolResult( - "toolu_direct_exec", - JSON.stringify({ status: "waiting", runId: "direct-exec" }), - ), - ], - })) as { - stop_reason: string; - content: Array>; - }; - expect(body.stop_reason).toBe("end_turn"); - expect(body.content.some((block) => block.type === "tool_use")).toBe(false); + ], + }, + ]; + for (const result of [ + { status: "waiting", runId: "direct-exec" }, + { + status: "completed", + value: { status: "waiting", runId: "direct-exec-completed-value" }, + }, + ]) { + const body = (await expectAnthropicMessagesJson(server, { + tools, + messages: [ + ...messages, + makeAnthropicToolResult("toolu_direct_exec", JSON.stringify(result)), + ], + })) as { + stop_reason: string; + content: Array>; + }; + expect(body.stop_reason).toBe("end_turn"); + expect(body.content.some((block) => block.type === "tool_use")).toBe(false); + } }); it("finishes Anthropic Code Mode fanout after the second wrapped spawn result", async () => { @@ -6914,6 +7444,16 @@ Update and merge these partial structured summaries.`, ).map((request) => requireRecord(request, "Anthropic debug request")); expect(debugRequests).toHaveLength(8); expect(debugRequests.every((request) => request.providerVariant === "anthropic")).toBe(true); + expect(debugRequests.map((request) => request.plannedToolCallId)).toEqual([ + readCallIds[0], + undefined, + readCallIds[1], + undefined, + readCallIds[2], + undefined, + readCallIds[3], + undefined, + ]); expect(debugRequests.map((request) => request.toolOutputCallId)).toEqual([ undefined, readCallIds[0], @@ -6941,6 +7481,16 @@ Update and merge these partial structured summaries.`, const response = await expectAnthropicMessages(server, { stream: true, + tools: [ + { + name: "read", + input_schema: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + }, + ], messages: [ makeAnthropicUserText( "Read the seeded docs and report worked, failed, blocked, and follow-up items.", @@ -6956,6 +7506,25 @@ Update and merge these partial structured summaries.`, expect(body).toContain("repo/docs/help/testing.md"); expect(body).toContain("event: message_delta"); expect(body).toContain("event: message_stop"); + const events = body + .split("\n") + .filter((line) => line.startsWith("data: ")) + .map((line) => + requireRecord(JSON.parse(line.slice("data: ".length)) as unknown, "Anthropic SSE event"), + ); + const toolUseStart = events.find( + (event) => + event.type === "content_block_start" && + requireRecord(event.content_block, "Anthropic SSE content block").type === "tool_use", + ); + const toolUse = requireRecord( + toolUseStart?.content_block, + "Anthropic SSE tool_use content block", + ); + expect(toolUse.id).toMatch(/^toolu_[A-Za-z0-9_]+$/); + expect(String(toolUse.id).length).toBeLessThanOrEqual(64); + const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request"); + expect(debug.plannedToolCallId).toBe(toolUse.id); }); it("streams Anthropic /v1/messages tool_result follow-ups as text deltas", async () => { diff --git a/extensions/qa-lab/src/providers/mock-openai/server.ts b/extensions/qa-lab/src/providers/mock-openai/server.ts index ee34564eb1bf..5a1ceffd6cb2 100644 --- a/extensions/qa-lab/src/providers/mock-openai/server.ts +++ b/extensions/qa-lab/src/providers/mock-openai/server.ts @@ -8,8 +8,11 @@ import { listMockCodexModelInfos, listMockOpenAiServerModelIds, } from "../shared/mock-model-config.js"; -import { buildMessagesPayload } from "./mock-anthropic-messages.js"; -import { convertAnthropicMessagesToResponsesInput } from "./mock-anthropic-wire.js"; +import { + buildMessagesPayload, + normalizeAnthropicMessagesRequest, +} from "./mock-anthropic-messages.js"; +import { adaptAnthropicToolCallIds } from "./mock-anthropic-wire.js"; import { buildAssistantText, isCanonicalCompactionRetryWriteResult, @@ -24,9 +27,12 @@ import { type MockOpenAiRequestKind, type MockCompactionSummaryFaultMode, type AnthropicMessagesRequest, + type QaMockProviderDispatchRequest, + type QaMockProviderDispatchResult, TINY_PNG_BASE64, QA_REASONING_ONLY_RECOVERY_PROMPT_RE, QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE, + QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE, QA_THINKING_VISIBILITY_OFF_PROMPT_RE, QA_THINKING_VISIBILITY_MAX_PROMPT_RE, QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE, @@ -125,7 +131,7 @@ import { buildReleaseAuditJson, buildReleaseHandoffMarkdown, extractPlannedToolName, - extractPlannedToolCallId, + extractPlannedToolIdentity, extractPlannedToolArgs, splitMockStreamingText, buildQaLongFinalText, @@ -139,6 +145,7 @@ import { extractLastMatchingUserTurn, hasToolOutput, extractToolOutput, + extractToolOutputValue, extractToolOutputStructuredError, extractToolOutputCallId, extractLatestToolOutput, @@ -146,7 +153,6 @@ import { extractUserTextAfterLatestToolOutput, extractSlackMpimRetainedBotNonce, extractAllUserTexts, - extractAllInputTexts, extractInstructionsText, extractAllRequestTexts, buildWhatsAppPendingHistoryReply, @@ -157,10 +163,7 @@ import { extractCurrentImageRequest, parseToolOutputJson, } from "./mock-openai-input.js"; -import { - attachQaMockResponsesWebSocketServer, - type QaMockResponsesDispatchResult, -} from "./mock-openai-responses-websocket.js"; +import { attachQaMockResponsesWebSocketServer } from "./mock-openai-responses-websocket.js"; import { readTargetFromPrompt, execCommandFromToolProgressPrompt, @@ -391,31 +394,40 @@ function findNamedToolDefinition( return null; } -function hasCodeModeExecSurface(body: Record) { +type CodeModeExecSurface = "native" | "guest"; + +function resolveCodeModeExecSurface(body: Record): CodeModeExecSurface | null { const tools = [ ...(Array.isArray(body.tools) ? body.tools : []), ...(Array.isArray(body.dynamicTools) ? body.dynamicTools : []), ]; const execDefinition = findNamedToolDefinition(tools, "exec"); if (!execDefinition || !hasToolDefinition(body, "wait")) { - return false; + return null; + } + if (execDefinition.type === "custom") { + return "native"; } const schema = (execDefinition.input_schema as Record | undefined) ?? (execDefinition.parameters as Record | undefined); if (!schema) { - return false; + return null; } const properties = schema.properties; const required = schema.required; - return ( - properties !== null && + return properties !== null && typeof properties === "object" && !Array.isArray(properties) && Object.hasOwn(properties, "code") && Array.isArray(required) && required.includes("code") - ); + ? "guest" + : null; +} + +function hasCodeModeExecSurface(body: Record) { + return resolveCodeModeExecSurface(body) !== null; } function resolveCurrentToolDeclarationSurface( @@ -456,20 +468,70 @@ function parseToolCallArguments(toolCall: ResponsesInputItem) { } } -function isGeneratedCodeModeExecCall(toolCall: ResponsesInputItem | undefined) { - if (!toolCall || toolCall.name !== "exec") { - return false; +function readGeneratedCodeModeExecSource(toolCall: ResponsesInputItem | undefined) { + if (toolCall?.type === "custom_tool_call" && typeof toolCall.input === "string") { + return toolCall.input; } - const args = parseToolCallArguments(toolCall); - return typeof args?.code === "string" && decodeCodeModeTarget(args.code) !== null; + const code = toolCall ? parseToolCallArguments(toolCall)?.code : undefined; + return typeof code === "string" ? code : undefined; +} + +function isGeneratedCodeModeExecCall(toolCall: ResponsesInputItem | undefined) { + const source = toolCall?.name === "exec" ? readGeneratedCodeModeExecSource(toolCall) : undefined; + return typeof source === "string" && decodeCodeModeTarget(source) !== null; +} + +function parseNativeCodeModeOutput( + output: unknown, +): { status: "waiting"; cellId: string } | { status: "completed"; value: unknown } | null { + if (!Array.isArray(output)) { + return null; + } + const readText = (item: unknown) => + typeof item === "string" + ? item + : item && + typeof item === "object" && + typeof (item as Record).text === "string" + ? String((item as Record).text) + : null; + const statusText = readText(output[0]); + if (!statusText) { + return null; + } + const cellId = /^Script running with cell ID ([^\s\n]+)/u.exec(statusText)?.[1]; + if (cellId) { + return { status: "waiting", cellId }; + } + if (!statusText.startsWith("Script completed\n")) { + return null; + } + for (const item of output.slice(1).toReversed()) { + const text = readText(item); + if (!text) { + continue; + } + try { + return { status: "completed", value: JSON.parse(text) as unknown }; + } catch { + // Native Code Mode may emit non-JSON content before the final value. + } + } + return null; } function isGeneratedCodeModeWaitCall(input: ResponsesInputItem[], toolCall: ResponsesInputItem) { if (toolCall.name !== "wait") { return false; } - const runId = parseToolCallArguments(toolCall)?.runId; - if (typeof runId !== "string") { + const args = parseToolCallArguments(toolCall); + const waitId = + typeof args?.cell_id === "string" + ? args.cell_id + : typeof args?.runId === "string" + ? args.runId + : undefined; + if (!waitId) { return false; } return input.some((item) => { @@ -479,11 +541,12 @@ function isGeneratedCodeModeWaitCall(input: ResponsesInputItem[], toolCall: Resp ) { return false; } - const output = stringifyScenarioToolOutput(item.output); - const parsed = parseToolOutputJson(output); + const native = parseNativeCodeModeOutput(item.output); + const parsed = native ?? parseToolOutputJson(stringifyScenarioToolOutput(item.output)); return ( parsed?.status === "waiting" && - parsed.runId === runId && + (("cellId" in parsed && parsed.cellId === waitId) || + ("runId" in parsed && parsed.runId === waitId)) && isGeneratedCodeModeExecCall(findToolCallByCallId(input, item.call_id)) ); }); @@ -537,6 +600,23 @@ function buildScenarioToolCallEvents( return buildRawToolCallEventsWithArgs(name, args, namespace); } const encodedTarget = encodeCodeModeTarget(name, args); + if (resolveCodeModeExecSurface(body) === "native") { + return buildCustomToolCallEventsWithInput( + "exec", + [ + `// ${QA_CODE_MODE_TARGET_MARKER}${encodedTarget}`, + `const targetName = ${JSON.stringify(name)};`, + `const targetArgs = ${JSON.stringify(args)};`, + "const target = ALL_TOOLS.find((entry) => entry.name === targetName);", + "if (!target) throw new Error(`QA mock target tool unavailable: ${targetName}`);", + "let value = await tools[target.name](targetArgs);", + 'if (targetName === "read" && value?.kind === "text" && typeof value.content === "string") {', + " value = { ...value, content: value.content.slice(0, 2048) };", + "}", + "text(JSON.stringify(value));", + ].join("\n"), + ); + } return buildRawToolCallEventsWithArgs("exec", { language: "javascript", code: [ @@ -557,10 +637,16 @@ function buildScenarioToolCallEvents( function extractScenarioPlannedTool(events: StreamEvent[]) { const wireName = extractPlannedToolName(events); const wireArgs = extractPlannedToolArgs(events); - if (wireName !== "exec" || typeof wireArgs?.code !== "string") { + const source = + typeof wireArgs?.input === "string" + ? wireArgs.input + : typeof wireArgs?.code === "string" + ? wireArgs.code + : undefined; + if (wireName !== "exec" || !source) { return { name: wireName, args: wireArgs, wireName }; } - const target = decodeCodeModeTarget(wireArgs.code); + const target = decodeCodeModeTarget(source); return target ? { name: target.name, args: target.args, wireName } : { name: wireName, args: wireArgs, wireName }; @@ -701,19 +787,25 @@ async function buildResponsesPayload( const prompt = extractLastUserText(input); const hasCompletedToolOutput = hasToolOutput(input); const rawToolOutput = extractToolOutput(input); - const codeModeControlJson = isCodeModeControlToolOutput(toolDeclarationBody, input) - ? parseToolOutputJson(rawToolOutput) + const codeModeSurface = resolveCodeModeExecSurface(toolDeclarationBody); + const hasCodeModeControlOutput = isCodeModeControlToolOutput(toolDeclarationBody, input); + const codeModeControlJson = hasCodeModeControlOutput + ? codeModeSurface === "native" + ? parseNativeCodeModeOutput(extractToolOutputValue(input)) + : parseToolOutputJson(rawToolOutput) : null; const toolOutput = codeModeControlJson?.status === "completed" && Object.hasOwn(codeModeControlJson, "value") ? stringifyScenarioToolOutput(codeModeControlJson.value) - : rawToolOutput; + : codeModeSurface === "native" && hasCodeModeControlOutput + ? "" + : rawToolOutput; const completedToolCall = findToolCallByCallId(input, extractToolOutputCallId(input)); const completedToolName = (() => { if (completedToolCall?.name !== "exec") { return completedToolCall?.name; } - const code = parseToolCallArguments(completedToolCall)?.code; + const code = readGeneratedCodeModeExecSource(completedToolCall); return typeof code === "string" ? decodeCodeModeTarget(code)?.name : undefined; })(); const buildToolCallEventsWithArgs = (name: string, args: Record) => @@ -759,12 +851,13 @@ async function buildResponsesPayload( ? extractLatestToolOutput(input) : ""); const toolJson = parseToolOutputJson(scenarioToolOutput); - if ( - codeModeControlJson?.status === "waiting" && - typeof codeModeControlJson.runId === "string" && - hasToolDefinition(toolDeclarationBody, "wait") - ) { - return buildRawToolCallEventsWithArgs("wait", { runId: codeModeControlJson.runId }); + if (codeModeControlJson?.status === "waiting" && hasToolDefinition(toolDeclarationBody, "wait")) { + if ("cellId" in codeModeControlJson && typeof codeModeControlJson.cellId === "string") { + return buildRawToolCallEventsWithArgs("wait", { cell_id: codeModeControlJson.cellId }); + } + if ("runId" in codeModeControlJson && typeof codeModeControlJson.runId === "string") { + return buildRawToolCallEventsWithArgs("wait", { runId: codeModeControlJson.runId }); + } } if (compactionRetryScenarioActive) { if (isCanonicalCompactionRetryWriteResult(toolOutput)) { @@ -1886,40 +1979,68 @@ async function buildResponsesPayload( return buildAssistantEvents("NONE"); } if (/thread memory check/i.test(allInputText)) { - if (!scenarioToolOutput) { + if (!hasCompletedToolOutput) { return buildToolCallEventsWithArgs("memory_search", { query: "hidden thread codename ORBIT-22", maxResults: 3, }); } - if (memoryToolUnavailable) { + const directThreadMemoryJson = + Array.isArray(toolJson?.results) || typeof toolJson?.text === "string" ? toolJson : null; + const completedMemoryValue = + toolJson?.status === "completed" && + (completedToolName === "memory_search" || completedToolName === "memory_get") && + toolJson.value !== null && + typeof toolJson.value === "object" && + !Array.isArray(toolJson.value) + ? (toolJson.value as Record) + : null; + const threadMemoryJson = completedMemoryValue ?? directThreadMemoryJson; + const threadMemoryToolName = completedMemoryValue + ? completedToolName + : Array.isArray(threadMemoryJson?.results) + ? "memory_search" + : typeof threadMemoryJson?.text === "string" + ? "memory_get" + : undefined; + const threadMemoryUnavailable = + threadMemoryJson?.unavailable === true || + threadMemoryJson?.disabled === true || + (typeof threadMemoryJson?.error === "string" && threadMemoryJson.error.trim().length > 0); + if (threadMemoryUnavailable) { return buildAssistantEvents("NONE"); } - const transcriptOrbitCode = extractOrbitCode(scenarioToolOutput); - if (transcriptOrbitCode) { - return buildAssistantEvents( - `Protocol note: I checked memory in-thread and the hidden thread codename is ${transcriptOrbitCode}.`, - ); + if (threadMemoryToolName === "memory_search") { + const results = Array.isArray(threadMemoryJson?.results) + ? (threadMemoryJson.results as Array>) + : []; + const first = results[0]; + if ( + typeof first?.path === "string" && + (typeof first.startLine === "number" || typeof first.endLine === "number") + ) { + const from = + typeof first.startLine === "number" + ? Math.max(1, first.startLine) + : typeof first.endLine === "number" + ? Math.max(1, first.endLine) + : 1; + return buildToolCallEventsWithArgs("memory_get", { + path: first.path, + from, + lines: 4, + }); + } } - const results = Array.isArray(toolJson?.results) - ? (toolJson.results as Array>) - : []; - const first = results[0]; - if ( - typeof first?.path === "string" && - (typeof first.startLine === "number" || typeof first.endLine === "number") - ) { - const from = - typeof first.startLine === "number" - ? Math.max(1, first.startLine) - : typeof first.endLine === "number" - ? Math.max(1, first.endLine) - : 1; - return buildToolCallEventsWithArgs("memory_get", { - path: first.path, - from, - lines: 4, - }); + const memoryGetText = + threadMemoryToolName === "memory_get" && typeof threadMemoryJson?.text === "string" + ? threadMemoryJson.text + : ""; + const memoryGetOrbitCode = extractOrbitCode(memoryGetText); + if (memoryGetOrbitCode) { + return buildAssistantEvents( + `Protocol note: I checked memory in-thread and the hidden thread codename is ${memoryGetOrbitCode}.`, + ); } return buildAssistantEvents("NONE"); } @@ -2185,13 +2306,7 @@ export async function startQaMockOpenAiServer(params?: { const scenarioStates = new Map(); const servedCompactionSummaryFaultMarkers = new Set(); const scenarioStateFor = (body: Record): MockScenarioState => { - const input = - typeof body.input === "string" || Array.isArray(body.input) - ? normalizeResponsesInput(body.input) - : convertAnthropicMessagesToResponsesInput({ - system: body.system as AnthropicMessagesRequest["system"], - messages: [], - }); + const input = normalizeResponsesInput(body.input); const sessionId = resolveQaRuntimeSessionId(input, body) ?? (body.client_metadata as { session_id?: unknown } | undefined)?.session_id; @@ -2224,18 +2339,25 @@ export async function startQaMockOpenAiServer(params?: { const inflightRequests = new Map(); let nextInflightRequestId = 1; const imageGenerationRequests: Array> = []; - const dispatchResponses = async (request: { - body: Record; - raw: string; - }): Promise => { - const input = normalizeResponsesInput(request.body.input); + const dispatchProvider = async ( + request: QaMockProviderDispatchRequest, + ): Promise => { + const normalized = + request.route === "anthropic-messages" + ? normalizeAnthropicMessagesRequest(request.body as AnthropicMessagesRequest) + : { + body: request.body, + input: normalizeResponsesInput(request.body.input), + model: typeof request.body.model === "string" ? request.body.model : "", + }; + const { body, input, model } = normalized; if (isRemoteCompactionV2Request(input)) { - return { events: buildRemoteCompactionV2Events() }; + return { events: buildRemoteCompactionV2Events(), model }; } const prompt = extractLastUserText(input); - const allInputText = extractAllRequestTexts(input, request.body); - const scenarioState = scenarioStateFor(request.body); - const requestKind = classifyMockOpenAiRequest(input, request.body); + const allInputText = extractAllRequestTexts(input, body); + const scenarioState = scenarioStateFor(body); + const requestKind = classifyMockOpenAiRequest(input, body); const compactionSummaryFaultMode = resolveCompactionSummaryFaultMode({ allInputText, requestKind, @@ -2248,16 +2370,15 @@ export async function startQaMockOpenAiServer(params?: { const compactionOverflowThresholdBytes = hasCompactionOutputRecoveryMarker(allInputText) ? QA_COMPACTION_OUTPUT_RECOVERY_OVERFLOW_THRESHOLD_BYTES : QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES; - const resolvedModel = typeof request.body.model === "string" ? request.body.model : ""; const requestSnapshotBase = { raw: request.raw, - body: request.body, + body, prompt, allInputText, - instructions: extractInstructionsText(request.body) || undefined, + instructions: extractInstructionsText(body) || undefined, toolOutput: extractToolOutput(input), - model: resolvedModel, - providerVariant: resolveProviderVariant(resolvedModel), + model, + providerVariant: resolveProviderVariant(model), imageInputCount: countImageInputs(input), requestKind, compactionSummaryFaultMode, @@ -2267,6 +2388,7 @@ export async function startQaMockOpenAiServer(params?: { | "outcome" | "errorCode" | "plannedToolCallId" + | "plannedToolItemId" | "plannedToolName" | "plannedWireToolName" | "plannedToolArgs" @@ -2288,6 +2410,7 @@ export async function startQaMockOpenAiServer(params?: { }); return { events: [], + model, failure: { status: 400, type: "invalid_request_error", @@ -2299,15 +2422,48 @@ export async function startQaMockOpenAiServer(params?: { const inflightRequestId = nextInflightRequestId++; inflightRequests.set(inflightRequestId, { prompt, allInputText }); let events: StreamEvent[]; + let injectedFailure: QaMockProviderDispatchResult["failure"]; try { - events = await buildResponsesPayload(request.body, scenarioState, { - waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled, - requestKind, - compactionSummaryFaultMode, - }); + if ( + request.route === "anthropic-messages" && + QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText) + ) { + const toolOutput = extractToolOutput(input); + const toolOutputCallId = extractToolOutputCallId(input); + const scenarioKey = `${model}\n${extractLastUserText(input)}`; + const shouldFail = + toolOutput.length > 0 && + toolOutputCallId.length > 0 && + !scenarioState.anthropicThinkingErrorScenarioKeys.has(scenarioKey); + if (shouldFail) { + scenarioState.anthropicThinkingErrorScenarioKeys.add(scenarioKey); + injectedFailure = { + status: 200, + type: "api_error", + message: "QA injected provider stream failure", + presentation: "anthropic-thinking", + }; + } + events = + toolOutput.length === 0 + ? buildRawToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" }) + : shouldFail + ? buildAssistantEvents("") + : buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK"); + } else { + events = await buildResponsesPayload(body, scenarioState, { + waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled, + requestKind, + compactionSummaryFaultMode, + }); + } } finally { inflightRequests.delete(inflightRequestId); } + if (request.route === "anthropic-messages") { + events = adaptAnthropicToolCallIds(events); + } + const plannedToolIdentity = extractPlannedToolIdentity(events); const plannedTool = extractScenarioPlannedTool(events); const terminalRequesterCase = extractLastMatchingUserTurn( input, @@ -2316,7 +2472,7 @@ export async function startQaMockOpenAiServer(params?: { ?.text.match(QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE)?.[1] ?.toLowerCase(); const settledTerminalRequester = - terminalRequesterCase && resolveQaRuntimeSessionId(input, request.body) + terminalRequesterCase && resolveQaRuntimeSessionId(input, body) ? { caseName: terminalRequesterCase, childSessionKey: resolveAcceptedChildSessionKey(input), @@ -2325,17 +2481,21 @@ export async function startQaMockOpenAiServer(params?: { const settledTerminalCaseName = settledTerminalRequester?.caseName; const settledChildSessionKey = settledTerminalRequester?.childSessionKey; const failure = - QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && hasToolOutput(input) + injectedFailure ?? + (QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && hasToolOutput(input) ? { status: 503, type: "server_error", message: "Service Unavailable", } - : undefined; + : undefined); recordRequest({ ...requestSnapshotBase, outcome: failure ? "error" : "success", - plannedToolCallId: extractPlannedToolCallId(events), + plannedToolCallId: plannedToolIdentity.callId, + ...(request.route === "responses" && plannedToolIdentity.itemId + ? { plannedToolItemId: plannedToolIdentity.itemId } + : {}), plannedToolName: plannedTool.name, ...(plannedTool.wireName && plannedTool.wireName !== plannedTool.name ? { plannedWireToolName: plannedTool.wireName } @@ -2346,6 +2506,7 @@ export async function startQaMockOpenAiServer(params?: { }); return { events, + model, ...(settledTerminalCaseName && settledChildSessionKey ? { onResponseSent: () => @@ -2361,6 +2522,8 @@ export async function startQaMockOpenAiServer(params?: { : {}), }; }; + const dispatchResponses = (request: Omit) => + dispatchProvider({ ...request, route: "responses" }); const server = createServer((req, res) => { void (async () => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); @@ -2492,16 +2655,6 @@ export async function startQaMockOpenAiServer(params?: { writeOpenAiMalformedJsonError(res, "OpenAI Responses"); return; } - const input = normalizeResponsesInput(body.input); - if (isRemoteCompactionV2Request(input)) { - const events = buildRemoteCompactionV2Events(); - if (body.stream === false) { - writeJson(res, 200, events[1].response); - } else { - writeSse(res, events); - } - return; - } const dispatched = await dispatchResponses({ body, raw }); if (dispatched.failure) { writeJson(res, dispatched.failure.status, { @@ -2545,48 +2698,25 @@ export async function startQaMockOpenAiServer(params?: { }); return; } - const scenarioState = scenarioStateFor(body as Record); - const { - events, - input, - responseBody, - streamEvents, - model: normalizedModel, - } = await buildMessagesPayload(body, scenarioState, buildResponsesPayload); - const plannedTool = extractScenarioPlannedTool(events); - // Record the adapted request snapshot so /debug/requests gives the QA - // suite the same plannedToolName / allInputText / toolOutput signals - // on the Anthropic route that the OpenAI route already exposes. This - // is what lets a single parity run diff assertions across both lanes. - // Reuse the normalized model so an empty-string body.model no longer - // leaks through to `lastRequest.model`. - recordRequest({ - raw, + const dispatched = await dispatchProvider({ + route: "anthropic-messages", body: body as Record, - prompt: extractLastUserText(input), - allInputText: extractAllInputTexts(input), - toolOutput: extractToolOutput(input), - model: normalizedModel, - providerVariant: resolveProviderVariant(normalizedModel), - imageInputCount: countImageInputs(input), - requestKind: classifyMockOpenAiRequest(input, body as Record), - compactionSummaryFaultMode: "none", - outcome: "success", - rawByteLength: Buffer.byteLength(raw), - plannedToolCallId: extractPlannedToolCallId(events), - plannedToolName: plannedTool.name, - ...(plannedTool.wireName && plannedTool.wireName !== plannedTool.name - ? { plannedWireToolName: plannedTool.wireName } - : {}), - plannedToolArgs: plannedTool.args, - toolOutputCallId: extractToolOutputCallId(input) || undefined, - ...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}), + raw, }); + const { responseBody, streamEvents } = buildMessagesPayload(dispatched); + if (dispatched.failure?.presentation !== "anthropic-thinking") { + if (dispatched.failure) { + writeJson(res, dispatched.failure.status, responseBody); + return; + } + } if (body.stream === true) { writeAnthropicSse(res, streamEvents); + dispatched.onResponseSent?.(); return; } - writeJson(res, 200, responseBody); + writeJson(res, dispatched.failure?.status ?? 200, responseBody); + dispatched.onResponseSent?.(); return; } writeJson(res, 404, { error: "not found" }); diff --git a/extensions/qa-lab/src/qa-gateway-config.test.ts b/extensions/qa-lab/src/qa-gateway-config.test.ts index 0b9374deb0e9..b309211e63d3 100644 --- a/extensions/qa-lab/src/qa-gateway-config.test.ts +++ b/extensions/qa-lab/src/qa-gateway-config.test.ts @@ -105,7 +105,7 @@ describe("buildQaGatewayConfig", () => { expect(cfg.models?.providers?.anthropic?.baseUrl).toBe("http://127.0.0.1:44080"); expect(cfg.models?.providers?.anthropic?.request).toEqual({ allowPrivateNetwork: true }); expect(cfg.memory?.search).toMatchObject({ - provider: "openai", + provider: "openai-compatible", model: "text-embedding-3-small", remote: { baseUrl: "http://127.0.0.1:44080/v1", diff --git a/extensions/qa-lab/src/qa-gateway-config.ts b/extensions/qa-lab/src/qa-gateway-config.ts index 27cc7b8f8bbf..976b7df2d4d4 100644 --- a/extensions/qa-lab/src/qa-gateway-config.ts +++ b/extensions/qa-lab/src/qa-gateway-config.ts @@ -202,7 +202,7 @@ export function buildQaGatewayConfig(params: { const mockMemorySearch = provider.kind === "mock" ? { - provider: "openai", + provider: "openai-compatible", model: "text-embedding-3-small", remote: { // Memory embeddings bypass the model runtime, so bind them to the diff --git a/extensions/qa-lab/src/scenario-catalog-causality.test.ts b/extensions/qa-lab/src/scenario-catalog-causality.test.ts new file mode 100644 index 000000000000..04f9f8431b52 --- /dev/null +++ b/extensions/qa-lab/src/scenario-catalog-causality.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it } from "vitest"; +import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js"; +import { readFlowAssertExpression, requireFlowScenario } from "./scenario-catalog.test-utils.js"; + +describe("qa scenario catalog causality", () => { + it("loads live gateway sentinel scenarios for harness self-health", () => { + const scenarioIds = [ + "plugin-hook-health-sentinel", + "plugin-manifest-contract-health", + "webchat-direct-reply-routing", + "long-context-progress-watchdog", + "gateway-restart-inflight-run", + "gateway-restart-multi-live", + "streaming-final-integrity", + ]; + + for (const scenarioId of scenarioIds) { + const scenario = readQaScenarioById(scenarioId); + expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0); + expect(scenario.coverage?.primary.length).toBeGreaterThan(0); + } + expect(readQaScenarioById("webchat-direct-reply-routing").sourcePath).toBe( + "qa/scenarios/channels/webchat-direct-reply-routing.yaml", + ); + expect(readQaScenarioById("long-context-progress-watchdog").sourcePath).toBe( + "qa/scenarios/runtime/long-context-progress-watchdog.yaml", + ); + const gatewayRestart = requireFlowScenario(readQaScenarioById("gateway-restart-inflight-run")); + const gatewayRestartFlow = gatewayRestart.execution.flow; + const gatewayRestartContract = JSON.stringify(gatewayRestartFlow); + const gatewayRestartActions = gatewayRestartFlow?.steps[0]?.actions ?? []; + const recoveryPollIndex = gatewayRestartActions.findIndex( + (action) => + (action as { call?: string }).call === "waitForCondition" && + (action as { saveAs?: string }).saveAs === "settledRecovery", + ); + const outboundIndex = gatewayRestartActions.findIndex( + (action) => + (action as { call?: string }).call === "waitForOutboundMessage" && + (action as { saveAs?: string }).saveAs === "outbound", + ); + const preOutboundRecoveryAssertIndex = gatewayRestartActions.findIndex((action) => + readFlowAssertExpression(action).includes( + "restartRecoveryRequestsBeforeOutbound.length === 1", + ), + ); + const preOutboundHeartbeatAssertIndex = gatewayRestartActions.findIndex((action) => + readFlowAssertExpression(action).includes( + "!String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('[OpenClaw heartbeat poll]')", + ), + ); + const settledRequestsIndex = gatewayRestartActions.findIndex( + (action) => (action as { set?: string }).set === "settledRecoveryRequests", + ); + const settledDedupeAssertIndex = gatewayRestartActions.findIndex((action) => + readFlowAssertExpression(action).includes("settledRestartRecoveryRequests.length === 1"), + ); + const recoveryPoll = gatewayRestartActions[recoveryPollIndex] as + | { args?: Array<{ lambda?: { expr?: string } }> } + | undefined; + const recoveryPollExpr = recoveryPoll?.args?.[0]?.lambda?.expr ?? ""; + expect(gatewayRestart.execution.retryCount).toBe(0); + expect(JSON.stringify(gatewayRestart.gatewayConfigPatch)).toContain( + '"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]', + ); + expect(gatewayRestartContract).toContain("plannedToolName === 'wait'"); + expect(gatewayRestartContract).toContain("lastAssistantToolNames?.includes('wait')"); + expect(gatewayRestartContract).toContain("restartRecoveryDeliveryContext"); + expect(gatewayRestartContract).toContain("sendInbound"); + expect(gatewayRestartContract).not.toContain("startAgentRun"); + expect(gatewayRestartContract).toContain('"restartGatewayWithConfigPatch"'); + expect(gatewayRestartContract).toContain("interruptedMatches.length === 1"); + expect(gatewayRestartContract).toContain("restartNotices.length === 0"); + expect(gatewayRestartContract).toContain("dispatching restart-safe recovery"); + expect(recoveryPollIndex).toBeGreaterThanOrEqual(0); + expect(recoveryPollExpr).toContain( + "String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart')", + ); + expect(recoveryPollExpr).toContain( + "String(request.allInputText ?? '').includes(config.interruptedMarker)", + ); + expect(recoveryPollExpr).toContain("restartRecoveryRequests.length >= 1"); + expect(recoveryPollExpr).toContain( + "String(transcript.finalText ?? '').includes(config.interruptedMarker)", + ); + expect(preOutboundRecoveryAssertIndex).toBeGreaterThan(recoveryPollIndex); + expect(preOutboundHeartbeatAssertIndex).toBeGreaterThan(preOutboundRecoveryAssertIndex); + expect(outboundIndex).toBeGreaterThan(preOutboundHeartbeatAssertIndex); + expect(settledRequestsIndex).toBeGreaterThan(outboundIndex); + expect(settledDedupeAssertIndex).toBeGreaterThan(settledRequestsIndex); + expect( + gatewayRestartActions.some((action) => (action as { call?: string }).call === "sleep"), + ).toBe(false); + expect(gatewayRestartContract).toContain("recoveryPromptHeartbeat=false"); + expect(gatewayRestartContract).toContain("liveTurnTimeoutMs(env, 180000)"); + expect(gatewayRestartContract).toContain("id: `dm:${conversationId}`"); + expect(gatewayRestartContract).toContain("dmScope: env.cfg.session?.dmScope"); + expect(gatewayRestart.gatewayConfigPatch).toMatchObject({ + plugins: { + slots: { memory: "none" }, + entries: { + acpx: { enabled: false }, + "memory-core": { enabled: false }, + }, + }, + }); + const liveMultiRestart = readQaScenarioById("gateway-restart-multi-live"); + const liveMultiRestartContract = JSON.stringify(liveMultiRestart.execution.flow); + expect(JSON.stringify(liveMultiRestart.gatewayConfigPatch)).toContain( + '"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]', + ); + expect(liveMultiRestartContract).toContain("assistantToolCallCounts.exec"); + expect(liveMultiRestartContract).toContain("checkpoint"); + expect(liveMultiRestartContract).toContain("restarts=3"); + expect(liveMultiRestartContract).toContain("dmScope: 'per-channel-peer'"); + expect(liveMultiRestartContract).toContain("dispatching restart-safe recovery"); + expect(readQaScenarioExecutionConfig("gateway-restart-multi-live")).toMatchObject({ + requiredProviderMode: "live-frontier", + requiredProvider: "openai", + requiredModel: "gpt-5.4", + }); + }); + + it("scopes prompt diagnostics to requests after each scenario cursor", () => { + for (const scenarioId of [ + "instruction-followthrough-repo-contract", + "subagent-handoff", + ] as const) { + const scenario = requireFlowScenario(readQaScenarioById(scenarioId)); + const flow = JSON.stringify(scenario.execution.flow); + const cursorIndex = flow.indexOf("/debug/request-cursor"); + const promptIndex = flow.indexOf('"call":"runAgentPrompt"'); + const requestsIndex = flow.indexOf("/debug/requests?after=${requestCursorBefore}"); + + expect(cursorIndex, scenarioId).toBeGreaterThanOrEqual(0); + expect(cursorIndex, scenarioId).toBeLessThan(promptIndex); + expect(requestsIndex, scenarioId).toBeGreaterThan(promptIndex); + expect(flow, scenarioId).not.toContain("`${env.mock.baseUrl}/debug/requests`"); + } + }); + + it.each([ + [ + "thread-memory-isolation", + "poll", + "finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId", + ], + [ + "memory-tools-channel-context", + "poll", + "finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId", + ], + [ + "agent-tool-consumption", + "immediate", + "getResultRequest.toolOutputCallId === searchResultRequest.plannedToolCallId", + ], + ] as const)( + "asserts the complete memory tool chain before %s delivery", + (scenarioId, requestCollectionMode, finalLinkNeedle) => { + const scenario = requireFlowScenario(readQaScenarioById(scenarioId)); + const actions = scenario.execution.flow?.steps[0]?.actions ?? []; + const outboundIndex = actions.findIndex( + (action) => (action as { call?: string }).call === "waitForOutboundMessage", + ); + const requestCollectionIndex = actions.findIndex((action) => + requestCollectionMode === "poll" + ? (action as { call?: string }).call === "waitForCondition" && + (action as { saveAs?: string }).saveAs === "scenarioRequests" + : (action as { set?: string }).set === "scenarioRequests", + ); + const requestCountAssertIndex = actions.findIndex((action) => + readFlowAssertExpression(action).includes("scenarioRequests.length === 3"), + ); + const searchPlanAssertIndex = actions.findIndex((action) => + readFlowAssertExpression(action).includes( + "searchPlanRequest.plannedToolName === 'memory_search'", + ), + ); + const searchResultAssertIndex = actions.findIndex((action) => + readFlowAssertExpression(action).includes( + "searchResultRequest.toolOutputCallId === searchPlanRequest.plannedToolCallId", + ), + ); + const finalRequestAssertIndex = actions.findIndex((action) => + readFlowAssertExpression(action).includes(finalLinkNeedle), + ); + + expect(requestCollectionIndex, scenarioId).toBeGreaterThanOrEqual(0); + expect(requestCountAssertIndex, scenarioId).toBeGreaterThan(requestCollectionIndex); + expect(searchPlanAssertIndex, scenarioId).toBeGreaterThan(requestCountAssertIndex); + expect(searchResultAssertIndex, scenarioId).toBeGreaterThan(searchPlanAssertIndex); + expect(finalRequestAssertIndex, scenarioId).toBeGreaterThan(searchResultAssertIndex); + expect(outboundIndex, scenarioId).toBeGreaterThan(finalRequestAssertIndex); + + if (requestCollectionMode === "poll") { + const requestPoll = actions[requestCollectionIndex] as + | { args?: Array<{ lambda?: { expr?: string } }> } + | undefined; + expect(requestPoll?.args?.[0]?.lambda?.expr, scenarioId).toContain( + "requests.length >= 3 ? requests : undefined", + ); + } else { + expect( + actions.some( + (action) => + (action as { call?: string }).call === "waitForCondition" && + (action as { saveAs?: string }).saveAs === "scenarioRequests", + ), + scenarioId, + ).toBe(false); + } + }, + ); +}); diff --git a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts index 3ba872955188..3c6859ea8983 100644 --- a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { readQaScenarioById } from "./scenario-catalog.js"; -import { requireFlowScenario } from "./scenario-catalog.test-utils.js"; +import { readFlowAssertExpression, requireFlowScenario } from "./scenario-catalog.test-utils.js"; describe("qa compaction scenario catalog", () => { it.each([ @@ -56,6 +56,47 @@ describe("qa compaction scenario catalog", () => { const scenario = requireFlowScenario(readQaScenarioById("compaction-retry-mutating-tool")); const flow = JSON.stringify(scenario.execution.flow); const serializedScenario = JSON.stringify(scenario); + const actions = scenario.execution.flow?.steps[0]?.actions ?? []; + const readSetExpression = (name: string) => { + const action = actions.find((candidate) => (candidate as { set?: string }).set === name) as + | { value?: { expr?: string } } + | undefined; + return action?.value?.expr ?? ""; + }; + const readAssertExpression = (needle: string) => + actions.map(readFlowAssertExpression).find((expression) => expression.includes(needle)) ?? ""; + const actionIndex = (predicate: (action: (typeof actions)[number]) => boolean) => + actions.findIndex(predicate); + const writeRequestsExpr = readSetExpression("writeRequests"); + const postWriteContinuationsExpr = readSetExpression("postWriteContinuations"); + const writeTranscriptToolCallIdExpr = readSetExpression("writeTranscriptToolCallId"); + const continuationChainExpr = readSetExpression("continuationChain"); + const continuationAssertIndex = actionIndex((action) => + readFlowAssertExpression(action).includes("continuationChain.valid === true"), + ); + const terminalAssertIndex = actionIndex((action) => + readFlowAssertExpression(action).includes("terminalContinuations.length === 1"), + ); + const distinctCallIdsAssertIndex = actionIndex((action) => + readFlowAssertExpression(action).includes("new Set([writeRequest.plannedToolCallId"), + ); + const stableCellIdAssertIndex = actionIndex((action) => + readFlowAssertExpression(action).includes("continuationChain.waits.length === 0"), + ); + const terminalEvidenceAssertIndex = actionIndex((action) => + readFlowAssertExpression(action).includes( + "terminalContinuations[0].providerVariant === 'openai'", + ), + ); + const outboundWaitIndex = actionIndex( + (action) => + (action as { call?: string }).call === "waitForCondition" && + (action as { saveAs?: string }).saveAs === "outbound", + ); + const stableCellIdAssertExpr = readAssertExpression("continuationChain.waits.length === 0"); + const terminalEvidenceAssertExpr = readAssertExpression( + "terminalContinuations[0].providerVariant === 'openai'", + ); const knownGap = "known-harness-gap compaction-retry-mutating-tool: provider-error recovery does not invoke Codex native compaction; native token-threshold compaction needs a separate scenario."; @@ -70,7 +111,7 @@ describe("qa compaction scenario catalog", () => { "One coded over-threshold provider overflow produces one persisted OpenClaw overflow compaction and one compacted retry retaining durable current context.", ); expect(scenario.successCriteria).toContain( - "OpenClaw performs exactly one successful write, one causal continuation, and returns the exact file content and final marker.", + "OpenClaw performs exactly one successful write, then one terminal continuation after zero-or-more causally linked waits, and returns the exact file content and final marker.", ); expect(scenario.successCriteria).toContain( "OpenClaw proves session-memory.pruning by retaining a nonempty contiguous suffix ending at block 15 while pruning marker block 10.", @@ -114,31 +155,129 @@ describe("qa compaction scenario catalog", () => { expect(flow).toContain("seedQaSessionTranscript"); expect(flow).toContain("sessions.compaction.branch"); expect(flow).toContain("env.runtimeId"); - expect(flow).toContain('"transcriptToolName":"write"'); - expect(flow).toContain('"requireSuccessfulTranscriptToolResult":true'); + expect(scenario.execution.retryCount).toBe(0); + expect(flow).not.toContain('"transcriptToolName":"write"'); + expect(flow).not.toContain('"requireSuccessfulTranscriptToolResult":true'); + expect(serializedScenario).not.toContain("expectedWriteToolResult"); expect(flow).toContain("outbound.text === config.finalMarker"); expect(flow).toContain("overflowRequests.length === 1"); expect(flow).toContain("overflowRequest.rawByteLength > config.overflowThresholdBytes"); expect(flow).toContain("writeRequests.length === 1"); - expect(flow).toContain("String(request.allInputText ?? '').includes(sessionId)"); + expect(writeRequestsExpr).toContain("request.plannedToolName === 'write'"); + expect(writeRequestsExpr).toContain("request.cursor > overflowRequest.cursor"); + expect(writeRequestsExpr).toContain("String(request.allInputText ?? '').includes(sessionId)"); + expect(writeRequestsExpr).toContain( + "String(request.allInputText ?? '').includes(config.promptSnippet)", + ); + expect(writeRequestsExpr).toContain( + "String(request.allInputText ?? '').includes(config.durableMarker)", + ); + expect(writeRequestsExpr).not.toContain("request.requestKind"); + expect(writeRequestsExpr).not.toContain("request.outcome"); + expect(writeRequestsExpr).not.toContain("request.toolOutput"); + expect(writeRequestsExpr).not.toContain("request.plannedToolArgs"); + expect(writeRequestsExpr).not.toContain("request.plannedWireToolName"); + expect(flow).toContain("writeRequest.requestKind === 'agent-initial'"); + expect(flow).toContain("writeRequest.outcome === 'success'"); + expect(flow).toContain("!writeRequest.toolOutput"); expect(flow).toContain( "String(writeRequest.allInputText ?? '').includes(config.durableMarker)", ); - expect(flow).toContain("request.cursor > overflowRequest.cursor"); - expect(flow).toContain("request.plannedToolArgs?.path === config.outputFile"); - expect(flow).toContain("request.plannedToolArgs?.content === config.expectedFileContent"); - expect(flow).toContain("request.toolOutputCallId === writeRequest.plannedToolCallId"); - expect(flow).toContain("request.cursor > writeRequest.cursor"); + expect(flow).not.toContain("request.plannedWireToolName === 'exec'"); expect(flow).toContain( - "String(request.toolOutput ?? '').trim() === config.expectedWriteToolResult", + "writeRequest.plannedWireToolName === undefined || writeRequest.plannedWireToolName === 'exec'", ); + expect(flow).toContain("writeRequest.plannedToolArgs?.path === config.outputFile"); + expect(flow).toContain("writeRequest.plannedToolArgs?.content === config.expectedFileContent"); + expect(flow).toContain("typeof writeRequest.plannedToolCallId === 'string'"); + expect(flow).not.toContain( + "writeRequest.plannedToolCallId.length > 0 && typeof writeRequest.plannedToolItemId", + ); + expect(flow).toContain( + 'writeWireToolName","value":{"expr":"writeRequest.plannedWireToolName ?? writeRequest.plannedToolName', + ); + expect(writeTranscriptToolCallIdExpr).toContain( + "typeof writeRequest.plannedToolItemId === 'string'", + ); + expect(writeTranscriptToolCallIdExpr).toContain("writeRequest.plannedToolItemId.length > 0"); + expect(writeTranscriptToolCallIdExpr).toContain( + "`${writeRequest.plannedToolCallId}|${writeRequest.plannedToolItemId}`", + ); + expect(writeTranscriptToolCallIdExpr).toContain(": writeRequest.plannedToolCallId"); + expect(flow).toContain( + "event.toolCallId === writeTranscriptToolCallId && event.name === writeWireToolName", + ); + expect(flow).toContain("successfulWriteTranscriptEvents.length === 1"); + expect(flow).toContain("transcript.successfulToolCallCounts[writeWireToolName] === 1"); + expect(flow).not.toContain("transcript.successfulToolCallCounts.write === 1"); + expect(postWriteContinuationsExpr).toContain("request.requestKind === 'tool-continuation'"); + expect(postWriteContinuationsExpr).toContain("request.cursor > writeRequest.cursor"); + expect(postWriteContinuationsExpr).toContain( + "String(request.allInputText ?? '').includes(sessionId)", + ); + expect(postWriteContinuationsExpr).not.toContain("request.outcome"); + expect(postWriteContinuationsExpr).not.toContain("request.plannedToolName"); + expect(postWriteContinuationsExpr).not.toContain("request.toolOutputCallId"); + expect(postWriteContinuationsExpr).not.toContain("request.toolOutputStructuredError"); + expect(flow).toContain("let currentCallId = writeRequest.plannedToolCallId"); + expect(flow).toContain("postWriteContinuations.filter"); + expect(flow).toContain("request.toolOutputCallId === currentCallId"); + expect(flow).toContain("request.plannedToolName === 'wait'"); + expect(flow).toContain("currentCallId = request.plannedToolCallId"); + expect(continuationChainExpr).toContain("request.toolOutputCallId === currentCallId"); + expect(continuationChainExpr).toContain("request.plannedToolName === 'wait'"); + expect(flow).toContain("continuationChain.requests.length === postWriteContinuations.length"); + expect(flow).toContain( + "continuationChain.requests.every((request, index) => request === postWriteContinuations[index])", + ); + expect(flow).toContain( + "continuationChain.requests.length === continuationChain.waits.length + 1", + ); + expect(flow).toContain("request.outcome === 'success'"); + expect(flow).toContain("request.toolOutputStructuredError !== true"); + expect(flow).toContain("terminalContinuations.length === 1"); + expect(flow).toContain("terminalContinuations[0] === continuationChain.terminal"); + expect(flow).toContain("String(terminalContinuations[0].toolOutput ?? '').trim().length > 0"); + expect(flow).toContain("new Set([writeRequest.plannedToolCallId"); + expect(stableCellIdAssertExpr).toContain("continuationChain.waits.length === 0 ||"); + expect(stableCellIdAssertExpr).toContain( + "typeof request.plannedToolArgs?.cell_id === 'string'", + ); + expect(stableCellIdAssertExpr).toContain( + "new Set(continuationChain.waits.map((request) => request.plannedToolArgs.cell_id)).size === 1", + ); + expect(terminalEvidenceAssertExpr).toContain("writeWireToolName !== 'exec'"); + const openAiEvidenceIndex = terminalEvidenceAssertExpr.indexOf( + "terminalContinuations[0].providerVariant === 'openai'", + ); + const anthropicEvidenceIndex = terminalEvidenceAssertExpr.indexOf( + "terminalContinuations[0].providerVariant === 'anthropic'", + ); + const unknownProviderFailClosedIndex = terminalEvidenceAssertExpr.lastIndexOf(": false"); + expect(openAiEvidenceIndex).toBeGreaterThanOrEqual(0); + expect(terminalEvidenceAssertExpr).toContain("startsWith('Script completed\\n')"); + expect(anthropicEvidenceIndex).toBeGreaterThan(openAiEvidenceIndex); + expect(terminalEvidenceAssertExpr).toContain( + "JSON.parse(String(terminalContinuations[0].toolOutput ?? ''))", + ); + expect(terminalEvidenceAssertExpr).toContain("parsed !== null"); + expect(terminalEvidenceAssertExpr).toContain("typeof parsed === 'object'"); + expect(terminalEvidenceAssertExpr).toContain("!Array.isArray(parsed)"); + expect(terminalEvidenceAssertExpr).toContain("parsed.status === 'completed'"); + expect(unknownProviderFailClosedIndex).toBeGreaterThan(anthropicEvidenceIndex); + expect(continuationAssertIndex).toBeGreaterThanOrEqual(0); + expect(terminalAssertIndex).toBeGreaterThan(continuationAssertIndex); + expect(distinctCallIdsAssertIndex).toBeGreaterThan(terminalAssertIndex); + expect(stableCellIdAssertIndex).toBeGreaterThan(distinctCallIdsAssertIndex); + expect(terminalEvidenceAssertIndex).toBeGreaterThan(stableCellIdAssertIndex); + expect(outboundWaitIndex).toBeGreaterThan(terminalEvidenceAssertIndex); expect(flow).not.toContain("config.expectedOpenClawToolResult"); expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`---"); expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`+++"); - expect(flow).toContain("postWriteContinuations.length === 1"); expect(flow).toContain( - "compactionSummaryRequests.every((request) => request.outcome === 'success' && request.plannedToolName === undefined)", + "compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true", ); + expect(flow).not.toContain("compactionSummaryRequests.every("); expect(flow).not.toContain("compactionSummaryRequests.length >= 1"); expect(flow).toContain( "writeRequest.rawByteLength < config.overflowThresholdBytes && writeRequest.rawByteLength < overflowRequest.rawByteLength", @@ -154,6 +293,23 @@ describe("qa compaction scenario catalog", () => { expect(flow).toContain("durable: String(request.allInputText ?? '')"); expect(flow).toContain("bulky: String(request.allInputText ?? '')"); expect(flow).toContain("inputChars: String(request.allInputText ?? '').length"); + expect(flow).toContain( + "resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null", + ); + expect(flow).toContain("callId: request.plannedToolCallId ?? null"); + expect(flow).toContain("itemId: request.plannedToolItemId ?? null"); + expect(flow).toContain( + "transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0", + ); + expect(flow).toContain(": request.plannedToolCallId ?? null"); + expect(flow).toContain("logicalWrites=${String(writeRequests.length)}"); + expect(flow).toContain("wireTool=${String(writeWireToolName)}"); + expect(flow).toContain("callId=${String(writeRequest.plannedToolCallId)}"); + expect(flow).toContain("itemId=${String(writeRequest.plannedToolItemId)}"); + expect(flow).toContain("transcriptId=${String(writeTranscriptToolCallId)}"); + expect(flow).toContain( + "wireSuccesses=${String(transcript.successfulToolCallCounts[writeWireToolName] ?? 0)}", + ); expect(flow).not.toContain("clientSessionId"); expect(flow).toContain("tailBlocks:"); expect(flow).toContain(".sort().slice(0, 16)"); diff --git a/extensions/qa-lab/src/scenario-catalog.test-utils.ts b/extensions/qa-lab/src/scenario-catalog.test-utils.ts index b451a1a65173..7c3325dc06c2 100644 --- a/extensions/qa-lab/src/scenario-catalog.test-utils.ts +++ b/extensions/qa-lab/src/scenario-catalog.test-utils.ts @@ -32,6 +32,20 @@ export function requireFlowScenario(scenario: CatalogScenario): FlowCatalogScena return scenario; } +export function readFlowAssertExpression(action: unknown): string { + if (!action || typeof action !== "object" || !("assert" in action)) { + return ""; + } + const assertion = action.assert; + if (typeof assertion === "string") { + return assertion; + } + if (!assertion || typeof assertion !== "object" || !("expr" in assertion)) { + return ""; + } + return typeof assertion.expr === "string" ? assertion.expr : ""; +} + export function flowContainsCall(value: unknown, callName: string): boolean { if (Array.isArray(value)) { return value.some((entry) => flowContainsCall(entry, callName)); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 2182f67725f0..db9119f886c7 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -592,72 +592,6 @@ describe("qa scenario catalog", () => { }); }); - it("loads live gateway sentinel scenarios for harness self-health", () => { - const scenarioIds = [ - "plugin-hook-health-sentinel", - "plugin-manifest-contract-health", - "webchat-direct-reply-routing", - "long-context-progress-watchdog", - "gateway-restart-inflight-run", - "gateway-restart-multi-live", - "streaming-final-integrity", - ]; - - for (const scenarioId of scenarioIds) { - const scenario = readQaScenarioById(scenarioId); - expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0); - expect(scenario.coverage?.primary.length).toBeGreaterThan(0); - } - expect(readQaScenarioById("webchat-direct-reply-routing").sourcePath).toBe( - "qa/scenarios/channels/webchat-direct-reply-routing.yaml", - ); - expect(readQaScenarioById("long-context-progress-watchdog").sourcePath).toBe( - "qa/scenarios/runtime/long-context-progress-watchdog.yaml", - ); - const gatewayRestartFlow = readQaScenarioById("gateway-restart-inflight-run").execution.flow; - const gatewayRestartContract = JSON.stringify(gatewayRestartFlow); - expect( - JSON.stringify(readQaScenarioById("gateway-restart-inflight-run").gatewayConfigPatch), - ).toContain('"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]'); - expect(gatewayRestartContract).toContain("plannedToolName === 'wait'"); - expect(gatewayRestartContract).toContain("lastAssistantToolNames?.includes('wait')"); - expect(gatewayRestartContract).toContain("restartRecoveryDeliveryContext"); - expect(gatewayRestartContract).toContain("sendInbound"); - expect(gatewayRestartContract).not.toContain("startAgentRun"); - expect(gatewayRestartContract).toContain('"restartGatewayWithConfigPatch"'); - expect(gatewayRestartContract).toContain("interruptedMatches.length === 1"); - expect(gatewayRestartContract).toContain("restartNotices.length === 0"); - expect(gatewayRestartContract).toContain("dispatching restart-safe recovery"); - expect(gatewayRestartContract).toContain("[OpenClaw heartbeat poll]"); - expect(gatewayRestartContract).toContain("liveTurnTimeoutMs(env, 180000)"); - expect(gatewayRestartContract).toContain("id: `dm:${conversationId}`"); - expect(gatewayRestartContract).toContain("dmScope: env.cfg.session?.dmScope"); - expect(readQaScenarioById("gateway-restart-inflight-run").gatewayConfigPatch).toMatchObject({ - plugins: { - slots: { memory: "none" }, - entries: { - acpx: { enabled: false }, - "memory-core": { enabled: false }, - }, - }, - }); - const liveMultiRestart = readQaScenarioById("gateway-restart-multi-live"); - const liveMultiRestartContract = JSON.stringify(liveMultiRestart.execution.flow); - expect(JSON.stringify(liveMultiRestart.gatewayConfigPatch)).toContain( - '"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]', - ); - expect(liveMultiRestartContract).toContain("assistantToolCallCounts.exec"); - expect(liveMultiRestartContract).toContain("checkpoint"); - expect(liveMultiRestartContract).toContain("restarts=3"); - expect(liveMultiRestartContract).toContain("dmScope: 'per-channel-peer'"); - expect(liveMultiRestartContract).toContain("dispatching restart-safe recovery"); - expect(readQaScenarioExecutionConfig("gateway-restart-multi-live")).toMatchObject({ - requiredProviderMode: "live-frontier", - requiredProvider: "openai", - requiredModel: "gpt-5.4", - }); - }); - it("loads the QA bus tool trace visibility harness scenario", () => { const scenario = readQaScenarioById("qa-bus-tool-trace-visibility"); const config = readQaScenarioExecutionConfig(scenario.id) as diff --git a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml index 00fbaeacb7e5..1c1a02d33094 100644 --- a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml +++ b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml @@ -89,6 +89,9 @@ flow: - set: artifactPath value: expr: "path.join(env.gateway.workspaceDir, 'repo-contract-summary.txt')" + - set: requestCursorBefore + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -128,7 +131,7 @@ flow: expr: "`repo contract followthrough bounced for permission or stalled: ${outbound.text}`" - set: followthroughDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].filter((request) => /repo contract followthrough check/i.test(String(request.allInputText ?? ''))) : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /repo contract followthrough check/i.test(String(request.allInputText ?? ''))) : []" - assert: expr: "!env.mock || followthroughDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 3" message: diff --git a/qa/scenarios/agents/subagent-handoff.yaml b/qa/scenarios/agents/subagent-handoff.yaml index 4f27fb3c6060..202d504c68ab 100644 --- a/qa/scenarios/agents/subagent-handoff.yaml +++ b/qa/scenarios/agents/subagent-handoff.yaml @@ -32,6 +32,9 @@ flow: - call: reset - try: actions: + - set: requestCursorBefore + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0" - call: runAgentPrompt args: - ref: env @@ -76,7 +79,7 @@ flow: # request after the tool runs has plannedToolName unset. - set: subagentDebugRequests value: - expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))] : []" + expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))] : []" - assert: expr: "!env.mock || subagentDebugRequests.some((request) => !request.toolOutput && /delegate one bounded qa task/i.test(String(request.allInputText ?? '')) && request.plannedToolName === 'sessions_spawn')" message: diff --git a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml index e60c8cddd755..49f9c4c0016d 100644 --- a/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml +++ b/qa/scenarios/channels/matrix-room-generated-image-delivery.yaml @@ -19,6 +19,7 @@ scenario: timeoutMs: 180000 retryCount: 0 config: + requiredChannelDriver: live matrixConfigOverrides: streaming: quiet matrixTopology: @@ -35,7 +36,17 @@ scenario: requireMention: true flow: - module: ./live-transports/matrix/scenarios/scenario-runtime-media.js - call: runGeneratedImageDeliveryScenario - args: - - expr: "scenarioContext" + steps: + - name: Matrix generated images deliver as real image attachments while streaming + actions: + - call: ensureImageGenerationConfigured + args: + - ref: env + - set: scenarioModule + value: + expr: "await qaImport('./live-transports/matrix/scenarios/scenario-runtime-media.js')" + - call: scenarioModule.runGeneratedImageDeliveryScenario + args: + - expr: "scenarioContext" + saveAs: result + detailsExpr: "result.details ?? (result.artifacts ? JSON.stringify(result.artifacts, null, 2) : undefined)" diff --git a/qa/scenarios/memory/memory-tools-channel-context.yaml b/qa/scenarios/memory/memory-tools-channel-context.yaml index f250ef055aa6..c160a5af9e45 100644 --- a/qa/scenarios/memory/memory-tools-channel-context.yaml +++ b/qa/scenarios/memory/memory-tools-channel-context.yaml @@ -90,19 +90,14 @@ flow: senderName: Alice text: expr: config.prompt - - call: waitForOutboundMessage - saveAs: outbound + - call: waitForCondition + saveAs: scenarioRequests args: - - ref: state - lambda: - params: [candidate] - expr: "candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && candidate.conversation.kind === 'channel' && candidate.text.includes(config.expectedNeedle)" + async: true + expr: "(async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeInbound}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)); return requests.length >= 3 ? requests : undefined; })()" - expr: liveTurnTimeoutMs(env, 30000) - - sinceIndex: - ref: outboundStartIndex - - set: scenarioRequests - value: - expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeInbound}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))" + - 100 - assert: expr: scenarioRequests.length === 3 message: @@ -128,6 +123,16 @@ flow: expr: "finalRequest.cursor > searchResultRequest.cursor && finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && finalRequest.toolOutputStructuredError !== true && String(finalRequest.toolOutput ?? '').includes(config.expectedMemoryPath) && String(finalRequest.toolOutput ?? '').includes(config.expectedNeedle) && !finalRequest.plannedToolName" message: expr: "`final request did not consume the matching successful memory_get result: ${JSON.stringify(finalRequest)}`" + - call: waitForOutboundMessage + saveAs: outbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && candidate.conversation.kind === 'channel' && candidate.text.includes(config.expectedNeedle)" + - expr: liveTurnTimeoutMs(env, 30000) + - sinceIndex: + ref: outboundStartIndex - call: sleep args: [4000] - set: visibleChannelOutbounds diff --git a/qa/scenarios/memory/thread-memory-isolation.yaml b/qa/scenarios/memory/thread-memory-isolation.yaml index aafa649ebb9a..724079c34bca 100644 --- a/qa/scenarios/memory/thread-memory-isolation.yaml +++ b/qa/scenarios/memory/thread-memory-isolation.yaml @@ -26,9 +26,12 @@ scenario: kind: flow summary: Verify a memory-backed answer requested inside a thread stays in-thread and does not leak into the root channel. channel: qa-channel + providerMode: mock-openai + retryCount: 0 config: memoryFact: "Thread-hidden codename: ORBIT-22." memoryQuery: "hidden thread codename ORBIT-22" + expectedMemoryPath: MEMORY.md expectedNeedle: "ORBIT-22" channelId: qa-room channelTitle: QA Room @@ -79,7 +82,10 @@ flow: - assert: expr: Boolean(threadId) message: missing thread id for memory isolation check - - set: beforeCursor + - set: requestCursorBefore + value: + expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor" + - set: outboundStartIndex value: expr: state.getSnapshot().messages.length - sendInbound: @@ -97,6 +103,39 @@ flow: ref: threadId threadTitle: expr: config.threadTitle + - call: waitForCondition + saveAs: scenarioRequests + args: + - lambda: + async: true + expr: "(async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)); return requests.length >= 3 ? requests : undefined; })()" + - expr: liveTurnTimeoutMs(env, 30000) + - 100 + - assert: + expr: scenarioRequests.length === 3 + message: + expr: "`expected one memory_search plan, one memory_get plan, and one final request: ${JSON.stringify(scenarioRequests)}`" + - set: searchPlanRequest + value: + expr: scenarioRequests[0] + - set: searchResultRequest + value: + expr: scenarioRequests[1] + - set: finalRequest + value: + expr: scenarioRequests[2] + - assert: + expr: "searchPlanRequest.plannedToolName === 'memory_search' && searchPlanRequest.plannedToolArgs?.query === config.memoryQuery && typeof searchPlanRequest.plannedToolCallId === 'string' && searchPlanRequest.plannedToolCallId.length > 0 && !searchPlanRequest.toolOutputCallId && !String(searchPlanRequest.allInputText ?? '').includes(config.expectedNeedle)" + message: + expr: "`initial thread request exposed the hidden fact, disabled memory_search, or routed incorrectly: ${JSON.stringify(searchPlanRequest)}`" + - assert: + expr: "searchResultRequest.cursor > searchPlanRequest.cursor && searchResultRequest.toolOutputCallId === searchPlanRequest.plannedToolCallId && searchResultRequest.toolOutputStructuredError !== true && String(searchResultRequest.toolOutput ?? '').includes(config.expectedMemoryPath) && String(searchResultRequest.toolOutput ?? '').includes(config.expectedNeedle) && searchResultRequest.plannedToolName === 'memory_get' && searchResultRequest.plannedToolArgs?.path === config.expectedMemoryPath && typeof searchResultRequest.plannedToolCallId === 'string' && searchResultRequest.plannedToolCallId.length > 0 && searchResultRequest.plannedToolCallId !== searchPlanRequest.plannedToolCallId" + message: + expr: "`memory_search result did not causally plan memory_get: ${JSON.stringify(searchResultRequest)}`" + - assert: + expr: "finalRequest.cursor > searchResultRequest.cursor && finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && finalRequest.toolOutputStructuredError !== true && String(finalRequest.toolOutput ?? '').includes(config.expectedNeedle) && !finalRequest.plannedToolName" + message: + expr: "`final thread request did not consume the matching memory_get result: ${JSON.stringify(finalRequest)}`" - call: waitForOutboundMessage saveAs: outbound args: @@ -105,10 +144,9 @@ flow: params: [candidate] expr: "((candidate.conversation.id === config.channelId && candidate.threadId === threadId) || candidate.conversation.id === threadId) && candidate.text.includes(config.expectedNeedle)" - expr: liveTurnTimeoutMs(env, 300000) + - sinceIndex: + ref: outboundStartIndex - assert: - expr: "!state.getSnapshot().messages.slice(beforeCursor).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && !candidate.threadId)" + expr: "!state.getSnapshot().messages.slice(outboundStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && !candidate.threadId)" message: threaded memory answer leaked into root channel - - assert: - expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'memory_search')" - message: expected memory_search in thread memory flow detailsExpr: outbound.text diff --git a/qa/scenarios/runtime/agent-tool-consumption.yaml b/qa/scenarios/runtime/agent-tool-consumption.yaml index 003262559af8..9ce2557597c8 100644 --- a/qa/scenarios/runtime/agent-tool-consumption.yaml +++ b/qa/scenarios/runtime/agent-tool-consumption.yaml @@ -89,16 +89,6 @@ flow: expr: config.prompt timeoutMs: expr: liveTurnTimeoutMs(env, 30000) - - call: waitForOutboundMessage - saveAs: outbound - args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(nonce)" - - expr: liveTurnTimeoutMs(env, 30000) - - sinceIndex: - ref: outboundStartIndex - set: scenarioRequests value: expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))" @@ -127,6 +117,16 @@ flow: expr: "getResultRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && getResultRequest.toolOutputStructuredError !== true && String(getResultRequest.toolOutput ?? '').includes(nonce) && String(getResultRequest.allInputText ?? '').includes(nonce) && !getResultRequest.plannedToolName" message: expr: "`memory_get result was not consumed before terminal generation: ${JSON.stringify(getResultRequest)}`" + - call: waitForOutboundMessage + saveAs: outbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(nonce)" + - expr: liveTurnTimeoutMs(env, 30000) + - sinceIndex: + ref: outboundStartIndex - set: terminalOutbounds value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)" diff --git a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml index d1de13188cc8..ce6b94690484 100644 --- a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml +++ b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml @@ -12,7 +12,7 @@ scenario: objective: Verify OpenClaw persists one overflow compaction, prunes bulky historical turns, retries from durable context, and performs the requested mutation exactly once. successCriteria: - One coded over-threshold provider overflow produces one persisted OpenClaw overflow compaction and one compacted retry retaining durable current context. - - OpenClaw performs exactly one successful write, one causal continuation, and returns the exact file content and final marker. + - OpenClaw performs exactly one successful write, then one terminal continuation after zero-or-more causally linked waits, and returns the exact file content and final marker. - OpenClaw proves session-memory.pruning by retaining a nonempty contiguous suffix ending at block 15 while pruning marker block 10. - OpenClaw records one overflow-retry checkpoint whose branch preserves the independent pre-compaction assistant marker. - The Codex runtime-pair cell reports a known harness gap before gateway, session, or provider work and makes no compaction coverage claim. @@ -25,6 +25,7 @@ scenario: execution: kind: flow summary: Force one OpenClaw context overflow, verify persisted pruning evidence, and prove exactly-once mutation after compaction. + retryCount: 0 config: outputFile: compaction-retry-summary.txt promptSnippet: Compaction retry mutating tool check @@ -33,7 +34,6 @@ scenario: checkpointMarker: QA-COMPACTION-PRECOMPACTION-ASSISTANT-MARKER overflowThresholdBytes: 262144 expectedFileContent: "Replay safety: unsafe after write.\n" - expectedWriteToolResult: Successfully wrote 35 bytes to compaction-retry-summary.txt finalMarker: "Protocol note: replay unsafe after write." prompt: |- Compaction retry mutating tool check. Current durable context marker: QA-COMPACTION-DURABLE-MARKER. @@ -97,25 +97,12 @@ flow: ref: config.prompt timeoutMs: expr: liveTurnTimeoutMs(env, 120000) - transcriptToolName: write - requireSuccessfulTranscriptToolResult: true - - call: waitForCondition - saveAs: outbound - args: - - lambda: - expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.finalMarker)).at(-1)" - - expr: liveTurnTimeoutMs(env, 45000) - - 100 - - assert: - expr: "outbound.text === config.finalMarker" - message: - expr: "`unexpected final compaction reply: ${JSON.stringify(outbound.text)}`" - set: scenarioRequests value: expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)" - set: requestEvidence value: - expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), tool: request.plannedToolName ?? null }))" + expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), tool: request.plannedToolName ?? null, resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null, callId: request.plannedToolCallId ?? null, itemId: request.plannedToolItemId ?? null, transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0 ? `${request.plannedToolCallId}|${request.plannedToolItemId}` : request.plannedToolCallId ?? null }))" - set: overflowRequests value: expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'error' && request.errorCode === 'context_length_exceeded' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" @@ -128,7 +115,7 @@ flow: expr: "overflowRequests[0]" - set: writeRequests value: - expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'success' && request.plannedToolName === 'write' && !request.toolOutput && request.cursor > overflowRequest.cursor && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker) && request.plannedToolArgs?.path === config.outputFile && request.plannedToolArgs?.content === config.expectedFileContent)" + expr: "scenarioRequests.filter((request) => request.cursor > overflowRequest.cursor && request.plannedToolName === 'write' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" - assert: expr: "writeRequests.length === 1" message: @@ -136,6 +123,16 @@ flow: - set: writeRequest value: expr: "writeRequests[0]" + - assert: + expr: "writeRequest.requestKind === 'agent-initial' && writeRequest.outcome === 'success' && !writeRequest.toolOutput && writeRequest.plannedToolArgs?.path === config.outputFile && writeRequest.plannedToolArgs?.content === config.expectedFileContent && (writeRequest.plannedWireToolName === undefined || writeRequest.plannedWireToolName === 'exec') && typeof writeRequest.plannedToolCallId === 'string' && writeRequest.plannedToolCallId.length > 0" + message: + expr: "`logical write did not have the expected successful direct-or-Code-Mode wire shape: ${JSON.stringify(requestEvidence.find((request) => request.cursor === writeRequest.cursor))}`" + - set: writeWireToolName + value: + expr: "writeRequest.plannedWireToolName ?? writeRequest.plannedToolName" + - set: writeTranscriptToolCallId + value: + expr: "typeof writeRequest.plannedToolItemId === 'string' && writeRequest.plannedToolItemId.length > 0 ? `${writeRequest.plannedToolCallId}|${writeRequest.plannedToolItemId}` : writeRequest.plannedToolCallId" - set: overflowEvidence value: expr: "requestEvidence.find((request) => request.cursor === overflowRequest.cursor)" @@ -147,10 +144,17 @@ flow: args: - ref: env - ref: sessionKey + - set: successfulWriteTranscriptEvents + value: + expr: "(transcript.successfulToolCallEvents ?? []).filter((event) => event.toolCallId === writeTranscriptToolCallId && event.name === writeWireToolName)" - assert: - expr: "transcript.successfulToolCallCounts.write === 1" + expr: "successfulWriteTranscriptEvents.length === 1" message: - expr: "`expected exactly one successful write, got ${JSON.stringify(transcript.successfulToolCallCounts)}`" + expr: "`expected one authenticated successful transcript event for the logical write's wire call: ${JSON.stringify({ write: writeEvidence, wireTool: writeWireToolName, callId: writeRequest.plannedToolCallId, itemId: writeRequest.plannedToolItemId, transcriptId: writeTranscriptToolCallId, events: transcript.successfulToolCallEvents ?? [] })}`" + - assert: + expr: "transcript.successfulToolCallCounts[writeWireToolName] === 1" + message: + expr: "`expected exactly one successful resolved wire call, got ${JSON.stringify({ wireTool: writeWireToolName, counts: transcript.successfulToolCallCounts })}`" - assert: expr: "overflowRequest.rawByteLength > config.overflowThresholdBytes" message: @@ -161,11 +165,44 @@ flow: expr: "`compacted retry did not retain durable current context: ${JSON.stringify({ overflow: overflowEvidence, write: writeEvidence })}`" - set: postWriteContinuations value: - expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.outcome === 'success' && request.plannedToolName === undefined && request.cursor > writeRequest.cursor && request.toolOutputCallId === writeRequest.plannedToolCallId && String(request.allInputText ?? '').includes(sessionId) && String(request.toolOutput ?? '').trim() === config.expectedWriteToolResult)" + expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.cursor > writeRequest.cursor && String(request.allInputText ?? '').includes(sessionId)).toSorted((left, right) => left.cursor - right.cursor)" + - set: continuationChain + value: + expr: "(() => { const requests = []; const waits = []; const followedCallIds = new Set(); let currentCallId = writeRequest.plannedToolCallId; let previousCursor = writeRequest.cursor; let terminal; let valid = typeof currentCallId === 'string' && currentCallId.length > 0; while (valid) { if (followedCallIds.has(currentCallId)) { valid = false; break; } followedCallIds.add(currentCallId); const matches = postWriteContinuations.filter((request) => request.cursor > previousCursor && request.toolOutputCallId === currentCallId); if (matches.length !== 1) { valid = false; break; } const request = matches[0]; requests.push(request); previousCursor = request.cursor; if (request.plannedToolName === 'wait') { if (typeof request.plannedToolCallId !== 'string' || request.plannedToolCallId.length === 0) { valid = false; break; } waits.push(request); currentCallId = request.plannedToolCallId; continue; } if (request.plannedToolName === undefined) { terminal = request; break; } valid = false; } return { valid, requests, waits, terminal }; })()" - assert: - expr: "postWriteContinuations.length === 1" + expr: "continuationChain.valid === true && continuationChain.requests.length === postWriteContinuations.length && continuationChain.requests.every((request, index) => request === postWriteContinuations[index]) && continuationChain.requests.length === continuationChain.waits.length + 1 && continuationChain.requests.every((request) => request.outcome === 'success' && request.toolOutputStructuredError !== true)" message: - expr: "`expected one post-write continuation and no replayed mutation: ${JSON.stringify(requestEvidence.filter((request) => request.cursor > writeRequest.cursor))}`" + expr: "`expected one exhaustive successful causal continuation chain after the logical write: ${JSON.stringify(postWriteContinuations)}`" + - set: terminalContinuations + value: + expr: "continuationChain.requests.filter((request) => request.plannedToolName === undefined)" + - assert: + expr: "terminalContinuations.length === 1 && terminalContinuations[0] === continuationChain.terminal && String(terminalContinuations[0].toolOutput ?? '').trim().length > 0 && terminalContinuations[0].toolOutputStructuredError !== true" + message: + expr: "`expected exactly one non-error terminal continuation after zero-or-more linked waits: ${JSON.stringify(postWriteContinuations)}`" + - assert: + expr: "continuationChain.waits.every((request) => request.plannedToolName === 'wait' && typeof request.plannedToolCallId === 'string' && request.plannedToolCallId.length > 0) && new Set([writeRequest.plannedToolCallId, ...continuationChain.waits.map((request) => request.plannedToolCallId)]).size === continuationChain.waits.length + 1" + message: + expr: "`linked wait calls did not preserve distinct causal call ids: ${JSON.stringify(postWriteContinuations)}`" + - assert: + expr: "continuationChain.waits.length === 0 || (continuationChain.waits.every((request) => typeof request.plannedToolArgs?.cell_id === 'string' && request.plannedToolArgs.cell_id.length > 0) && new Set(continuationChain.waits.map((request) => request.plannedToolArgs.cell_id)).size === 1)" + message: + expr: "`linked waits did not preserve one nonempty Code Mode cell id: ${JSON.stringify(postWriteContinuations)}`" + - assert: + expr: "writeWireToolName !== 'exec' || (terminalContinuations[0].providerVariant === 'openai' ? String(terminalContinuations[0].toolOutput ?? '').startsWith('Script completed\\n') : terminalContinuations[0].providerVariant === 'anthropic' ? (() => { try { const parsed = JSON.parse(String(terminalContinuations[0].toolOutput ?? '')); return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.status === 'completed'; } catch { return false; } })() : false)" + message: + expr: "`Code Mode terminal continuation did not report successful completion: ${JSON.stringify(terminalContinuations[0])}`" + - call: waitForCondition + saveAs: outbound + args: + - lambda: + expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.finalMarker)).at(-1)" + - expr: liveTurnTimeoutMs(env, 45000) + - 100 + - assert: + expr: "outbound.text === config.finalMarker" + message: + expr: "`unexpected final compaction reply: ${JSON.stringify(outbound.text)}`" - call: fs.readFile saveAs: writtenSummary args: @@ -191,9 +228,9 @@ flow: value: expr: "scenarioRequests.filter((request) => request.requestKind === 'compaction-summary')" - assert: - expr: "compactionSummaryRequests.every((request) => request.outcome === 'success' && request.plannedToolName === undefined)" + expr: "compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true" message: - expr: "`OpenClaw compaction summary requests included a failure or tool plan: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`" + expr: "`expected exactly one successful OpenClaw compaction summary request: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`" - call: readRawQaSessionStore saveAs: store args: @@ -227,4 +264,4 @@ flow: expr: "branchSummary.finalText === config.checkpointMarker" message: expr: "`checkpoint branch did not preserve pre-compaction assistant marker: ${JSON.stringify({ key: branchResult.key, finalText: branchSummary.finalText })}`" - detailsExpr: "`${outbound.text}\\nOpenClaw originalBytes=${String(overflowRequest.rawByteLength)} retryBytes=${String(writeRequest.rawByteLength)} writes=${String(transcript.successfulToolCallCounts.write ?? 0)} compactions=${String(sessionEntry.compactionCount)} checkpoints=${String(overflowCheckpoints.length)}`" + detailsExpr: "`${outbound.text}\\nOpenClaw originalBytes=${String(overflowRequest.rawByteLength)} retryBytes=${String(writeRequest.rawByteLength)} logicalWrites=${String(writeRequests.length)} wireTool=${String(writeWireToolName)} callId=${String(writeRequest.plannedToolCallId)} itemId=${String(writeRequest.plannedToolItemId)} transcriptId=${String(writeTranscriptToolCallId)} wireSuccesses=${String(transcript.successfulToolCallCounts[writeWireToolName] ?? 0)} compactions=${String(sessionEntry.compactionCount)} checkpoints=${String(overflowCheckpoints.length)}`" diff --git a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml index 2d43fa80938e..785bea42acea 100644 --- a/qa/scenarios/runtime/gateway-restart-inflight-run.yaml +++ b/qa/scenarios/runtime/gateway-restart-inflight-run.yaml @@ -44,6 +44,7 @@ scenario: kind: flow runtime: openclaw timeoutMs: 420000 + retryCount: 0 summary: Restart while replay-safe Code Mode wait is executing, then verify automatic reconstruction and delivery. config: requiredProviderMode: mock-openai @@ -136,18 +137,34 @@ flow: args: - ref: env - 180000 - - call: sleep + - call: waitForCondition + saveAs: settledRecovery args: - - 6500 - - call: readSessionTranscriptSummary - saveAs: postRestartTranscript - args: - - ref: env - - ref: sessionKey + - lambda: + async: true + expr: "(async () => { const recoveryRequests = env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []; const restartRecoveryRequests = recoveryRequests.filter((request) => String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(request.allInputText ?? '').includes(config.interruptedMarker)); const transcript = await readSessionTranscriptSummary(env, sessionKey); return restartRecoveryRequests.length >= 1 && transcript.lastMessageRole === 'assistant' && String(transcript.finalText ?? '').includes(config.interruptedMarker) ? { recoveryRequests, restartRecoveryRequests, transcript } : undefined; })()" + - expr: liveTurnTimeoutMs(env, 120000) + - 25 + - set: recoveryRequestsBeforeOutbound + value: + expr: settledRecovery.recoveryRequests + - set: restartRecoveryRequestsBeforeOutbound + value: + expr: settledRecovery.restartRecoveryRequests + - set: postRestartTranscript + value: + expr: settledRecovery.transcript - assert: expr: "postRestartTranscript.lastAssistantStopReason !== 'error' && postRestartTranscript.lastAssistantStopReason !== 'aborted'" message: expr: "`restart recovery left an abort artifact: ${JSON.stringify(postRestartTranscript)}`" + - assert: + expr: "restartRecoveryRequestsBeforeOutbound.length === 1 && String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(restartRecoveryRequestsBeforeOutbound[0].allInputText ?? '').includes(config.interruptedMarker)" + message: + expr: "`expected exactly one scoped restart recovery provider request before delivery, got ${restartRecoveryRequestsBeforeOutbound.length}; total post-cursor requests=${recoveryRequestsBeforeOutbound.length}`" + - assert: + expr: "!String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('[OpenClaw heartbeat poll]')" + message: restart recovery provider request was replaced by a heartbeat poll - call: waitForOutboundMessage saveAs: outbound args: @@ -158,6 +175,16 @@ flow: - expr: liveTurnTimeoutMs(env, 180000) - sinceIndex: ref: startIndex + - set: settledRecoveryRequests + value: + expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []" + - set: settledRestartRecoveryRequests + value: + expr: "settledRecoveryRequests.filter((request) => String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(request.allInputText ?? '').includes(config.interruptedMarker))" + - assert: + expr: "settledRestartRecoveryRequests.length === 1 && settledRestartRecoveryRequests[0].cursor === restartRecoveryRequestsBeforeOutbound[0].cursor" + message: + expr: "`restart recovery provider request was duplicated after delivery: before=${JSON.stringify(restartRecoveryRequestsBeforeOutbound)} settled=${JSON.stringify(settledRestartRecoveryRequests)}`" - set: interruptedMatches value: expr: "state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId && candidate.text.includes(config.interruptedMarker))" @@ -172,15 +199,6 @@ flow: expr: "restartNotices.length === 0" message: expr: "`automatic recovery emitted ${restartNotices.length} resend notice(s); outbound=${recentOutboundSummary(state)}`" - - set: recoveryRequests - value: - expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []" - - assert: - expr: "recoveryRequests.some((request) => String(request.allInputText ?? '').includes('Your previous turn was interrupted by a gateway restart'))" - message: restart recovery prompt did not reach the provider - - assert: - expr: "recoveryRequests.every((request) => !String(request.allInputText ?? '').includes('[OpenClaw heartbeat poll]'))" - message: heartbeat poll raced the restart recovery proof - set: recoveryLogs value: expr: readGatewayLogs().slice(gatewayLogCursor) @@ -188,4 +206,4 @@ flow: expr: "recoveryLogs.includes('dispatching restart-safe recovery') && recoveryLogs.includes('restart-safe recovery tool policy retained')" message: expr: "`restart-safe host policy was not observed; logs=${recoveryLogs}`" - detailsExpr: "`session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true heartbeatPoll=false\\n${outbound.text}`" + detailsExpr: "`session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true recoveryPromptHeartbeat=false\\n${outbound.text}`"