mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(qa): repair release validation scenarios (#119150)
* fix(qa): repair parity and matrix media setup Punchcard-Session: silver-valley-valley-dt * test(qa): accept token-budgeted compaction suffix Punchcard-Session: silver-valley-valley-dt * test(qa): scope request diagnostics Punchcard-Session: silver-valley-valley-dt * test(qa): prove thread-memory causality Punchcard-Session: silver-valley-valley-dt * test(qa): configure Matrix image generation and fail fast Punchcard-Session: silver-valley-valley-dt * fix(qa): require memory get before thread recall Punchcard-Session: silver-valley-valley-dt * fix(qa): unify mock provider dispatch Punchcard-Session: silver-valley-valley-dt * fix(qa): repair Anthropic IDs and compaction causality Punchcard-Session: silver-valley-valley-dt * fix(qa): authenticate compaction wire causality Punchcard-Session: silver-valley-valley-dt * fix(qa): preserve Responses tool item identity Punchcard-Session: silver-valley-valley-dt * fix(qa): scope restart heartbeat proof Punchcard-Session: silver-valley-valley-dt * test(qa): assert causal traces before delivery Punchcard-Session: silver-valley-valley-dt * fix(qa): validate code mode completion evidence Punchcard-Session: silver-valley-valley-dt * test(qa): split causal catalog checks Punchcard-Session: silver-valley-valley-dt * style(qa): format matrix scenario tests Punchcard-Session: silver-valley-valley-dt
This commit is contained in:
Vincent Koc
@@ -5,6 +5,7 @@ import {
|
||||
readQaScenarioExecutionConfig,
|
||||
} from "../../scenario-catalog.js";
|
||||
import { requireFlowScenario } from "../../scenario-catalog.test-utils.js";
|
||||
import { collectQaSuitePluginIds } from "../../suite-planning.js";
|
||||
|
||||
const MATRIX_MENTION_GATE_PRIMARY_SCENARIOS = [
|
||||
"matrix-allowbots-default-block",
|
||||
@@ -221,14 +222,17 @@ describe("Matrix QA Lab scenario flows", () => {
|
||||
});
|
||||
|
||||
it("loads the voice preflight provider and media overrides", () => {
|
||||
expect(readQaScenarioById("matrix-voice-preflight-mention").plugins).toEqual(["openai"]);
|
||||
expect(readQaScenarioById("matrix-voice-preflight-mention").execution).toMatchObject({
|
||||
const scenario = readQaScenarioById("matrix-voice-preflight-mention");
|
||||
|
||||
expect(scenario.execution).toMatchObject({
|
||||
kind: "flow",
|
||||
providerMode: "mock-openai",
|
||||
retryCount: 0,
|
||||
timeoutMs: 90_000,
|
||||
});
|
||||
expect(readQaScenarioById("matrix-voice-preflight-mention").gatewayConfigPatch).toMatchObject({
|
||||
expect(scenario.plugins).toEqual(["openai"]);
|
||||
expect(collectQaSuitePluginIds([scenario])).toEqual(["openai"]);
|
||||
expect(scenario.gatewayConfigPatch).toMatchObject({
|
||||
tools: {
|
||||
media: {
|
||||
models: [{ capabilities: ["audio"], model: "gpt-4o-transcribe", provider: "openai" }],
|
||||
@@ -274,4 +278,37 @@ describe("Matrix QA Lab scenario flows", () => {
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("configures image generation before the single generated-image flow call", () => {
|
||||
const scenario = requireFlowScenario(
|
||||
readQaScenarioById("matrix-room-generated-image-delivery"),
|
||||
);
|
||||
const actions = scenario.execution.flow?.steps[0]?.actions ?? [];
|
||||
|
||||
expect(scenario.execution).toMatchObject({
|
||||
channel: "matrix",
|
||||
retryCount: 0,
|
||||
timeoutMs: 180_000,
|
||||
config: {
|
||||
requiredChannelDriver: "live",
|
||||
},
|
||||
});
|
||||
expect(actions).toEqual([
|
||||
{
|
||||
call: "ensureImageGenerationConfigured",
|
||||
args: [{ ref: "env" }],
|
||||
},
|
||||
{
|
||||
set: "scenarioModule",
|
||||
value: {
|
||||
expr: "await qaImport('./live-transports/matrix/scenarios/scenario-runtime-media.js')",
|
||||
},
|
||||
},
|
||||
{
|
||||
call: "scenarioModule.runGeneratedImageDeliveryScenario",
|
||||
args: [{ expr: "scenarioContext" }],
|
||||
saveAs: "result",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
+130
-2
@@ -1,5 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { testing } from "./scenario-runtime-media.js";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const advanceMatrixQaActorCursor = vi.hoisted(() => vi.fn());
|
||||
const primeMatrixQaActorCursor = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./scenario-runtime-shared.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("./scenario-runtime-shared.js")>()),
|
||||
advanceMatrixQaActorCursor,
|
||||
primeMatrixQaActorCursor,
|
||||
}));
|
||||
|
||||
import { runGeneratedImageDeliveryScenario, testing } from "./scenario-runtime-media.js";
|
||||
import type { MatrixQaScenarioContext } from "./scenario-runtime-shared.js";
|
||||
|
||||
describe("Matrix voice preflight reply matching", () => {
|
||||
it("accepts punctuation differences in the transcribed marker", () => {
|
||||
@@ -10,3 +21,120 @@ describe("Matrix voice preflight reply matching", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
const roomId = "!media:matrix-qa.test";
|
||||
|
||||
function createGeneratedImageContext(
|
||||
observedEvents: MatrixQaScenarioContext["observedEvents"] = [],
|
||||
): MatrixQaScenarioContext {
|
||||
return {
|
||||
baseUrl: "http://127.0.0.1:28008",
|
||||
driverAccessToken: "driver-token",
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
observedEvents,
|
||||
observerAccessToken: "observer-token",
|
||||
observerUserId: "@observer:matrix-qa.test",
|
||||
roomId,
|
||||
sutAccessToken: "sut-token",
|
||||
sutUserId: "@sut:matrix-qa.test",
|
||||
syncState: {},
|
||||
timeoutMs: 180_000,
|
||||
topology: {
|
||||
defaultRoomId: roomId,
|
||||
defaultRoomKey: "media",
|
||||
rooms: [
|
||||
{
|
||||
encrypted: false,
|
||||
key: "media",
|
||||
kind: "group",
|
||||
memberRoles: ["driver", "observer", "sut"],
|
||||
memberUserIds: [
|
||||
"@driver:matrix-qa.test",
|
||||
"@observer:matrix-qa.test",
|
||||
"@sut:matrix-qa.test",
|
||||
],
|
||||
name: "Matrix QA Media Room",
|
||||
requireMention: true,
|
||||
roomId,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("Matrix generated image delivery", () => {
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("sends one trigger and accepts one fresh top-level image", async () => {
|
||||
const sendTextMessage = vi.fn(async () => "$driver-trigger");
|
||||
const waitForOptionalRoomEvent = vi.fn(async ({ predicate }) => {
|
||||
const event = {
|
||||
attachment: { filename: "generated.png", kind: "image" as const },
|
||||
eventId: "$generated-image",
|
||||
kind: "message" as const,
|
||||
msgtype: "m.image",
|
||||
originServerTs: Date.now() + 1,
|
||||
roomId,
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
};
|
||||
expect(predicate(event)).toBe(true);
|
||||
return { event, matched: true as const, since: "next" };
|
||||
});
|
||||
primeMatrixQaActorCursor.mockResolvedValue({
|
||||
client: { sendTextMessage, waitForOptionalRoomEvent },
|
||||
startSince: "start",
|
||||
});
|
||||
|
||||
const result = await runGeneratedImageDeliveryScenario(createGeneratedImageContext());
|
||||
|
||||
expect(sendTextMessage).toHaveBeenCalledTimes(1);
|
||||
expect(waitForOptionalRoomEvent).toHaveBeenCalledTimes(1);
|
||||
expect(result.artifacts?.driverEventId).toBe("$driver-trigger");
|
||||
expect(result.artifacts?.attachmentEventId).toBe("$generated-image");
|
||||
});
|
||||
|
||||
it("bounds timeout diagnostics to the last eight room events", async () => {
|
||||
const observedEvents = Array.from({ length: 10 }, (_, index) => ({
|
||||
body: `event body ${index + 1}`,
|
||||
eventId: `$event-${index + 1}`,
|
||||
kind: "message" as const,
|
||||
roomId,
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
}));
|
||||
observedEvents.push({
|
||||
body: "other room",
|
||||
eventId: "$other-room",
|
||||
kind: "message",
|
||||
roomId: "!other:matrix-qa.test",
|
||||
sender: "@sut:matrix-qa.test",
|
||||
type: "m.room.message",
|
||||
});
|
||||
const sendTextMessage = vi.fn(async () => "$driver-trigger");
|
||||
primeMatrixQaActorCursor.mockResolvedValue({
|
||||
client: {
|
||||
sendTextMessage,
|
||||
waitForOptionalRoomEvent: vi.fn(async () => ({
|
||||
matched: false as const,
|
||||
since: "next",
|
||||
})),
|
||||
},
|
||||
startSince: "start",
|
||||
});
|
||||
|
||||
const error = await runGeneratedImageDeliveryScenario(
|
||||
createGeneratedImageContext(observedEvents),
|
||||
).catch((cause: unknown) => cause);
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
|
||||
expect(sendTextMessage).toHaveBeenCalledTimes(1);
|
||||
expect(message).toContain("$event-3");
|
||||
expect(message).toContain("$event-10");
|
||||
expect(message).not.toContain('$event-1"');
|
||||
expect(message).not.toContain('$event-2"');
|
||||
expect(message).not.toContain("$other-room");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -425,41 +425,43 @@ export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenari
|
||||
const roomId = resolveMatrixQaScenarioRoomId(context, MATRIX_QA_MEDIA_ROOM_KEY);
|
||||
const { client, startSince } = await primeMatrixQaDriverMediaClient(context);
|
||||
const triggerBody = buildMatrixQaImageGenerationPrompt(context.sutUserId);
|
||||
const driverEventIds: string[] = [];
|
||||
const triggerSentAt = Date.now();
|
||||
const driverEventId = await client.sendTextMessage({
|
||||
body: triggerBody,
|
||||
mentionUserIds: [context.sutUserId],
|
||||
roomId,
|
||||
});
|
||||
const isGeneratedImageEvent = (event: MatrixQaObservedEvent) =>
|
||||
event.roomId === roomId &&
|
||||
event.sender === context.sutUserId &&
|
||||
event.type === "m.room.message" &&
|
||||
event.relatesTo === undefined &&
|
||||
event.msgtype === "m.image" &&
|
||||
event.attachment?.kind === "image";
|
||||
let matched: Awaited<ReturnType<typeof client.waitForOptionalRoomEvent>> | undefined;
|
||||
for (let attempt = 1; attempt <= 2; attempt += 1) {
|
||||
const triggerSentAt = Date.now();
|
||||
const driverEventId = await client.sendTextMessage({
|
||||
body: triggerBody,
|
||||
mentionUserIds: [context.sutUserId],
|
||||
roomId,
|
||||
});
|
||||
driverEventIds.push(driverEventId);
|
||||
matched = await client.waitForOptionalRoomEvent({
|
||||
observedEvents: context.observedEvents,
|
||||
// The start cursor can still receive delayed images from an earlier run.
|
||||
predicate: (event) =>
|
||||
isGeneratedImageEvent(event) &&
|
||||
typeof event.originServerTs === "number" &&
|
||||
event.originServerTs >= triggerSentAt,
|
||||
roomId,
|
||||
since: matched?.since ?? startSince,
|
||||
timeoutMs: context.timeoutMs,
|
||||
});
|
||||
if (matched.matched) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!matched?.matched) {
|
||||
event.attachment?.kind === "image" &&
|
||||
typeof event.originServerTs === "number" &&
|
||||
event.originServerTs >= triggerSentAt;
|
||||
const matched = await client.waitForOptionalRoomEvent({
|
||||
observedEvents: context.observedEvents,
|
||||
// The start cursor can still receive delayed images from an earlier run.
|
||||
predicate: isGeneratedImageEvent,
|
||||
roomId,
|
||||
since: startSince,
|
||||
timeoutMs: context.timeoutMs,
|
||||
});
|
||||
if (!matched.matched) {
|
||||
const recentRoomEvents = context.observedEvents
|
||||
.filter((event) => event.roomId === roomId)
|
||||
.slice(-8)
|
||||
.map((event) => ({
|
||||
body: truncateMatrixQaPreview(event.body),
|
||||
eventId: event.eventId,
|
||||
kind: event.kind,
|
||||
msgtype: event.msgtype,
|
||||
sender: event.sender,
|
||||
type: event.type,
|
||||
}));
|
||||
throw new Error(
|
||||
`timed out after ${context.timeoutMs}ms waiting for Matrix generated image after ${driverEventIds.length} attempt(s)`,
|
||||
`timed out after ${context.timeoutMs}ms waiting for Matrix generated image; recent room events: ${JSON.stringify(recentRoomEvents)}`,
|
||||
);
|
||||
}
|
||||
const matchedEvent = matched.event;
|
||||
@@ -480,14 +482,13 @@ export async function runGeneratedImageDeliveryScenario(context: MatrixQaScenari
|
||||
attachmentFilename: attachment.filename,
|
||||
attachmentKind: attachment.kind,
|
||||
attachmentMsgtype: matchedEvent.msgtype,
|
||||
driverEventId: driverEventIds[0],
|
||||
driverEventIds,
|
||||
driverEventId,
|
||||
roomId,
|
||||
triggerBody,
|
||||
},
|
||||
details: [
|
||||
`room id: ${roomId}`,
|
||||
`driver events: ${driverEventIds.join(", ")}`,
|
||||
`driver event: ${driverEventId}`,
|
||||
...buildMatrixQaAttachmentDetailLines({
|
||||
attachmentEvent: matchedEvent,
|
||||
label: "generated image",
|
||||
|
||||
@@ -83,6 +83,35 @@ describe("matrix qa config", () => {
|
||||
expect(next.messages?.groupChat?.visibleReplies).toBe("automatic");
|
||||
});
|
||||
|
||||
it("preserves the scenario provider plugin without enabling unrelated plugins", () => {
|
||||
const next = buildMatrixQaConfig(
|
||||
{
|
||||
plugins: {
|
||||
allow: ["acpx", "memory-core", "qa-lab", "openai"],
|
||||
entries: {
|
||||
openai: { enabled: true },
|
||||
unrelated: { enabled: true },
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig,
|
||||
{
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
homeserver: "http://127.0.0.1:28008/",
|
||||
observerUserId: "@observer:matrix-qa.test",
|
||||
sutAccessToken: "sut-token",
|
||||
sutAccountId: "sut",
|
||||
sutUserId: "@sut:matrix-qa.test",
|
||||
topology,
|
||||
},
|
||||
);
|
||||
|
||||
expect(next.plugins?.allow).toEqual(["acpx", "memory-core", "qa-lab", "openai", "matrix"]);
|
||||
expect(next.plugins?.allow).not.toContain("unrelated");
|
||||
expect(next.plugins?.allow).not.toContain("anthropic");
|
||||
expect(next.plugins?.entries?.matrix).toEqual({ enabled: true });
|
||||
expect(next.plugins?.entries?.openai).toEqual({ enabled: true });
|
||||
});
|
||||
|
||||
it("honors an explicit DM disable with a provisioned DM room", () => {
|
||||
const next = buildMatrixQaConfig({} as OpenClawConfig, {
|
||||
driverUserId: "@driver:matrix-qa.test",
|
||||
|
||||
@@ -1,109 +1,70 @@
|
||||
// QA Lab Anthropic Messages wire adapter.
|
||||
import {
|
||||
convertAnthropicMessagesToResponsesInput,
|
||||
type ExtractedAssistantOutput,
|
||||
extractFinalAssistantOutputFromEvents,
|
||||
buildAnthropicFailureResponse,
|
||||
buildAnthropicMessageResponse,
|
||||
buildAnthropicMessageStreamEvents,
|
||||
buildAnthropicThinkingErrorResponse,
|
||||
buildAnthropicThinkingErrorStreamEvents,
|
||||
buildAnthropicMessageStreamEvents,
|
||||
convertAnthropicMessagesToResponsesInput,
|
||||
extractFinalAssistantOutputFromEvents,
|
||||
normalizeAnthropicSystemToString,
|
||||
} from "./mock-anthropic-wire.js";
|
||||
// QA Lab Anthropic Messages request dispatcher.
|
||||
import {
|
||||
type ResponsesInputItem,
|
||||
type StreamEvent,
|
||||
type AnthropicMessagesRequest,
|
||||
QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE,
|
||||
type MockScenarioState,
|
||||
type AnthropicStreamEvent,
|
||||
import type {
|
||||
AnthropicMessagesRequest,
|
||||
AnthropicStreamEvent,
|
||||
QaMockProviderDispatchResult,
|
||||
ResponsesInputItem,
|
||||
} from "./mock-openai-contracts.js";
|
||||
import { buildAssistantEvents } from "./mock-openai-events.js";
|
||||
import {
|
||||
extractAllRequestTexts,
|
||||
extractLastUserText,
|
||||
extractToolOutput,
|
||||
extractToolOutputCallId,
|
||||
} from "./mock-openai-input.js";
|
||||
import { buildToolCallEventsWithArgs } from "./mock-openai-tooling.js";
|
||||
|
||||
export async function buildMessagesPayload(
|
||||
body: AnthropicMessagesRequest,
|
||||
scenarioState: MockScenarioState,
|
||||
dispatchResponses: (
|
||||
body: Record<string, unknown>,
|
||||
scenarioState: MockScenarioState,
|
||||
) => Promise<StreamEvent[]>,
|
||||
): Promise<{
|
||||
events: StreamEvent[];
|
||||
export function normalizeAnthropicMessagesRequest(body: AnthropicMessagesRequest): {
|
||||
body: Record<string, unknown>;
|
||||
input: ResponsesInputItem[];
|
||||
extracted: ExtractedAssistantOutput;
|
||||
model: string;
|
||||
} {
|
||||
const model =
|
||||
typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8";
|
||||
const input = convertAnthropicMessagesToResponsesInput({
|
||||
messages: Array.isArray(body.messages) ? body.messages : [],
|
||||
});
|
||||
const instructions = normalizeAnthropicSystemToString(body.system);
|
||||
return {
|
||||
body: {
|
||||
input,
|
||||
model,
|
||||
stream: false,
|
||||
...(instructions ? { instructions } : {}),
|
||||
...(Array.isArray(body.tools) ? { tools: body.tools } : {}),
|
||||
},
|
||||
input,
|
||||
model,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildMessagesPayload(dispatched: QaMockProviderDispatchResult): {
|
||||
responseBody: Record<string, unknown>;
|
||||
streamEvents: AnthropicStreamEvent[];
|
||||
model: string;
|
||||
}> {
|
||||
const messages = Array.isArray(body.messages) ? body.messages : [];
|
||||
const input = convertAnthropicMessagesToResponsesInput({
|
||||
system: body.system,
|
||||
messages,
|
||||
});
|
||||
// Treat empty-string model the same as absent. A bare typeof check lets
|
||||
// `""` leak through to `responseBody.model` and `lastRequest.model`,
|
||||
// which then confuses parity consumers that assume the mock always
|
||||
// echoes the real provider label. Normalize once and reuse everywhere.
|
||||
const normalizedModel =
|
||||
typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-8";
|
||||
// Dispatch through the same scenario logic the /v1/responses route uses.
|
||||
// Preserve declared tools so route-specific adapters mirror what the
|
||||
// real provider request made available to the model.
|
||||
const dispatchBody: Record<string, unknown> = {
|
||||
input,
|
||||
model: normalizedModel,
|
||||
stream: false,
|
||||
...(Array.isArray(body.tools) ? { tools: body.tools } : {}),
|
||||
};
|
||||
const allInputText = extractAllRequestTexts(input, dispatchBody);
|
||||
if (QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText)) {
|
||||
const toolOutput = extractToolOutput(input);
|
||||
const toolOutputCallId = extractToolOutputCallId(input);
|
||||
const scenarioKey = `${normalizedModel}\n${extractLastUserText(input)}`;
|
||||
const shouldEmitThinkingError =
|
||||
toolOutput.length > 0 &&
|
||||
toolOutputCallId.length > 0 &&
|
||||
!scenarioState.anthropicThinkingErrorScenarioKeys.has(scenarioKey);
|
||||
// Safe retries generate fresh read call IDs. The original user prompt stays
|
||||
// stable, so fail once per model and nonce-bearing logical scenario instead.
|
||||
if (shouldEmitThinkingError) {
|
||||
scenarioState.anthropicThinkingErrorScenarioKeys.add(scenarioKey);
|
||||
}
|
||||
const events =
|
||||
toolOutput.length === 0
|
||||
? buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" })
|
||||
: shouldEmitThinkingError
|
||||
? buildAssistantEvents("")
|
||||
: buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK");
|
||||
const extracted = extractFinalAssistantOutputFromEvents(events);
|
||||
const responseBody = shouldEmitThinkingError
|
||||
? buildAnthropicThinkingErrorResponse({ model: normalizedModel })
|
||||
: buildAnthropicMessageResponse({
|
||||
model: normalizedModel,
|
||||
extracted,
|
||||
});
|
||||
const streamEvents = shouldEmitThinkingError
|
||||
? buildAnthropicThinkingErrorStreamEvents({ model: normalizedModel })
|
||||
: buildAnthropicMessageStreamEvents({
|
||||
model: normalizedModel,
|
||||
extracted,
|
||||
});
|
||||
return { events, input, extracted, responseBody, streamEvents, model: normalizedModel };
|
||||
} {
|
||||
if (dispatched.failure?.presentation === "anthropic-thinking") {
|
||||
return {
|
||||
responseBody: buildAnthropicThinkingErrorResponse({ model: dispatched.model }),
|
||||
streamEvents: buildAnthropicThinkingErrorStreamEvents({ model: dispatched.model }),
|
||||
};
|
||||
}
|
||||
const events = await dispatchResponses(dispatchBody, scenarioState);
|
||||
const extracted = extractFinalAssistantOutputFromEvents(events);
|
||||
const responseBody = buildAnthropicMessageResponse({
|
||||
model: normalizedModel,
|
||||
extracted,
|
||||
});
|
||||
const streamEvents = buildAnthropicMessageStreamEvents({
|
||||
model: normalizedModel,
|
||||
extracted,
|
||||
});
|
||||
return { events, input, extracted, responseBody, streamEvents, model: normalizedModel };
|
||||
if (dispatched.failure) {
|
||||
return {
|
||||
responseBody: buildAnthropicFailureResponse(dispatched.failure),
|
||||
streamEvents: [],
|
||||
};
|
||||
}
|
||||
const extracted = extractFinalAssistantOutputFromEvents(dispatched.events);
|
||||
return {
|
||||
responseBody: buildAnthropicMessageResponse({
|
||||
model: dispatched.model,
|
||||
extracted,
|
||||
}),
|
||||
streamEvents: buildAnthropicMessageStreamEvents({
|
||||
model: dispatched.model,
|
||||
extracted,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// QA Lab Anthropic Messages wire conversion and response events.
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
type ResponsesInputItem,
|
||||
type StreamEvent,
|
||||
@@ -6,13 +7,14 @@ import {
|
||||
type AnthropicMessage,
|
||||
type AnthropicMessagesRequest,
|
||||
type AnthropicStreamEvent,
|
||||
type QaMockProviderFailure,
|
||||
countApproxTokens,
|
||||
} from "./mock-openai-contracts.js";
|
||||
|
||||
// Anthropic Messages conversion preserves role and tool ordering while reusing
|
||||
// the shared Responses scenario dispatcher for provider parity.
|
||||
|
||||
function normalizeAnthropicSystemToString(
|
||||
export function normalizeAnthropicSystemToString(
|
||||
system: AnthropicMessagesRequest["system"],
|
||||
): string | undefined {
|
||||
if (typeof system === "string") {
|
||||
@@ -145,11 +147,62 @@ export function convertAnthropicMessagesToResponsesInput(params: {
|
||||
return items;
|
||||
}
|
||||
|
||||
export type ExtractedAssistantOutput = {
|
||||
type ExtractedAssistantOutput = {
|
||||
text: string;
|
||||
toolCalls: Array<{ id: string; name: string; input: Record<string, unknown> }>;
|
||||
};
|
||||
|
||||
const NATIVE_ANTHROPIC_TOOL_USE_ID_RE = /^toolu_[A-Za-z0-9_]+$/;
|
||||
const ANTHROPIC_TOOL_USE_ID_MAX_LENGTH = 64;
|
||||
|
||||
function isNativeAnthropicToolUseId(id: string): boolean {
|
||||
return id.length <= ANTHROPIC_TOOL_USE_ID_MAX_LENGTH && NATIVE_ANTHROPIC_TOOL_USE_ID_RE.test(id);
|
||||
}
|
||||
|
||||
export function adaptAnthropicToolCallIds(events: StreamEvent[]): StreamEvent[] {
|
||||
const adaptedIds = new Map<string, string>();
|
||||
const adaptId = (id: string) => {
|
||||
if (isNativeAnthropicToolUseId(id)) {
|
||||
return id;
|
||||
}
|
||||
const existing = adaptedIds.get(id);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const adapted = `toolu_${createHash("sha256").update(id).digest("hex").slice(0, 48)}`;
|
||||
adaptedIds.set(id, adapted);
|
||||
return adapted;
|
||||
};
|
||||
const adaptItem = (item: Record<string, unknown>) => {
|
||||
if (
|
||||
(item.type === "function_call" || item.type === "custom_tool_call") &&
|
||||
typeof item.call_id === "string"
|
||||
) {
|
||||
return { ...item, call_id: adaptId(item.call_id) };
|
||||
}
|
||||
return item;
|
||||
};
|
||||
|
||||
return events.map((event) => {
|
||||
if (event.type === "response.output_item.added" || event.type === "response.output_item.done") {
|
||||
return { ...event, item: adaptItem(event.item) };
|
||||
}
|
||||
if (event.type === "response.custom_tool_call_input.delta") {
|
||||
return { ...event, call_id: adaptId(event.call_id) };
|
||||
}
|
||||
if (event.type === "response.completed") {
|
||||
return {
|
||||
...event,
|
||||
response: {
|
||||
...event.response,
|
||||
output: event.response.output.map(adaptItem),
|
||||
},
|
||||
};
|
||||
}
|
||||
return event;
|
||||
});
|
||||
}
|
||||
|
||||
export function extractFinalAssistantOutputFromEvents(
|
||||
events: StreamEvent[],
|
||||
): ExtractedAssistantOutput {
|
||||
@@ -237,6 +290,19 @@ export function buildAnthropicMessageResponse(params: {
|
||||
};
|
||||
}
|
||||
|
||||
export function buildAnthropicFailureResponse(
|
||||
failure: QaMockProviderFailure,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
type: "error",
|
||||
error: {
|
||||
type: failure.type,
|
||||
...(failure.code ? { code: failure.code } : {}),
|
||||
message: failure.message,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const QA_ANTHROPIC_THINKING_ERROR_TEXT =
|
||||
"QA replay-safe read completed, but the provider stream failed after signed thinking.";
|
||||
const QA_ANTHROPIC_THINKING_ERROR_SIGNATURE = "qa_signed_thinking_block_91953";
|
||||
|
||||
@@ -15,6 +15,28 @@ export type MockCompactionSummaryFaultMode =
|
||||
|
||||
type MockOpenAiRequestOutcome = "success" | "error";
|
||||
|
||||
export type QaMockProviderDispatchRequest = {
|
||||
route: "responses" | "anthropic-messages";
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
};
|
||||
|
||||
export type QaMockProviderFailure = {
|
||||
status: number;
|
||||
type: string;
|
||||
code?: string;
|
||||
message: string;
|
||||
presentation?: "anthropic-thinking";
|
||||
};
|
||||
|
||||
export type QaMockProviderDispatchResult = {
|
||||
events: StreamEvent[];
|
||||
model: string;
|
||||
failure?: QaMockProviderFailure;
|
||||
onResponseSent?: () => void;
|
||||
previewPauseMs?: number;
|
||||
};
|
||||
|
||||
export type StreamEvent =
|
||||
| { type: "response.created"; response: { id: string } }
|
||||
| {
|
||||
@@ -136,6 +158,7 @@ export type MockOpenAiRequestSnapshot = {
|
||||
errorCode?: string;
|
||||
rawByteLength: number;
|
||||
plannedToolCallId?: string;
|
||||
plannedToolItemId?: string;
|
||||
plannedToolName?: string;
|
||||
plannedWireToolName?: string;
|
||||
plannedToolArgs?: Record<string, unknown>;
|
||||
|
||||
@@ -108,20 +108,26 @@ export function extractPlannedToolName(events: StreamEvent[]) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function extractPlannedToolCallId(events: StreamEvent[]) {
|
||||
export function extractPlannedToolIdentity(events: StreamEvent[]): {
|
||||
callId?: string;
|
||||
itemId?: string;
|
||||
} {
|
||||
for (const event of events) {
|
||||
if (event.type !== "response.output_item.done") {
|
||||
continue;
|
||||
}
|
||||
const item = event.item as { type?: unknown; call_id?: unknown };
|
||||
const item = event.item as { type?: unknown; id?: unknown; call_id?: unknown };
|
||||
if (
|
||||
(item.type === "function_call" || item.type === "custom_tool_call") &&
|
||||
typeof item.call_id === "string"
|
||||
) {
|
||||
return item.call_id;
|
||||
return {
|
||||
callId: item.call_id,
|
||||
itemId: typeof item.id === "string" ? item.id : undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
return {};
|
||||
}
|
||||
|
||||
export function extractPlannedToolArgs(events: StreamEvent[]) {
|
||||
|
||||
@@ -158,6 +158,9 @@ export function extractToolOutput(input: ResponsesInputItem[]) {
|
||||
return item ? stringifyFunctionCallOutput(item.output) : "";
|
||||
}
|
||||
|
||||
export const extractToolOutputValue = (input: ResponsesInputItem[]) =>
|
||||
findCurrentToolOutput(input)?.output;
|
||||
|
||||
export function extractToolOutputStructuredError(input: ResponsesInputItem[]) {
|
||||
const item = findCurrentToolOutput(input);
|
||||
return item?.is_error === true || item?.isError === true;
|
||||
@@ -264,7 +267,7 @@ export function extractSlackMpimRetainedBotNonce(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function extractAllInputTexts(input: ResponsesInputItem[]) {
|
||||
function extractAllInputTexts(input: ResponsesInputItem[]) {
|
||||
const texts: string[] = [];
|
||||
for (const item of input) {
|
||||
if (typeof item.output === "string" && item.output.trim()) {
|
||||
|
||||
@@ -1,24 +1,12 @@
|
||||
import type { Server } from "node:http";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import type { ResponsesInputItem, StreamEvent } from "./mock-openai-contracts.js";
|
||||
|
||||
export type QaMockResponsesDispatchResult = {
|
||||
events: StreamEvent[];
|
||||
failure?: {
|
||||
status: number;
|
||||
type: string;
|
||||
code?: string;
|
||||
message: string;
|
||||
};
|
||||
onResponseSent?: () => void;
|
||||
previewPauseMs?: number;
|
||||
};
|
||||
import type { QaMockProviderDispatchResult, ResponsesInputItem } from "./mock-openai-contracts.js";
|
||||
|
||||
type QaMockResponsesWebSocketDispatch = (params: {
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
}) => Promise<QaMockResponsesDispatchResult>;
|
||||
}) => Promise<QaMockProviderDispatchResult>;
|
||||
|
||||
type QaMockResponsesWebSocketHistory = {
|
||||
id: string;
|
||||
|
||||
@@ -3,6 +3,8 @@ import { once } from "node:events";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket } from "ws";
|
||||
import { readQaMockRequestCursor } from "../shared/debug-request-cursor.js";
|
||||
import { adaptAnthropicToolCallIds } from "./mock-anthropic-wire.js";
|
||||
import type { StreamEvent } from "./mock-openai-contracts.js";
|
||||
import { readTargetFromPrompt } from "./mock-openai-tooling.js";
|
||||
import { startQaMockOpenAiServer } from "./server.js";
|
||||
|
||||
@@ -2315,6 +2317,65 @@ describe("qa mock openai server", () => {
|
||||
expect(requests.every((request) => request.errorCode === "context_length_exceeded")).toBe(true);
|
||||
});
|
||||
|
||||
it("injects one Anthropic overflow per session before planning the logical write", async () => {
|
||||
const server = await startMockServer();
|
||||
const bodyFor = (sessionId: string) => ({
|
||||
system: `Runtime: embedded | sessionId=${sessionId}`,
|
||||
tools: [
|
||||
{
|
||||
name: "exec",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
language: { type: "string" },
|
||||
code: { type: "string" },
|
||||
},
|
||||
required: ["code"],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wait",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { runId: { type: "string" } },
|
||||
required: ["runId"],
|
||||
},
|
||||
},
|
||||
],
|
||||
messages: [
|
||||
makeAnthropicUserText(
|
||||
`${QA_COMPACTION_RETRY_PROMPT}\n${QA_COMPACTION_RETRY_OVERFLOW_PADDING}`,
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
const first = await postAnthropicMessages(server, bodyFor("anthropic-overflow-a"));
|
||||
expect(first.status).toBe(400);
|
||||
expect(await first.json()).toEqual({
|
||||
type: "error",
|
||||
error: {
|
||||
type: "invalid_request_error",
|
||||
code: "context_length_exceeded",
|
||||
message: "This model's maximum context length was exceeded.",
|
||||
},
|
||||
});
|
||||
|
||||
const second = await postAnthropicMessages(server, bodyFor("anthropic-overflow-a"));
|
||||
expect(second.status).toBe(200);
|
||||
const content = requireArray(
|
||||
requireRecord(await second.json(), "Anthropic response").content,
|
||||
"content",
|
||||
);
|
||||
expect(content).toContainEqual(expect.objectContaining({ type: "tool_use", name: "exec" }));
|
||||
expect(await getJson(server, "/debug/last-request")).toMatchObject({
|
||||
plannedToolName: "write",
|
||||
plannedWireToolName: "exec",
|
||||
});
|
||||
|
||||
const independent = await postAnthropicMessages(server, bodyFor("anthropic-overflow-b"));
|
||||
expect(independent.status).toBe(400);
|
||||
});
|
||||
|
||||
it("excludes compaction summary requests from overflow injection", async () => {
|
||||
const server = await startMockServer();
|
||||
const initial = await postNonStreamingResponses(server, {
|
||||
@@ -2496,6 +2557,28 @@ describe("qa mock openai server", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("excludes Anthropic compaction summary requests from overflow injection", async () => {
|
||||
const server = await startMockServer();
|
||||
const response = await postAnthropicMessages(server, {
|
||||
system: QA_COMPACTION_SUMMARY_INSTRUCTIONS,
|
||||
messages: [
|
||||
makeAnthropicUserText(
|
||||
`<conversation>\n${QA_COMPACTION_RETRY_OVERFLOW_PADDING}\n</conversation>`,
|
||||
),
|
||||
],
|
||||
});
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
const body = requireRecord(await response.json(), "Anthropic summary response");
|
||||
expect(requireArray(body.content, "content")).toContainEqual(
|
||||
expect.objectContaining({ type: "text", text: expect.stringContaining("## Goal") }),
|
||||
);
|
||||
expect(await getJson(server, "/debug/last-request")).toMatchObject({
|
||||
requestKind: "compaction-summary",
|
||||
outcome: "success",
|
||||
});
|
||||
});
|
||||
|
||||
it("handles staged scalar compaction summaries and promotes the durable merge without state leakage", async () => {
|
||||
const server = await startMockServer();
|
||||
const genericChunkPayload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
@@ -3549,6 +3632,115 @@ Update and merge these partial structured summaries.`,
|
||||
expect(imageRequest.size).toBe("1024x1024");
|
||||
});
|
||||
|
||||
it("requires memory_get before answering thread recall in Code Mode", async () => {
|
||||
const server = await startMockServer();
|
||||
const prompt =
|
||||
"@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.";
|
||||
const codeModeTools = [
|
||||
{
|
||||
type: "function",
|
||||
name: "exec",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
language: { type: "string" },
|
||||
code: { type: "string" },
|
||||
},
|
||||
required: ["code"],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "function",
|
||||
name: "wait",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { runId: { type: "string" } },
|
||||
required: ["runId"],
|
||||
},
|
||||
},
|
||||
];
|
||||
const initialInput: Array<Record<string, unknown>> = [
|
||||
{
|
||||
type: "additional_tools",
|
||||
role: "developer",
|
||||
tools: codeModeTools,
|
||||
},
|
||||
makeUserInput(prompt),
|
||||
];
|
||||
|
||||
const searchPlan = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
input: initialInput,
|
||||
});
|
||||
const searchCall = outputToolCall(searchPlan, "exec");
|
||||
const searchCallId = outputToolCallId(searchCall, "call_mock_memory_search");
|
||||
const continuationInput: Array<Record<string, unknown>> = [
|
||||
makeUserInput(prompt),
|
||||
searchCall,
|
||||
makeToolOutputWithCallId(
|
||||
searchCallId,
|
||||
JSON.stringify({
|
||||
status: "completed",
|
||||
value: {
|
||||
results: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
snippet: "Thread-hidden codename: ORBIT-21.",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
const getPlan = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
input: continuationInput,
|
||||
});
|
||||
const getCall = outputToolCall(getPlan, "memory_get");
|
||||
const getCallId = outputToolCallId(getCall, "call_mock_memory_get");
|
||||
expect(getCallId).not.toBe(searchCallId);
|
||||
expect(outputItems(getPlan).some((item) => item.type === "message")).toBe(false);
|
||||
expect(JSON.stringify(getPlan)).not.toContain("hidden thread codename is ORBIT-21");
|
||||
continuationInput.push(
|
||||
getCall,
|
||||
makeToolOutputWithCallId(
|
||||
getCallId,
|
||||
JSON.stringify({
|
||||
status: "completed",
|
||||
value: { text: "Thread-hidden codename: ORBIT-22." },
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const final = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
input: continuationInput,
|
||||
});
|
||||
expect(outputText(final)).toContain("ORBIT-22");
|
||||
expect(outputText(final)).not.toContain("ORBIT-21");
|
||||
expect(outputItems(final).some((item) => item.type === "function_call")).toBe(false);
|
||||
|
||||
const requests = requireArray(await getJson(server, "/debug/requests"), "debug requests").map(
|
||||
(request, index) => requireRecord(request, `debug request ${index}`),
|
||||
);
|
||||
expect(requests).toHaveLength(3);
|
||||
expect(requests[0]).toMatchObject({
|
||||
plannedToolName: "memory_search",
|
||||
plannedWireToolName: "exec",
|
||||
plannedToolCallId: searchCallId,
|
||||
});
|
||||
expect(requests[1]).toMatchObject({
|
||||
toolOutputCallId: searchCallId,
|
||||
plannedToolName: "memory_get",
|
||||
plannedToolCallId: getCallId,
|
||||
});
|
||||
expect(requests[1]).not.toHaveProperty("plannedWireToolName");
|
||||
expect(requests[2]).toMatchObject({
|
||||
toolOutputCallId: getCallId,
|
||||
});
|
||||
expect(requests[2]).not.toHaveProperty("plannedToolName");
|
||||
});
|
||||
|
||||
it("supports advanced QA memory and subagent recovery prompts", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
@@ -3570,6 +3762,29 @@ Update and merge these partial structured summaries.`,
|
||||
expect(threadMemorySearchText).toContain('"name":"memory_search"');
|
||||
expect(threadMemorySearchText).toContain("ORBIT-22");
|
||||
|
||||
const threadMemoryGetText = await expectStreamingResponsesText(server, {
|
||||
instructions:
|
||||
"@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.",
|
||||
input: [
|
||||
makeToolOutput(
|
||||
JSON.stringify({
|
||||
results: [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 1,
|
||||
endLine: 1,
|
||||
snippet: "Thread-hidden codename: ORBIT-22.",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
makeUserInput("Protocol note: acknowledged. Continue with the QA scenario plan."),
|
||||
],
|
||||
});
|
||||
expect(threadMemoryGetText).toContain('"name":"memory_get"');
|
||||
expect(threadMemoryGetText).toContain('\\"path\\":\\"MEMORY.md\\"');
|
||||
expect(threadMemoryGetText).not.toContain("hidden thread codename is ORBIT-22");
|
||||
|
||||
const threadMemorySummary = await expectNonStreamingResponses(server, {
|
||||
instructions:
|
||||
"@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.",
|
||||
@@ -3584,17 +3799,17 @@ Update and merge these partial structured summaries.`,
|
||||
});
|
||||
expect(JSON.stringify(await threadMemorySummary.json())).toContain("ORBIT-22");
|
||||
|
||||
const structuredThreadMemorySummary = await expectNonStreamingResponses(server, {
|
||||
const rawThreadMemorySummary = await expectNonStreamingResponses(server, {
|
||||
instructions:
|
||||
"@openclaw Thread memory check: what is the hidden thread codename stored only in memory? Use memory tools first and reply only in this thread.",
|
||||
input: [
|
||||
makeToolOutput({
|
||||
text: "Thread-hidden codename: ORBIT-22.",
|
||||
}),
|
||||
makeToolOutput("Thread-hidden codename: ORBIT-23."),
|
||||
makeUserInput("Protocol note: acknowledged. Continue with the QA scenario plan."),
|
||||
],
|
||||
});
|
||||
expect(JSON.stringify(await structuredThreadMemorySummary.json())).toContain("ORBIT-22");
|
||||
const rawThreadMemoryText = JSON.stringify(await rawThreadMemorySummary.json());
|
||||
expect(rawThreadMemoryText).toContain("NONE");
|
||||
expect(rawThreadMemoryText).not.toContain("ORBIT-23");
|
||||
|
||||
const unavailableThreadMemorySummary = await expectNonStreamingResponses(server, {
|
||||
input: [
|
||||
@@ -5213,7 +5428,8 @@ Update and merge these partial structured summaries.`,
|
||||
});
|
||||
|
||||
const payload = await response.json();
|
||||
expect(outputItem(payload)).toMatchObject({ type: "function_call", name: "apply_patch" });
|
||||
const item = outputItem(payload);
|
||||
expect(item).toMatchObject({ type: "function_call", name: "apply_patch" });
|
||||
const args = outputToolArgs(payload);
|
||||
expect(args).not.toHaveProperty("__qaFailureMode");
|
||||
expect(args.input).toBeTypeOf("string");
|
||||
@@ -5223,6 +5439,9 @@ Update and merge these partial structured summaries.`,
|
||||
expect(args.input).toContain("\n@@\n-runtime-tool-fixture-denied-original\n");
|
||||
}
|
||||
expect(args.input).toContain("\n*** End Patch\n");
|
||||
const debug = requireRecord(await getJson(server, "/debug/last-request"), "function plan");
|
||||
expect(debug.plannedToolCallId).toBe(item.call_id);
|
||||
expect(debug.plannedToolItemId).toBe(item.id);
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -5265,6 +5484,7 @@ Update and merge these partial structured summaries.`,
|
||||
);
|
||||
expect(debug.plannedToolName).toBe("apply_patch");
|
||||
expect(debug.plannedToolCallId).toBe(item.call_id);
|
||||
expect(debug.plannedToolItemId).toBe(item.id);
|
||||
expect(debug.plannedToolArgs).toEqual({ input: item.input });
|
||||
});
|
||||
|
||||
@@ -6081,6 +6301,16 @@ Update and merge these partial structured summaries.`,
|
||||
const server = await startMockServer();
|
||||
|
||||
const body = (await expectAnthropicMessagesJson(server, {
|
||||
tools: [
|
||||
{
|
||||
name: "read",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { path: { type: "string" } },
|
||||
required: ["path"],
|
||||
},
|
||||
},
|
||||
],
|
||||
messages: [
|
||||
makeAnthropicUserText(
|
||||
"Read the seeded docs and report worked, failed, blocked, and follow-up items.",
|
||||
@@ -6098,8 +6328,10 @@ Update and merge these partial structured summaries.`,
|
||||
expect(body.model).toBe("claude-opus-4-8");
|
||||
expect(body.stop_reason).toBe("tool_use");
|
||||
const toolUseBlock = body.content.find((block) => block.type === "tool_use") as
|
||||
| { name: string; input: Record<string, unknown> }
|
||||
| { id: string; name: string; input: Record<string, unknown> }
|
||||
| undefined;
|
||||
expect(toolUseBlock?.id).toMatch(/^toolu_[A-Za-z0-9_]+$/);
|
||||
expect(toolUseBlock?.id.length).toBeLessThanOrEqual(64);
|
||||
expect(toolUseBlock?.name).toBe("read");
|
||||
expect(toolUseBlock?.input).toEqual({ path: "repo/docs/help/testing.md" });
|
||||
|
||||
@@ -6108,9 +6340,85 @@ Update and merge these partial structured summaries.`,
|
||||
"debug request",
|
||||
);
|
||||
expect(debugPayload.model).toBe("claude-opus-4-8");
|
||||
expect(debugPayload.plannedToolCallId).toBe(toolUseBlock?.id);
|
||||
expect(debugPayload).not.toHaveProperty("plannedToolItemId");
|
||||
expect(debugPayload.plannedToolName).toBe("read");
|
||||
});
|
||||
|
||||
it("preserves already-native Anthropic tool IDs while adapting shared generated IDs", () => {
|
||||
const nativeId = "toolu_native_123";
|
||||
const generatedId = "call_mock_read_generated_1";
|
||||
const events: StreamEvent[] = [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "function_call", name: "read", call_id: nativeId, arguments: "{}" },
|
||||
},
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: { type: "function_call", name: "read", call_id: nativeId, arguments: "{}" },
|
||||
},
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
item: { type: "function_call", name: "read", call_id: generatedId, arguments: "{}" },
|
||||
},
|
||||
{
|
||||
type: "response.output_item.done",
|
||||
item: { type: "function_call", name: "read", call_id: generatedId, arguments: "{}" },
|
||||
},
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "response_mock",
|
||||
status: "completed",
|
||||
output: [
|
||||
{ type: "function_call", name: "read", call_id: nativeId, arguments: "{}" },
|
||||
{ type: "function_call", name: "read", call_id: generatedId, arguments: "{}" },
|
||||
],
|
||||
usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 },
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const adapted = adaptAnthropicToolCallIds(events);
|
||||
const callIds = adapted.flatMap((event) => {
|
||||
if (
|
||||
event.type === "response.output_item.added" ||
|
||||
event.type === "response.output_item.done"
|
||||
) {
|
||||
return typeof event.item.call_id === "string" ? [event.item.call_id] : [];
|
||||
}
|
||||
if (event.type === "response.completed") {
|
||||
return event.response.output.flatMap((item) =>
|
||||
typeof item.call_id === "string" ? [item.call_id] : [],
|
||||
);
|
||||
}
|
||||
return [];
|
||||
});
|
||||
const adaptedGeneratedIds = callIds.filter((id) => id !== nativeId);
|
||||
const repeatedGeneratedIds = adaptAnthropicToolCallIds(events).flatMap((event) => {
|
||||
if (
|
||||
event.type === "response.output_item.added" ||
|
||||
event.type === "response.output_item.done"
|
||||
) {
|
||||
return event.item.call_id === nativeId || typeof event.item.call_id !== "string"
|
||||
? []
|
||||
: [event.item.call_id];
|
||||
}
|
||||
if (event.type === "response.completed") {
|
||||
return event.response.output.flatMap((item) =>
|
||||
item.call_id === nativeId || typeof item.call_id !== "string" ? [] : [item.call_id],
|
||||
);
|
||||
}
|
||||
return [];
|
||||
});
|
||||
|
||||
expect(callIds.filter((id) => id === nativeId)).toHaveLength(3);
|
||||
expect(new Set(adaptedGeneratedIds).size).toBe(1);
|
||||
expect(repeatedGeneratedIds).toEqual(adaptedGeneratedIds);
|
||||
expect(adaptedGeneratedIds[0]).toMatch(/^toolu_[A-Za-z0-9_]+$/);
|
||||
expect(adaptedGeneratedIds[0]?.length).toBeLessThanOrEqual(64);
|
||||
});
|
||||
|
||||
it("routes Anthropic hidden tools through Code Mode and preserves scenario evidence", async () => {
|
||||
const server = await startMockServer();
|
||||
const prompt =
|
||||
@@ -6137,16 +6445,24 @@ Update and merge these partial structured summaries.`,
|
||||
},
|
||||
];
|
||||
const messages: Array<Record<string, unknown>> = [makeAnthropicUserText(prompt)];
|
||||
const emittedToolUseIds: string[] = [];
|
||||
|
||||
const request = async () => {
|
||||
const request = async (expectedToolResultId?: string) => {
|
||||
const response = await expectAnthropicMessages(server, {
|
||||
tools,
|
||||
messages,
|
||||
});
|
||||
return (await response.json()) as {
|
||||
const body = (await response.json()) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request");
|
||||
if (expectedToolResultId) {
|
||||
expect(debug.toolOutputCallId).toBe(expectedToolResultId);
|
||||
} else {
|
||||
expect(debug).not.toHaveProperty("toolOutputCallId");
|
||||
}
|
||||
return body;
|
||||
};
|
||||
const readToolUse = (body: {
|
||||
stop_reason: string;
|
||||
@@ -6157,6 +6473,9 @@ Update and merge these partial structured summaries.`,
|
||||
if (!toolUse || typeof toolUse.id !== "string" || typeof toolUse.name !== "string") {
|
||||
throw new Error("Expected Anthropic tool_use block");
|
||||
}
|
||||
expect(toolUse.id).toMatch(/^toolu_[A-Za-z0-9_]+$/);
|
||||
expect(toolUse.id.length).toBeLessThanOrEqual(64);
|
||||
emittedToolUseIds.push(toolUse.id);
|
||||
return toolUse;
|
||||
};
|
||||
const appendToolResult = (
|
||||
@@ -6168,8 +6487,14 @@ Update and merge these partial structured summaries.`,
|
||||
makeAnthropicToolResult(toolUse.id, JSON.stringify(result)),
|
||||
);
|
||||
};
|
||||
const expectPlan = async (name: string, args: Record<string, unknown>, wireName = "exec") => {
|
||||
const expectPlan = async (
|
||||
name: string,
|
||||
args: Record<string, unknown>,
|
||||
callId: string,
|
||||
wireName = "exec",
|
||||
) => {
|
||||
const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request");
|
||||
expect(debug.plannedToolCallId).toBe(callId);
|
||||
expect(debug.plannedToolName).toBe(name);
|
||||
expect(debug.plannedWireToolName).toBe(wireName);
|
||||
expect(debug.plannedToolArgs).toEqual(args);
|
||||
@@ -6180,15 +6505,16 @@ Update and merge these partial structured summaries.`,
|
||||
const readAgentCode = String(requireRecord(readAgent.input, "exec input").code);
|
||||
expect(readAgentCode).toContain("tools.callValue(target.id, targetArgs)");
|
||||
expect(readAgentCode).toContain("value.content.slice(0, 2048)");
|
||||
await expectPlan("read", { path: "AGENT.md" });
|
||||
await expectPlan("read", { path: "AGENT.md" }, String(readAgent.id));
|
||||
|
||||
appendToolResult(readAgent, { status: "waiting", runId: "qa-code-mode-read-agent" });
|
||||
const waitForAgent = readToolUse(await request());
|
||||
const waitForAgent = readToolUse(await request(String(readAgent.id)));
|
||||
expect(waitForAgent.name).toBe("wait");
|
||||
const waitDebug = requireRecord(
|
||||
await fetch(`${server.baseUrl}/debug/last-request`).then((response) => response.json()),
|
||||
"wait debug request",
|
||||
);
|
||||
expect(waitDebug.plannedToolCallId).toBe(waitForAgent.id);
|
||||
expect(waitDebug.plannedToolName).toBe("wait");
|
||||
expect(waitDebug).not.toHaveProperty("plannedWireToolName");
|
||||
expect(waitDebug.plannedToolArgs).toEqual({ runId: "qa-code-mode-read-agent" });
|
||||
@@ -6197,17 +6523,17 @@ Update and merge these partial structured summaries.`,
|
||||
status: "completed",
|
||||
value: { kind: "text", content: "# Repo contract\nDo not stop after planning." },
|
||||
});
|
||||
const readSoul = readToolUse(await request());
|
||||
const readSoul = readToolUse(await request(String(waitForAgent.id)));
|
||||
expect(readSoul.name).toBe("exec");
|
||||
await expectPlan("read", { path: "SOUL.md" });
|
||||
await expectPlan("read", { path: "SOUL.md" }, String(readSoul.id));
|
||||
|
||||
appendToolResult(readSoul, {
|
||||
status: "completed",
|
||||
value: { kind: "text", content: "# Execution style\nStay action-first." },
|
||||
});
|
||||
const readInput = readToolUse(await request());
|
||||
const readInput = readToolUse(await request(String(readSoul.id)));
|
||||
expect(readInput.name).toBe("exec");
|
||||
await expectPlan("read", { path: "FOLLOWTHROUGH_INPUT.md" });
|
||||
await expectPlan("read", { path: "FOLLOWTHROUGH_INPUT.md" }, String(readInput.id));
|
||||
|
||||
appendToolResult(readInput, {
|
||||
status: "completed",
|
||||
@@ -6217,24 +6543,215 @@ Update and merge these partial structured summaries.`,
|
||||
"Mission: prove you followed the repo contract.\nEvidence path: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md -> repo-contract-summary.txt",
|
||||
},
|
||||
});
|
||||
const writeSummary = readToolUse(await request());
|
||||
const writeSummary = readToolUse(await request(String(readInput.id)));
|
||||
expect(writeSummary.name).toBe("exec");
|
||||
await expectPlan("write", {
|
||||
path: "repo-contract-summary.txt",
|
||||
content:
|
||||
"Mission: prove you followed the repo contract.\nEvidence: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md\nStatus: complete",
|
||||
});
|
||||
await expectPlan(
|
||||
"write",
|
||||
{
|
||||
path: "repo-contract-summary.txt",
|
||||
content:
|
||||
"Mission: prove you followed the repo contract.\nEvidence: AGENT.md -> SOUL.md -> FOLLOWTHROUGH_INPUT.md\nStatus: complete",
|
||||
},
|
||||
String(writeSummary.id),
|
||||
);
|
||||
|
||||
appendToolResult(writeSummary, {
|
||||
status: "completed",
|
||||
value: "Successfully wrote 146 bytes to repo-contract-summary.txt.",
|
||||
});
|
||||
const final = await request();
|
||||
const final = await request(String(writeSummary.id));
|
||||
expect(final.stop_reason).toBe("end_turn");
|
||||
const text = final.content.find((block) => block.type === "text")?.text;
|
||||
expect(text).toBe(
|
||||
"Read: AGENT.md, SOUL.md, FOLLOWTHROUGH_INPUT.md\nWrote: repo-contract-summary.txt\nStatus: complete",
|
||||
);
|
||||
expect(new Set(emittedToolUseIds).size).toBe(emittedToolUseIds.length);
|
||||
});
|
||||
|
||||
it("uses native Codex custom exec, output arrays, and cell_id waits", async () => {
|
||||
const server = await startMockServer();
|
||||
const tools = [
|
||||
{ type: "custom", name: "exec", format: { type: "grammar", syntax: "lark", definition: "" } },
|
||||
{
|
||||
type: "function",
|
||||
name: "wait",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { cell_id: { type: "string" } },
|
||||
required: ["cell_id"],
|
||||
},
|
||||
},
|
||||
];
|
||||
const prompt = QA_COMPACTION_RETRY_PROMPT;
|
||||
const execPayload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
tools,
|
||||
input: [makeUserInput(prompt)],
|
||||
});
|
||||
const execCall = outputItem(execPayload);
|
||||
expect(execCall).toMatchObject({ type: "custom_tool_call", name: "exec" });
|
||||
const source = String(execCall.input);
|
||||
expect(source).toContain("tools[target.name](targetArgs)");
|
||||
expect(source).toContain("text(JSON.stringify(value));");
|
||||
expect(source).not.toContain("tools.callValue");
|
||||
expect(source).not.toContain("target.id");
|
||||
expect(source).not.toMatch(/ALL_TOOLS[^\n]*\.id/);
|
||||
expect(source).not.toContain("return value");
|
||||
|
||||
const execCallId = outputToolCallId(execCall, "native-exec");
|
||||
const waitPayload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
tools,
|
||||
input: [
|
||||
makeUserInput(prompt),
|
||||
execCall,
|
||||
{
|
||||
type: "custom_tool_call_output",
|
||||
call_id: execCallId,
|
||||
output: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: "Script running with cell ID cell-write-1\nLive output:\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const waitCall = outputToolCall(waitPayload, "wait");
|
||||
expect(outputToolArgsFromItem(waitCall)).toEqual({ cell_id: "cell-write-1" });
|
||||
|
||||
const finalPayload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
tools,
|
||||
input: [
|
||||
makeUserInput(prompt),
|
||||
execCall,
|
||||
{
|
||||
type: "custom_tool_call_output",
|
||||
call_id: execCallId,
|
||||
output: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: "Script running with cell ID cell-write-1\nLive output:\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
waitCall,
|
||||
{
|
||||
type: "function_call_output",
|
||||
call_id: outputToolCallId(waitCall, "native-wait"),
|
||||
output: [
|
||||
{ type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify({ status: "completed", value: { changed: false } }),
|
||||
},
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(outputText(finalPayload)).toBe("Protocol note: replay unsafe after write.");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "failed header with canonical JSON",
|
||||
output: [
|
||||
{ type: "input_text", text: "Script failed\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "terminated header with canonical JSON",
|
||||
output: [
|
||||
{ type: "input_text", text: "Script terminated\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "unknown header with canonical JSON",
|
||||
output: [
|
||||
{ type: "input_text", text: "Script paused\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "failed header followed by a running marker",
|
||||
output: [
|
||||
{ type: "input_text", text: "Script failed\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
{
|
||||
type: "input_text",
|
||||
text: "Script running with cell ID cell-write-late\nLive output:\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "missing header with canonical JSON",
|
||||
output: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "reordered completed header and canonical JSON",
|
||||
output: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: JSON.stringify(QA_COMPACTION_RETRY_CODE_MODE_WRITE_RESULT),
|
||||
},
|
||||
{ type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" },
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "completed header without JSON",
|
||||
output: [{ type: "input_text", text: "Script completed\nWall time: 0.1 seconds\nOutput:\n" }],
|
||||
},
|
||||
])("rejects native Code Mode compaction evidence with $label", async ({ output }) => {
|
||||
const server = await startMockServer();
|
||||
const tools = [
|
||||
{ type: "custom", name: "exec", format: { type: "grammar", syntax: "lark", definition: "" } },
|
||||
{
|
||||
type: "function",
|
||||
name: "wait",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: { cell_id: { type: "string" } },
|
||||
required: ["cell_id"],
|
||||
},
|
||||
},
|
||||
];
|
||||
const execPayload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
tools,
|
||||
input: [makeUserInput(QA_COMPACTION_RETRY_PROMPT)],
|
||||
});
|
||||
const execCall = outputItem(execPayload);
|
||||
const payload = await expectOpenAiNonStreamingResponsesJson(server, {
|
||||
tools,
|
||||
input: [
|
||||
makeUserInput(QA_COMPACTION_RETRY_PROMPT),
|
||||
execCall,
|
||||
{
|
||||
type: "custom_tool_call_output",
|
||||
call_id: outputToolCallId(execCall, "native-exec"),
|
||||
output,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(outputItems(payload).some((item) => item.type === "function_call")).toBe(false);
|
||||
expect(outputText(payload)).not.toBe("Protocol note: replay unsafe after write.");
|
||||
});
|
||||
|
||||
it("routes Anthropic image generation through Code Mode when only exec and wait are visible", async () => {
|
||||
@@ -6336,70 +6853,83 @@ Update and merge these partial structured summaries.`,
|
||||
if (!readToolUse || typeof readToolUse.id !== "string") {
|
||||
throw new Error("Expected Anthropic read tool_use block");
|
||||
}
|
||||
messages.push(
|
||||
{ role: "assistant", content: [readToolUse] },
|
||||
makeAnthropicToolResult(
|
||||
readToolUse.id,
|
||||
JSON.stringify({ status: "waiting", runId: "ordinary-read" }),
|
||||
),
|
||||
);
|
||||
|
||||
const body = (await expectAnthropicMessagesJson(server, {
|
||||
tools,
|
||||
messages,
|
||||
})) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.stop_reason).toBe("end_turn");
|
||||
expect(body.content.some((block) => block.type === "tool_use")).toBe(false);
|
||||
for (const result of [
|
||||
{ status: "waiting", runId: "ordinary-read" },
|
||||
{
|
||||
status: "completed",
|
||||
value: { status: "waiting", runId: "ordinary-read-completed-value" },
|
||||
},
|
||||
]) {
|
||||
const body = (await expectAnthropicMessagesJson(server, {
|
||||
tools,
|
||||
messages: [
|
||||
...messages,
|
||||
{ role: "assistant", content: [readToolUse] },
|
||||
makeAnthropicToolResult(readToolUse.id, JSON.stringify(result)),
|
||||
],
|
||||
})) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.stop_reason).toBe("end_turn");
|
||||
expect(body.content.some((block) => block.type === "tool_use")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not interpret unmarked direct exec results as Code Mode control envelopes", async () => {
|
||||
const server = await startMockServer();
|
||||
const body = (await expectAnthropicMessagesJson(server, {
|
||||
tools: [
|
||||
{
|
||||
name: "exec",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { code: { type: "string" } },
|
||||
required: ["code"],
|
||||
const tools = [
|
||||
{
|
||||
name: "exec",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { code: { type: "string" } },
|
||||
required: ["code"],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wait",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { runId: { type: "string" } },
|
||||
required: ["runId"],
|
||||
},
|
||||
},
|
||||
];
|
||||
const messages = [
|
||||
makeAnthropicUserText("Direct exec envelope isolation check."),
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool_use",
|
||||
id: "toolu_direct_exec",
|
||||
name: "exec",
|
||||
input: { language: "javascript", code: "return 1;" },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wait",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { runId: { type: "string" } },
|
||||
required: ["runId"],
|
||||
},
|
||||
},
|
||||
],
|
||||
messages: [
|
||||
makeAnthropicUserText("Direct exec envelope isolation check."),
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool_use",
|
||||
id: "toolu_direct_exec",
|
||||
name: "exec",
|
||||
input: { language: "javascript", code: "return 1;" },
|
||||
},
|
||||
],
|
||||
},
|
||||
makeAnthropicToolResult(
|
||||
"toolu_direct_exec",
|
||||
JSON.stringify({ status: "waiting", runId: "direct-exec" }),
|
||||
),
|
||||
],
|
||||
})) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.stop_reason).toBe("end_turn");
|
||||
expect(body.content.some((block) => block.type === "tool_use")).toBe(false);
|
||||
],
|
||||
},
|
||||
];
|
||||
for (const result of [
|
||||
{ status: "waiting", runId: "direct-exec" },
|
||||
{
|
||||
status: "completed",
|
||||
value: { status: "waiting", runId: "direct-exec-completed-value" },
|
||||
},
|
||||
]) {
|
||||
const body = (await expectAnthropicMessagesJson(server, {
|
||||
tools,
|
||||
messages: [
|
||||
...messages,
|
||||
makeAnthropicToolResult("toolu_direct_exec", JSON.stringify(result)),
|
||||
],
|
||||
})) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.stop_reason).toBe("end_turn");
|
||||
expect(body.content.some((block) => block.type === "tool_use")).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("finishes Anthropic Code Mode fanout after the second wrapped spawn result", async () => {
|
||||
@@ -6914,6 +7444,16 @@ Update and merge these partial structured summaries.`,
|
||||
).map((request) => requireRecord(request, "Anthropic debug request"));
|
||||
expect(debugRequests).toHaveLength(8);
|
||||
expect(debugRequests.every((request) => request.providerVariant === "anthropic")).toBe(true);
|
||||
expect(debugRequests.map((request) => request.plannedToolCallId)).toEqual([
|
||||
readCallIds[0],
|
||||
undefined,
|
||||
readCallIds[1],
|
||||
undefined,
|
||||
readCallIds[2],
|
||||
undefined,
|
||||
readCallIds[3],
|
||||
undefined,
|
||||
]);
|
||||
expect(debugRequests.map((request) => request.toolOutputCallId)).toEqual([
|
||||
undefined,
|
||||
readCallIds[0],
|
||||
@@ -6941,6 +7481,16 @@ Update and merge these partial structured summaries.`,
|
||||
|
||||
const response = await expectAnthropicMessages(server, {
|
||||
stream: true,
|
||||
tools: [
|
||||
{
|
||||
name: "read",
|
||||
input_schema: {
|
||||
type: "object",
|
||||
properties: { path: { type: "string" } },
|
||||
required: ["path"],
|
||||
},
|
||||
},
|
||||
],
|
||||
messages: [
|
||||
makeAnthropicUserText(
|
||||
"Read the seeded docs and report worked, failed, blocked, and follow-up items.",
|
||||
@@ -6956,6 +7506,25 @@ Update and merge these partial structured summaries.`,
|
||||
expect(body).toContain("repo/docs/help/testing.md");
|
||||
expect(body).toContain("event: message_delta");
|
||||
expect(body).toContain("event: message_stop");
|
||||
const events = body
|
||||
.split("\n")
|
||||
.filter((line) => line.startsWith("data: "))
|
||||
.map((line) =>
|
||||
requireRecord(JSON.parse(line.slice("data: ".length)) as unknown, "Anthropic SSE event"),
|
||||
);
|
||||
const toolUseStart = events.find(
|
||||
(event) =>
|
||||
event.type === "content_block_start" &&
|
||||
requireRecord(event.content_block, "Anthropic SSE content block").type === "tool_use",
|
||||
);
|
||||
const toolUse = requireRecord(
|
||||
toolUseStart?.content_block,
|
||||
"Anthropic SSE tool_use content block",
|
||||
);
|
||||
expect(toolUse.id).toMatch(/^toolu_[A-Za-z0-9_]+$/);
|
||||
expect(String(toolUse.id).length).toBeLessThanOrEqual(64);
|
||||
const debug = requireRecord(await getJson(server, "/debug/last-request"), "debug request");
|
||||
expect(debug.plannedToolCallId).toBe(toolUse.id);
|
||||
});
|
||||
|
||||
it("streams Anthropic /v1/messages tool_result follow-ups as text deltas", async () => {
|
||||
|
||||
@@ -8,8 +8,11 @@ import {
|
||||
listMockCodexModelInfos,
|
||||
listMockOpenAiServerModelIds,
|
||||
} from "../shared/mock-model-config.js";
|
||||
import { buildMessagesPayload } from "./mock-anthropic-messages.js";
|
||||
import { convertAnthropicMessagesToResponsesInput } from "./mock-anthropic-wire.js";
|
||||
import {
|
||||
buildMessagesPayload,
|
||||
normalizeAnthropicMessagesRequest,
|
||||
} from "./mock-anthropic-messages.js";
|
||||
import { adaptAnthropicToolCallIds } from "./mock-anthropic-wire.js";
|
||||
import {
|
||||
buildAssistantText,
|
||||
isCanonicalCompactionRetryWriteResult,
|
||||
@@ -24,9 +27,12 @@ import {
|
||||
type MockOpenAiRequestKind,
|
||||
type MockCompactionSummaryFaultMode,
|
||||
type AnthropicMessagesRequest,
|
||||
type QaMockProviderDispatchRequest,
|
||||
type QaMockProviderDispatchResult,
|
||||
TINY_PNG_BASE64,
|
||||
QA_REASONING_ONLY_RECOVERY_PROMPT_RE,
|
||||
QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE,
|
||||
QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE,
|
||||
QA_THINKING_VISIBILITY_OFF_PROMPT_RE,
|
||||
QA_THINKING_VISIBILITY_MAX_PROMPT_RE,
|
||||
QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE,
|
||||
@@ -125,7 +131,7 @@ import {
|
||||
buildReleaseAuditJson,
|
||||
buildReleaseHandoffMarkdown,
|
||||
extractPlannedToolName,
|
||||
extractPlannedToolCallId,
|
||||
extractPlannedToolIdentity,
|
||||
extractPlannedToolArgs,
|
||||
splitMockStreamingText,
|
||||
buildQaLongFinalText,
|
||||
@@ -139,6 +145,7 @@ import {
|
||||
extractLastMatchingUserTurn,
|
||||
hasToolOutput,
|
||||
extractToolOutput,
|
||||
extractToolOutputValue,
|
||||
extractToolOutputStructuredError,
|
||||
extractToolOutputCallId,
|
||||
extractLatestToolOutput,
|
||||
@@ -146,7 +153,6 @@ import {
|
||||
extractUserTextAfterLatestToolOutput,
|
||||
extractSlackMpimRetainedBotNonce,
|
||||
extractAllUserTexts,
|
||||
extractAllInputTexts,
|
||||
extractInstructionsText,
|
||||
extractAllRequestTexts,
|
||||
buildWhatsAppPendingHistoryReply,
|
||||
@@ -157,10 +163,7 @@ import {
|
||||
extractCurrentImageRequest,
|
||||
parseToolOutputJson,
|
||||
} from "./mock-openai-input.js";
|
||||
import {
|
||||
attachQaMockResponsesWebSocketServer,
|
||||
type QaMockResponsesDispatchResult,
|
||||
} from "./mock-openai-responses-websocket.js";
|
||||
import { attachQaMockResponsesWebSocketServer } from "./mock-openai-responses-websocket.js";
|
||||
import {
|
||||
readTargetFromPrompt,
|
||||
execCommandFromToolProgressPrompt,
|
||||
@@ -391,31 +394,40 @@ function findNamedToolDefinition(
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasCodeModeExecSurface(body: Record<string, unknown>) {
|
||||
type CodeModeExecSurface = "native" | "guest";
|
||||
|
||||
function resolveCodeModeExecSurface(body: Record<string, unknown>): CodeModeExecSurface | null {
|
||||
const tools = [
|
||||
...(Array.isArray(body.tools) ? body.tools : []),
|
||||
...(Array.isArray(body.dynamicTools) ? body.dynamicTools : []),
|
||||
];
|
||||
const execDefinition = findNamedToolDefinition(tools, "exec");
|
||||
if (!execDefinition || !hasToolDefinition(body, "wait")) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
if (execDefinition.type === "custom") {
|
||||
return "native";
|
||||
}
|
||||
const schema =
|
||||
(execDefinition.input_schema as Record<string, unknown> | undefined) ??
|
||||
(execDefinition.parameters as Record<string, unknown> | undefined);
|
||||
if (!schema) {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
const properties = schema.properties;
|
||||
const required = schema.required;
|
||||
return (
|
||||
properties !== null &&
|
||||
return properties !== null &&
|
||||
typeof properties === "object" &&
|
||||
!Array.isArray(properties) &&
|
||||
Object.hasOwn(properties, "code") &&
|
||||
Array.isArray(required) &&
|
||||
required.includes("code")
|
||||
);
|
||||
? "guest"
|
||||
: null;
|
||||
}
|
||||
|
||||
function hasCodeModeExecSurface(body: Record<string, unknown>) {
|
||||
return resolveCodeModeExecSurface(body) !== null;
|
||||
}
|
||||
|
||||
function resolveCurrentToolDeclarationSurface(
|
||||
@@ -456,20 +468,70 @@ function parseToolCallArguments(toolCall: ResponsesInputItem) {
|
||||
}
|
||||
}
|
||||
|
||||
function isGeneratedCodeModeExecCall(toolCall: ResponsesInputItem | undefined) {
|
||||
if (!toolCall || toolCall.name !== "exec") {
|
||||
return false;
|
||||
function readGeneratedCodeModeExecSource(toolCall: ResponsesInputItem | undefined) {
|
||||
if (toolCall?.type === "custom_tool_call" && typeof toolCall.input === "string") {
|
||||
return toolCall.input;
|
||||
}
|
||||
const args = parseToolCallArguments(toolCall);
|
||||
return typeof args?.code === "string" && decodeCodeModeTarget(args.code) !== null;
|
||||
const code = toolCall ? parseToolCallArguments(toolCall)?.code : undefined;
|
||||
return typeof code === "string" ? code : undefined;
|
||||
}
|
||||
|
||||
function isGeneratedCodeModeExecCall(toolCall: ResponsesInputItem | undefined) {
|
||||
const source = toolCall?.name === "exec" ? readGeneratedCodeModeExecSource(toolCall) : undefined;
|
||||
return typeof source === "string" && decodeCodeModeTarget(source) !== null;
|
||||
}
|
||||
|
||||
function parseNativeCodeModeOutput(
|
||||
output: unknown,
|
||||
): { status: "waiting"; cellId: string } | { status: "completed"; value: unknown } | null {
|
||||
if (!Array.isArray(output)) {
|
||||
return null;
|
||||
}
|
||||
const readText = (item: unknown) =>
|
||||
typeof item === "string"
|
||||
? item
|
||||
: item &&
|
||||
typeof item === "object" &&
|
||||
typeof (item as Record<string, unknown>).text === "string"
|
||||
? String((item as Record<string, unknown>).text)
|
||||
: null;
|
||||
const statusText = readText(output[0]);
|
||||
if (!statusText) {
|
||||
return null;
|
||||
}
|
||||
const cellId = /^Script running with cell ID ([^\s\n]+)/u.exec(statusText)?.[1];
|
||||
if (cellId) {
|
||||
return { status: "waiting", cellId };
|
||||
}
|
||||
if (!statusText.startsWith("Script completed\n")) {
|
||||
return null;
|
||||
}
|
||||
for (const item of output.slice(1).toReversed()) {
|
||||
const text = readText(item);
|
||||
if (!text) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
return { status: "completed", value: JSON.parse(text) as unknown };
|
||||
} catch {
|
||||
// Native Code Mode may emit non-JSON content before the final value.
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isGeneratedCodeModeWaitCall(input: ResponsesInputItem[], toolCall: ResponsesInputItem) {
|
||||
if (toolCall.name !== "wait") {
|
||||
return false;
|
||||
}
|
||||
const runId = parseToolCallArguments(toolCall)?.runId;
|
||||
if (typeof runId !== "string") {
|
||||
const args = parseToolCallArguments(toolCall);
|
||||
const waitId =
|
||||
typeof args?.cell_id === "string"
|
||||
? args.cell_id
|
||||
: typeof args?.runId === "string"
|
||||
? args.runId
|
||||
: undefined;
|
||||
if (!waitId) {
|
||||
return false;
|
||||
}
|
||||
return input.some((item) => {
|
||||
@@ -479,11 +541,12 @@ function isGeneratedCodeModeWaitCall(input: ResponsesInputItem[], toolCall: Resp
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const output = stringifyScenarioToolOutput(item.output);
|
||||
const parsed = parseToolOutputJson(output);
|
||||
const native = parseNativeCodeModeOutput(item.output);
|
||||
const parsed = native ?? parseToolOutputJson(stringifyScenarioToolOutput(item.output));
|
||||
return (
|
||||
parsed?.status === "waiting" &&
|
||||
parsed.runId === runId &&
|
||||
(("cellId" in parsed && parsed.cellId === waitId) ||
|
||||
("runId" in parsed && parsed.runId === waitId)) &&
|
||||
isGeneratedCodeModeExecCall(findToolCallByCallId(input, item.call_id))
|
||||
);
|
||||
});
|
||||
@@ -537,6 +600,23 @@ function buildScenarioToolCallEvents(
|
||||
return buildRawToolCallEventsWithArgs(name, args, namespace);
|
||||
}
|
||||
const encodedTarget = encodeCodeModeTarget(name, args);
|
||||
if (resolveCodeModeExecSurface(body) === "native") {
|
||||
return buildCustomToolCallEventsWithInput(
|
||||
"exec",
|
||||
[
|
||||
`// ${QA_CODE_MODE_TARGET_MARKER}${encodedTarget}`,
|
||||
`const targetName = ${JSON.stringify(name)};`,
|
||||
`const targetArgs = ${JSON.stringify(args)};`,
|
||||
"const target = ALL_TOOLS.find((entry) => entry.name === targetName);",
|
||||
"if (!target) throw new Error(`QA mock target tool unavailable: ${targetName}`);",
|
||||
"let value = await tools[target.name](targetArgs);",
|
||||
'if (targetName === "read" && value?.kind === "text" && typeof value.content === "string") {',
|
||||
" value = { ...value, content: value.content.slice(0, 2048) };",
|
||||
"}",
|
||||
"text(JSON.stringify(value));",
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
return buildRawToolCallEventsWithArgs("exec", {
|
||||
language: "javascript",
|
||||
code: [
|
||||
@@ -557,10 +637,16 @@ function buildScenarioToolCallEvents(
|
||||
function extractScenarioPlannedTool(events: StreamEvent[]) {
|
||||
const wireName = extractPlannedToolName(events);
|
||||
const wireArgs = extractPlannedToolArgs(events);
|
||||
if (wireName !== "exec" || typeof wireArgs?.code !== "string") {
|
||||
const source =
|
||||
typeof wireArgs?.input === "string"
|
||||
? wireArgs.input
|
||||
: typeof wireArgs?.code === "string"
|
||||
? wireArgs.code
|
||||
: undefined;
|
||||
if (wireName !== "exec" || !source) {
|
||||
return { name: wireName, args: wireArgs, wireName };
|
||||
}
|
||||
const target = decodeCodeModeTarget(wireArgs.code);
|
||||
const target = decodeCodeModeTarget(source);
|
||||
return target
|
||||
? { name: target.name, args: target.args, wireName }
|
||||
: { name: wireName, args: wireArgs, wireName };
|
||||
@@ -701,19 +787,25 @@ async function buildResponsesPayload(
|
||||
const prompt = extractLastUserText(input);
|
||||
const hasCompletedToolOutput = hasToolOutput(input);
|
||||
const rawToolOutput = extractToolOutput(input);
|
||||
const codeModeControlJson = isCodeModeControlToolOutput(toolDeclarationBody, input)
|
||||
? parseToolOutputJson(rawToolOutput)
|
||||
const codeModeSurface = resolveCodeModeExecSurface(toolDeclarationBody);
|
||||
const hasCodeModeControlOutput = isCodeModeControlToolOutput(toolDeclarationBody, input);
|
||||
const codeModeControlJson = hasCodeModeControlOutput
|
||||
? codeModeSurface === "native"
|
||||
? parseNativeCodeModeOutput(extractToolOutputValue(input))
|
||||
: parseToolOutputJson(rawToolOutput)
|
||||
: null;
|
||||
const toolOutput =
|
||||
codeModeControlJson?.status === "completed" && Object.hasOwn(codeModeControlJson, "value")
|
||||
? stringifyScenarioToolOutput(codeModeControlJson.value)
|
||||
: rawToolOutput;
|
||||
: codeModeSurface === "native" && hasCodeModeControlOutput
|
||||
? ""
|
||||
: rawToolOutput;
|
||||
const completedToolCall = findToolCallByCallId(input, extractToolOutputCallId(input));
|
||||
const completedToolName = (() => {
|
||||
if (completedToolCall?.name !== "exec") {
|
||||
return completedToolCall?.name;
|
||||
}
|
||||
const code = parseToolCallArguments(completedToolCall)?.code;
|
||||
const code = readGeneratedCodeModeExecSource(completedToolCall);
|
||||
return typeof code === "string" ? decodeCodeModeTarget(code)?.name : undefined;
|
||||
})();
|
||||
const buildToolCallEventsWithArgs = (name: string, args: Record<string, unknown>) =>
|
||||
@@ -759,12 +851,13 @@ async function buildResponsesPayload(
|
||||
? extractLatestToolOutput(input)
|
||||
: "");
|
||||
const toolJson = parseToolOutputJson(scenarioToolOutput);
|
||||
if (
|
||||
codeModeControlJson?.status === "waiting" &&
|
||||
typeof codeModeControlJson.runId === "string" &&
|
||||
hasToolDefinition(toolDeclarationBody, "wait")
|
||||
) {
|
||||
return buildRawToolCallEventsWithArgs("wait", { runId: codeModeControlJson.runId });
|
||||
if (codeModeControlJson?.status === "waiting" && hasToolDefinition(toolDeclarationBody, "wait")) {
|
||||
if ("cellId" in codeModeControlJson && typeof codeModeControlJson.cellId === "string") {
|
||||
return buildRawToolCallEventsWithArgs("wait", { cell_id: codeModeControlJson.cellId });
|
||||
}
|
||||
if ("runId" in codeModeControlJson && typeof codeModeControlJson.runId === "string") {
|
||||
return buildRawToolCallEventsWithArgs("wait", { runId: codeModeControlJson.runId });
|
||||
}
|
||||
}
|
||||
if (compactionRetryScenarioActive) {
|
||||
if (isCanonicalCompactionRetryWriteResult(toolOutput)) {
|
||||
@@ -1886,40 +1979,68 @@ async function buildResponsesPayload(
|
||||
return buildAssistantEvents("NONE");
|
||||
}
|
||||
if (/thread memory check/i.test(allInputText)) {
|
||||
if (!scenarioToolOutput) {
|
||||
if (!hasCompletedToolOutput) {
|
||||
return buildToolCallEventsWithArgs("memory_search", {
|
||||
query: "hidden thread codename ORBIT-22",
|
||||
maxResults: 3,
|
||||
});
|
||||
}
|
||||
if (memoryToolUnavailable) {
|
||||
const directThreadMemoryJson =
|
||||
Array.isArray(toolJson?.results) || typeof toolJson?.text === "string" ? toolJson : null;
|
||||
const completedMemoryValue =
|
||||
toolJson?.status === "completed" &&
|
||||
(completedToolName === "memory_search" || completedToolName === "memory_get") &&
|
||||
toolJson.value !== null &&
|
||||
typeof toolJson.value === "object" &&
|
||||
!Array.isArray(toolJson.value)
|
||||
? (toolJson.value as Record<string, unknown>)
|
||||
: null;
|
||||
const threadMemoryJson = completedMemoryValue ?? directThreadMemoryJson;
|
||||
const threadMemoryToolName = completedMemoryValue
|
||||
? completedToolName
|
||||
: Array.isArray(threadMemoryJson?.results)
|
||||
? "memory_search"
|
||||
: typeof threadMemoryJson?.text === "string"
|
||||
? "memory_get"
|
||||
: undefined;
|
||||
const threadMemoryUnavailable =
|
||||
threadMemoryJson?.unavailable === true ||
|
||||
threadMemoryJson?.disabled === true ||
|
||||
(typeof threadMemoryJson?.error === "string" && threadMemoryJson.error.trim().length > 0);
|
||||
if (threadMemoryUnavailable) {
|
||||
return buildAssistantEvents("NONE");
|
||||
}
|
||||
const transcriptOrbitCode = extractOrbitCode(scenarioToolOutput);
|
||||
if (transcriptOrbitCode) {
|
||||
return buildAssistantEvents(
|
||||
`Protocol note: I checked memory in-thread and the hidden thread codename is ${transcriptOrbitCode}.`,
|
||||
);
|
||||
if (threadMemoryToolName === "memory_search") {
|
||||
const results = Array.isArray(threadMemoryJson?.results)
|
||||
? (threadMemoryJson.results as Array<Record<string, unknown>>)
|
||||
: [];
|
||||
const first = results[0];
|
||||
if (
|
||||
typeof first?.path === "string" &&
|
||||
(typeof first.startLine === "number" || typeof first.endLine === "number")
|
||||
) {
|
||||
const from =
|
||||
typeof first.startLine === "number"
|
||||
? Math.max(1, first.startLine)
|
||||
: typeof first.endLine === "number"
|
||||
? Math.max(1, first.endLine)
|
||||
: 1;
|
||||
return buildToolCallEventsWithArgs("memory_get", {
|
||||
path: first.path,
|
||||
from,
|
||||
lines: 4,
|
||||
});
|
||||
}
|
||||
}
|
||||
const results = Array.isArray(toolJson?.results)
|
||||
? (toolJson.results as Array<Record<string, unknown>>)
|
||||
: [];
|
||||
const first = results[0];
|
||||
if (
|
||||
typeof first?.path === "string" &&
|
||||
(typeof first.startLine === "number" || typeof first.endLine === "number")
|
||||
) {
|
||||
const from =
|
||||
typeof first.startLine === "number"
|
||||
? Math.max(1, first.startLine)
|
||||
: typeof first.endLine === "number"
|
||||
? Math.max(1, first.endLine)
|
||||
: 1;
|
||||
return buildToolCallEventsWithArgs("memory_get", {
|
||||
path: first.path,
|
||||
from,
|
||||
lines: 4,
|
||||
});
|
||||
const memoryGetText =
|
||||
threadMemoryToolName === "memory_get" && typeof threadMemoryJson?.text === "string"
|
||||
? threadMemoryJson.text
|
||||
: "";
|
||||
const memoryGetOrbitCode = extractOrbitCode(memoryGetText);
|
||||
if (memoryGetOrbitCode) {
|
||||
return buildAssistantEvents(
|
||||
`Protocol note: I checked memory in-thread and the hidden thread codename is ${memoryGetOrbitCode}.`,
|
||||
);
|
||||
}
|
||||
return buildAssistantEvents("NONE");
|
||||
}
|
||||
@@ -2185,13 +2306,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const scenarioStates = new Map<string, MockScenarioState>();
|
||||
const servedCompactionSummaryFaultMarkers = new Set<string>();
|
||||
const scenarioStateFor = (body: Record<string, unknown>): MockScenarioState => {
|
||||
const input =
|
||||
typeof body.input === "string" || Array.isArray(body.input)
|
||||
? normalizeResponsesInput(body.input)
|
||||
: convertAnthropicMessagesToResponsesInput({
|
||||
system: body.system as AnthropicMessagesRequest["system"],
|
||||
messages: [],
|
||||
});
|
||||
const input = normalizeResponsesInput(body.input);
|
||||
const sessionId =
|
||||
resolveQaRuntimeSessionId(input, body) ??
|
||||
(body.client_metadata as { session_id?: unknown } | undefined)?.session_id;
|
||||
@@ -2224,18 +2339,25 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const inflightRequests = new Map<number, { prompt: string; allInputText: string }>();
|
||||
let nextInflightRequestId = 1;
|
||||
const imageGenerationRequests: Array<Record<string, unknown>> = [];
|
||||
const dispatchResponses = async (request: {
|
||||
body: Record<string, unknown>;
|
||||
raw: string;
|
||||
}): Promise<QaMockResponsesDispatchResult> => {
|
||||
const input = normalizeResponsesInput(request.body.input);
|
||||
const dispatchProvider = async (
|
||||
request: QaMockProviderDispatchRequest,
|
||||
): Promise<QaMockProviderDispatchResult> => {
|
||||
const normalized =
|
||||
request.route === "anthropic-messages"
|
||||
? normalizeAnthropicMessagesRequest(request.body as AnthropicMessagesRequest)
|
||||
: {
|
||||
body: request.body,
|
||||
input: normalizeResponsesInput(request.body.input),
|
||||
model: typeof request.body.model === "string" ? request.body.model : "",
|
||||
};
|
||||
const { body, input, model } = normalized;
|
||||
if (isRemoteCompactionV2Request(input)) {
|
||||
return { events: buildRemoteCompactionV2Events() };
|
||||
return { events: buildRemoteCompactionV2Events(), model };
|
||||
}
|
||||
const prompt = extractLastUserText(input);
|
||||
const allInputText = extractAllRequestTexts(input, request.body);
|
||||
const scenarioState = scenarioStateFor(request.body);
|
||||
const requestKind = classifyMockOpenAiRequest(input, request.body);
|
||||
const allInputText = extractAllRequestTexts(input, body);
|
||||
const scenarioState = scenarioStateFor(body);
|
||||
const requestKind = classifyMockOpenAiRequest(input, body);
|
||||
const compactionSummaryFaultMode = resolveCompactionSummaryFaultMode({
|
||||
allInputText,
|
||||
requestKind,
|
||||
@@ -2248,16 +2370,15 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const compactionOverflowThresholdBytes = hasCompactionOutputRecoveryMarker(allInputText)
|
||||
? QA_COMPACTION_OUTPUT_RECOVERY_OVERFLOW_THRESHOLD_BYTES
|
||||
: QA_COMPACTION_RETRY_OVERFLOW_THRESHOLD_BYTES;
|
||||
const resolvedModel = typeof request.body.model === "string" ? request.body.model : "";
|
||||
const requestSnapshotBase = {
|
||||
raw: request.raw,
|
||||
body: request.body,
|
||||
body,
|
||||
prompt,
|
||||
allInputText,
|
||||
instructions: extractInstructionsText(request.body) || undefined,
|
||||
instructions: extractInstructionsText(body) || undefined,
|
||||
toolOutput: extractToolOutput(input),
|
||||
model: resolvedModel,
|
||||
providerVariant: resolveProviderVariant(resolvedModel),
|
||||
model,
|
||||
providerVariant: resolveProviderVariant(model),
|
||||
imageInputCount: countImageInputs(input),
|
||||
requestKind,
|
||||
compactionSummaryFaultMode,
|
||||
@@ -2267,6 +2388,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
| "outcome"
|
||||
| "errorCode"
|
||||
| "plannedToolCallId"
|
||||
| "plannedToolItemId"
|
||||
| "plannedToolName"
|
||||
| "plannedWireToolName"
|
||||
| "plannedToolArgs"
|
||||
@@ -2288,6 +2410,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
});
|
||||
return {
|
||||
events: [],
|
||||
model,
|
||||
failure: {
|
||||
status: 400,
|
||||
type: "invalid_request_error",
|
||||
@@ -2299,15 +2422,48 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const inflightRequestId = nextInflightRequestId++;
|
||||
inflightRequests.set(inflightRequestId, { prompt, allInputText });
|
||||
let events: StreamEvent[];
|
||||
let injectedFailure: QaMockProviderDispatchResult["failure"];
|
||||
try {
|
||||
events = await buildResponsesPayload(request.body, scenarioState, {
|
||||
waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled,
|
||||
requestKind,
|
||||
compactionSummaryFaultMode,
|
||||
});
|
||||
if (
|
||||
request.route === "anthropic-messages" &&
|
||||
QA_ANTHROPIC_THINKING_ERROR_RECOVERY_PROMPT_RE.test(allInputText)
|
||||
) {
|
||||
const toolOutput = extractToolOutput(input);
|
||||
const toolOutputCallId = extractToolOutputCallId(input);
|
||||
const scenarioKey = `${model}\n${extractLastUserText(input)}`;
|
||||
const shouldFail =
|
||||
toolOutput.length > 0 &&
|
||||
toolOutputCallId.length > 0 &&
|
||||
!scenarioState.anthropicThinkingErrorScenarioKeys.has(scenarioKey);
|
||||
if (shouldFail) {
|
||||
scenarioState.anthropicThinkingErrorScenarioKeys.add(scenarioKey);
|
||||
injectedFailure = {
|
||||
status: 200,
|
||||
type: "api_error",
|
||||
message: "QA injected provider stream failure",
|
||||
presentation: "anthropic-thinking",
|
||||
};
|
||||
}
|
||||
events =
|
||||
toolOutput.length === 0
|
||||
? buildRawToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" })
|
||||
: shouldFail
|
||||
? buildAssistantEvents("")
|
||||
: buildAssistantEvents("ANTHROPIC-THINKING-ERROR-RECOVERED-OK");
|
||||
} else {
|
||||
events = await buildResponsesPayload(body, scenarioState, {
|
||||
waitForTerminalRequesterSettled: terminalRequesterSettleGate.waitUntilSettled,
|
||||
requestKind,
|
||||
compactionSummaryFaultMode,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
inflightRequests.delete(inflightRequestId);
|
||||
}
|
||||
if (request.route === "anthropic-messages") {
|
||||
events = adaptAnthropicToolCallIds(events);
|
||||
}
|
||||
const plannedToolIdentity = extractPlannedToolIdentity(events);
|
||||
const plannedTool = extractScenarioPlannedTool(events);
|
||||
const terminalRequesterCase = extractLastMatchingUserTurn(
|
||||
input,
|
||||
@@ -2316,7 +2472,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
?.text.match(QA_SUBAGENT_TERMINAL_MATRIX_PROMPT_RE)?.[1]
|
||||
?.toLowerCase();
|
||||
const settledTerminalRequester =
|
||||
terminalRequesterCase && resolveQaRuntimeSessionId(input, request.body)
|
||||
terminalRequesterCase && resolveQaRuntimeSessionId(input, body)
|
||||
? {
|
||||
caseName: terminalRequesterCase,
|
||||
childSessionKey: resolveAcceptedChildSessionKey(input),
|
||||
@@ -2325,17 +2481,21 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
const settledTerminalCaseName = settledTerminalRequester?.caseName;
|
||||
const settledChildSessionKey = settledTerminalRequester?.childSessionKey;
|
||||
const failure =
|
||||
QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && hasToolOutput(input)
|
||||
injectedFailure ??
|
||||
(QA_PROVIDER_HTTP_503_AFTER_TOOL_PROMPT_RE.test(allInputText) && hasToolOutput(input)
|
||||
? {
|
||||
status: 503,
|
||||
type: "server_error",
|
||||
message: "Service Unavailable",
|
||||
}
|
||||
: undefined;
|
||||
: undefined);
|
||||
recordRequest({
|
||||
...requestSnapshotBase,
|
||||
outcome: failure ? "error" : "success",
|
||||
plannedToolCallId: extractPlannedToolCallId(events),
|
||||
plannedToolCallId: plannedToolIdentity.callId,
|
||||
...(request.route === "responses" && plannedToolIdentity.itemId
|
||||
? { plannedToolItemId: plannedToolIdentity.itemId }
|
||||
: {}),
|
||||
plannedToolName: plannedTool.name,
|
||||
...(plannedTool.wireName && plannedTool.wireName !== plannedTool.name
|
||||
? { plannedWireToolName: plannedTool.wireName }
|
||||
@@ -2346,6 +2506,7 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
});
|
||||
return {
|
||||
events,
|
||||
model,
|
||||
...(settledTerminalCaseName && settledChildSessionKey
|
||||
? {
|
||||
onResponseSent: () =>
|
||||
@@ -2361,6 +2522,8 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
const dispatchResponses = (request: Omit<QaMockProviderDispatchRequest, "route">) =>
|
||||
dispatchProvider({ ...request, route: "responses" });
|
||||
const server = createServer((req, res) => {
|
||||
void (async () => {
|
||||
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
||||
@@ -2492,16 +2655,6 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
writeOpenAiMalformedJsonError(res, "OpenAI Responses");
|
||||
return;
|
||||
}
|
||||
const input = normalizeResponsesInput(body.input);
|
||||
if (isRemoteCompactionV2Request(input)) {
|
||||
const events = buildRemoteCompactionV2Events();
|
||||
if (body.stream === false) {
|
||||
writeJson(res, 200, events[1].response);
|
||||
} else {
|
||||
writeSse(res, events);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const dispatched = await dispatchResponses({ body, raw });
|
||||
if (dispatched.failure) {
|
||||
writeJson(res, dispatched.failure.status, {
|
||||
@@ -2545,48 +2698,25 @@ export async function startQaMockOpenAiServer(params?: {
|
||||
});
|
||||
return;
|
||||
}
|
||||
const scenarioState = scenarioStateFor(body as Record<string, unknown>);
|
||||
const {
|
||||
events,
|
||||
input,
|
||||
responseBody,
|
||||
streamEvents,
|
||||
model: normalizedModel,
|
||||
} = await buildMessagesPayload(body, scenarioState, buildResponsesPayload);
|
||||
const plannedTool = extractScenarioPlannedTool(events);
|
||||
// Record the adapted request snapshot so /debug/requests gives the QA
|
||||
// suite the same plannedToolName / allInputText / toolOutput signals
|
||||
// on the Anthropic route that the OpenAI route already exposes. This
|
||||
// is what lets a single parity run diff assertions across both lanes.
|
||||
// Reuse the normalized model so an empty-string body.model no longer
|
||||
// leaks through to `lastRequest.model`.
|
||||
recordRequest({
|
||||
raw,
|
||||
const dispatched = await dispatchProvider({
|
||||
route: "anthropic-messages",
|
||||
body: body as Record<string, unknown>,
|
||||
prompt: extractLastUserText(input),
|
||||
allInputText: extractAllInputTexts(input),
|
||||
toolOutput: extractToolOutput(input),
|
||||
model: normalizedModel,
|
||||
providerVariant: resolveProviderVariant(normalizedModel),
|
||||
imageInputCount: countImageInputs(input),
|
||||
requestKind: classifyMockOpenAiRequest(input, body as Record<string, unknown>),
|
||||
compactionSummaryFaultMode: "none",
|
||||
outcome: "success",
|
||||
rawByteLength: Buffer.byteLength(raw),
|
||||
plannedToolCallId: extractPlannedToolCallId(events),
|
||||
plannedToolName: plannedTool.name,
|
||||
...(plannedTool.wireName && plannedTool.wireName !== plannedTool.name
|
||||
? { plannedWireToolName: plannedTool.wireName }
|
||||
: {}),
|
||||
plannedToolArgs: plannedTool.args,
|
||||
toolOutputCallId: extractToolOutputCallId(input) || undefined,
|
||||
...(extractToolOutputStructuredError(input) ? { toolOutputStructuredError: true } : {}),
|
||||
raw,
|
||||
});
|
||||
const { responseBody, streamEvents } = buildMessagesPayload(dispatched);
|
||||
if (dispatched.failure?.presentation !== "anthropic-thinking") {
|
||||
if (dispatched.failure) {
|
||||
writeJson(res, dispatched.failure.status, responseBody);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (body.stream === true) {
|
||||
writeAnthropicSse(res, streamEvents);
|
||||
dispatched.onResponseSent?.();
|
||||
return;
|
||||
}
|
||||
writeJson(res, 200, responseBody);
|
||||
writeJson(res, dispatched.failure?.status ?? 200, responseBody);
|
||||
dispatched.onResponseSent?.();
|
||||
return;
|
||||
}
|
||||
writeJson(res, 404, { error: "not found" });
|
||||
|
||||
@@ -105,7 +105,7 @@ describe("buildQaGatewayConfig", () => {
|
||||
expect(cfg.models?.providers?.anthropic?.baseUrl).toBe("http://127.0.0.1:44080");
|
||||
expect(cfg.models?.providers?.anthropic?.request).toEqual({ allowPrivateNetwork: true });
|
||||
expect(cfg.memory?.search).toMatchObject({
|
||||
provider: "openai",
|
||||
provider: "openai-compatible",
|
||||
model: "text-embedding-3-small",
|
||||
remote: {
|
||||
baseUrl: "http://127.0.0.1:44080/v1",
|
||||
|
||||
@@ -202,7 +202,7 @@ export function buildQaGatewayConfig(params: {
|
||||
const mockMemorySearch =
|
||||
provider.kind === "mock"
|
||||
? {
|
||||
provider: "openai",
|
||||
provider: "openai-compatible",
|
||||
model: "text-embedding-3-small",
|
||||
remote: {
|
||||
// Memory embeddings bypass the model runtime, so bind them to the
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readQaScenarioById, readQaScenarioExecutionConfig } from "./scenario-catalog.js";
|
||||
import { readFlowAssertExpression, requireFlowScenario } from "./scenario-catalog.test-utils.js";
|
||||
|
||||
describe("qa scenario catalog causality", () => {
|
||||
it("loads live gateway sentinel scenarios for harness self-health", () => {
|
||||
const scenarioIds = [
|
||||
"plugin-hook-health-sentinel",
|
||||
"plugin-manifest-contract-health",
|
||||
"webchat-direct-reply-routing",
|
||||
"long-context-progress-watchdog",
|
||||
"gateway-restart-inflight-run",
|
||||
"gateway-restart-multi-live",
|
||||
"streaming-final-integrity",
|
||||
];
|
||||
|
||||
for (const scenarioId of scenarioIds) {
|
||||
const scenario = readQaScenarioById(scenarioId);
|
||||
expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0);
|
||||
expect(scenario.coverage?.primary.length).toBeGreaterThan(0);
|
||||
}
|
||||
expect(readQaScenarioById("webchat-direct-reply-routing").sourcePath).toBe(
|
||||
"qa/scenarios/channels/webchat-direct-reply-routing.yaml",
|
||||
);
|
||||
expect(readQaScenarioById("long-context-progress-watchdog").sourcePath).toBe(
|
||||
"qa/scenarios/runtime/long-context-progress-watchdog.yaml",
|
||||
);
|
||||
const gatewayRestart = requireFlowScenario(readQaScenarioById("gateway-restart-inflight-run"));
|
||||
const gatewayRestartFlow = gatewayRestart.execution.flow;
|
||||
const gatewayRestartContract = JSON.stringify(gatewayRestartFlow);
|
||||
const gatewayRestartActions = gatewayRestartFlow?.steps[0]?.actions ?? [];
|
||||
const recoveryPollIndex = gatewayRestartActions.findIndex(
|
||||
(action) =>
|
||||
(action as { call?: string }).call === "waitForCondition" &&
|
||||
(action as { saveAs?: string }).saveAs === "settledRecovery",
|
||||
);
|
||||
const outboundIndex = gatewayRestartActions.findIndex(
|
||||
(action) =>
|
||||
(action as { call?: string }).call === "waitForOutboundMessage" &&
|
||||
(action as { saveAs?: string }).saveAs === "outbound",
|
||||
);
|
||||
const preOutboundRecoveryAssertIndex = gatewayRestartActions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(
|
||||
"restartRecoveryRequestsBeforeOutbound.length === 1",
|
||||
),
|
||||
);
|
||||
const preOutboundHeartbeatAssertIndex = gatewayRestartActions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(
|
||||
"!String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('[OpenClaw heartbeat poll]')",
|
||||
),
|
||||
);
|
||||
const settledRequestsIndex = gatewayRestartActions.findIndex(
|
||||
(action) => (action as { set?: string }).set === "settledRecoveryRequests",
|
||||
);
|
||||
const settledDedupeAssertIndex = gatewayRestartActions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("settledRestartRecoveryRequests.length === 1"),
|
||||
);
|
||||
const recoveryPoll = gatewayRestartActions[recoveryPollIndex] as
|
||||
| { args?: Array<{ lambda?: { expr?: string } }> }
|
||||
| undefined;
|
||||
const recoveryPollExpr = recoveryPoll?.args?.[0]?.lambda?.expr ?? "";
|
||||
expect(gatewayRestart.execution.retryCount).toBe(0);
|
||||
expect(JSON.stringify(gatewayRestart.gatewayConfigPatch)).toContain(
|
||||
'"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]',
|
||||
);
|
||||
expect(gatewayRestartContract).toContain("plannedToolName === 'wait'");
|
||||
expect(gatewayRestartContract).toContain("lastAssistantToolNames?.includes('wait')");
|
||||
expect(gatewayRestartContract).toContain("restartRecoveryDeliveryContext");
|
||||
expect(gatewayRestartContract).toContain("sendInbound");
|
||||
expect(gatewayRestartContract).not.toContain("startAgentRun");
|
||||
expect(gatewayRestartContract).toContain('"restartGatewayWithConfigPatch"');
|
||||
expect(gatewayRestartContract).toContain("interruptedMatches.length === 1");
|
||||
expect(gatewayRestartContract).toContain("restartNotices.length === 0");
|
||||
expect(gatewayRestartContract).toContain("dispatching restart-safe recovery");
|
||||
expect(recoveryPollIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(recoveryPollExpr).toContain(
|
||||
"String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart')",
|
||||
);
|
||||
expect(recoveryPollExpr).toContain(
|
||||
"String(request.allInputText ?? '').includes(config.interruptedMarker)",
|
||||
);
|
||||
expect(recoveryPollExpr).toContain("restartRecoveryRequests.length >= 1");
|
||||
expect(recoveryPollExpr).toContain(
|
||||
"String(transcript.finalText ?? '').includes(config.interruptedMarker)",
|
||||
);
|
||||
expect(preOutboundRecoveryAssertIndex).toBeGreaterThan(recoveryPollIndex);
|
||||
expect(preOutboundHeartbeatAssertIndex).toBeGreaterThan(preOutboundRecoveryAssertIndex);
|
||||
expect(outboundIndex).toBeGreaterThan(preOutboundHeartbeatAssertIndex);
|
||||
expect(settledRequestsIndex).toBeGreaterThan(outboundIndex);
|
||||
expect(settledDedupeAssertIndex).toBeGreaterThan(settledRequestsIndex);
|
||||
expect(
|
||||
gatewayRestartActions.some((action) => (action as { call?: string }).call === "sleep"),
|
||||
).toBe(false);
|
||||
expect(gatewayRestartContract).toContain("recoveryPromptHeartbeat=false");
|
||||
expect(gatewayRestartContract).toContain("liveTurnTimeoutMs(env, 180000)");
|
||||
expect(gatewayRestartContract).toContain("id: `dm:${conversationId}`");
|
||||
expect(gatewayRestartContract).toContain("dmScope: env.cfg.session?.dmScope");
|
||||
expect(gatewayRestart.gatewayConfigPatch).toMatchObject({
|
||||
plugins: {
|
||||
slots: { memory: "none" },
|
||||
entries: {
|
||||
acpx: { enabled: false },
|
||||
"memory-core": { enabled: false },
|
||||
},
|
||||
},
|
||||
});
|
||||
const liveMultiRestart = readQaScenarioById("gateway-restart-multi-live");
|
||||
const liveMultiRestartContract = JSON.stringify(liveMultiRestart.execution.flow);
|
||||
expect(JSON.stringify(liveMultiRestart.gatewayConfigPatch)).toContain(
|
||||
'"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]',
|
||||
);
|
||||
expect(liveMultiRestartContract).toContain("assistantToolCallCounts.exec");
|
||||
expect(liveMultiRestartContract).toContain("checkpoint");
|
||||
expect(liveMultiRestartContract).toContain("restarts=3");
|
||||
expect(liveMultiRestartContract).toContain("dmScope: 'per-channel-peer'");
|
||||
expect(liveMultiRestartContract).toContain("dispatching restart-safe recovery");
|
||||
expect(readQaScenarioExecutionConfig("gateway-restart-multi-live")).toMatchObject({
|
||||
requiredProviderMode: "live-frontier",
|
||||
requiredProvider: "openai",
|
||||
requiredModel: "gpt-5.4",
|
||||
});
|
||||
});
|
||||
|
||||
it("scopes prompt diagnostics to requests after each scenario cursor", () => {
|
||||
for (const scenarioId of [
|
||||
"instruction-followthrough-repo-contract",
|
||||
"subagent-handoff",
|
||||
] as const) {
|
||||
const scenario = requireFlowScenario(readQaScenarioById(scenarioId));
|
||||
const flow = JSON.stringify(scenario.execution.flow);
|
||||
const cursorIndex = flow.indexOf("/debug/request-cursor");
|
||||
const promptIndex = flow.indexOf('"call":"runAgentPrompt"');
|
||||
const requestsIndex = flow.indexOf("/debug/requests?after=${requestCursorBefore}");
|
||||
|
||||
expect(cursorIndex, scenarioId).toBeGreaterThanOrEqual(0);
|
||||
expect(cursorIndex, scenarioId).toBeLessThan(promptIndex);
|
||||
expect(requestsIndex, scenarioId).toBeGreaterThan(promptIndex);
|
||||
expect(flow, scenarioId).not.toContain("`${env.mock.baseUrl}/debug/requests`");
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"thread-memory-isolation",
|
||||
"poll",
|
||||
"finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId",
|
||||
],
|
||||
[
|
||||
"memory-tools-channel-context",
|
||||
"poll",
|
||||
"finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId",
|
||||
],
|
||||
[
|
||||
"agent-tool-consumption",
|
||||
"immediate",
|
||||
"getResultRequest.toolOutputCallId === searchResultRequest.plannedToolCallId",
|
||||
],
|
||||
] as const)(
|
||||
"asserts the complete memory tool chain before %s delivery",
|
||||
(scenarioId, requestCollectionMode, finalLinkNeedle) => {
|
||||
const scenario = requireFlowScenario(readQaScenarioById(scenarioId));
|
||||
const actions = scenario.execution.flow?.steps[0]?.actions ?? [];
|
||||
const outboundIndex = actions.findIndex(
|
||||
(action) => (action as { call?: string }).call === "waitForOutboundMessage",
|
||||
);
|
||||
const requestCollectionIndex = actions.findIndex((action) =>
|
||||
requestCollectionMode === "poll"
|
||||
? (action as { call?: string }).call === "waitForCondition" &&
|
||||
(action as { saveAs?: string }).saveAs === "scenarioRequests"
|
||||
: (action as { set?: string }).set === "scenarioRequests",
|
||||
);
|
||||
const requestCountAssertIndex = actions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("scenarioRequests.length === 3"),
|
||||
);
|
||||
const searchPlanAssertIndex = actions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(
|
||||
"searchPlanRequest.plannedToolName === 'memory_search'",
|
||||
),
|
||||
);
|
||||
const searchResultAssertIndex = actions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(
|
||||
"searchResultRequest.toolOutputCallId === searchPlanRequest.plannedToolCallId",
|
||||
),
|
||||
);
|
||||
const finalRequestAssertIndex = actions.findIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(finalLinkNeedle),
|
||||
);
|
||||
|
||||
expect(requestCollectionIndex, scenarioId).toBeGreaterThanOrEqual(0);
|
||||
expect(requestCountAssertIndex, scenarioId).toBeGreaterThan(requestCollectionIndex);
|
||||
expect(searchPlanAssertIndex, scenarioId).toBeGreaterThan(requestCountAssertIndex);
|
||||
expect(searchResultAssertIndex, scenarioId).toBeGreaterThan(searchPlanAssertIndex);
|
||||
expect(finalRequestAssertIndex, scenarioId).toBeGreaterThan(searchResultAssertIndex);
|
||||
expect(outboundIndex, scenarioId).toBeGreaterThan(finalRequestAssertIndex);
|
||||
|
||||
if (requestCollectionMode === "poll") {
|
||||
const requestPoll = actions[requestCollectionIndex] as
|
||||
| { args?: Array<{ lambda?: { expr?: string } }> }
|
||||
| undefined;
|
||||
expect(requestPoll?.args?.[0]?.lambda?.expr, scenarioId).toContain(
|
||||
"requests.length >= 3 ? requests : undefined",
|
||||
);
|
||||
} else {
|
||||
expect(
|
||||
actions.some(
|
||||
(action) =>
|
||||
(action as { call?: string }).call === "waitForCondition" &&
|
||||
(action as { saveAs?: string }).saveAs === "scenarioRequests",
|
||||
),
|
||||
scenarioId,
|
||||
).toBe(false);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readQaScenarioById } from "./scenario-catalog.js";
|
||||
import { requireFlowScenario } from "./scenario-catalog.test-utils.js";
|
||||
import { readFlowAssertExpression, requireFlowScenario } from "./scenario-catalog.test-utils.js";
|
||||
|
||||
describe("qa compaction scenario catalog", () => {
|
||||
it.each([
|
||||
@@ -56,6 +56,47 @@ describe("qa compaction scenario catalog", () => {
|
||||
const scenario = requireFlowScenario(readQaScenarioById("compaction-retry-mutating-tool"));
|
||||
const flow = JSON.stringify(scenario.execution.flow);
|
||||
const serializedScenario = JSON.stringify(scenario);
|
||||
const actions = scenario.execution.flow?.steps[0]?.actions ?? [];
|
||||
const readSetExpression = (name: string) => {
|
||||
const action = actions.find((candidate) => (candidate as { set?: string }).set === name) as
|
||||
| { value?: { expr?: string } }
|
||||
| undefined;
|
||||
return action?.value?.expr ?? "";
|
||||
};
|
||||
const readAssertExpression = (needle: string) =>
|
||||
actions.map(readFlowAssertExpression).find((expression) => expression.includes(needle)) ?? "";
|
||||
const actionIndex = (predicate: (action: (typeof actions)[number]) => boolean) =>
|
||||
actions.findIndex(predicate);
|
||||
const writeRequestsExpr = readSetExpression("writeRequests");
|
||||
const postWriteContinuationsExpr = readSetExpression("postWriteContinuations");
|
||||
const writeTranscriptToolCallIdExpr = readSetExpression("writeTranscriptToolCallId");
|
||||
const continuationChainExpr = readSetExpression("continuationChain");
|
||||
const continuationAssertIndex = actionIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("continuationChain.valid === true"),
|
||||
);
|
||||
const terminalAssertIndex = actionIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("terminalContinuations.length === 1"),
|
||||
);
|
||||
const distinctCallIdsAssertIndex = actionIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("new Set([writeRequest.plannedToolCallId"),
|
||||
);
|
||||
const stableCellIdAssertIndex = actionIndex((action) =>
|
||||
readFlowAssertExpression(action).includes("continuationChain.waits.length === 0"),
|
||||
);
|
||||
const terminalEvidenceAssertIndex = actionIndex((action) =>
|
||||
readFlowAssertExpression(action).includes(
|
||||
"terminalContinuations[0].providerVariant === 'openai'",
|
||||
),
|
||||
);
|
||||
const outboundWaitIndex = actionIndex(
|
||||
(action) =>
|
||||
(action as { call?: string }).call === "waitForCondition" &&
|
||||
(action as { saveAs?: string }).saveAs === "outbound",
|
||||
);
|
||||
const stableCellIdAssertExpr = readAssertExpression("continuationChain.waits.length === 0");
|
||||
const terminalEvidenceAssertExpr = readAssertExpression(
|
||||
"terminalContinuations[0].providerVariant === 'openai'",
|
||||
);
|
||||
const knownGap =
|
||||
"known-harness-gap compaction-retry-mutating-tool: provider-error recovery does not invoke Codex native compaction; native token-threshold compaction needs a separate scenario.";
|
||||
|
||||
@@ -70,7 +111,7 @@ describe("qa compaction scenario catalog", () => {
|
||||
"One coded over-threshold provider overflow produces one persisted OpenClaw overflow compaction and one compacted retry retaining durable current context.",
|
||||
);
|
||||
expect(scenario.successCriteria).toContain(
|
||||
"OpenClaw performs exactly one successful write, one causal continuation, and returns the exact file content and final marker.",
|
||||
"OpenClaw performs exactly one successful write, then one terminal continuation after zero-or-more causally linked waits, and returns the exact file content and final marker.",
|
||||
);
|
||||
expect(scenario.successCriteria).toContain(
|
||||
"OpenClaw proves session-memory.pruning by retaining a nonempty contiguous suffix ending at block 15 while pruning marker block 10.",
|
||||
@@ -114,31 +155,129 @@ describe("qa compaction scenario catalog", () => {
|
||||
expect(flow).toContain("seedQaSessionTranscript");
|
||||
expect(flow).toContain("sessions.compaction.branch");
|
||||
expect(flow).toContain("env.runtimeId");
|
||||
expect(flow).toContain('"transcriptToolName":"write"');
|
||||
expect(flow).toContain('"requireSuccessfulTranscriptToolResult":true');
|
||||
expect(scenario.execution.retryCount).toBe(0);
|
||||
expect(flow).not.toContain('"transcriptToolName":"write"');
|
||||
expect(flow).not.toContain('"requireSuccessfulTranscriptToolResult":true');
|
||||
expect(serializedScenario).not.toContain("expectedWriteToolResult");
|
||||
expect(flow).toContain("outbound.text === config.finalMarker");
|
||||
expect(flow).toContain("overflowRequests.length === 1");
|
||||
expect(flow).toContain("overflowRequest.rawByteLength > config.overflowThresholdBytes");
|
||||
expect(flow).toContain("writeRequests.length === 1");
|
||||
expect(flow).toContain("String(request.allInputText ?? '').includes(sessionId)");
|
||||
expect(writeRequestsExpr).toContain("request.plannedToolName === 'write'");
|
||||
expect(writeRequestsExpr).toContain("request.cursor > overflowRequest.cursor");
|
||||
expect(writeRequestsExpr).toContain("String(request.allInputText ?? '').includes(sessionId)");
|
||||
expect(writeRequestsExpr).toContain(
|
||||
"String(request.allInputText ?? '').includes(config.promptSnippet)",
|
||||
);
|
||||
expect(writeRequestsExpr).toContain(
|
||||
"String(request.allInputText ?? '').includes(config.durableMarker)",
|
||||
);
|
||||
expect(writeRequestsExpr).not.toContain("request.requestKind");
|
||||
expect(writeRequestsExpr).not.toContain("request.outcome");
|
||||
expect(writeRequestsExpr).not.toContain("request.toolOutput");
|
||||
expect(writeRequestsExpr).not.toContain("request.plannedToolArgs");
|
||||
expect(writeRequestsExpr).not.toContain("request.plannedWireToolName");
|
||||
expect(flow).toContain("writeRequest.requestKind === 'agent-initial'");
|
||||
expect(flow).toContain("writeRequest.outcome === 'success'");
|
||||
expect(flow).toContain("!writeRequest.toolOutput");
|
||||
expect(flow).toContain(
|
||||
"String(writeRequest.allInputText ?? '').includes(config.durableMarker)",
|
||||
);
|
||||
expect(flow).toContain("request.cursor > overflowRequest.cursor");
|
||||
expect(flow).toContain("request.plannedToolArgs?.path === config.outputFile");
|
||||
expect(flow).toContain("request.plannedToolArgs?.content === config.expectedFileContent");
|
||||
expect(flow).toContain("request.toolOutputCallId === writeRequest.plannedToolCallId");
|
||||
expect(flow).toContain("request.cursor > writeRequest.cursor");
|
||||
expect(flow).not.toContain("request.plannedWireToolName === 'exec'");
|
||||
expect(flow).toContain(
|
||||
"String(request.toolOutput ?? '').trim() === config.expectedWriteToolResult",
|
||||
"writeRequest.plannedWireToolName === undefined || writeRequest.plannedWireToolName === 'exec'",
|
||||
);
|
||||
expect(flow).toContain("writeRequest.plannedToolArgs?.path === config.outputFile");
|
||||
expect(flow).toContain("writeRequest.plannedToolArgs?.content === config.expectedFileContent");
|
||||
expect(flow).toContain("typeof writeRequest.plannedToolCallId === 'string'");
|
||||
expect(flow).not.toContain(
|
||||
"writeRequest.plannedToolCallId.length > 0 && typeof writeRequest.plannedToolItemId",
|
||||
);
|
||||
expect(flow).toContain(
|
||||
'writeWireToolName","value":{"expr":"writeRequest.plannedWireToolName ?? writeRequest.plannedToolName',
|
||||
);
|
||||
expect(writeTranscriptToolCallIdExpr).toContain(
|
||||
"typeof writeRequest.plannedToolItemId === 'string'",
|
||||
);
|
||||
expect(writeTranscriptToolCallIdExpr).toContain("writeRequest.plannedToolItemId.length > 0");
|
||||
expect(writeTranscriptToolCallIdExpr).toContain(
|
||||
"`${writeRequest.plannedToolCallId}|${writeRequest.plannedToolItemId}`",
|
||||
);
|
||||
expect(writeTranscriptToolCallIdExpr).toContain(": writeRequest.plannedToolCallId");
|
||||
expect(flow).toContain(
|
||||
"event.toolCallId === writeTranscriptToolCallId && event.name === writeWireToolName",
|
||||
);
|
||||
expect(flow).toContain("successfulWriteTranscriptEvents.length === 1");
|
||||
expect(flow).toContain("transcript.successfulToolCallCounts[writeWireToolName] === 1");
|
||||
expect(flow).not.toContain("transcript.successfulToolCallCounts.write === 1");
|
||||
expect(postWriteContinuationsExpr).toContain("request.requestKind === 'tool-continuation'");
|
||||
expect(postWriteContinuationsExpr).toContain("request.cursor > writeRequest.cursor");
|
||||
expect(postWriteContinuationsExpr).toContain(
|
||||
"String(request.allInputText ?? '').includes(sessionId)",
|
||||
);
|
||||
expect(postWriteContinuationsExpr).not.toContain("request.outcome");
|
||||
expect(postWriteContinuationsExpr).not.toContain("request.plannedToolName");
|
||||
expect(postWriteContinuationsExpr).not.toContain("request.toolOutputCallId");
|
||||
expect(postWriteContinuationsExpr).not.toContain("request.toolOutputStructuredError");
|
||||
expect(flow).toContain("let currentCallId = writeRequest.plannedToolCallId");
|
||||
expect(flow).toContain("postWriteContinuations.filter");
|
||||
expect(flow).toContain("request.toolOutputCallId === currentCallId");
|
||||
expect(flow).toContain("request.plannedToolName === 'wait'");
|
||||
expect(flow).toContain("currentCallId = request.plannedToolCallId");
|
||||
expect(continuationChainExpr).toContain("request.toolOutputCallId === currentCallId");
|
||||
expect(continuationChainExpr).toContain("request.plannedToolName === 'wait'");
|
||||
expect(flow).toContain("continuationChain.requests.length === postWriteContinuations.length");
|
||||
expect(flow).toContain(
|
||||
"continuationChain.requests.every((request, index) => request === postWriteContinuations[index])",
|
||||
);
|
||||
expect(flow).toContain(
|
||||
"continuationChain.requests.length === continuationChain.waits.length + 1",
|
||||
);
|
||||
expect(flow).toContain("request.outcome === 'success'");
|
||||
expect(flow).toContain("request.toolOutputStructuredError !== true");
|
||||
expect(flow).toContain("terminalContinuations.length === 1");
|
||||
expect(flow).toContain("terminalContinuations[0] === continuationChain.terminal");
|
||||
expect(flow).toContain("String(terminalContinuations[0].toolOutput ?? '').trim().length > 0");
|
||||
expect(flow).toContain("new Set([writeRequest.plannedToolCallId");
|
||||
expect(stableCellIdAssertExpr).toContain("continuationChain.waits.length === 0 ||");
|
||||
expect(stableCellIdAssertExpr).toContain(
|
||||
"typeof request.plannedToolArgs?.cell_id === 'string'",
|
||||
);
|
||||
expect(stableCellIdAssertExpr).toContain(
|
||||
"new Set(continuationChain.waits.map((request) => request.plannedToolArgs.cell_id)).size === 1",
|
||||
);
|
||||
expect(terminalEvidenceAssertExpr).toContain("writeWireToolName !== 'exec'");
|
||||
const openAiEvidenceIndex = terminalEvidenceAssertExpr.indexOf(
|
||||
"terminalContinuations[0].providerVariant === 'openai'",
|
||||
);
|
||||
const anthropicEvidenceIndex = terminalEvidenceAssertExpr.indexOf(
|
||||
"terminalContinuations[0].providerVariant === 'anthropic'",
|
||||
);
|
||||
const unknownProviderFailClosedIndex = terminalEvidenceAssertExpr.lastIndexOf(": false");
|
||||
expect(openAiEvidenceIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(terminalEvidenceAssertExpr).toContain("startsWith('Script completed\\n')");
|
||||
expect(anthropicEvidenceIndex).toBeGreaterThan(openAiEvidenceIndex);
|
||||
expect(terminalEvidenceAssertExpr).toContain(
|
||||
"JSON.parse(String(terminalContinuations[0].toolOutput ?? ''))",
|
||||
);
|
||||
expect(terminalEvidenceAssertExpr).toContain("parsed !== null");
|
||||
expect(terminalEvidenceAssertExpr).toContain("typeof parsed === 'object'");
|
||||
expect(terminalEvidenceAssertExpr).toContain("!Array.isArray(parsed)");
|
||||
expect(terminalEvidenceAssertExpr).toContain("parsed.status === 'completed'");
|
||||
expect(unknownProviderFailClosedIndex).toBeGreaterThan(anthropicEvidenceIndex);
|
||||
expect(continuationAssertIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(terminalAssertIndex).toBeGreaterThan(continuationAssertIndex);
|
||||
expect(distinctCallIdsAssertIndex).toBeGreaterThan(terminalAssertIndex);
|
||||
expect(stableCellIdAssertIndex).toBeGreaterThan(distinctCallIdsAssertIndex);
|
||||
expect(terminalEvidenceAssertIndex).toBeGreaterThan(stableCellIdAssertIndex);
|
||||
expect(outboundWaitIndex).toBeGreaterThan(terminalEvidenceAssertIndex);
|
||||
expect(flow).not.toContain("config.expectedOpenClawToolResult");
|
||||
expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`---");
|
||||
expect(flow).not.toContain("String(request.toolOutput ?? '').includes(`+++");
|
||||
expect(flow).toContain("postWriteContinuations.length === 1");
|
||||
expect(flow).toContain(
|
||||
"compactionSummaryRequests.every((request) => request.outcome === 'success' && request.plannedToolName === undefined)",
|
||||
"compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true",
|
||||
);
|
||||
expect(flow).not.toContain("compactionSummaryRequests.every(");
|
||||
expect(flow).not.toContain("compactionSummaryRequests.length >= 1");
|
||||
expect(flow).toContain(
|
||||
"writeRequest.rawByteLength < config.overflowThresholdBytes && writeRequest.rawByteLength < overflowRequest.rawByteLength",
|
||||
@@ -154,6 +293,23 @@ describe("qa compaction scenario catalog", () => {
|
||||
expect(flow).toContain("durable: String(request.allInputText ?? '')");
|
||||
expect(flow).toContain("bulky: String(request.allInputText ?? '')");
|
||||
expect(flow).toContain("inputChars: String(request.allInputText ?? '').length");
|
||||
expect(flow).toContain(
|
||||
"resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null",
|
||||
);
|
||||
expect(flow).toContain("callId: request.plannedToolCallId ?? null");
|
||||
expect(flow).toContain("itemId: request.plannedToolItemId ?? null");
|
||||
expect(flow).toContain(
|
||||
"transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0",
|
||||
);
|
||||
expect(flow).toContain(": request.plannedToolCallId ?? null");
|
||||
expect(flow).toContain("logicalWrites=${String(writeRequests.length)}");
|
||||
expect(flow).toContain("wireTool=${String(writeWireToolName)}");
|
||||
expect(flow).toContain("callId=${String(writeRequest.plannedToolCallId)}");
|
||||
expect(flow).toContain("itemId=${String(writeRequest.plannedToolItemId)}");
|
||||
expect(flow).toContain("transcriptId=${String(writeTranscriptToolCallId)}");
|
||||
expect(flow).toContain(
|
||||
"wireSuccesses=${String(transcript.successfulToolCallCounts[writeWireToolName] ?? 0)}",
|
||||
);
|
||||
expect(flow).not.toContain("clientSessionId");
|
||||
expect(flow).toContain("tailBlocks:");
|
||||
expect(flow).toContain(".sort().slice(0, 16)");
|
||||
|
||||
@@ -32,6 +32,20 @@ export function requireFlowScenario(scenario: CatalogScenario): FlowCatalogScena
|
||||
return scenario;
|
||||
}
|
||||
|
||||
export function readFlowAssertExpression(action: unknown): string {
|
||||
if (!action || typeof action !== "object" || !("assert" in action)) {
|
||||
return "";
|
||||
}
|
||||
const assertion = action.assert;
|
||||
if (typeof assertion === "string") {
|
||||
return assertion;
|
||||
}
|
||||
if (!assertion || typeof assertion !== "object" || !("expr" in assertion)) {
|
||||
return "";
|
||||
}
|
||||
return typeof assertion.expr === "string" ? assertion.expr : "";
|
||||
}
|
||||
|
||||
export function flowContainsCall(value: unknown, callName: string): boolean {
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((entry) => flowContainsCall(entry, callName));
|
||||
|
||||
@@ -592,72 +592,6 @@ describe("qa scenario catalog", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("loads live gateway sentinel scenarios for harness self-health", () => {
|
||||
const scenarioIds = [
|
||||
"plugin-hook-health-sentinel",
|
||||
"plugin-manifest-contract-health",
|
||||
"webchat-direct-reply-routing",
|
||||
"long-context-progress-watchdog",
|
||||
"gateway-restart-inflight-run",
|
||||
"gateway-restart-multi-live",
|
||||
"streaming-final-integrity",
|
||||
];
|
||||
|
||||
for (const scenarioId of scenarioIds) {
|
||||
const scenario = readQaScenarioById(scenarioId);
|
||||
expect(scenario.execution.flow?.steps.length).toBeGreaterThan(0);
|
||||
expect(scenario.coverage?.primary.length).toBeGreaterThan(0);
|
||||
}
|
||||
expect(readQaScenarioById("webchat-direct-reply-routing").sourcePath).toBe(
|
||||
"qa/scenarios/channels/webchat-direct-reply-routing.yaml",
|
||||
);
|
||||
expect(readQaScenarioById("long-context-progress-watchdog").sourcePath).toBe(
|
||||
"qa/scenarios/runtime/long-context-progress-watchdog.yaml",
|
||||
);
|
||||
const gatewayRestartFlow = readQaScenarioById("gateway-restart-inflight-run").execution.flow;
|
||||
const gatewayRestartContract = JSON.stringify(gatewayRestartFlow);
|
||||
expect(
|
||||
JSON.stringify(readQaScenarioById("gateway-restart-inflight-run").gatewayConfigPatch),
|
||||
).toContain('"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]');
|
||||
expect(gatewayRestartContract).toContain("plannedToolName === 'wait'");
|
||||
expect(gatewayRestartContract).toContain("lastAssistantToolNames?.includes('wait')");
|
||||
expect(gatewayRestartContract).toContain("restartRecoveryDeliveryContext");
|
||||
expect(gatewayRestartContract).toContain("sendInbound");
|
||||
expect(gatewayRestartContract).not.toContain("startAgentRun");
|
||||
expect(gatewayRestartContract).toContain('"restartGatewayWithConfigPatch"');
|
||||
expect(gatewayRestartContract).toContain("interruptedMatches.length === 1");
|
||||
expect(gatewayRestartContract).toContain("restartNotices.length === 0");
|
||||
expect(gatewayRestartContract).toContain("dispatching restart-safe recovery");
|
||||
expect(gatewayRestartContract).toContain("[OpenClaw heartbeat poll]");
|
||||
expect(gatewayRestartContract).toContain("liveTurnTimeoutMs(env, 180000)");
|
||||
expect(gatewayRestartContract).toContain("id: `dm:${conversationId}`");
|
||||
expect(gatewayRestartContract).toContain("dmScope: env.cfg.session?.dmScope");
|
||||
expect(readQaScenarioById("gateway-restart-inflight-run").gatewayConfigPatch).toMatchObject({
|
||||
plugins: {
|
||||
slots: { memory: "none" },
|
||||
entries: {
|
||||
acpx: { enabled: false },
|
||||
"memory-core": { enabled: false },
|
||||
},
|
||||
},
|
||||
});
|
||||
const liveMultiRestart = readQaScenarioById("gateway-restart-multi-live");
|
||||
const liveMultiRestartContract = JSON.stringify(liveMultiRestart.execution.flow);
|
||||
expect(JSON.stringify(liveMultiRestart.gatewayConfigPatch)).toContain(
|
||||
'"alsoAllow":["qa_restart_wait","qa_restart_unsafe_probe"]',
|
||||
);
|
||||
expect(liveMultiRestartContract).toContain("assistantToolCallCounts.exec");
|
||||
expect(liveMultiRestartContract).toContain("checkpoint");
|
||||
expect(liveMultiRestartContract).toContain("restarts=3");
|
||||
expect(liveMultiRestartContract).toContain("dmScope: 'per-channel-peer'");
|
||||
expect(liveMultiRestartContract).toContain("dispatching restart-safe recovery");
|
||||
expect(readQaScenarioExecutionConfig("gateway-restart-multi-live")).toMatchObject({
|
||||
requiredProviderMode: "live-frontier",
|
||||
requiredProvider: "openai",
|
||||
requiredModel: "gpt-5.4",
|
||||
});
|
||||
});
|
||||
|
||||
it("loads the QA bus tool trace visibility harness scenario", () => {
|
||||
const scenario = readQaScenarioById("qa-bus-tool-trace-visibility");
|
||||
const config = readQaScenarioExecutionConfig(scenario.id) as
|
||||
|
||||
@@ -89,6 +89,9 @@ flow:
|
||||
- set: artifactPath
|
||||
value:
|
||||
expr: "path.join(env.gateway.workspaceDir, 'repo-contract-summary.txt')"
|
||||
- set: requestCursorBefore
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
@@ -128,7 +131,7 @@ flow:
|
||||
expr: "`repo contract followthrough bounced for permission or stalled: ${outbound.text}`"
|
||||
- set: followthroughDebugRequests
|
||||
value:
|
||||
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))].filter((request) => /repo contract followthrough check/i.test(String(request.allInputText ?? ''))) : []"
|
||||
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))].filter((request) => /repo contract followthrough check/i.test(String(request.allInputText ?? ''))) : []"
|
||||
- assert:
|
||||
expr: "!env.mock || followthroughDebugRequests.filter((request) => request.plannedToolName === 'read').length >= 3"
|
||||
message:
|
||||
|
||||
@@ -32,6 +32,9 @@ flow:
|
||||
- call: reset
|
||||
- try:
|
||||
actions:
|
||||
- set: requestCursorBefore
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor : 0"
|
||||
- call: runAgentPrompt
|
||||
args:
|
||||
- ref: env
|
||||
@@ -76,7 +79,7 @@ flow:
|
||||
# request after the tool runs has plannedToolName unset.
|
||||
- set: subagentDebugRequests
|
||||
value:
|
||||
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests`))] : []"
|
||||
expr: "env.mock ? [...(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`))] : []"
|
||||
- assert:
|
||||
expr: "!env.mock || subagentDebugRequests.some((request) => !request.toolOutput && /delegate one bounded qa task/i.test(String(request.allInputText ?? '')) && request.plannedToolName === 'sessions_spawn')"
|
||||
message:
|
||||
|
||||
@@ -19,6 +19,7 @@ scenario:
|
||||
timeoutMs: 180000
|
||||
retryCount: 0
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
matrixConfigOverrides:
|
||||
streaming: quiet
|
||||
matrixTopology:
|
||||
@@ -35,7 +36,17 @@ scenario:
|
||||
requireMention: true
|
||||
|
||||
flow:
|
||||
module: ./live-transports/matrix/scenarios/scenario-runtime-media.js
|
||||
call: runGeneratedImageDeliveryScenario
|
||||
args:
|
||||
- expr: "scenarioContext"
|
||||
steps:
|
||||
- name: Matrix generated images deliver as real image attachments while streaming
|
||||
actions:
|
||||
- call: ensureImageGenerationConfigured
|
||||
args:
|
||||
- ref: env
|
||||
- set: scenarioModule
|
||||
value:
|
||||
expr: "await qaImport('./live-transports/matrix/scenarios/scenario-runtime-media.js')"
|
||||
- call: scenarioModule.runGeneratedImageDeliveryScenario
|
||||
args:
|
||||
- expr: "scenarioContext"
|
||||
saveAs: result
|
||||
detailsExpr: "result.details ?? (result.artifacts ? JSON.stringify(result.artifacts, null, 2) : undefined)"
|
||||
|
||||
@@ -90,19 +90,14 @@ flow:
|
||||
senderName: Alice
|
||||
text:
|
||||
expr: config.prompt
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
- call: waitForCondition
|
||||
saveAs: scenarioRequests
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && candidate.conversation.kind === 'channel' && candidate.text.includes(config.expectedNeedle)"
|
||||
async: true
|
||||
expr: "(async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeInbound}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)); return requests.length >= 3 ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- sinceIndex:
|
||||
ref: outboundStartIndex
|
||||
- set: scenarioRequests
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBeforeInbound}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))"
|
||||
- 100
|
||||
- assert:
|
||||
expr: scenarioRequests.length === 3
|
||||
message:
|
||||
@@ -128,6 +123,16 @@ flow:
|
||||
expr: "finalRequest.cursor > searchResultRequest.cursor && finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && finalRequest.toolOutputStructuredError !== true && String(finalRequest.toolOutput ?? '').includes(config.expectedMemoryPath) && String(finalRequest.toolOutput ?? '').includes(config.expectedNeedle) && !finalRequest.plannedToolName"
|
||||
message:
|
||||
expr: "`final request did not consume the matching successful memory_get result: ${JSON.stringify(finalRequest)}`"
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && candidate.conversation.kind === 'channel' && candidate.text.includes(config.expectedNeedle)"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- sinceIndex:
|
||||
ref: outboundStartIndex
|
||||
- call: sleep
|
||||
args: [4000]
|
||||
- set: visibleChannelOutbounds
|
||||
|
||||
@@ -26,9 +26,12 @@ scenario:
|
||||
kind: flow
|
||||
summary: Verify a memory-backed answer requested inside a thread stays in-thread and does not leak into the root channel.
|
||||
channel: qa-channel
|
||||
providerMode: mock-openai
|
||||
retryCount: 0
|
||||
config:
|
||||
memoryFact: "Thread-hidden codename: ORBIT-22."
|
||||
memoryQuery: "hidden thread codename ORBIT-22"
|
||||
expectedMemoryPath: MEMORY.md
|
||||
expectedNeedle: "ORBIT-22"
|
||||
channelId: qa-room
|
||||
channelTitle: QA Room
|
||||
@@ -79,7 +82,10 @@ flow:
|
||||
- assert:
|
||||
expr: Boolean(threadId)
|
||||
message: missing thread id for memory isolation check
|
||||
- set: beforeCursor
|
||||
- set: requestCursorBefore
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/request-cursor`)).cursor"
|
||||
- set: outboundStartIndex
|
||||
value:
|
||||
expr: state.getSnapshot().messages.length
|
||||
- sendInbound:
|
||||
@@ -97,6 +103,39 @@ flow:
|
||||
ref: threadId
|
||||
threadTitle:
|
||||
expr: config.threadTitle
|
||||
- call: waitForCondition
|
||||
saveAs: scenarioRequests
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "(async () => { const requests = (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)); return requests.length >= 3 ? requests : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- 100
|
||||
- assert:
|
||||
expr: scenarioRequests.length === 3
|
||||
message:
|
||||
expr: "`expected one memory_search plan, one memory_get plan, and one final request: ${JSON.stringify(scenarioRequests)}`"
|
||||
- set: searchPlanRequest
|
||||
value:
|
||||
expr: scenarioRequests[0]
|
||||
- set: searchResultRequest
|
||||
value:
|
||||
expr: scenarioRequests[1]
|
||||
- set: finalRequest
|
||||
value:
|
||||
expr: scenarioRequests[2]
|
||||
- assert:
|
||||
expr: "searchPlanRequest.plannedToolName === 'memory_search' && searchPlanRequest.plannedToolArgs?.query === config.memoryQuery && typeof searchPlanRequest.plannedToolCallId === 'string' && searchPlanRequest.plannedToolCallId.length > 0 && !searchPlanRequest.toolOutputCallId && !String(searchPlanRequest.allInputText ?? '').includes(config.expectedNeedle)"
|
||||
message:
|
||||
expr: "`initial thread request exposed the hidden fact, disabled memory_search, or routed incorrectly: ${JSON.stringify(searchPlanRequest)}`"
|
||||
- assert:
|
||||
expr: "searchResultRequest.cursor > searchPlanRequest.cursor && searchResultRequest.toolOutputCallId === searchPlanRequest.plannedToolCallId && searchResultRequest.toolOutputStructuredError !== true && String(searchResultRequest.toolOutput ?? '').includes(config.expectedMemoryPath) && String(searchResultRequest.toolOutput ?? '').includes(config.expectedNeedle) && searchResultRequest.plannedToolName === 'memory_get' && searchResultRequest.plannedToolArgs?.path === config.expectedMemoryPath && typeof searchResultRequest.plannedToolCallId === 'string' && searchResultRequest.plannedToolCallId.length > 0 && searchResultRequest.plannedToolCallId !== searchPlanRequest.plannedToolCallId"
|
||||
message:
|
||||
expr: "`memory_search result did not causally plan memory_get: ${JSON.stringify(searchResultRequest)}`"
|
||||
- assert:
|
||||
expr: "finalRequest.cursor > searchResultRequest.cursor && finalRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && finalRequest.toolOutputStructuredError !== true && String(finalRequest.toolOutput ?? '').includes(config.expectedNeedle) && !finalRequest.plannedToolName"
|
||||
message:
|
||||
expr: "`final thread request did not consume the matching memory_get result: ${JSON.stringify(finalRequest)}`"
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
args:
|
||||
@@ -105,10 +144,9 @@ flow:
|
||||
params: [candidate]
|
||||
expr: "((candidate.conversation.id === config.channelId && candidate.threadId === threadId) || candidate.conversation.id === threadId) && candidate.text.includes(config.expectedNeedle)"
|
||||
- expr: liveTurnTimeoutMs(env, 300000)
|
||||
- sinceIndex:
|
||||
ref: outboundStartIndex
|
||||
- assert:
|
||||
expr: "!state.getSnapshot().messages.slice(beforeCursor).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && !candidate.threadId)"
|
||||
expr: "!state.getSnapshot().messages.slice(outboundStartIndex).some((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.channelId && !candidate.threadId)"
|
||||
message: threaded memory answer leaked into root channel
|
||||
- assert:
|
||||
expr: "!env.mock || (await fetchJson(`${env.mock.baseUrl}/debug/requests`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet)).some((request) => request.plannedToolName === 'memory_search')"
|
||||
message: expected memory_search in thread memory flow
|
||||
detailsExpr: outbound.text
|
||||
|
||||
@@ -89,16 +89,6 @@ flow:
|
||||
expr: config.prompt
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 30000)
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(nonce)"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- sinceIndex:
|
||||
ref: outboundStartIndex
|
||||
- set: scenarioRequests
|
||||
value:
|
||||
expr: "(await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.promptSnippet))"
|
||||
@@ -127,6 +117,16 @@ flow:
|
||||
expr: "getResultRequest.toolOutputCallId === searchResultRequest.plannedToolCallId && getResultRequest.toolOutputStructuredError !== true && String(getResultRequest.toolOutput ?? '').includes(nonce) && String(getResultRequest.allInputText ?? '').includes(nonce) && !getResultRequest.plannedToolName"
|
||||
message:
|
||||
expr: "`memory_get result was not consumed before terminal generation: ${JSON.stringify(getResultRequest)}`"
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
args:
|
||||
- ref: state
|
||||
- lambda:
|
||||
params: [candidate]
|
||||
expr: "candidate.conversation.id === 'qa-operator' && candidate.text.includes(nonce)"
|
||||
- expr: liveTurnTimeoutMs(env, 30000)
|
||||
- sinceIndex:
|
||||
ref: outboundStartIndex
|
||||
- set: terminalOutbounds
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(outboundStartIndex)"
|
||||
|
||||
@@ -12,7 +12,7 @@ scenario:
|
||||
objective: Verify OpenClaw persists one overflow compaction, prunes bulky historical turns, retries from durable context, and performs the requested mutation exactly once.
|
||||
successCriteria:
|
||||
- One coded over-threshold provider overflow produces one persisted OpenClaw overflow compaction and one compacted retry retaining durable current context.
|
||||
- OpenClaw performs exactly one successful write, one causal continuation, and returns the exact file content and final marker.
|
||||
- OpenClaw performs exactly one successful write, then one terminal continuation after zero-or-more causally linked waits, and returns the exact file content and final marker.
|
||||
- OpenClaw proves session-memory.pruning by retaining a nonempty contiguous suffix ending at block 15 while pruning marker block 10.
|
||||
- OpenClaw records one overflow-retry checkpoint whose branch preserves the independent pre-compaction assistant marker.
|
||||
- The Codex runtime-pair cell reports a known harness gap before gateway, session, or provider work and makes no compaction coverage claim.
|
||||
@@ -25,6 +25,7 @@ scenario:
|
||||
execution:
|
||||
kind: flow
|
||||
summary: Force one OpenClaw context overflow, verify persisted pruning evidence, and prove exactly-once mutation after compaction.
|
||||
retryCount: 0
|
||||
config:
|
||||
outputFile: compaction-retry-summary.txt
|
||||
promptSnippet: Compaction retry mutating tool check
|
||||
@@ -33,7 +34,6 @@ scenario:
|
||||
checkpointMarker: QA-COMPACTION-PRECOMPACTION-ASSISTANT-MARKER
|
||||
overflowThresholdBytes: 262144
|
||||
expectedFileContent: "Replay safety: unsafe after write.\n"
|
||||
expectedWriteToolResult: Successfully wrote 35 bytes to compaction-retry-summary.txt
|
||||
finalMarker: "Protocol note: replay unsafe after write."
|
||||
prompt: |-
|
||||
Compaction retry mutating tool check. Current durable context marker: QA-COMPACTION-DURABLE-MARKER.
|
||||
@@ -97,25 +97,12 @@ flow:
|
||||
ref: config.prompt
|
||||
timeoutMs:
|
||||
expr: liveTurnTimeoutMs(env, 120000)
|
||||
transcriptToolName: write
|
||||
requireSuccessfulTranscriptToolResult: true
|
||||
- call: waitForCondition
|
||||
saveAs: outbound
|
||||
args:
|
||||
- lambda:
|
||||
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.finalMarker)).at(-1)"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- 100
|
||||
- assert:
|
||||
expr: "outbound.text === config.finalMarker"
|
||||
message:
|
||||
expr: "`unexpected final compaction reply: ${JSON.stringify(outbound.text)}`"
|
||||
- set: scenarioRequests
|
||||
value:
|
||||
expr: "await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)"
|
||||
- set: requestEvidence
|
||||
value:
|
||||
expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), tool: request.plannedToolName ?? null }))"
|
||||
expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`post-marker historical user block ${id}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), tool: request.plannedToolName ?? null, resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null, callId: request.plannedToolCallId ?? null, itemId: request.plannedToolItemId ?? null, transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0 ? `${request.plannedToolCallId}|${request.plannedToolItemId}` : request.plannedToolCallId ?? null }))"
|
||||
- set: overflowRequests
|
||||
value:
|
||||
expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'error' && request.errorCode === 'context_length_exceeded' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))"
|
||||
@@ -128,7 +115,7 @@ flow:
|
||||
expr: "overflowRequests[0]"
|
||||
- set: writeRequests
|
||||
value:
|
||||
expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'success' && request.plannedToolName === 'write' && !request.toolOutput && request.cursor > overflowRequest.cursor && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker) && request.plannedToolArgs?.path === config.outputFile && request.plannedToolArgs?.content === config.expectedFileContent)"
|
||||
expr: "scenarioRequests.filter((request) => request.cursor > overflowRequest.cursor && request.plannedToolName === 'write' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))"
|
||||
- assert:
|
||||
expr: "writeRequests.length === 1"
|
||||
message:
|
||||
@@ -136,6 +123,16 @@ flow:
|
||||
- set: writeRequest
|
||||
value:
|
||||
expr: "writeRequests[0]"
|
||||
- assert:
|
||||
expr: "writeRequest.requestKind === 'agent-initial' && writeRequest.outcome === 'success' && !writeRequest.toolOutput && writeRequest.plannedToolArgs?.path === config.outputFile && writeRequest.plannedToolArgs?.content === config.expectedFileContent && (writeRequest.plannedWireToolName === undefined || writeRequest.plannedWireToolName === 'exec') && typeof writeRequest.plannedToolCallId === 'string' && writeRequest.plannedToolCallId.length > 0"
|
||||
message:
|
||||
expr: "`logical write did not have the expected successful direct-or-Code-Mode wire shape: ${JSON.stringify(requestEvidence.find((request) => request.cursor === writeRequest.cursor))}`"
|
||||
- set: writeWireToolName
|
||||
value:
|
||||
expr: "writeRequest.plannedWireToolName ?? writeRequest.plannedToolName"
|
||||
- set: writeTranscriptToolCallId
|
||||
value:
|
||||
expr: "typeof writeRequest.plannedToolItemId === 'string' && writeRequest.plannedToolItemId.length > 0 ? `${writeRequest.plannedToolCallId}|${writeRequest.plannedToolItemId}` : writeRequest.plannedToolCallId"
|
||||
- set: overflowEvidence
|
||||
value:
|
||||
expr: "requestEvidence.find((request) => request.cursor === overflowRequest.cursor)"
|
||||
@@ -147,10 +144,17 @@ flow:
|
||||
args:
|
||||
- ref: env
|
||||
- ref: sessionKey
|
||||
- set: successfulWriteTranscriptEvents
|
||||
value:
|
||||
expr: "(transcript.successfulToolCallEvents ?? []).filter((event) => event.toolCallId === writeTranscriptToolCallId && event.name === writeWireToolName)"
|
||||
- assert:
|
||||
expr: "transcript.successfulToolCallCounts.write === 1"
|
||||
expr: "successfulWriteTranscriptEvents.length === 1"
|
||||
message:
|
||||
expr: "`expected exactly one successful write, got ${JSON.stringify(transcript.successfulToolCallCounts)}`"
|
||||
expr: "`expected one authenticated successful transcript event for the logical write's wire call: ${JSON.stringify({ write: writeEvidence, wireTool: writeWireToolName, callId: writeRequest.plannedToolCallId, itemId: writeRequest.plannedToolItemId, transcriptId: writeTranscriptToolCallId, events: transcript.successfulToolCallEvents ?? [] })}`"
|
||||
- assert:
|
||||
expr: "transcript.successfulToolCallCounts[writeWireToolName] === 1"
|
||||
message:
|
||||
expr: "`expected exactly one successful resolved wire call, got ${JSON.stringify({ wireTool: writeWireToolName, counts: transcript.successfulToolCallCounts })}`"
|
||||
- assert:
|
||||
expr: "overflowRequest.rawByteLength > config.overflowThresholdBytes"
|
||||
message:
|
||||
@@ -161,11 +165,44 @@ flow:
|
||||
expr: "`compacted retry did not retain durable current context: ${JSON.stringify({ overflow: overflowEvidence, write: writeEvidence })}`"
|
||||
- set: postWriteContinuations
|
||||
value:
|
||||
expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.outcome === 'success' && request.plannedToolName === undefined && request.cursor > writeRequest.cursor && request.toolOutputCallId === writeRequest.plannedToolCallId && String(request.allInputText ?? '').includes(sessionId) && String(request.toolOutput ?? '').trim() === config.expectedWriteToolResult)"
|
||||
expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.cursor > writeRequest.cursor && String(request.allInputText ?? '').includes(sessionId)).toSorted((left, right) => left.cursor - right.cursor)"
|
||||
- set: continuationChain
|
||||
value:
|
||||
expr: "(() => { const requests = []; const waits = []; const followedCallIds = new Set(); let currentCallId = writeRequest.plannedToolCallId; let previousCursor = writeRequest.cursor; let terminal; let valid = typeof currentCallId === 'string' && currentCallId.length > 0; while (valid) { if (followedCallIds.has(currentCallId)) { valid = false; break; } followedCallIds.add(currentCallId); const matches = postWriteContinuations.filter((request) => request.cursor > previousCursor && request.toolOutputCallId === currentCallId); if (matches.length !== 1) { valid = false; break; } const request = matches[0]; requests.push(request); previousCursor = request.cursor; if (request.plannedToolName === 'wait') { if (typeof request.plannedToolCallId !== 'string' || request.plannedToolCallId.length === 0) { valid = false; break; } waits.push(request); currentCallId = request.plannedToolCallId; continue; } if (request.plannedToolName === undefined) { terminal = request; break; } valid = false; } return { valid, requests, waits, terminal }; })()"
|
||||
- assert:
|
||||
expr: "postWriteContinuations.length === 1"
|
||||
expr: "continuationChain.valid === true && continuationChain.requests.length === postWriteContinuations.length && continuationChain.requests.every((request, index) => request === postWriteContinuations[index]) && continuationChain.requests.length === continuationChain.waits.length + 1 && continuationChain.requests.every((request) => request.outcome === 'success' && request.toolOutputStructuredError !== true)"
|
||||
message:
|
||||
expr: "`expected one post-write continuation and no replayed mutation: ${JSON.stringify(requestEvidence.filter((request) => request.cursor > writeRequest.cursor))}`"
|
||||
expr: "`expected one exhaustive successful causal continuation chain after the logical write: ${JSON.stringify(postWriteContinuations)}`"
|
||||
- set: terminalContinuations
|
||||
value:
|
||||
expr: "continuationChain.requests.filter((request) => request.plannedToolName === undefined)"
|
||||
- assert:
|
||||
expr: "terminalContinuations.length === 1 && terminalContinuations[0] === continuationChain.terminal && String(terminalContinuations[0].toolOutput ?? '').trim().length > 0 && terminalContinuations[0].toolOutputStructuredError !== true"
|
||||
message:
|
||||
expr: "`expected exactly one non-error terminal continuation after zero-or-more linked waits: ${JSON.stringify(postWriteContinuations)}`"
|
||||
- assert:
|
||||
expr: "continuationChain.waits.every((request) => request.plannedToolName === 'wait' && typeof request.plannedToolCallId === 'string' && request.plannedToolCallId.length > 0) && new Set([writeRequest.plannedToolCallId, ...continuationChain.waits.map((request) => request.plannedToolCallId)]).size === continuationChain.waits.length + 1"
|
||||
message:
|
||||
expr: "`linked wait calls did not preserve distinct causal call ids: ${JSON.stringify(postWriteContinuations)}`"
|
||||
- assert:
|
||||
expr: "continuationChain.waits.length === 0 || (continuationChain.waits.every((request) => typeof request.plannedToolArgs?.cell_id === 'string' && request.plannedToolArgs.cell_id.length > 0) && new Set(continuationChain.waits.map((request) => request.plannedToolArgs.cell_id)).size === 1)"
|
||||
message:
|
||||
expr: "`linked waits did not preserve one nonempty Code Mode cell id: ${JSON.stringify(postWriteContinuations)}`"
|
||||
- assert:
|
||||
expr: "writeWireToolName !== 'exec' || (terminalContinuations[0].providerVariant === 'openai' ? String(terminalContinuations[0].toolOutput ?? '').startsWith('Script completed\\n') : terminalContinuations[0].providerVariant === 'anthropic' ? (() => { try { const parsed = JSON.parse(String(terminalContinuations[0].toolOutput ?? '')); return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed) && parsed.status === 'completed'; } catch { return false; } })() : false)"
|
||||
message:
|
||||
expr: "`Code Mode terminal continuation did not report successful completion: ${JSON.stringify(terminalContinuations[0])}`"
|
||||
- call: waitForCondition
|
||||
saveAs: outbound
|
||||
args:
|
||||
- lambda:
|
||||
expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && candidate.text.includes(config.finalMarker)).at(-1)"
|
||||
- expr: liveTurnTimeoutMs(env, 45000)
|
||||
- 100
|
||||
- assert:
|
||||
expr: "outbound.text === config.finalMarker"
|
||||
message:
|
||||
expr: "`unexpected final compaction reply: ${JSON.stringify(outbound.text)}`"
|
||||
- call: fs.readFile
|
||||
saveAs: writtenSummary
|
||||
args:
|
||||
@@ -191,9 +228,9 @@ flow:
|
||||
value:
|
||||
expr: "scenarioRequests.filter((request) => request.requestKind === 'compaction-summary')"
|
||||
- assert:
|
||||
expr: "compactionSummaryRequests.every((request) => request.outcome === 'success' && request.plannedToolName === undefined)"
|
||||
expr: "compactionSummaryRequests.length === 1 && compactionSummaryRequests[0].outcome === 'success' && compactionSummaryRequests[0].plannedToolName === undefined && compactionSummaryRequests[0].toolOutputStructuredError !== true"
|
||||
message:
|
||||
expr: "`OpenClaw compaction summary requests included a failure or tool plan: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`"
|
||||
expr: "`expected exactly one successful OpenClaw compaction summary request: ${JSON.stringify(requestEvidence.filter((request) => request.kind === 'compaction-summary'))}`"
|
||||
- call: readRawQaSessionStore
|
||||
saveAs: store
|
||||
args:
|
||||
@@ -227,4 +264,4 @@ flow:
|
||||
expr: "branchSummary.finalText === config.checkpointMarker"
|
||||
message:
|
||||
expr: "`checkpoint branch did not preserve pre-compaction assistant marker: ${JSON.stringify({ key: branchResult.key, finalText: branchSummary.finalText })}`"
|
||||
detailsExpr: "`${outbound.text}\\nOpenClaw originalBytes=${String(overflowRequest.rawByteLength)} retryBytes=${String(writeRequest.rawByteLength)} writes=${String(transcript.successfulToolCallCounts.write ?? 0)} compactions=${String(sessionEntry.compactionCount)} checkpoints=${String(overflowCheckpoints.length)}`"
|
||||
detailsExpr: "`${outbound.text}\\nOpenClaw originalBytes=${String(overflowRequest.rawByteLength)} retryBytes=${String(writeRequest.rawByteLength)} logicalWrites=${String(writeRequests.length)} wireTool=${String(writeWireToolName)} callId=${String(writeRequest.plannedToolCallId)} itemId=${String(writeRequest.plannedToolItemId)} transcriptId=${String(writeTranscriptToolCallId)} wireSuccesses=${String(transcript.successfulToolCallCounts[writeWireToolName] ?? 0)} compactions=${String(sessionEntry.compactionCount)} checkpoints=${String(overflowCheckpoints.length)}`"
|
||||
|
||||
@@ -44,6 +44,7 @@ scenario:
|
||||
kind: flow
|
||||
runtime: openclaw
|
||||
timeoutMs: 420000
|
||||
retryCount: 0
|
||||
summary: Restart while replay-safe Code Mode wait is executing, then verify automatic reconstruction and delivery.
|
||||
config:
|
||||
requiredProviderMode: mock-openai
|
||||
@@ -136,18 +137,34 @@ flow:
|
||||
args:
|
||||
- ref: env
|
||||
- 180000
|
||||
- call: sleep
|
||||
- call: waitForCondition
|
||||
saveAs: settledRecovery
|
||||
args:
|
||||
- 6500
|
||||
- call: readSessionTranscriptSummary
|
||||
saveAs: postRestartTranscript
|
||||
args:
|
||||
- ref: env
|
||||
- ref: sessionKey
|
||||
- lambda:
|
||||
async: true
|
||||
expr: "(async () => { const recoveryRequests = env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []; const restartRecoveryRequests = recoveryRequests.filter((request) => String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(request.allInputText ?? '').includes(config.interruptedMarker)); const transcript = await readSessionTranscriptSummary(env, sessionKey); return restartRecoveryRequests.length >= 1 && transcript.lastMessageRole === 'assistant' && String(transcript.finalText ?? '').includes(config.interruptedMarker) ? { recoveryRequests, restartRecoveryRequests, transcript } : undefined; })()"
|
||||
- expr: liveTurnTimeoutMs(env, 120000)
|
||||
- 25
|
||||
- set: recoveryRequestsBeforeOutbound
|
||||
value:
|
||||
expr: settledRecovery.recoveryRequests
|
||||
- set: restartRecoveryRequestsBeforeOutbound
|
||||
value:
|
||||
expr: settledRecovery.restartRecoveryRequests
|
||||
- set: postRestartTranscript
|
||||
value:
|
||||
expr: settledRecovery.transcript
|
||||
- assert:
|
||||
expr: "postRestartTranscript.lastAssistantStopReason !== 'error' && postRestartTranscript.lastAssistantStopReason !== 'aborted'"
|
||||
message:
|
||||
expr: "`restart recovery left an abort artifact: ${JSON.stringify(postRestartTranscript)}`"
|
||||
- assert:
|
||||
expr: "restartRecoveryRequestsBeforeOutbound.length === 1 && String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(restartRecoveryRequestsBeforeOutbound[0].allInputText ?? '').includes(config.interruptedMarker)"
|
||||
message:
|
||||
expr: "`expected exactly one scoped restart recovery provider request before delivery, got ${restartRecoveryRequestsBeforeOutbound.length}; total post-cursor requests=${recoveryRequestsBeforeOutbound.length}`"
|
||||
- assert:
|
||||
expr: "!String(restartRecoveryRequestsBeforeOutbound[0].prompt ?? '').includes('[OpenClaw heartbeat poll]')"
|
||||
message: restart recovery provider request was replaced by a heartbeat poll
|
||||
- call: waitForOutboundMessage
|
||||
saveAs: outbound
|
||||
args:
|
||||
@@ -158,6 +175,16 @@ flow:
|
||||
- expr: liveTurnTimeoutMs(env, 180000)
|
||||
- sinceIndex:
|
||||
ref: startIndex
|
||||
- set: settledRecoveryRequests
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []"
|
||||
- set: settledRestartRecoveryRequests
|
||||
value:
|
||||
expr: "settledRecoveryRequests.filter((request) => String(request.prompt ?? '').includes('Your previous turn was interrupted by a gateway restart') && String(request.allInputText ?? '').includes(config.interruptedMarker))"
|
||||
- assert:
|
||||
expr: "settledRestartRecoveryRequests.length === 1 && settledRestartRecoveryRequests[0].cursor === restartRecoveryRequestsBeforeOutbound[0].cursor"
|
||||
message:
|
||||
expr: "`restart recovery provider request was duplicated after delivery: before=${JSON.stringify(restartRecoveryRequestsBeforeOutbound)} settled=${JSON.stringify(settledRestartRecoveryRequests)}`"
|
||||
- set: interruptedMatches
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.slice(startIndex).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === conversationId && candidate.text.includes(config.interruptedMarker))"
|
||||
@@ -172,15 +199,6 @@ flow:
|
||||
expr: "restartNotices.length === 0"
|
||||
message:
|
||||
expr: "`automatic recovery emitted ${restartNotices.length} resend notice(s); outbound=${recentOutboundSummary(state)}`"
|
||||
- set: recoveryRequests
|
||||
value:
|
||||
expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)) : []"
|
||||
- assert:
|
||||
expr: "recoveryRequests.some((request) => String(request.allInputText ?? '').includes('Your previous turn was interrupted by a gateway restart'))"
|
||||
message: restart recovery prompt did not reach the provider
|
||||
- assert:
|
||||
expr: "recoveryRequests.every((request) => !String(request.allInputText ?? '').includes('[OpenClaw heartbeat poll]'))"
|
||||
message: heartbeat poll raced the restart recovery proof
|
||||
- set: recoveryLogs
|
||||
value:
|
||||
expr: readGatewayLogs().slice(gatewayLogCursor)
|
||||
@@ -188,4 +206,4 @@ flow:
|
||||
expr: "recoveryLogs.includes('dispatching restart-safe recovery') && recoveryLogs.includes('restart-safe recovery tool policy retained')"
|
||||
message:
|
||||
expr: "`restart-safe host policy was not observed; logs=${recoveryLogs}`"
|
||||
detailsExpr: "`session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true heartbeatPoll=false\\n${outbound.text}`"
|
||||
detailsExpr: "`session=${sessionKey} plannedTool=${plannedWait.plannedToolName} persistedTail=${interruptedTranscript.lastMessageRole}/${interruptedTranscript.lastAssistantStopReason}/${interruptedTranscript.lastAssistantToolNames?.join(',')} recoveredTail=${postRestartTranscript.lastMessageRole}/${postRestartTranscript.lastAssistantStopReason}/${postRestartTranscript.lastAssistantContentTypes?.join(',')} recoveredMarkers=${interruptedMatches.length} resendNotices=${restartNotices.length} restartSafePolicy=true recoveryPromptHeartbeat=false\\n${outbound.text}`"
|
||||
|
||||
Reference in New Issue
Block a user