diff --git a/packages/ai/src/providers/anthropic.test.ts b/packages/ai/src/providers/anthropic.test.ts index fd4034640e89..96dd86c651ec 100644 --- a/packages/ai/src/providers/anthropic.test.ts +++ b/packages/ai/src/providers/anthropic.test.ts @@ -36,6 +36,16 @@ function createSseResponse(events: Record[] = []): Response { }); } +function createAnthropicSseClient(events: Record[]) { + return { + messages: { + create: vi.fn(() => ({ + asResponse: () => Promise.resolve(createSseResponse(events)), + })), + }, + }; +} + function makeAnthropicModel(overrides: Partial> = {}) { return { id: "claude-sonnet-4-6", @@ -52,6 +62,57 @@ function makeAnthropicModel(overrides: Partial> = {} } satisfies Model<"anthropic-messages">; } +type SimpleAnthropicTestOptions = Omit< + NonNullable[2]>, + "onPayload" +> & { + injectPayload?: Record; + stopBeforeNetwork?: boolean; +}; + +type AnthropicAuthenticationTestCase = { + name: string; + model: Partial>; + key: string; + expected: { apiKey: string | null; authToken: string | null }; + headers?: Record; + absent?: string[]; + useHostFetch?: boolean; + resolveSentinel?: boolean; +}; + +type AnthropicAdaptiveThinkingTestCase = { + name: string; + model: Partial>; + options: SimpleAnthropicTestOptions; + context?: Context; + expected: Record; + absent?: string[]; +}; + +async function captureSimpleAnthropicPayload( + model: Partial>, + options: SimpleAnthropicTestOptions = {}, + context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, +) { + const { injectPayload, stopBeforeNetwork, ...streamOptions } = options; + let capturedPayload: unknown; + const result = await streamSimpleAnthropic(makeAnthropicModel(model), context, { + apiKey: "sk-ant-provider", + ...streamOptions, + onPayload: (payload) => { + capturedPayload = injectPayload + ? { ...(payload as Record), ...injectPayload } + : payload; + if (stopBeforeNetwork) { + throw new Error("stop before network"); + } + return capturedPayload; + }, + }).result(); + return { payload: capturedPayload as Record, result }; +} + function makeSonnet5PrefillContext(): Context { return { messages: [ @@ -103,25 +164,75 @@ describe("Anthropic provider", () => { configureAiTransportHost({}); }); - it("keeps Cloudflare AI Gateway upstream provider auth on the Anthropic API key", async () => { - // Prove the Cloudflare client receives the host-built model fetch. - const hostFetch: typeof fetch = async () => new Response(null, { status: 500 }); - configureAiTransportHost({ buildModelFetch: () => hostFetch }); - const model = makeAnthropicModel({ - provider: "cloudflare-ai-gateway", - baseUrl: "https://gateway.ai.cloudflare.com/v1/account/gateway/anthropic/v1/messages", - headers: { - "cf-aig-authorization": "Bearer gateway-token", + const foundrySentinel = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; + + const authenticationCases: AnthropicAuthenticationTestCase[] = [ + { + name: "keeps Cloudflare AI Gateway upstream provider auth on the Anthropic API key", + model: { + provider: "cloudflare-ai-gateway", + baseUrl: "https://gateway.ai.cloudflare.com/v1/account/gateway/anthropic/v1/messages", + headers: { "cf-aig-authorization": "Bearer gateway-token" }, }, - }); - const context = { - messages: [{ role: "user", content: "hello", timestamp: 1 }], - } satisfies Context; - - streamAnthropic(model, context, { - apiKey: "sk-ant-provider", - }); + key: "sk-ant-provider", + expected: { apiKey: "sk-ant-provider", authToken: null }, + headers: { "cf-aig-authorization": "Bearer gateway-token" }, + absent: ["x-api-key"], + useHostFetch: true, + }, + { + name: "uses bearer auth for Microsoft Foundry Anthropic requests", + model: { + provider: "microsoft-foundry", + baseUrl: "https://example.services.ai.azure.com/anthropic", + authHeader: true, + headers: { "api-key": "stale-foundry-key", "x-api-key": "stale-resource-key" }, + }, + key: "entra-access-token", + expected: { apiKey: null, authToken: "entra-access-token" }, + absent: ["Authorization", "api-key", "x-api-key"], + }, + { + name: "keeps sentinel-backed Foundry Authorization headers on bearer routing", + model: { + provider: "microsoft-foundry", + baseUrl: "https://example.services.ai.azure.com/anthropic", + headers: { Authorization: foundrySentinel }, + }, + key: foundrySentinel, + expected: { apiKey: null, authToken: foundrySentinel }, + resolveSentinel: true, + }, + { + name: "keeps Microsoft Foundry API-key profiles on Anthropic API key auth", + model: { + provider: "microsoft-foundry", + baseUrl: "https://example.services.ai.azure.com/anthropic", + headers: { "api-key": "foundry-resource-key" }, + }, + key: "foundry-resource-key", + expected: { apiKey: "foundry-resource-key", authToken: null }, + }, + ]; + it.each(authenticationCases)("$name", async (testCase) => { + const hostFetch: typeof fetch = async () => new Response(null, { status: 500 }); + if (testCase.useHostFetch || testCase.resolveSentinel) { + configureAiTransportHost({ + buildModelFetch: () => hostFetch, + ...(testCase.resolveSentinel + ? { + resolveSecretSentinel: (value: string) => + value.replaceAll(foundrySentinel, "Bearer entra-access-token"), + } + : {}), + }); + } + streamAnthropic( + makeAnthropicModel(testCase.model), + { messages: [{ role: "user", content: "hello", timestamp: 1 }] }, + { apiKey: testCase.key }, + ); await vi.waitFor(() => expect(anthropicMockState.configs).toHaveLength(1)); const config = anthropicMockState.configs[0] as { apiKey?: string | null; @@ -129,97 +240,16 @@ describe("Anthropic provider", () => { defaultHeaders?: Record; fetch?: unknown; }; - - expect(config.apiKey).toBe("sk-ant-provider"); - expect(config.authToken).toBeNull(); - expect(config.defaultHeaders?.["x-api-key"]).toBeUndefined(); - expect(config.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer gateway-token"); - expect(config.fetch).toBe(hostFetch); - }); - - it("uses bearer auth for Microsoft Foundry Anthropic requests", async () => { - const model = makeAnthropicModel({ - provider: "microsoft-foundry", - baseUrl: "https://example.services.ai.azure.com/anthropic", - authHeader: true, - headers: { - "api-key": "stale-foundry-key", - "x-api-key": "stale-resource-key", - }, - }); - const context = { - messages: [{ role: "user", content: "hello", timestamp: 1 }], - } satisfies Context; - - streamAnthropic(model, context, { - apiKey: "entra-access-token", - }); - - await vi.waitFor(() => expect(anthropicMockState.configs).toHaveLength(1)); - const config = anthropicMockState.configs[0] as { - apiKey?: string | null; - authToken?: string | null; - defaultHeaders?: Record; - }; - - expect(config.apiKey).toBeNull(); - expect(config.authToken).toBe("entra-access-token"); - expect(config.defaultHeaders?.Authorization).toBeUndefined(); - expect(config.defaultHeaders?.["api-key"]).toBeUndefined(); - expect(config.defaultHeaders?.["x-api-key"]).toBeUndefined(); - }); - - it("keeps sentinel-backed Foundry Authorization headers on bearer routing", async () => { - const sentinel = "oc-sent-v2.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.end"; - configureAiTransportHost({ - buildModelFetch: () => async () => new Response(null, { status: 500 }), - resolveSecretSentinel: (value) => value.replaceAll(sentinel, "Bearer entra-access-token"), - }); - const model = makeAnthropicModel({ - provider: "microsoft-foundry", - baseUrl: "https://example.services.ai.azure.com/anthropic", - headers: { Authorization: sentinel }, - }); - - streamAnthropic( - model, - { messages: [{ role: "user", content: "hello", timestamp: 1 }] }, - { - apiKey: sentinel, - }, - ); - - await vi.waitFor(() => expect(anthropicMockState.configs).toHaveLength(1)); - const config = anthropicMockState.configs[0] as { - apiKey?: string | null; - authToken?: string | null; - }; - expect(config.apiKey).toBeNull(); - expect(config.authToken).toBe(sentinel); - }); - - it("keeps Microsoft Foundry API-key profiles on Anthropic API key auth", async () => { - const model = makeAnthropicModel({ - provider: "microsoft-foundry", - baseUrl: "https://example.services.ai.azure.com/anthropic", - headers: { "api-key": "foundry-resource-key" }, - }); - const context = { - messages: [{ role: "user", content: "hello", timestamp: 1 }], - } satisfies Context; - - streamAnthropic(model, context, { - apiKey: "foundry-resource-key", - }); - - await vi.waitFor(() => expect(anthropicMockState.configs).toHaveLength(1)); - const config = anthropicMockState.configs[0] as { - apiKey?: string | null; - authToken?: string | null; - }; - - expect(config.apiKey).toBe("foundry-resource-key"); - expect(config.authToken).toBeNull(); + expect(config).toMatchObject(testCase.expected); + for (const [key, value] of Object.entries(testCase.headers ?? {})) { + expect(config.defaultHeaders?.[key]).toBe(value); + } + for (const key of testCase.absent ?? []) { + expect(config.defaultHeaders?.[key]).toBeUndefined(); + } + if (testCase.useHostFetch) { + expect(config.fetch).toBe(hostFetch); + } }); it("puts Claude subscription billing identity first for OAuth requests", async () => { @@ -254,68 +284,59 @@ describe("Anthropic provider", () => { }); it("keeps aggregate cache billing buckets out of the context total", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_usage", - model: "claude-fable-5", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: null, - }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "Done." }, - }, - { type: "content_block_stop", index: 0 }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 819_661, - cache_creation_input_tokens: 93_130, - iterations: [ - { - type: "compaction", - input_tokens: 12, - output_tokens: 1_000, - cache_read_input_tokens: 819_661, - cache_creation_input_tokens: 93_130, - }, - { - type: "message", - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: 0, - }, - ], - }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_usage", + model: "claude-fable-5", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: null, + }, + }, }, - }; + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "Done." }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 819_661, + cache_creation_input_tokens: 93_130, + iterations: [ + { + type: "compaction", + input_tokens: 12, + output_tokens: 1_000, + cache_read_input_tokens: 819_661, + cache_creation_input_tokens: 93_130, + }, + { + type: "message", + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: 0, + }, + ], + }, + }, + { type: "message_stop" }, + ]); const result = await streamAnthropic( makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), @@ -338,32 +359,23 @@ describe("Anthropic provider", () => { }); it("ignores a message_delta whose usage object is omitted", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_no_delta_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 3, - cache_creation_input_tokens: 4, - }, - }, - }, - { type: "message_delta", delta: { stop_reason: "end_turn" } }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_no_delta_usage", + model: "claude-sonnet-4-6", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 3, + cache_creation_input_tokens: 4, + }, + }, }, - }; + { type: "message_delta", delta: { stop_reason: "end_turn" } }, + { type: "message_stop" }, + ]); const result = await streamAnthropic( makeAnthropicModel({ @@ -386,40 +398,31 @@ describe("Anthropic provider", () => { }); it("prices reported 1-hour cache writes at twice the input rate", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_cache_ttl_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 100, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 1_000_000, - cache_creation: { - ephemeral_5m_input_tokens: 600_000, - ephemeral_1h_input_tokens: 400_000, - }, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { output_tokens: 5 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_cache_ttl_usage", + model: "claude-sonnet-4-6", + usage: { + input_tokens: 100, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 1_000_000, + cache_creation: { + ephemeral_5m_input_tokens: 600_000, + ephemeral_1h_input_tokens: 400_000, + }, + }, + }, }, - }; + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { output_tokens: 5 }, + }, + { type: "message_stop" }, + ]); const result = await streamAnthropic( makeAnthropicModel({ @@ -509,284 +512,141 @@ describe("Anthropic provider", () => { }, ); - it("does not fall back to aggregate usage when the final iteration is malformed", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_invalid_iteration", - model: "claude-fable-5", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 819_661, - cache_creation_input_tokens: 93_130, - iterations: [ - { - type: "message", - input_tokens: "malformed", - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: 0, - }, - ], - }, - }, - { type: "message_stop" }, - ]), - ), - })), + it.each([ + { + name: "does not fall back to aggregate usage when the final iteration is malformed", + id: "msg_invalid_iteration", + model: "claude-fable-5", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: 0, }, - }; - + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 819_661, + cache_creation_input_tokens: 93_130, + iterations: [ + { + type: "message", + input_tokens: "malformed", + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: 0, + }, + ], + }, + expected: { totalTokens: 927_907 }, + context: { state: "unavailable" }, + }, + { + name: "uses complete final usage when message-start prompt buckets are zero placeholders", + id: "msg_zero_start", + model: "claude-fable-5", + initial: { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: 0, + }, + context: { state: "available", promptTokens: 148_874, totalTokens: 163_978 }, + }, + { + name: "does not treat zero start placeholders as complete final prompt usage", + id: "msg_zero_start_partial_delta", + model: "claude-fable-5", + initial: { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + final: { output_tokens: 15_104 }, + context: { state: "unavailable" }, + }, + { + name: "uses accumulated prompt buckets when the final usage update is partial", + id: "msg_partial_final_usage", + model: "claude-sonnet-4-6", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: 500, + }, + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: null, + }, + context: { state: "available", promptTokens: 149_374, totalTokens: 164_478 }, + }, + { + name: "preserves valid message-start billing buckets when a sibling is malformed", + id: "msg_malformed_usage", + model: "claude-sonnet-4-6", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: "malformed", + cache_creation_input_tokens: 500, + }, + final: { input_tokens: 12, output_tokens: 15_104, cache_creation_input_tokens: null }, + expected: { input: 12, output: 15_104, cacheRead: 0, cacheWrite: 500, totalTokens: 15_616 }, + context: { state: "unavailable" }, + }, + ])("$name", async (testCase) => { + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { id: testCase.id, model: testCase.model, usage: testCase.initial }, + }, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: testCase.final }, + { type: "message_stop" }, + ]); const result = await streamAnthropic( - makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), + makeAnthropicModel({ + id: testCase.model, + name: testCase.model === "claude-fable-5" ? "Claude Fable 5" : "Claude Sonnet 4.6", + }), { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, { apiKey: "sk-ant-provider", client: client as never }, ).result(); - - expect(result.usage.totalTokens).toBe(927_907); - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); - }); - - it("uses complete final usage when message-start prompt buckets are zero placeholders", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_zero_start", - model: "claude-fable-5", - usage: { - input_tokens: 0, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: 0, - }, - }, - { type: "message_stop" }, - ]), - ), - })), - }, - }; - - const result = await streamAnthropic( - makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { apiKey: "sk-ant-provider", client: client as never }, - ).result(); - - expect(result.usage.contextUsage).toEqual({ - state: "available", - promptTokens: 148_874, - totalTokens: 163_978, - }); - }); - - it("does not treat zero start placeholders as complete final prompt usage", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_zero_start_partial_delta", - model: "claude-fable-5", - usage: { - input_tokens: 0, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { output_tokens: 15_104 }, - }, - { type: "message_stop" }, - ]), - ), - })), - }, - }; - - const result = await streamAnthropic( - makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { apiKey: "sk-ant-provider", client: client as never }, - ).result(); - - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); - }); - - it("uses accumulated prompt buckets when the final usage update is partial", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_partial_final_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: 500, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: null, - }, - }, - { type: "message_stop" }, - ]), - ), - })), - }, - }; - - const result = await streamAnthropic( - makeAnthropicModel(), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { apiKey: "sk-ant-provider", client: client as never }, - ).result(); - - expect(result.usage.contextUsage).toEqual({ - state: "available", - promptTokens: 149_374, - totalTokens: 164_478, - }); - }); - - it("preserves valid message-start billing buckets when a sibling is malformed", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_malformed_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: "malformed", - cache_creation_input_tokens: 500, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_creation_input_tokens: null, - }, - }, - { type: "message_stop" }, - ]), - ), - })), - }, - }; - - const result = await streamAnthropic( - makeAnthropicModel(), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { apiKey: "sk-ant-provider", client: client as never }, - ).result(); - - expect(result.usage).toMatchObject({ - input: 12, - output: 15_104, - cacheRead: 0, - cacheWrite: 500, - totalTokens: 15_616, - }); - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); + if (testCase.expected) { + expect(result.usage).toMatchObject(testCase.expected); + } + expect(result.usage.contextUsage).toEqual(testCase.context); }); it("preserves provider-signed Anthropic thinking and drops reasoning_content placeholders", async () => { const highSurrogate = String.fromCharCode(0xd83d); const signedThinking = `keep${highSurrogate}signed`; let capturedPayload: unknown; - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_1", - model: "claude-fable-5", - usage: { input_tokens: 1, output_tokens: 0 }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { input_tokens: 1, output_tokens: 1 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_1", + model: "claude-fable-5", + usage: { input_tokens: 1, output_tokens: 0 }, + }, }, - }; + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 1, output_tokens: 1 }, + }, + { type: "message_stop" }, + ]); const stream = streamAnthropic( makeAnthropicModel({ @@ -1509,45 +1369,36 @@ describe("Anthropic provider", () => { ["claude-sonnet-5", "Claude Sonnet 5", "anthropic", "sk-ant-provider"], ["claude-sonnet-5", "Claude Sonnet 5", "anthropic-vertex", "vertex-token"], ])("surfaces structured %s streaming refusals for %s", async (id, name, provider, apiKey) => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { id: "msg_refusal", usage: { input_tokens: 3, output_tokens: 0 } }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "discard this partial output" }, - }, - { type: "content_block_stop", index: 0 }, - { - type: "message_delta", - delta: { - stop_reason: "refusal", - stop_details: { - type: "refusal", - category: "cyber", - explanation: "This request is not allowed.", - }, - }, - usage: { input_tokens: 3, output_tokens: 2 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { id: "msg_refusal", usage: { input_tokens: 3, output_tokens: 0 } }, }, - }; + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "discard this partial output" }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { + stop_reason: "refusal", + stop_details: { + type: "refusal", + category: "cyber", + explanation: "This request is not allowed.", + }, + }, + usage: { input_tokens: 3, output_tokens: 2 }, + }, + { type: "message_stop" }, + ]); const stream = streamAnthropic( makeAnthropicModel({ @@ -1650,80 +1501,71 @@ describe("Anthropic provider", () => { }); it("rebuilds Fable output at a mid-stream server-side fallback boundary", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_fallback", - model: "claude-fable-5", - usage: { input_tokens: 5, output_tokens: 0 }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "thinking", thinking: "" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "thinking_delta", thinking: "pre-boundary reasoning" }, - }, - { type: "content_block_stop", index: 0 }, - { - type: "content_block_start", - index: 1, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 1, - delta: { type: "text_delta", text: "partial " }, - }, - { type: "content_block_stop", index: 1 }, - { - type: "content_block_start", - index: 2, - content_block: { type: "tool_use", id: "call_1", name: "lookup", input: {} }, - }, - { type: "content_block_stop", index: 2 }, - { - type: "content_block_start", - index: 3, - content_block: { - type: "fallback", - from: { model: "claude-fable-5" }, - to: { model: "claude-opus-4-8" }, - }, - }, - { type: "content_block_stop", index: 3 }, - { - type: "content_block_start", - index: 4, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 4, - delta: { type: "text_delta", text: "continued" }, - }, - { type: "content_block_stop", index: 4 }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { input_tokens: 5, output_tokens: 9 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_fallback", + model: "claude-fable-5", + usage: { input_tokens: 5, output_tokens: 0 }, + }, }, - }; + { + type: "content_block_start", + index: 0, + content_block: { type: "thinking", thinking: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "thinking_delta", thinking: "pre-boundary reasoning" }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 1, + delta: { type: "text_delta", text: "partial " }, + }, + { type: "content_block_stop", index: 1 }, + { + type: "content_block_start", + index: 2, + content_block: { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + }, + { type: "content_block_stop", index: 2 }, + { + type: "content_block_start", + index: 3, + content_block: { + type: "fallback", + from: { model: "claude-fable-5" }, + to: { model: "claude-opus-4-8" }, + }, + }, + { type: "content_block_stop", index: 3 }, + { + type: "content_block_start", + index: 4, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 4, + delta: { type: "text_delta", text: "continued" }, + }, + { type: "content_block_stop", index: 4 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 5, output_tokens: 9 }, + }, + { type: "message_stop" }, + ]); const stream = streamAnthropic( makeAnthropicModel({ @@ -1769,52 +1611,43 @@ describe("Anthropic provider", () => { }); it("records a pre-output server-side fallback and keeps the continuation", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_fallback", - model: "claude-fable-5", - usage: { input_tokens: 5, output_tokens: 0 }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { - type: "fallback", - from: { model: "claude-fable-5" }, - to: { model: "claude-opus-4-8" }, - }, - }, - { type: "content_block_stop", index: 0 }, - { - type: "content_block_start", - index: 1, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 1, - delta: { type: "text_delta", text: "Hi!" }, - }, - { type: "content_block_stop", index: 1 }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { input_tokens: 5, output_tokens: 2 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { + id: "msg_fallback", + model: "claude-fable-5", + usage: { input_tokens: 5, output_tokens: 0 }, + }, }, - }; + { + type: "content_block_start", + index: 0, + content_block: { + type: "fallback", + from: { model: "claude-fable-5" }, + to: { model: "claude-opus-4-8" }, + }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 1, + delta: { type: "text_delta", text: "Hi!" }, + }, + { type: "content_block_stop", index: 1 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 5, output_tokens: 2 }, + }, + { type: "message_stop" }, + ]); const stream = streamAnthropic( makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), @@ -1840,49 +1673,40 @@ describe("Anthropic provider", () => { }); it("routes interleaved active content blocks by their event indexes", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "message_start", - message: { id: "msg_interleaved", usage: { input_tokens: 1, output_tokens: 0 } }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "text" }, - }, - { - type: "content_block_start", - index: 1, - content_block: { type: "text" }, - }, - { - type: "content_block_delta", - index: 1, - delta: { type: "text_delta", text: "second" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "first" }, - }, - { type: "content_block_stop", index: 1 }, - { type: "content_block_stop", index: 0 }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { input_tokens: 1, output_tokens: 2 }, - }, - { type: "message_stop" }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "message_start", + message: { id: "msg_interleaved", usage: { input_tokens: 1, output_tokens: 0 } }, }, - }; + { + type: "content_block_start", + index: 0, + content_block: { type: "text" }, + }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text" }, + }, + { + type: "content_block_delta", + index: 1, + delta: { type: "text_delta", text: "second" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "first" }, + }, + { type: "content_block_stop", index: 1 }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 1, output_tokens: 2 }, + }, + { type: "message_stop" }, + ]); const result = await streamAnthropic( makeAnthropicModel(), @@ -1897,27 +1721,18 @@ describe("Anthropic provider", () => { }); it("discards buffered Fable output when the stream fails before terminal status", async () => { - const client = { - messages: { - create: vi.fn(() => ({ - asResponse: () => - Promise.resolve( - createSseResponse([ - { - type: "content_block_start", - index: 0, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "unsafe partial output" }, - }, - ]), - ), - })), + const client = createAnthropicSseClient([ + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, }, - }; + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "unsafe partial output" }, + }, + ]); const stream = streamAnthropic( makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, @@ -1985,29 +1800,11 @@ describe("Anthropic provider", () => { { reasoning: "xhigh", expectedEffort: "high" }, { reasoning: "max", expectedEffort: "max" }, ] as const)("maps Claude 4.6 $reasoning effort", async ({ reasoning, expectedEffort }) => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ - id: "claude-sonnet-4-6", - name: "Claude Sonnet 4.6", - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], - }, - { - apiKey: "sk-ant-provider", - reasoning, - onPayload: (payload) => { - capturedPayload = payload; - }, - }, + const { payload } = await captureSimpleAnthropicPayload( + { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, + { reasoning }, ); - - await stream.result(); - - expect((capturedPayload as { output_config?: unknown }).output_config).toEqual({ - effort: expectedEffort, - }); + expect(payload.output_config).toEqual({ effort: expectedEffort }); }); it.each([ @@ -2026,35 +1823,18 @@ describe("Anthropic provider", () => { ] as const)( "honors proxy effort restrictions for $id", async ({ id, reasoning, thinkingLevelMap, expectedEffort }) => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ - id, - provider: "github-copilot", - thinkingLevelMap, - }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "copilot-token", - reasoning, - onPayload: (payload) => { - capturedPayload = payload; - }, - }, + const { payload } = await captureSimpleAnthropicPayload( + { id, provider: "github-copilot", thinkingLevelMap }, + { apiKey: "copilot-token", reasoning }, ); - - await stream.result(); - - expect((capturedPayload as { output_config?: unknown }).output_config).toEqual({ - effort: expectedEffort, - }); + expect(payload.output_config).toEqual({ effort: expectedEffort }); }, ); - it("uses the Claude Opus 5 adaptive-thinking request contract", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + const adaptiveThinkingCases: AnthropicAdaptiveThinkingTestCase[] = [ + { + name: "uses the Claude Opus 5 adaptive-thinking request contract", + model: { id: "prod-opus", name: "Production Claude", provider: "microsoft-foundry", @@ -2062,77 +1842,41 @@ describe("Anthropic provider", () => { reasoning: false, baseUrl: "https://example.services.ai.azure.com/anthropic", maxTokens: 128_000, - }), - { + }, + options: { temperature: 0.2, injectPayload: { service_tier: "auto", top_p: 0.9, top_k: 40 } }, + context: { messages: [ { role: "user", content: "hello", timestamp: 0 }, { role: "assistant", content: [{ type: "text", text: "prefill" }], timestamp: 0 }, ], } as unknown as Context, - { - apiKey: "sk-ant-provider", - temperature: 0.2, - onPayload: (payload) => { - capturedPayload = { - ...(payload as Record), - service_tier: "auto", - top_p: 0.9, - top_k: 40, - }; - return capturedPayload; - }, + expected: { + messages: [{ role: "user" }], + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "high" }, }, - ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - messages: [{ role: "user" }], - thinking: { type: "adaptive", display: "summarized" }, - output_config: { effort: "high" }, - }); - expect(capturedPayload).not.toHaveProperty("temperature"); - expect(capturedPayload).not.toHaveProperty("top_p"); - expect(capturedPayload).not.toHaveProperty("top_k"); - expect(capturedPayload).not.toHaveProperty("service_tier"); - }); - - it("uses always-on adaptive thinking for Claude Fable 5", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + absent: ["temperature", "top_p", "top_k", "service_tier"], + }, + { + name: "uses always-on adaptive thinking for Claude Fable 5", + model: { id: "prod-primary", name: "Production Claude", provider: "microsoft-foundry", params: { canonicalModelId: "claude-fable-5" }, reasoning: false, baseUrl: "https://example.services.ai.azure.com/anthropic", - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], }, - { - apiKey: "sk-ant-provider", - temperature: 0.2, - onPayload: (payload) => { - capturedPayload = payload; - }, + options: { temperature: 0.2 }, + expected: { + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "high" }, }, - ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - thinking: { type: "adaptive", display: "summarized" }, - output_config: { effort: "high" }, - }); - expect(capturedPayload).not.toHaveProperty("temperature"); - }); - - it("uses mandatory adaptive thinking and default sampling for Claude Mythos 5", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + absent: ["temperature"], + }, + { + name: "uses mandatory adaptive thinking and default sampling for Claude Mythos 5", + model: { id: "prod-mythos", name: "Production Claude", provider: "microsoft-foundry", @@ -2140,206 +1884,87 @@ describe("Anthropic provider", () => { reasoning: false, baseUrl: "https://example.services.ai.azure.com/anthropic", maxTokens: 128_000, - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], }, - { - apiKey: "sk-ant-provider", - reasoning: "off", - temperature: 0.2, - onPayload: (payload) => { - capturedPayload = { - ...(payload as Record), - top_p: 0.9, - top_k: 40, - }; - return capturedPayload; - }, + options: { reasoning: "off", temperature: 0.2, injectPayload: { top_p: 0.9, top_k: 40 } }, + expected: { + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "low" }, }, - ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - thinking: { type: "adaptive", display: "summarized" }, - output_config: { effort: "low" }, - }); - expect(capturedPayload).not.toHaveProperty("temperature"); - expect(capturedPayload).not.toHaveProperty("top_p"); - expect(capturedPayload).not.toHaveProperty("top_k"); - }); - - it("preserves native max effort for Claude Mythos Preview", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + absent: ["temperature", "top_p", "top_k"], + }, + { + name: "preserves native max effort for Claude Mythos Preview", + model: { id: "claude-mythos-preview", name: "Claude Mythos Preview", reasoning: true, maxTokens: 128_000, thinkingLevelMap: { max: "max" }, - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], }, - { - apiKey: "sk-ant-provider", - reasoning: "max", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, - ); - - await stream.result(); - - expect((capturedPayload as { output_config?: unknown }).output_config).toEqual({ - effort: "max", - }); - }); - - it("uses mandatory adaptive thinking for Foundry Mythos Preview", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + options: { reasoning: "max", stopBeforeNetwork: true }, + expected: { output_config: { effort: "max" } }, + }, + { + name: "uses mandatory adaptive thinking for Foundry Mythos Preview", + model: { id: "prod-mythos-preview", name: "Production Claude", provider: "microsoft-foundry", params: { canonicalModelId: "claude-mythos-preview" }, reasoning: false, - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], }, - { - apiKey: "sk-ant-provider", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, - ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - thinking: { type: "adaptive" }, - output_config: { effort: "high" }, - }); - }); - - it("uses adaptive high effort for Foundry Mythos Preview without native max metadata", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + options: { stopBeforeNetwork: true }, + expected: { thinking: { type: "adaptive" }, output_config: { effort: "high" } }, + }, + { + name: "uses adaptive high effort for Foundry Mythos Preview without native max metadata", + model: { id: "prod-mythos-preview", name: "Production Claude", provider: "microsoft-foundry", params: { canonicalModelId: "claude-mythos-preview" }, reasoning: true, - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], }, - { - apiKey: "sk-ant-provider", - reasoning: "max", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, - ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - thinking: { type: "adaptive" }, - output_config: { effort: "high" }, - }); - }); - - it("does not infer adaptive thinking from forward-compatible effort maps", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + options: { reasoning: "max", stopBeforeNetwork: true }, + expected: { thinking: { type: "adaptive" }, output_config: { effort: "high" } }, + }, + { + name: "does not infer adaptive thinking from forward-compatible effort maps", + model: { id: "claude-future", name: "Future Claude", provider: "github-copilot", reasoning: true, thinkingLevelMap: { xhigh: null, max: "max" }, - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], - }, - { - apiKey: "copilot-token", - reasoning: "max", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, }, + options: { apiKey: "copilot-token", reasoning: "max", stopBeforeNetwork: true }, + expected: { thinking: { type: "enabled" } }, + absent: ["output_config"], + }, + { + name: "resolves thinking as disabled when the legacy budget collapses below 1024", + model: { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", reasoning: true, maxTokens: 1024 }, + options: { reasoning: "minimal", stopBeforeNetwork: true }, + expected: { thinking: { type: "disabled" } }, + }, + { + name: "resolves thinking as disabled when the legacy budget is positive but sub-minimum", + model: { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", reasoning: true, maxTokens: 1500 }, + options: { reasoning: "low", stopBeforeNetwork: true }, + expected: { thinking: { type: "disabled" } }, + }, + ]; + + it.each(adaptiveThinkingCases)("$name", async (testCase) => { + const { payload } = await captureSimpleAnthropicPayload( + testCase.model, + testCase.options, + testCase.context, ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ - thinking: { type: "enabled" }, - }); - expect((capturedPayload as { output_config?: unknown }).output_config).toBeUndefined(); - }); - - it("resolves thinking as disabled when the legacy budget collapses below 1024", async () => { - // reasoning:true so the builder enters the thinking block, but an id that - // does not match the adaptive-thinking regex so the budget-based path is used. - const model = makeAnthropicModel({ - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", - reasoning: true, - maxTokens: 1024, - }); - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - model, - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "sk-ant-provider", - reasoning: "minimal", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, - ); - await stream.result(); - expect((capturedPayload as { thinking?: unknown }).thinking).toEqual({ type: "disabled" }); - }); - - it("resolves thinking as disabled when the legacy budget is positive but sub-minimum", async () => { - const model = makeAnthropicModel({ - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", - reasoning: true, - maxTokens: 1500, - }); - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - model, - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "sk-ant-provider", - reasoning: "low", - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, - ); - await stream.result(); - expect((capturedPayload as { thinking?: unknown }).thinking).toEqual({ type: "disabled" }); + expect(payload).toMatchObject(testCase.expected); + for (const property of testCase.absent ?? []) { + expect(payload).not.toHaveProperty(property); + } }); it.each([ @@ -2449,34 +2074,22 @@ describe("Anthropic provider", () => { }); it("uses canonical Claude policy for deployment aliases", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + const { payload } = await captureSimpleAnthropicPayload( + { id: "production-claude", name: "Production Claude", params: { canonicalModelId: "claude-opus-4-8" }, reasoning: false, thinkingLevelMap: { xhigh: "xhigh", max: "max" }, - }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "sk-ant-provider", - reasoning: "xhigh", - temperature: 0.2, - onPayload: (payload) => { - capturedPayload = payload; - }, }, + { reasoning: "xhigh", temperature: 0.2 }, ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ + expect(payload).toMatchObject({ model: "production-claude", thinking: { type: "adaptive" }, output_config: { effort: "xhigh" }, }); - expect(capturedPayload).not.toHaveProperty("temperature"); + expect(payload).not.toHaveProperty("temperature"); }); it.each([ @@ -2486,27 +2099,16 @@ describe("Anthropic provider", () => { ] as const)( "normalizes temperature for canonical $canonicalModelId aliases when thinking is off", async ({ canonicalModelId, expectedTemperature }) => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + const { payload } = await captureSimpleAnthropicPayload( + { id: "production-claude", params: { canonicalModelId }, reasoning: false, thinkingLevelMap: { xhigh: "xhigh", max: "max" }, - }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "sk-ant-provider", - temperature: 0.2, - onPayload: (payload) => { - capturedPayload = payload; - }, }, + { temperature: 0.2 }, ); - - await stream.result(); - - expect((capturedPayload as { temperature?: number }).temperature).toBe(expectedTemperature); + expect(payload.temperature).toBe(expectedTemperature); }, ); @@ -2575,82 +2177,40 @@ describe("Anthropic provider", () => { }); it("honors provider effort restrictions for Claude Fable 5", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ + const { payload } = await captureSimpleAnthropicPayload( + { id: "claude-fable-5", name: "Claude Fable 5", provider: "github-copilot", reasoning: false, thinkingLevelMap: { xhigh: null, max: null }, - }), - { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, - { - apiKey: "copilot-token", - reasoning: "xhigh", - onPayload: (payload) => { - capturedPayload = payload; - }, }, + { apiKey: "copilot-token", reasoning: "xhigh" }, ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ + expect(payload).toMatchObject({ thinking: { type: "adaptive", display: "summarized" }, output_config: { effort: "high" }, }); }); it("uses the Claude Fable 5 contract on Anthropic Vertex", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel({ - id: "claude-fable-5", - name: "Claude Fable 5", - provider: "anthropic-vertex", - }), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], - }, - { - apiKey: "vertex-token", - reasoning: "high", - onPayload: (payload) => { - capturedPayload = payload; - }, - }, + const { payload } = await captureSimpleAnthropicPayload( + { id: "claude-fable-5", name: "Claude Fable 5", provider: "anthropic-vertex" }, + { apiKey: "vertex-token", reasoning: "high" }, ); - - await stream.result(); - - expect(capturedPayload).toMatchObject({ + expect(payload).toMatchObject({ thinking: { type: "adaptive", display: "summarized" }, output_config: { effort: "high" }, }); }); it("forwards simple stop sequences to Anthropic stop_sequences", async () => { - let capturedPayload: unknown; - const stream = streamSimpleAnthropic( - makeAnthropicModel(), - { - messages: [{ role: "user", content: "hello", timestamp: 0 }], - }, - { - apiKey: "sk-ant-provider", - stop: ["STOP"], - onPayload: (payload) => { - capturedPayload = payload; - throw new Error("stop before network"); - }, - }, + const { payload, result } = await captureSimpleAnthropicPayload( + {}, + { stop: ["STOP"], stopBeforeNetwork: true }, ); - - const result = await stream.result(); - expect(result.stopReason).toBe("error"); - expect((capturedPayload as { stop_sequences?: unknown }).stop_sequences).toEqual(["STOP"]); + expect(payload.stop_sequences).toEqual(["STOP"]); }); it("skips unreadable Anthropic provider tools while preserving healthy siblings", async () => { diff --git a/packages/ai/src/transports/anthropic-transport-stream.test.ts b/packages/ai/src/transports/anthropic-transport-stream.test.ts index 4b76f8288c36..b96fc4c89ba8 100644 --- a/packages/ai/src/transports/anthropic-transport-stream.test.ts +++ b/packages/ai/src/transports/anthropic-transport-stream.test.ts @@ -363,295 +363,171 @@ describe("anthropic transport stream", () => { configureAiTransportHost(coreTransportHost); }); - it("keeps aggregate cache billing buckets out of the context total", async () => { - guardedFetchMock.mockResolvedValueOnce( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_usage", - model: "claude-fable-5", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: null, - }, - }, - }, - { - type: "content_block_start", - index: 0, - content_block: { type: "text", text: "" }, - }, - { - type: "content_block_delta", - index: 0, - delta: { type: "text_delta", text: "Done." }, - }, - { type: "content_block_stop", index: 0 }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 819_661, - cache_creation_input_tokens: 93_130, - iterations: [ - { - type: "compaction", - input_tokens: 12, - output_tokens: 1_000, - cache_read_input_tokens: 819_661, - cache_creation_input_tokens: 93_130, - }, - { - type: "message", - input_tokens: 12, - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: 0, - }, - ], - }, - }, - { type: "message_stop" }, - ]), - ); - - const result = await runTransportStream( - makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, - { apiKey: "sk-ant-api" } as AnthropicStreamOptions, - ); - - expect(result.usage).toMatchObject({ - input: 12, - output: 15_104, - cacheRead: 819_661, - cacheWrite: 93_130, - contextUsage: { - state: "available", - promptTokens: 148_874, - totalTokens: 163_978, + it.each([ + { + name: "keeps aggregate cache billing buckets out of the context total", + id: "msg_usage", + model: "claude-fable-5", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: null, }, - totalTokens: 927_907, - }); - }); - - it("does not fall back to aggregate usage when the final iteration is malformed", async () => { - guardedFetchMock.mockResolvedValueOnce( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_invalid_iteration", - model: "claude-fable-5", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 819_661, + cache_creation_input_tokens: 93_130, + iterations: [ + { + type: "compaction", input_tokens: 12, - output_tokens: 15_104, + output_tokens: 1_000, cache_read_input_tokens: 819_661, cache_creation_input_tokens: 93_130, - iterations: [ - { - type: "message", - input_tokens: "malformed", - output_tokens: 15_104, - cache_read_input_tokens: 148_862, - cache_creation_input_tokens: 0, - }, - ], }, - }, - { type: "message_stop" }, - ]), - ); - - const result = await runTransportStream( - makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, - { apiKey: "sk-ant-api" } as AnthropicStreamOptions, - ); - - expect(result.usage.totalTokens).toBe(927_907); - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); - }); - - it("uses complete final usage when message-start prompt buckets are zero placeholders", async () => { - guardedFetchMock.mockResolvedValueOnce( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_zero_start", - model: "claude-fable-5", - usage: { - input_tokens: 0, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { + { + type: "message", input_tokens: 12, output_tokens: 15_104, cache_read_input_tokens: 148_862, cache_creation_input_tokens: 0, }, - }, - { type: "message_stop" }, - ]), - ); - - const result = await runTransportStream( - makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, - { apiKey: "sk-ant-api" } as AnthropicStreamOptions, - ); - - expect(result.usage.contextUsage).toEqual({ - state: "available", - promptTokens: 148_874, - totalTokens: 163_978, - }); - }); - - it("does not treat zero start placeholders as complete final prompt usage", async () => { - guardedFetchMock.mockResolvedValueOnce( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_zero_start_partial_delta", - model: "claude-fable-5", - usage: { - input_tokens: 0, - output_tokens: 0, - cache_read_input_tokens: 0, - cache_creation_input_tokens: 0, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { output_tokens: 15_104 }, - }, - { type: "message_stop" }, - ]), - ); - - const result = await runTransportStream( - makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }), - { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, - { apiKey: "sk-ant-api" } as AnthropicStreamOptions, - ); - - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); - }); - - it("uses accumulated prompt buckets when the final usage update is partial", async () => { - guardedFetchMock.mockResolvedValueOnce( - createSseResponse([ - { - type: "message_start", - message: { - id: "msg_partial_final_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: 120_000, - cache_creation_input_tokens: 500, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, + ], + }, + content: true, + expected: { + input: 12, + output: 15_104, + cacheRead: 819_661, + cacheWrite: 93_130, + totalTokens: 927_907, + }, + context: { state: "available", promptTokens: 148_874, totalTokens: 163_978 }, + }, + { + name: "does not fall back to aggregate usage when the final iteration is malformed", + id: "msg_invalid_iteration", + model: "claude-fable-5", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: 0, + }, + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 819_661, + cache_creation_input_tokens: 93_130, + iterations: [ + { + type: "message", + input_tokens: "malformed", output_tokens: 15_104, cache_read_input_tokens: 148_862, - cache_creation_input_tokens: null, + cache_creation_input_tokens: 0, }, - }, - { type: "message_stop" }, - ]), - ); - - const result = await runTransportStream( - makeAnthropicTransportModel(), - { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, - { apiKey: "sk-ant-api" } as AnthropicStreamOptions, - ); - - expect(result.usage.contextUsage).toEqual({ - state: "available", - promptTokens: 149_374, - totalTokens: 164_478, - }); - }); - - it("preserves valid message-start billing buckets when a sibling is malformed", async () => { + ], + }, + expected: { totalTokens: 927_907 }, + context: { state: "unavailable" }, + }, + { + name: "uses complete final usage when message-start prompt buckets are zero placeholders", + id: "msg_zero_start", + model: "claude-fable-5", + initial: { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: 0, + }, + context: { state: "available", promptTokens: 148_874, totalTokens: 163_978 }, + }, + { + name: "does not treat zero start placeholders as complete final prompt usage", + id: "msg_zero_start_partial_delta", + model: "claude-fable-5", + initial: { + input_tokens: 0, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + final: { output_tokens: 15_104 }, + context: { state: "unavailable" }, + }, + { + name: "uses accumulated prompt buckets when the final usage update is partial", + id: "msg_partial_final_usage", + model: "claude-sonnet-4-6", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 120_000, + cache_creation_input_tokens: 500, + }, + final: { + input_tokens: 12, + output_tokens: 15_104, + cache_read_input_tokens: 148_862, + cache_creation_input_tokens: null, + }, + context: { state: "available", promptTokens: 149_374, totalTokens: 164_478 }, + }, + { + name: "preserves valid message-start billing buckets when a sibling is malformed", + id: "msg_malformed_usage", + model: "claude-sonnet-4-6", + initial: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: "malformed", + cache_creation_input_tokens: 500, + }, + final: { input_tokens: 12, output_tokens: 15_104, cache_creation_input_tokens: null }, + expected: { input: 12, output: 15_104, cacheRead: 0, cacheWrite: 500, totalTokens: 15_616 }, + context: { state: "unavailable" }, + }, + ])("$name", async (testCase) => { + const textEvents = testCase.content + ? [ + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Done." } }, + { type: "content_block_stop", index: 0 }, + ] + : []; guardedFetchMock.mockResolvedValueOnce( createSseResponse([ { type: "message_start", - message: { - id: "msg_malformed_usage", - model: "claude-sonnet-4-6", - usage: { - input_tokens: 12, - output_tokens: 0, - cache_read_input_tokens: "malformed", - cache_creation_input_tokens: 500, - }, - }, - }, - { - type: "message_delta", - delta: { stop_reason: "end_turn" }, - usage: { - input_tokens: 12, - output_tokens: 15_104, - cache_creation_input_tokens: null, - }, + message: { id: testCase.id, model: testCase.model, usage: testCase.initial }, }, + ...textEvents, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: testCase.final }, { type: "message_stop" }, ]), ); - const result = await runTransportStream( - makeAnthropicTransportModel(), + makeAnthropicTransportModel({ + id: testCase.model, + name: testCase.model === "claude-fable-5" ? "Claude Fable 5" : "Claude Sonnet 4.6", + }), { messages: [{ role: "user", content: "hello" }] } as AnthropicStreamContext, { apiKey: "sk-ant-api" } as AnthropicStreamOptions, ); - - expect(result.usage).toMatchObject({ - input: 12, - output: 15_104, - cacheRead: 0, - cacheWrite: 500, - totalTokens: 15_616, - }); - expect(result.usage.contextUsage).toEqual({ state: "unavailable" }); + if (testCase.expected) { + expect(result.usage).toMatchObject(testCase.expected); + } + expect(result.usage.contextUsage).toEqual(testCase.context); }); it("tags pre-tool narration as commentary when a proxy mislabels stop_reason (pioneer/Bedrock)", async () => { @@ -2887,501 +2763,248 @@ describe("anthropic transport stream", () => { expect(toolUse.input).toEqual({}); }); - it("replays reasoning_content from compatible Anthropic thinking blocks", async () => { - const highSurrogate = String.fromCharCode(0xd83d); - await runTransportStream( - makeAnthropicTransportModel({ + it.each([ + { + name: "replays reasoning_content from compatible Anthropic thinking blocks", + model: { id: "mimo-v2.6-pro", name: "MiMo V2.6 Pro", provider: "xiaomi", baseUrl: "https://token-plan-ams.xiaomimimo.com/anthropic", - }), - { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "xiaomi", - api: "anthropic-messages", - model: "mimo-v2.6-pro", - stopReason: "stop", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: `Need${highSurrogate} to answer politely.`, - thinkingSignature: "reasoning_content", - }, - { type: "text", text: "Hello!" }, - { - type: "thinking", - thinking: "Then ask a follow-up.", - thinkingSignature: "reasoning_content", - }, - ], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-xiaomi-test", - reasoning: "high", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage.reasoning_content).toBe( - "Need to answer politely.\nThen ask a follow-up.", - ); - expect(assistantMessage).not.toHaveProperty("reasoning"); - expect(assistantMessage).not.toHaveProperty("reasoning_text"); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: "Need to answer politely.", - signature: "reasoning_content", }, - { type: "text", text: "Hello!" }, - { - type: "thinking", - thinking: "Then ask a follow-up.", - signature: "reasoning_content", - }, - ]); - }); - - it("preserves provider-signed Anthropic thinking text on replay", async () => { - const highSurrogate = String.fromCharCode(0xd83d); - const signedThinking = `keep${highSurrogate}signed`; - await runTransportStream( - makeAnthropicTransportModel({ - id: "claude-fable-5", - name: "Claude Fable 5", - }), - { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "anthropic", - api: "anthropic-messages", - model: "claude-fable-5", - stopReason: "stop", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: signedThinking, - thinkingSignature: "sig_1", - }, - { - type: "thinking", - thinking: "", - thinkingSignature: "sig_omitted", - }, - ], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - reasoning: "high", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: signedThinking, - signature: "sig_1", - }, - { - type: "thinking", - thinking: "", - signature: "sig_omitted", - }, - ]); - }); - - it("replaces a completed thinking-only turn when the current request disables thinking", async () => { - await runTransportStream( - makeAnthropicTransportModel(), - { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "anthropic", - api: "anthropic-messages", - model: "claude-sonnet-4-6", - stopReason: "stop", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: "private reasoning", - thinkingSignature: "sig_1", - }, - { - type: "thinking", - thinking: "[Reasoning redacted]", - thinkingSignature: "opaque_1", - redacted: true, - }, - ], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - } as AnthropicStreamOptions, - ); - - const payload = latestAnthropicRequest().payload; - const assistantMessage = findRecord(payload.messages, (record) => record.role === "assistant"); - expect(payload.thinking).toEqual({ type: "disabled" }); - expect(assistantMessage.content).toEqual([ - { type: "text", text: "[assistant reasoning omitted]" }, - ]); - }); - - it("preserves signed thinking for an active tool turn when new thinking is disabled", async () => { - await runTransportStream( - makeAnthropicTransportModel(), - { - messages: [ - { role: "user", content: "look it up" }, - { - role: "assistant", - provider: "anthropic", - api: "anthropic-messages", - model: "claude-sonnet-4-6", - stopReason: "toolUse", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: "call lookup", - thinkingSignature: "sig_tool", - }, - { type: "toolCall", id: "call_1", name: "lookup", arguments: {} }, - ], - }, - { - role: "toolResult", - toolCallId: "call_1", - toolName: "lookup", - content: [{ type: "text", text: "42" }], - isError: false, - }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage.content).toEqual([ - { type: "thinking", thinking: "call lookup", signature: "sig_tool" }, - { type: "tool_use", id: "call_1", name: "lookup", input: {} }, - ]); - }); - - it("backfills empty reasoning_content thinking blocks for compatible Anthropic tool-use replays", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + content: [ + { + type: "thinking", + thinking: `Need${String.fromCharCode(0xd83d)} to answer politely.`, + thinkingSignature: "reasoning_content", + }, + { type: "text", text: "Hello!" }, + { + type: "thinking", + thinking: "Then ask a follow-up.", + thinkingSignature: "reasoning_content", + }, + ], + options: { apiKey: "sk-xiaomi-test", reasoning: "high" }, + expectedContent: [ + { type: "thinking", thinking: "Need to answer politely.", signature: "reasoning_content" }, + { type: "text", text: "Hello!" }, + { type: "thinking", thinking: "Then ask a follow-up.", signature: "reasoning_content" }, + ], + expectedReasoning: "Need to answer politely.\nThen ask a follow-up.", + absent: ["reasoning", "reasoning_text"], + }, + { + name: "preserves provider-signed Anthropic thinking text on replay", + model: { id: "claude-fable-5", name: "Claude Fable 5" }, + content: [ + { + type: "thinking", + thinking: `keep${String.fromCharCode(0xd83d)}signed`, + thinkingSignature: "sig_1", + }, + { type: "thinking", thinking: "", thinkingSignature: "sig_omitted" }, + ], + options: { apiKey: "sk-ant-api", reasoning: "high" }, + expectedContent: [ + { + type: "thinking", + thinking: `keep${String.fromCharCode(0xd83d)}signed`, + signature: "sig_1", + }, + { type: "thinking", thinking: "", signature: "sig_omitted" }, + ], + }, + { + name: "replaces a completed thinking-only turn when the current request disables thinking", + model: { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, + content: [ + { type: "thinking", thinking: "private reasoning", thinkingSignature: "sig_1" }, + { + type: "thinking", + thinking: "[Reasoning redacted]", + thinkingSignature: "opaque_1", + redacted: true, + }, + ], + options: { apiKey: "sk-ant-api" }, + expectedContent: [{ type: "text", text: "[assistant reasoning omitted]" }], + expectedThinking: { type: "disabled" }, + }, + { + name: "preserves signed thinking for an active tool turn when new thinking is disabled", + model: { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6" }, + firstUser: "look it up", + content: [ + { type: "thinking", thinking: "call lookup", thinkingSignature: "sig_tool" }, + { type: "toolCall", id: "call_1", name: "lookup", arguments: {} }, + ], + toolResult: { toolName: "lookup", text: "42" }, + options: { apiKey: "sk-ant-api" }, + expectedContent: [ + { type: "thinking", thinking: "call lookup", signature: "sig_tool" }, + { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + ], + }, + { + name: "backfills empty reasoning_content thinking blocks for compatible Anthropic tool-use replays", + model: { id: "mimo-v2.6-pro", name: "MiMo V2.6 Pro", provider: "xiaomi", baseUrl: "https://token-plan-ams.xiaomimimo.com/anthropic", - }), - { - messages: [ - { role: "user", content: "look this up" }, - { - role: "assistant", - provider: "xiaomi", - api: "anthropic-messages", - model: "mimo-v2.6-pro", - stopReason: "toolUse", - timestamp: 0, - content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], - }, - { - role: "toolResult", - toolCallId: "call_1", - content: [{ type: "text", text: "found" }], - isError: false, - }, - { role: "user", content: "continue" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-xiaomi-test", - reasoning: "high", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage).not.toHaveProperty("reasoning_content"); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: "", - signature: "reasoning_content", }, - { type: "tool_use", id: "call_1", name: "lookup", input: {} }, - ]); - }); - - it("backfills MiMo v2-flash tool-use replay when OpenClaw thinking is off", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + firstUser: "look this up", + content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], + toolResult: { text: "found", continueAfter: true }, + options: { apiKey: "sk-xiaomi-test", reasoning: "high" }, + expectedContent: [ + { type: "thinking", thinking: "", signature: "reasoning_content" }, + { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + ], + absent: ["reasoning_content"], + }, + { + name: "backfills MiMo v2-flash tool-use replay when OpenClaw thinking is off", + model: { id: "mimo-v2-flash", name: "MiMo V2 Flash", provider: "xiaomi", baseUrl: "https://api.xiaomimimo.com/anthropic", reasoning: false, - }), - { - messages: [ - { role: "user", content: "look this up" }, - { - role: "assistant", - provider: "xiaomi", - api: "anthropic-messages", - model: "mimo-v2-flash", - stopReason: "toolUse", - timestamp: 0, - content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], - }, - { - role: "toolResult", - toolCallId: "call_1", - content: [{ type: "text", text: "found" }], - isError: false, - }, - { role: "user", content: "continue" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-xiaomi-test", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(latestAnthropicRequest().payload).not.toHaveProperty("thinking"); - expect(assistantMessage).not.toHaveProperty("reasoning_content"); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: "", - signature: "reasoning_content", }, - { type: "tool_use", id: "call_1", name: "lookup", input: {} }, - ]); - }); - - it("backfills empty reasoning_content thinking blocks for compatible Anthropic text replays", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + firstUser: "look this up", + content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], + toolResult: { text: "found", continueAfter: true }, + options: { apiKey: "sk-xiaomi-test" }, + expectedContent: [ + { type: "thinking", thinking: "", signature: "reasoning_content" }, + { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + ], + absent: ["reasoning_content"], + omitThinking: true, + }, + { + name: "backfills empty reasoning_content thinking blocks for compatible Anthropic text replays", + model: { id: "mimo-v2.6-pro", name: "MiMo V2.6 Pro", provider: "xiaomi", baseUrl: "https://token-plan-ams.xiaomimimo.com/anthropic", - }), - { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "xiaomi", - api: "anthropic-messages", - model: "mimo-v2.6-pro", - stopReason: "stop", - timestamp: 0, - content: [{ type: "text", text: "Hello!" }], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-xiaomi-test", - reasoning: "high", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage).not.toHaveProperty("reasoning_content"); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: "", - signature: "reasoning_content", }, - { type: "text", text: "Hello!" }, - ]); - }); - - it("does not backfill reasoning_content for generic Anthropic-compatible tool-use replays", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + content: [{ type: "text", text: "Hello!" }], + options: { apiKey: "sk-xiaomi-test", reasoning: "high" }, + expectedContent: [ + { type: "thinking", thinking: "", signature: "reasoning_content" }, + { type: "text", text: "Hello!" }, + ], + absent: ["reasoning_content"], + }, + { + name: "does not backfill reasoning_content for generic Anthropic-compatible tool-use replays", + model: { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6", provider: "gateway", baseUrl: "https://gateway.example.com/anthropic", - }), - { - messages: [ - { role: "user", content: "look this up" }, - { - role: "assistant", - provider: "gateway", - api: "anthropic-messages", - model: "claude-sonnet-4-6", - stopReason: "toolUse", - timestamp: 0, - content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], - }, - { - role: "toolResult", - toolCallId: "call_1", - content: [{ type: "text", text: "found" }], - isError: false, - }, - { role: "user", content: "continue" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-gateway-test", - reasoning: "high", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage).not.toHaveProperty("reasoning_content"); - expect(assistantMessage.content).toEqual([ - { type: "tool_use", id: "call_1", name: "lookup", input: {} }, - ]); - }); - - it("replays observed reasoning_content for compatible Anthropic routes when thinking is disabled", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + }, + firstUser: "look this up", + content: [{ type: "toolCall", id: "call_1", name: "lookup", arguments: {} }], + toolResult: { text: "found", continueAfter: true }, + options: { apiKey: "sk-gateway-test", reasoning: "high" }, + expectedContent: [{ type: "tool_use", id: "call_1", name: "lookup", input: {} }], + absent: ["reasoning_content"], + }, + { + name: "replays observed reasoning_content for compatible Anthropic routes when thinking is disabled", + model: { id: "mimo-v2.6-pro", name: "MiMo V2.6 Pro", provider: "xiaomi", baseUrl: "https://token-plan-ams.xiaomimimo.com/anthropic", - }), - { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "xiaomi", - api: "anthropic-messages", - model: "mimo-v2.6-pro", - stopReason: "stop", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: "Need to answer politely.", - thinkingSignature: "reasoning_content", - }, - { type: "text", text: "Hello!" }, - ], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-xiaomi-test", - } as AnthropicStreamOptions, - ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(latestAnthropicRequest().payload.thinking).toEqual({ type: "disabled" }); - expect(assistantMessage.reasoning_content).toBe("Need to answer politely."); - expect(assistantMessage.content).toEqual([ - { - type: "thinking", - thinking: "Need to answer politely.", - signature: "reasoning_content", }, - { type: "text", text: "Hello!" }, - ]); - }); - - it("does not replay synthetic reasoning_content to native Anthropic models", async () => { - await runTransportStream( - makeAnthropicTransportModel({ + content: [ + { + type: "thinking", + thinking: "Need to answer politely.", + thinkingSignature: "reasoning_content", + }, + { type: "text", text: "Hello!" }, + ], + options: { apiKey: "sk-xiaomi-test" }, + expectedContent: [ + { type: "thinking", thinking: "Need to answer politely.", signature: "reasoning_content" }, + { type: "text", text: "Hello!" }, + ], + expectedReasoning: "Need to answer politely.", + expectedThinking: { type: "disabled" }, + }, + { + name: "does not replay synthetic reasoning_content to native Anthropic models", + model: { id: "claude-sonnet-4-6", name: "Claude Sonnet 4.6", provider: "anthropic", baseUrl: "https://api.anthropic.com", - }), + }, + content: [ + { + type: "thinking", + thinking: "Private replay text.", + thinkingSignature: "reasoning_content", + }, + { type: "text", text: "Visible reply." }, + ], + options: { apiKey: "sk-ant-api" }, + expectedContent: [{ type: "text", text: "Visible reply." }], + absent: ["reasoning_content"], + }, + ])("$name", async (testCase) => { + const provider = testCase.model.provider ?? "anthropic"; + const messages: Record[] = [ + { role: "user", content: testCase.firstUser ?? "hello" }, { - messages: [ - { role: "user", content: "hello" }, - { - role: "assistant", - provider: "anthropic", - api: "anthropic-messages", - model: "claude-sonnet-4-6", - stopReason: "stop", - timestamp: 0, - content: [ - { - type: "thinking", - thinking: "Private replay text.", - thinkingSignature: "reasoning_content", - }, - { type: "text", text: "Visible reply." }, - ], - }, - { role: "user", content: "again" }, - ], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - } as AnthropicStreamOptions, + role: "assistant", + provider, + api: "anthropic-messages", + model: testCase.model.id, + stopReason: testCase.toolResult ? "toolUse" : "stop", + timestamp: 0, + content: testCase.content, + }, + ]; + if (testCase.toolResult) { + messages.push({ + role: "toolResult", + toolCallId: "call_1", + ...(testCase.toolResult.toolName ? { toolName: testCase.toolResult.toolName } : {}), + content: [{ type: "text", text: testCase.toolResult.text }], + isError: false, + }); + } + if (!testCase.toolResult || testCase.toolResult.continueAfter) { + messages.push({ role: "user", content: testCase.toolResult ? "continue" : "again" }); + } + await runTransportStream( + makeAnthropicTransportModel(testCase.model), + { messages } as unknown as AnthropicStreamContext, + testCase.options as AnthropicStreamOptions, ); - - const assistantMessage = findRecord( - latestAnthropicRequest().payload.messages, - (record) => record.role === "assistant", - ); - expect(assistantMessage).not.toHaveProperty("reasoning_content"); - expect(assistantMessage.content).toEqual([{ type: "text", text: "Visible reply." }]); + const payload = latestAnthropicRequest().payload; + const assistantMessage = findRecord(payload.messages, (record) => record.role === "assistant"); + expect(assistantMessage.content).toEqual(testCase.expectedContent); + if (testCase.expectedReasoning) { + expect(assistantMessage.reasoning_content).toBe(testCase.expectedReasoning); + } + if (testCase.expectedThinking) { + expect(payload.thinking).toEqual(testCase.expectedThinking); + } + if (testCase.omitThinking) { + expect(payload).not.toHaveProperty("thinking"); + } + for (const property of testCase.absent ?? []) { + expect(assistantMessage).not.toHaveProperty(property); + } }); it.each([ @@ -3929,153 +3552,89 @@ describe("anthropic transport stream", () => { expect(cancelCalled).toBe(true); }); - it("maps unsupported xhigh to high effort for Claude 4.6 transport runs", async () => { - const model = makeAnthropicTransportModel({ - id: "claude-opus-4-6", - name: "Claude Opus 4.6", - maxTokens: 8192, - }); - + it.each([ + { + name: "maps unsupported xhigh to high effort for Claude 4.6 transport runs", + model: { id: "claude-opus-4-6", name: "Claude Opus 4.6", maxTokens: 8192 }, + message: "Think deeply.", + options: { apiKey: "sk-ant-api", reasoning: "xhigh" }, + expected: { + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "high" }, + }, + absent: ["tool_choice"], + }, + { + name: "does not infer adaptive thinking from forward-compatible effort maps", + model: { + id: "claude-future", + name: "Future Claude", + provider: "github-copilot", + reasoning: true, + thinkingLevelMap: { xhigh: null, max: "max" }, + }, + message: "Think as much as supported.", + options: { apiKey: "copilot-token", reasoning: "max" }, + expected: { thinking: { type: "enabled", budget_tokens: 7168 } }, + absent: ["output_config"], + }, + { + name: "resolves thinking as disabled when the legacy budget collapses to zero", + model: { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", reasoning: true, maxTokens: 1024 }, + message: "hello", + options: { apiKey: "test-token", reasoning: "minimal" }, + expected: { thinking: { type: "disabled" } }, + }, + { + name: "resolves thinking as disabled when the legacy budget is positive but sub-minimum", + model: { id: "claude-haiku-4-5", name: "Claude Haiku 4.5", reasoning: true, maxTokens: 1500 }, + message: "hello", + options: { apiKey: "test-token", reasoning: "low" }, + expected: { thinking: { type: "disabled" } }, + }, + { + name: "honors provider effort restrictions for transport runs", + model: { + id: "claude-opus-4.7-1m-internal", + name: "Claude Opus 4.7", + provider: "github-copilot", + maxTokens: 64_000, + thinkingLevelMap: { xhigh: "xhigh" }, + }, + message: "Think as much as supported.", + options: { apiKey: "copilot-token", reasoning: "max" }, + expected: { output_config: { effort: "xhigh" } }, + }, + { + name: "uses canonical Claude policy for transport deployment aliases", + model: { + id: "production-claude", + name: "Production Claude", + params: { canonicalModelId: "claude-opus-4-8" }, + reasoning: false, + thinkingLevelMap: { xhigh: "xhigh", max: "max" }, + maxTokens: 8192, + }, + message: "Think extra hard.", + options: { apiKey: "sk-ant-api", reasoning: "xhigh", temperature: 0.2 }, + expected: { + model: "production-claude", + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "xhigh" }, + }, + absent: ["temperature"], + }, + ])("$name", async (testCase) => { await runTransportStream( - model, - { - messages: [{ role: "user", content: "Think deeply." }], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - reasoning: "xhigh", - } as AnthropicStreamOptions, + makeAnthropicTransportModel(testCase.model), + { messages: [{ role: "user", content: testCase.message }] } as AnthropicStreamContext, + testCase.options as AnthropicStreamOptions, ); - const payload = latestAnthropicRequest().payload; - expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); - expect(payload.output_config).toEqual({ effort: "high" }); - expect(payload.tool_choice).toBeUndefined(); - }); - - it("does not infer adaptive thinking from forward-compatible effort maps", async () => { - const model = makeAnthropicTransportModel({ - id: "claude-future", - name: "Future Claude", - provider: "github-copilot", - reasoning: true, - thinkingLevelMap: { xhigh: null, max: "max" }, - }); - - await runTransportStream( - model, - { - messages: [{ role: "user", content: "Think as much as supported." }], - } as AnthropicStreamContext, - { - apiKey: "copilot-token", - reasoning: "max", - } as AnthropicStreamOptions, - ); - - const payload = latestAnthropicRequest().payload; - expect(payload.thinking).toEqual({ type: "enabled", budget_tokens: 7168 }); - expect(payload.output_config).toBeUndefined(); - }); - - it("resolves thinking as disabled when the legacy budget collapses to zero", async () => { - // reasoning:true so the builder enters the thinking block, but an id that - // does not match the adaptive-thinking regex so the budget-based path is used. - const model = makeAnthropicTransportModel({ - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", - reasoning: true, - maxTokens: 1024, - }); - - await runTransportStream( - model, - { - messages: [{ role: "user", content: "hello" }], - } as AnthropicStreamContext, - { - apiKey: "test-token", - reasoning: "minimal", - } as AnthropicStreamOptions, - ); - - const payload = latestAnthropicRequest().payload; - expect(payload.thinking).toEqual({ type: "disabled" }); - }); - - it("resolves thinking as disabled when the legacy budget is positive but sub-minimum", async () => { - const model = makeAnthropicTransportModel({ - id: "claude-haiku-4-5", - name: "Claude Haiku 4.5", - reasoning: true, - maxTokens: 1500, - }); - - await runTransportStream( - model, - { - messages: [{ role: "user", content: "hello" }], - } as AnthropicStreamContext, - { - apiKey: "test-token", - reasoning: "low", - } as AnthropicStreamOptions, - ); - - const payload = latestAnthropicRequest().payload; - expect(payload.thinking).toEqual({ type: "disabled" }); - }); - - it("honors provider effort restrictions for transport runs", async () => { - const model = makeAnthropicTransportModel({ - id: "claude-opus-4.7-1m-internal", - name: "Claude Opus 4.7", - provider: "github-copilot", - maxTokens: 64_000, - thinkingLevelMap: { xhigh: "xhigh" }, - }); - - await runTransportStream( - model, - { - messages: [{ role: "user", content: "Think as much as supported." }], - } as AnthropicStreamContext, - { - apiKey: "copilot-token", - reasoning: "max", - } as AnthropicStreamOptions, - ); - - expect(latestAnthropicRequest().payload.output_config).toEqual({ effort: "xhigh" }); - }); - - it("uses canonical Claude policy for transport deployment aliases", async () => { - const model = makeAnthropicTransportModel({ - id: "production-claude", - name: "Production Claude", - params: { canonicalModelId: "claude-opus-4-8" }, - reasoning: false, - thinkingLevelMap: { xhigh: "xhigh", max: "max" }, - maxTokens: 8192, - }); - - await runTransportStream( - model, - { - messages: [{ role: "user", content: "Think extra hard." }], - } as AnthropicStreamContext, - { - apiKey: "sk-ant-api", - reasoning: "xhigh", - temperature: 0.2, - } as AnthropicStreamOptions, - ); - - const payload = latestAnthropicRequest().payload; - expect(payload.model).toBe("production-claude"); - expect(payload.thinking).toEqual({ type: "adaptive", display: "summarized" }); - expect(payload.output_config).toEqual({ effort: "xhigh" }); - expect(payload).not.toHaveProperty("temperature"); + expect(payload).toMatchObject(testCase.expected); + for (const property of testCase.absent ?? []) { + expect(payload).not.toHaveProperty(property); + } }); it.each([ @@ -4106,77 +3665,56 @@ describe("anthropic transport stream", () => { it.each([ { - name: "defaults to adaptive high", - reasoning: undefined, - thinking: { type: "adaptive", display: "summarized" }, - effort: { effort: "high" }, - toolChoice: { type: "auto" }, - }, - { - name: "allows explicit off", - reasoning: "off" as const, - thinking: { type: "disabled" }, - effort: undefined, - toolChoice: { type: "any" }, - }, - ])("supports Claude Opus 5 transport: $name", async (testCase) => { - const model = makeAnthropicTransportModel({ + name: "supports Claude Opus 5 transport: defaults to adaptive high", id: "claude-opus-5", - name: "Claude Opus 5", - maxTokens: 128_000, - }); - - await runTransportStream(model, makeSonnet5PrefillContext(), { - apiKey: "sk-ant-api", - reasoning: testCase.reasoning, - temperature: 0.2, - toolChoice: "any", - } as AnthropicStreamOptions); - - const payload = latestAnthropicRequest().payload; - expect(payload).toMatchObject({ - max_tokens: 128_000, - messages: [{ role: "user" }], - thinking: testCase.thinking, - tool_choice: testCase.toolChoice, - }); - expect(payload).not.toHaveProperty("temperature"); - if (testCase.effort) { - expect(payload.output_config).toEqual(testCase.effort); - } else { - expect(payload).not.toHaveProperty("output_config"); - } - }); - - it.each([ - { - name: "defaults to adaptive high", + modelName: "Claude Opus 5", reasoning: undefined, thinking: { type: "adaptive", display: "summarized" }, effort: { effort: "high" }, toolChoice: { type: "auto" }, }, { - name: "allows explicit off", + name: "supports Claude Opus 5 transport: allows explicit off", + id: "claude-opus-5", + modelName: "Claude Opus 5", reasoning: "off" as const, thinking: { type: "disabled" }, effort: undefined, toolChoice: { type: "any" }, }, - ])("supports Claude Sonnet 5 transport: $name", async (testCase) => { - const model = makeAnthropicTransportModel({ + { + name: "supports Claude Sonnet 5 transport: defaults to adaptive high", id: "claude-sonnet-5", - name: "Claude Sonnet 5", - maxTokens: 128_000, - }); - - await runTransportStream(model, makeSonnet5PrefillContext(), { - apiKey: "sk-ant-api", - reasoning: testCase.reasoning, - temperature: 0.2, - toolChoice: "any", - } as AnthropicStreamOptions); - + modelName: "Claude Sonnet 5", + reasoning: undefined, + thinking: { type: "adaptive", display: "summarized" }, + effort: { effort: "high" }, + toolChoice: { type: "auto" }, + }, + { + name: "supports Claude Sonnet 5 transport: allows explicit off", + id: "claude-sonnet-5", + modelName: "Claude Sonnet 5", + reasoning: "off" as const, + thinking: { type: "disabled" }, + effort: undefined, + toolChoice: { type: "any" }, + }, + ])("$name", async (testCase) => { + await runTransportStream( + makeAnthropicTransportModel({ + id: testCase.id, + name: testCase.modelName, + maxTokens: 128_000, + }), + makeSonnet5PrefillContext(), + { + apiKey: "sk-ant-api", + reasoning: testCase.reasoning, + temperature: 0.2, + toolChoice: "any", + } as AnthropicStreamOptions, + ); const payload = latestAnthropicRequest().payload; expect(payload).toMatchObject({ max_tokens: 128_000,