fix(cloudflare-ai-gateway): strip anthropic thinking prefill

This commit is contained in:
Peter Steinberger
2026-04-27 21:36:35 +01:00
parent 013939cfc7
commit 599b1b8462
12 changed files with 409 additions and 57 deletions
+1
View File
@@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- Providers/Cloudflare AI Gateway: strip assistant prefill turns from Anthropic Messages payloads when thinking is enabled, so Claude requests through Cloudflare AI Gateway no longer fail Anthropic conversation-ending validation. Fixes #72905; carries forward #73005. Thanks @AaronFaby and @sahilsatralkar.
- Channels/sessions: prevent guarded inbound session recording from creating route-only phantom sessions while still allowing last-route updates for sessions that already exist. Carries forward #73009. Thanks @jzakirov.
- Plugins/runtime deps: stage bundled plugin dependencies imported by mirrored root dist chunks, so packaged memory and status commands do not miss `chokidar` or similar root-chunk dependencies after update. Fixes #72882 and #72970; carries forward #72992. Thanks @shrimpy8, @colin-chang, and @Schnup03.
- Agents/runtime context: deliver hidden runtime context through prompt-local system context while keeping the transcript-only custom entry out of provider user turns, and strip stale copied runtime-context prefaces from user-facing replies. Fixes #72386; carries forward #72969. Thanks @jhsmith409.
+1 -1
View File
@@ -340,7 +340,7 @@ API key auth, and dynamic model resolution.
Each family builder is composed from lower-level public helpers exported from the same package, which you can reach for when a provider needs to go off the common pattern:
- `openclaw/plugin-sdk/provider-model-shared` — `ProviderReplayFamily`, `buildProviderReplayFamilyHooks(...)`, and the raw replay builders (`buildOpenAICompatibleReplayPolicy`, `buildAnthropicReplayPolicyForModel`, `buildGoogleGeminiReplayPolicy`, `buildHybridAnthropicOrOpenAIReplayPolicy`). Also exports Gemini replay helpers (`sanitizeGoogleGeminiReplayHistory`, `resolveTaggedReasoningOutputMode`) and endpoint/model helpers (`resolveProviderEndpoint`, `normalizeProviderId`, `normalizeGooglePreviewModelId`, `normalizeNativeXaiModelId`).
- `openclaw/plugin-sdk/provider-stream` — `ProviderStreamFamily`, `buildProviderStreamFamilyHooks(...)`, `composeProviderStreamWrappers(...)`, plus the shared OpenAI/Codex wrappers (`createOpenAIAttributionHeadersWrapper`, `createOpenAIFastModeWrapper`, `createOpenAIServiceTierWrapper`, `createOpenAIResponsesContextManagementWrapper`, `createCodexNativeWebSearchWrapper`), DeepSeek V4 OpenAI-compatible wrapper (`createDeepSeekV4OpenAICompatibleThinkingWrapper`), and shared proxy/provider wrappers (`createOpenRouterWrapper`, `createToolStreamWrapper`, `createMinimaxFastModeWrapper`).
- `openclaw/plugin-sdk/provider-stream` — `ProviderStreamFamily`, `buildProviderStreamFamilyHooks(...)`, `composeProviderStreamWrappers(...)`, plus the shared OpenAI/Codex wrappers (`createOpenAIAttributionHeadersWrapper`, `createOpenAIFastModeWrapper`, `createOpenAIServiceTierWrapper`, `createOpenAIResponsesContextManagementWrapper`, `createCodexNativeWebSearchWrapper`), DeepSeek V4 OpenAI-compatible wrapper (`createDeepSeekV4OpenAICompatibleThinkingWrapper`), Anthropic Messages thinking prefill cleanup (`createAnthropicThinkingPrefillPayloadWrapper`), and shared proxy/provider wrappers (`createOpenRouterWrapper`, `createToolStreamWrapper`, `createMinimaxFastModeWrapper`).
- `openclaw/plugin-sdk/provider-tools` — `ProviderToolCompatFamily`, `buildProviderToolCompatFamilyHooks("gemini")`, underlying Gemini schema helpers (`normalizeGeminiToolSchemas`, `inspectGeminiToolSchemas`), and xAI compat helpers (`resolveXaiModelCompatPatch()`, `applyXaiModelCompat(model)`). The bundled xAI plugin uses `normalizeResolvedModel` + `contributeResolvedModelCompat` with these to keep xAI rules owned by the provider.
Some stream helpers stay provider-local on purpose. `@openclaw/anthropic-provider` keeps `wrapAnthropicProviderStream`, `resolveAnthropicBetas`, `resolveAnthropicFastMode`, `resolveAnthropicServiceTier`, and the lower-level Anthropic wrapper builders in its own public `api.ts` / `contract-api.ts` seam because they encode Claude OAuth beta handling and `context1m` gating. The xAI plugin similarly keeps native xAI Responses shaping in its own `wrapStreamFn` (`/fast` aliases, default `tool_stream`, unsupported strict-tool cleanup, xAI-specific reasoning-payload removal).
+5
View File
@@ -19,6 +19,11 @@ Cloudflare AI Gateway sits in front of provider APIs and lets you add analytics,
For Anthropic models routed through Cloudflare AI Gateway, use your **Anthropic API key** as the provider key.
</Note>
When thinking is enabled for Anthropic Messages models, OpenClaw strips trailing
assistant prefill turns before sending the payload through Cloudflare AI Gateway.
Anthropic rejects response prefilling with extended thinking, while ordinary
non-thinking prefill remains available.
## Getting started
<Steps>
+2
View File
@@ -137,6 +137,8 @@ external end-user instructions.
- Tool result pairing repair and synthetic tool results.
- Turn validation (merge consecutive user turns to satisfy strict alternation).
- Trailing assistant prefill turns are stripped from outgoing Anthropic Messages
payloads when thinking is enabled, including Cloudflare AI Gateway routes.
- Thinking blocks with missing, empty, or blank replay signatures are stripped
before provider conversion. If that empties an assistant turn, OpenClaw keeps
turn shape with non-empty omitted-reasoning text.
+11 -56
View File
@@ -4,7 +4,9 @@ import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-ent
import {
applyAnthropicPayloadPolicyToParams,
composeProviderStreamWrappers,
createAnthropicThinkingPrefillPayloadWrapper,
resolveAnthropicPayloadPolicy,
stripTrailingAnthropicAssistantPrefillWhenThinking,
streamWithPayloadPatch,
} from "openclaw/plugin-sdk/provider-stream-shared";
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
@@ -30,51 +32,6 @@ const PI_AI_OAUTH_ANTHROPIC_BETAS = [
type AnthropicServiceTier = "auto" | "standard_only";
function isAnthropicThinkingEnabled(payloadObj: Record<string, unknown>): boolean {
const thinking = payloadObj.thinking;
if (!thinking || typeof thinking !== "object") {
return false;
}
return (thinking as { type?: unknown }).type !== "disabled";
}
function assistantMessageHasToolUse(message: Record<string, unknown>): boolean {
if (Array.isArray(message.tool_calls) && message.tool_calls.length > 0) {
return true;
}
const content = message.content;
if (!Array.isArray(content)) {
return false;
}
return content.some(
(block) =>
block &&
typeof block === "object" &&
((block as { type?: unknown }).type === "tool_use" ||
(block as { type?: unknown }).type === "toolCall"),
);
}
function stripTrailingAssistantPrefillWhenThinking(payloadObj: Record<string, unknown>): number {
if (!isAnthropicThinkingEnabled(payloadObj) || !Array.isArray(payloadObj.messages)) {
return 0;
}
let stripped = 0;
while (payloadObj.messages.length > 0) {
const last = payloadObj.messages[payloadObj.messages.length - 1];
if (!last || typeof last !== "object") {
break;
}
const message = last as Record<string, unknown>;
if (message.role !== "assistant" || assistantMessageHasToolUse(message)) {
break;
}
payloadObj.messages.pop();
stripped += 1;
}
return stripped;
}
function isAnthropic1MModel(modelId: string): boolean {
const normalized = normalizeLowercaseStringOrEmpty(modelId);
return ANTHROPIC_1M_MODEL_PREFIXES.some((prefix) => normalized.startsWith(prefix));
@@ -216,16 +173,11 @@ export function createAnthropicServiceTierWrapper(
export function createAnthropicThinkingPrefillWrapper(
baseStreamFn: StreamFn | undefined,
): StreamFn {
const underlying = baseStreamFn ?? streamSimple;
return (model, context, options) =>
streamWithPayloadPatch(underlying, model, context, options, (payloadObj) => {
const stripped = stripTrailingAssistantPrefillWhenThinking(payloadObj);
if (stripped > 0) {
log.warn(
`removed ${stripped} trailing assistant prefill message${stripped === 1 ? "" : "s"} because Anthropic extended thinking requires conversations to end with a user turn`,
);
}
});
return createAnthropicThinkingPrefillPayloadWrapper(baseStreamFn, (stripped) => {
log.warn(
`removed ${stripped} trailing assistant prefill message${stripped === 1 ? "" : "s"} because Anthropic extended thinking requires conversations to end with a user turn`,
);
});
}
export function resolveAnthropicFastMode(
@@ -269,4 +221,7 @@ export function wrapAnthropicProviderStream(
);
}
export const __testing = { log, stripTrailingAssistantPrefillWhenThinking };
export const __testing = {
log,
stripTrailingAssistantPrefillWhenThinking: stripTrailingAnthropicAssistantPrefillWhenThinking,
};
@@ -0,0 +1,47 @@
import type { StreamFn } from "@mariozechner/pi-agent-core";
import { capturePluginRegistration } from "openclaw/plugin-sdk/testing";
import { describe, expect, it } from "vitest";
import plugin from "./index.js";
function registerProvider() {
const captured = capturePluginRegistration(plugin);
const provider = captured.providers[0];
expect(provider?.id).toBe("cloudflare-ai-gateway");
return provider;
}
describe("cloudflare-ai-gateway plugin", () => {
it("registers a stream wrapper that strips Anthropic thinking assistant prefill", () => {
const provider = registerProvider();
expect(provider?.wrapStreamFn).toBeTypeOf("function");
let capturedPayload: Record<string, unknown> | undefined;
const baseStreamFn: StreamFn = (_model, _context, options) => {
const payload: Record<string, unknown> = {
thinking: { type: "enabled", budget_tokens: 1024 },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
};
options?.onPayload?.(payload as never, _model as never);
capturedPayload = payload;
return {} as ReturnType<StreamFn>;
};
const wrapped = provider?.wrapStreamFn?.({
provider: "cloudflare-ai-gateway",
modelId: "claude-sonnet-4-6",
model: { api: "anthropic-messages" },
streamFn: baseStreamFn,
} as never);
void wrapped?.(
{ provider: "cloudflare-ai-gateway", api: "anthropic-messages" } as never,
{} as never,
{},
);
expect(capturedPayload?.messages).toEqual([{ role: "user", content: "Return JSON." }]);
});
});
@@ -14,6 +14,7 @@ import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
import { buildCloudflareAiGatewayCatalogProvider } from "./catalog-provider.js";
import { CLOUDFLARE_AI_GATEWAY_DEFAULT_MODEL_REF } from "./models.js";
import { applyCloudflareAiGatewayConfig, buildCloudflareAiGatewayConfigPatch } from "./onboard.js";
import { wrapCloudflareAiGatewayProviderStream } from "./stream-wrappers.js";
const PROVIDER_ID = "cloudflare-ai-gateway";
const PROVIDER_ENV_VAR = "CLOUDFLARE_AI_GATEWAY_API_KEY";
@@ -216,6 +217,7 @@ export default definePluginEntry({
},
classifyFailoverReason: ({ errorMessage }) =>
/\bworkers?_ai\b.*\b(?:rate|limit|quota)\b/i.test(errorMessage) ? "rate_limit" : undefined,
wrapStreamFn: wrapCloudflareAiGatewayProviderStream,
});
},
});
@@ -0,0 +1,155 @@
import type { StreamFn } from "@mariozechner/pi-agent-core";
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
__testing,
createCloudflareAiGatewayAnthropicThinkingPrefillWrapper,
wrapCloudflareAiGatewayProviderStream,
} from "./stream-wrappers.js";
const { warnMock } = vi.hoisted(() => ({
warnMock: vi.fn(),
}));
vi.mock("openclaw/plugin-sdk/runtime-env", () => ({
createSubsystemLogger: () => ({
debug: vi.fn(),
error: vi.fn(),
info: vi.fn(),
warn: warnMock,
}),
}));
function createPayloadBaseStream(payload: Record<string, unknown>): StreamFn {
return ((model, _context, options) => {
options?.onPayload?.(payload as never, model as never);
return {} as ReturnType<StreamFn>;
}) as StreamFn;
}
function runWrapper(payload: Record<string, unknown>): Record<string, unknown> {
const wrapper = createCloudflareAiGatewayAnthropicThinkingPrefillWrapper(
createPayloadBaseStream(payload),
);
void wrapper(
{ provider: "cloudflare-ai-gateway", api: "anthropic-messages" } as never,
{} as never,
{},
);
return payload;
}
describe("createCloudflareAiGatewayAnthropicThinkingPrefillWrapper", () => {
beforeEach(() => {
warnMock.mockClear();
});
it("removes trailing assistant prefill when thinking is enabled", () => {
const payload = runWrapper({
thinking: { type: "enabled", budget_tokens: 1024 },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
});
expect(payload.messages).toEqual([{ role: "user", content: "Return JSON." }]);
expect(warnMock).toHaveBeenCalledWith(
"removed 1 trailing assistant prefill message because Anthropic extended thinking requires conversations to end with a user turn",
);
});
it("removes multiple trailing assistant prefill messages until the conversation ends with user", () => {
const payload = runWrapper({
thinking: { type: "adaptive" },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
{ role: "assistant", content: '"status"' },
],
});
expect(payload.messages).toEqual([{ role: "user", content: "Return JSON." }]);
expect(warnMock).toHaveBeenCalledWith(
"removed 2 trailing assistant prefill messages because Anthropic extended thinking requires conversations to end with a user turn",
);
});
it("keeps assistant prefill when thinking is disabled", () => {
const payload = runWrapper({
thinking: { type: "disabled" },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
});
expect(payload.messages).toHaveLength(2);
expect(warnMock).not.toHaveBeenCalled();
});
it("keeps trailing assistant tool use turns when thinking is enabled", () => {
const payload = runWrapper({
thinking: { type: "enabled", budget_tokens: 1024 },
messages: [
{ role: "user", content: "Read a file." },
{
role: "assistant",
content: [{ type: "tool_use", id: "toolu_1", name: "Read" }],
},
],
});
expect(payload.messages).toHaveLength(2);
expect(warnMock).not.toHaveBeenCalled();
});
});
describe("wrapCloudflareAiGatewayProviderStream", () => {
beforeEach(() => {
warnMock.mockClear();
});
it("patches Anthropic Messages models", () => {
const payload = {
thinking: { type: "enabled" },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
};
const wrapped = wrapCloudflareAiGatewayProviderStream({
model: { api: "anthropic-messages" },
streamFn: createPayloadBaseStream(payload),
} as never);
void wrapped?.(
{ provider: "cloudflare-ai-gateway", api: "anthropic-messages" } as never,
{} as never,
{},
);
expect(payload.messages).toEqual([{ role: "user", content: "Return JSON." }]);
});
it("leaves non-Anthropic model APIs on the original stream path", () => {
let onPayloadWasInstalled = false;
const baseStreamFn: StreamFn = (_model, _context, options) => {
onPayloadWasInstalled = typeof options?.onPayload === "function";
return {} as ReturnType<StreamFn>;
};
const wrapped = wrapCloudflareAiGatewayProviderStream({
model: { api: "openai-completions" },
streamFn: baseStreamFn,
} as never);
void wrapped?.({ api: "openai-completions" } as never, {} as never, {});
expect(wrapped).toBe(baseStreamFn);
expect(onPayloadWasInstalled).toBe(false);
expect(warnMock).not.toHaveBeenCalled();
});
it("treats missing model API as the plugin's default Anthropic Messages route", () => {
expect(__testing.shouldPatchAnthropicMessagesPayload({} as never)).toBe(true);
});
});
@@ -0,0 +1,31 @@
import type { StreamFn } from "@mariozechner/pi-agent-core";
import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry";
import { createAnthropicThinkingPrefillPayloadWrapper } from "openclaw/plugin-sdk/provider-stream-shared";
import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env";
const log = createSubsystemLogger("cloudflare-ai-gateway-stream");
function shouldPatchAnthropicMessagesPayload(model: ProviderWrapStreamFnContext["model"]): boolean {
return model?.api === undefined || model.api === "anthropic-messages";
}
export function createCloudflareAiGatewayAnthropicThinkingPrefillWrapper(
baseStreamFn: StreamFn | undefined,
): StreamFn {
return createAnthropicThinkingPrefillPayloadWrapper(baseStreamFn, (stripped) => {
log.warn(
`removed ${stripped} trailing assistant prefill message${stripped === 1 ? "" : "s"} because Anthropic extended thinking requires conversations to end with a user turn`,
);
});
}
export function wrapCloudflareAiGatewayProviderStream(
ctx: ProviderWrapStreamFnContext,
): StreamFn | undefined {
if (!shouldPatchAnthropicMessagesPayload(ctx.model)) {
return ctx.streamFn;
}
return createCloudflareAiGatewayAnthropicThinkingPrefillWrapper(ctx.streamFn);
}
export const __testing = { log, shouldPatchAnthropicMessagesPayload };
@@ -3,11 +3,13 @@ import { describe, expect, it } from "vitest";
import {
buildCopilotDynamicHeaders,
createHtmlEntityToolCallArgumentDecodingWrapper,
createAnthropicThinkingPrefillPayloadWrapper,
createPayloadPatchStreamWrapper,
defaultToolStreamExtraParams,
decodeHtmlEntitiesInObject,
hasCopilotVisionInput,
isOpenAICompatibleThinkingEnabled,
stripTrailingAnthropicAssistantPrefillWhenThinking,
} from "./provider-stream-shared.js";
type FakeWrappedStream = {
@@ -265,3 +267,86 @@ describe("createPayloadPatchStreamWrapper", () => {
expect(onPayloadWasInstalled).toBe(false);
});
});
describe("stripTrailingAnthropicAssistantPrefillWhenThinking", () => {
it("removes trailing assistant text turns when Anthropic thinking is enabled", () => {
const payload = {
thinking: { type: "enabled", budget_tokens: 1024 },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
{ role: "assistant", content: '"status"' },
],
};
expect(stripTrailingAnthropicAssistantPrefillWhenThinking(payload)).toBe(2);
expect(payload.messages).toEqual([{ role: "user", content: "Return JSON." }]);
});
it("preserves assistant tool-use turns across Anthropic and OpenAI-shaped payloads", () => {
const anthropicPayload = {
thinking: { type: "adaptive" },
messages: [
{ role: "user", content: "Read a file." },
{ role: "assistant", content: [{ type: "tool_use", id: "toolu_1", name: "Read" }] },
],
};
const openAiPayload = {
thinking: { type: "adaptive" },
messages: [
{ role: "user", content: "Read a file." },
{ role: "assistant", content: [{ type: "toolCall", id: "call_1", name: "Read" }] },
],
};
const toolCallsPayload = {
thinking: { type: "adaptive" },
messages: [{ role: "assistant", tool_calls: [{ id: "call_1", name: "Read" }] }],
};
expect(stripTrailingAnthropicAssistantPrefillWhenThinking(anthropicPayload)).toBe(0);
expect(stripTrailingAnthropicAssistantPrefillWhenThinking(openAiPayload)).toBe(0);
expect(stripTrailingAnthropicAssistantPrefillWhenThinking(toolCallsPayload)).toBe(0);
});
it("keeps assistant prefill when Anthropic thinking is disabled", () => {
const payload = {
thinking: { type: "disabled" },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
};
expect(stripTrailingAnthropicAssistantPrefillWhenThinking(payload)).toBe(0);
expect(payload.messages).toHaveLength(2);
});
});
describe("createAnthropicThinkingPrefillPayloadWrapper", () => {
it("reports stripped assistant prefill count", () => {
const payload = {
thinking: { type: "enabled" },
messages: [
{ role: "user", content: "Return JSON." },
{ role: "assistant", content: "{" },
],
};
let strippedCount = 0;
const baseStreamFn: StreamFn = (_model, _context, options) => {
options?.onPayload?.(payload as never, _model as never);
return {} as ReturnType<StreamFn>;
};
const wrapped = createAnthropicThinkingPrefillPayloadWrapper(
baseStreamFn,
(stripped) => {
strippedCount = stripped;
},
{ shouldPatch: ({ model }) => model.api === "anthropic-messages" },
);
void wrapped({ api: "anthropic-messages" } as never, {} as never, {});
expect(payload.messages).toEqual([{ role: "user", content: "Return JSON." }]);
expect(strippedCount).toBe(1);
});
});
+67
View File
@@ -154,6 +154,73 @@ export function createPayloadPatchStreamWrapper(
};
}
function isAnthropicThinkingEnabled(payload: Record<string, unknown>): boolean {
const thinking = payload.thinking;
if (!thinking || typeof thinking !== "object") {
return false;
}
return (thinking as { type?: unknown }).type !== "disabled";
}
function assistantMessageHasAnthropicToolUse(message: Record<string, unknown>): boolean {
if (Array.isArray(message.tool_calls) && message.tool_calls.length > 0) {
return true;
}
const content = message.content;
if (!Array.isArray(content)) {
return false;
}
return content.some(
(block) =>
block &&
typeof block === "object" &&
((block as { type?: unknown }).type === "tool_use" ||
(block as { type?: unknown }).type === "toolCall"),
);
}
export function stripTrailingAnthropicAssistantPrefillWhenThinking(
payload: Record<string, unknown>,
): number {
if (!isAnthropicThinkingEnabled(payload) || !Array.isArray(payload.messages)) {
return 0;
}
let stripped = 0;
while (payload.messages.length > 0) {
const finalMessage = payload.messages[payload.messages.length - 1];
if (!finalMessage || typeof finalMessage !== "object") {
break;
}
const message = finalMessage as Record<string, unknown>;
if (message.role !== "assistant" || assistantMessageHasAnthropicToolUse(message)) {
break;
}
payload.messages.pop();
stripped += 1;
}
return stripped;
}
export function createAnthropicThinkingPrefillPayloadWrapper(
baseStreamFn: StreamFn | undefined,
onStripped?: (stripped: number) => void,
wrapperOptions?: Parameters<typeof createPayloadPatchStreamWrapper>[2],
): StreamFn {
return createPayloadPatchStreamWrapper(
baseStreamFn,
({ payload }) => {
const stripped = stripTrailingAnthropicAssistantPrefillWhenThinking(payload);
if (stripped > 0) {
onStripped?.(stripped);
}
},
wrapperOptions,
);
}
export type OpenAICompatibleThinkingLevel = ProviderWrapStreamFnContext["thinkingLevel"];
export function isOpenAICompatibleThinkingEnabled(params: {
+2
View File
@@ -36,6 +36,7 @@ export {
applyAnthropicPayloadPolicyToParams,
buildCopilotDynamicHeaders,
composeProviderStreamWrappers,
createAnthropicThinkingPrefillPayloadWrapper,
createBedrockNoCacheWrapper,
createMoonshotThinkingWrapper,
createToolStreamWrapper,
@@ -48,6 +49,7 @@ export {
resolveAnthropicPayloadPolicy,
resolveMoonshotThinkingType,
streamWithPayloadPatch,
stripTrailingAnthropicAssistantPrefillWhenThinking,
} from "./provider-stream-shared.js";
export type ProviderStreamFamily =