From 087fb56f77df2bc68deed53413b11abdcf9d3070 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 11 Aug 2026 16:49:42 -0700 Subject: [PATCH] feat(moonshot): add native K3 video input (#122337) --- extensions/moonshot/index.test.ts | 65 ++++ extensions/moonshot/index.ts | 63 ++-- .../moonshot/media-understanding-provider.ts | 4 +- extensions/moonshot/moonshot.live.test.ts | 91 +++++ extensions/moonshot/native-video.test.ts | 312 ++++++++++++++++ extensions/moonshot/native-video.ts | 93 +++++ extensions/moonshot/onboard.ts | 24 +- extensions/moonshot/provider-catalog.ts | 8 +- extensions/moonshot/provider-contract-api.ts | 31 +- extensions/moonshot/provider-policy-api.ts | 41 ++- .../src/openai-completions-messages.test.ts | 35 ++ .../ai/src/openai-completions-messages.ts | 22 +- .../ai/src/provider-transcript-transform.ts | 20 +- .../src/providers/transform-messages.test.ts | 13 +- .../stream-wrappers/moonshot-thinking.ts | 347 ++++++------------ src/plugin-sdk/provider-stream-shared.ts | 1 + 16 files changed, 842 insertions(+), 328 deletions(-) create mode 100644 extensions/moonshot/native-video.test.ts create mode 100644 extensions/moonshot/native-video.ts diff --git a/extensions/moonshot/index.test.ts b/extensions/moonshot/index.test.ts index 5942295b283e..07e14cdfad11 100644 --- a/extensions/moonshot/index.test.ts +++ b/extensions/moonshot/index.test.ts @@ -5,6 +5,7 @@ import { registerSingleProviderPlugin } from "openclaw/plugin-sdk/plugin-test-ru import { createCapturedThinkingConfigStream } from "openclaw/plugin-sdk/provider-test-contracts"; import { describe, expect, it } from "vitest"; import plugin from "./index.js"; +import { MOONSHOT_BASE_URL, MOONSHOT_CN_BASE_URL } from "./provider-catalog.js"; import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js"; type MoonshotManifest = { @@ -24,6 +25,70 @@ function readManifest(): MoonshotManifest { } describe("moonshot provider plugin", () => { + it.each([ + ["international", "moonshot", "kimi-k3", "openai-completions", MOONSHOT_BASE_URL, true], + [ + "international slash", + "moonshot", + "kimi-k3", + "openai-completions", + `${MOONSHOT_BASE_URL}/`, + true, + ], + ["China", "moonshot", "kimi-k3", "openai-completions", MOONSHOT_CN_BASE_URL, true], + ["China slash", "moonshot", "kimi-k3", "openai-completions", `${MOONSHOT_CN_BASE_URL}/`, true], + ["K2.7", "moonshot", "kimi-k2.7-code", "openai-completions", MOONSHOT_BASE_URL, false], + ["K2.6", "moonshot", "kimi-k2.6", "openai-completions", MOONSHOT_BASE_URL, false], + ["model alias", "moonshot", "moonshot/kimi-k3", "openai-completions", MOONSHOT_BASE_URL, false], + ["unknown model", "moonshot", "kimi-k3-latest", "openai-completions", MOONSHOT_BASE_URL, false], + ["Responses", "moonshot", "kimi-k3", "openai-responses", MOONSHOT_BASE_URL, false], + ["proxy", "moonshot", "kimi-k3", "openai-completions", "https://proxy.example/v1", false], + ["query", "moonshot", "kimi-k3", "openai-completions", `${MOONSHOT_BASE_URL}?x=1`, false], + ["fragment", "moonshot", "kimi-k3", "openai-completions", `${MOONSHOT_BASE_URL}#x`, false], + [ + "userinfo", + "moonshot", + "kimi-k3", + "openai-completions", + "https://u@api.moonshot.ai/v1", + false, + ], + [ + "different path", + "moonshot", + "kimi-k3", + "openai-completions", + "https://api.moonshot.ai/v1/chat", + false, + ], + ["HTTP", "moonshot", "kimi-k3", "openai-completions", "http://api.moonshot.ai/v1", false], + ["provider alias", "moonshotai", "kimi-k3", "openai-completions", MOONSHOT_BASE_URL, false], + ] as const)( + "enables native video only for the exact %s route", + async (_name, providerId, modelId, api, baseUrl, expected) => { + const provider = await registerSingleProviderPlugin(plugin); + const model = { + id: modelId, + name: modelId, + provider: providerId, + api, + baseUrl, + reasoning: true, + input: ["text", "image", "video"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_000_000, + maxTokens: 1_000_000, + } as unknown as Model; + const normalized = provider.normalizeResolvedModel?.({ + provider: providerId, + modelId, + model, + } as never); + + expect(((normalized ?? model).input as string[]).includes("video")).toBe(expected); + }, + ); + it("mirrors Kimi web-search env credentials in manifest metadata", () => { const manifestEnvVars = readManifest().setup?.providers?.find((provider) => provider.id === "moonshot")?.envVars ?? diff --git a/extensions/moonshot/index.ts b/extensions/moonshot/index.ts index 1490a8693304..f1b9a7392f63 100644 --- a/extensions/moonshot/index.ts +++ b/extensions/moonshot/index.ts @@ -1,41 +1,34 @@ +import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key"; // Moonshot plugin entrypoint registers its OpenClaw integration. import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry"; import { buildOpenAICompatibleReplayPolicy } from "openclaw/plugin-sdk/provider-model-shared"; -import { buildProviderStreamFamilyHooks } from "openclaw/plugin-sdk/provider-stream-family"; import { applyMoonshotNativeStreamingUsageCompat } from "./api.js"; import { moonshotMediaUnderstandingProvider } from "./media-understanding-provider.js"; +import { wrapMoonshotStream } from "./native-video.js"; import { applyMoonshotConfig, applyMoonshotConfigCn } from "./onboard.js"; +import manifest from "./openclaw.plugin.json" with { type: "json" }; import { buildMoonshotProvider, MOONSHOT_DEFAULT_MODEL_REF } from "./provider-catalog.js"; -import { isMoonshotAlwaysThinkingModelId, resolveThinkingProfile } from "./provider-policy-api.js"; +import { + isMoonshotAlwaysThinkingModelId, + isMoonshotK3NativeVideoRoute, + resolveThinkingProfile, +} from "./provider-policy-api.js"; import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js"; const PROVIDER_ID = "moonshot"; -const moonshotThinkingStreamHooks = buildProviderStreamFamilyHooks("moonshot-thinking"); - export default defineSingleProviderPluginEntry({ id: PROVIDER_ID, name: "Moonshot Provider", description: "Bundled Moonshot provider plugin", + manifest, provider: { label: "Moonshot", docsPath: "/providers/moonshot", aliases: ["moonshotai", "moonshot-ai"], - auth: [ - { - methodId: "api-key", - label: "Kimi API key (.ai)", - hint: "Kimi API models · https://platform.kimi.ai/docs/pricing/chat", - optionKey: "moonshotApiKey", - flagName: "--moonshot-api-key", - envVar: "MOONSHOT_API_KEY", - promptMessage: "Enter Moonshot API key", - defaultModel: MOONSHOT_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyMoonshotConfig(cfg), - wizard: { - groupLabel: "Moonshot AI (Kimi)", - }, - }, - { + manifestAuth: { applyConfig: applyMoonshotConfig }, + extraAuth: [ + createProviderApiKeyAuthMethod({ + providerId: PROVIDER_ID, methodId: "api-key-cn", label: "Kimi API key (.cn)", hint: "Kimi API models · https://platform.kimi.ai/docs/pricing/chat", @@ -44,11 +37,9 @@ export default defineSingleProviderPluginEntry({ envVar: "MOONSHOT_API_KEY", promptMessage: "Enter Moonshot API key (.cn)", defaultModel: MOONSHOT_DEFAULT_MODEL_REF, - applyConfig: (cfg) => applyMoonshotConfigCn(cfg), - wizard: { - groupLabel: "Moonshot AI (Kimi)", - }, - }, + applyConfig: applyMoonshotConfigCn, + wizard: { groupLabel: "Moonshot AI (Kimi)" }, + }), ], catalog: { buildProvider: buildMoonshotProvider, @@ -58,6 +49,21 @@ export default defineSingleProviderPluginEntry({ }, applyNativeStreamingUsageCompat: ({ providerConfig }) => applyMoonshotNativeStreamingUsageCompat(providerConfig), + normalizeResolvedModel: (ctx) => + ({ + ...ctx.model, + input: (ctx.model.input as string[]) + .filter((type) => type !== "video") + .concat( + isMoonshotK3NativeVideoRoute({ + ...ctx.model, + provider: ctx.provider, + modelId: ctx.modelId, + }) + ? "video" + : [], + ), + }) as typeof ctx.model, buildReplayPolicy: ({ modelApi, modelId }) => buildOpenAICompatibleReplayPolicy(modelApi, { modelId, @@ -65,11 +71,8 @@ export default defineSingleProviderPluginEntry({ duplicateToolCallIdStyle: "openai", dropReasoningFromHistory: false, }), - ...moonshotThinkingStreamHooks, - wrapSimpleCompletionStreamFn: (ctx) => - isMoonshotAlwaysThinkingModelId(ctx.modelId) - ? moonshotThinkingStreamHooks.wrapStreamFn?.(ctx) - : ctx.streamFn, + wrapStreamFn: (ctx) => wrapMoonshotStream(ctx), + wrapSimpleCompletionStreamFn: (ctx) => wrapMoonshotStream(ctx, true), resolveThinkingProfile, isModernModelRef: ({ modelId }) => isMoonshotAlwaysThinkingModelId(modelId), }, diff --git a/extensions/moonshot/media-understanding-provider.ts b/extensions/moonshot/media-understanding-provider.ts index 2893c79f0875..599224ac6c6e 100644 --- a/extensions/moonshot/media-understanding-provider.ts +++ b/extensions/moonshot/media-understanding-provider.ts @@ -17,8 +17,8 @@ import { resolveProviderHttpRequestConfig, } from "openclaw/plugin-sdk/provider-http"; import manifest from "./openclaw.plugin.json" with { type: "json" }; +import { MOONSHOT_BASE_URL } from "./provider-catalog.js"; -const DEFAULT_MOONSHOT_VIDEO_BASE_URL = "https://api.moonshot.ai/v1"; // Media defaults are capability-specific and intentionally independent from chat onboarding. const DEFAULT_MOONSHOT_IMAGE_MODEL = manifest.mediaUnderstandingProviderMetadata.moonshot.defaultModels.image; @@ -36,7 +36,7 @@ async function describeMoonshotVideo( const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = resolveProviderHttpRequestConfig({ baseUrl: params.baseUrl, - defaultBaseUrl: DEFAULT_MOONSHOT_VIDEO_BASE_URL, + defaultBaseUrl: MOONSHOT_BASE_URL, headers: params.headers, request: params.request, defaultHeaders: { diff --git a/extensions/moonshot/moonshot.live.test.ts b/extensions/moonshot/moonshot.live.test.ts index 3aeb8a566f20..d67dcd3199c2 100644 --- a/extensions/moonshot/moonshot.live.test.ts +++ b/extensions/moonshot/moonshot.live.test.ts @@ -5,6 +5,7 @@ import { type AssistantMessage, type Context, type Model, + type ProviderContext, type Tool, } from "openclaw/plugin-sdk/llm"; import { registerSingleProviderPlugin } from "openclaw/plugin-sdk/plugin-test-runtime"; @@ -18,10 +19,31 @@ import { createKimiWebSearchProvider } from "./src/kimi-web-search-provider.js"; const KIMI_SEARCH_KEY = process.env.KIMI_API_KEY?.trim() || process.env.MOONSHOT_API_KEY?.trim() || ""; const MOONSHOT_API_KEY = process.env.MOONSHOT_API_KEY?.trim() || ""; +const MOONSHOT_CN_API_KEY = process.env.MOONSHOT_CN_API_KEY?.trim() || ""; const describeLive = isLiveTestEnabled() && KIMI_SEARCH_KEY.length > 0 ? describe : describe.skip; const describeModelLive = isLiveTestEnabled() && MOONSHOT_API_KEY.length > 0 ? describe : describe.skip; const KIMI_LIVE_SEARCH_TIMEOUT_SECONDS = 60; +const itInternationalVideoLive = isLiveTestEnabled() && MOONSHOT_API_KEY.length > 0 ? it : it.skip; +const itChinaVideoLive = isLiveTestEnabled() && MOONSHOT_CN_API_KEY.length > 0 ? it : it.skip; +// Two 64x64 solid-red H.264 frames keep regional native-video proof deterministic. +const KIMI_K3_LIVE_RED_VIDEO_BASE64 = [ + "AAAAJGZ0eXBpc29tAAACAGlzb21pc282aXNvMmF2YzFtcDQxAAAC5m1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAAA", + "AAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "AAAAAAAAAAIAAAHodHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAA", + "AAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAABAAAAAQAAAAAABhG1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAQAAAAAAA", + "VcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAS9taW5mAAAAFHZtaGQAAAABAAAA", + "AAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAADvc3RibAAAAKNzdHNkAAAAAAAAAAEAAACTYXZj", + "MQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAABAAEAASAAAAEgAAAAAAAAAARVMYXZjNjIuMjguMTAyIGxpYngyNjQAAAAAAAAA", + "AAAAABj//wAAAC1hdmNDAULACv/hABZnQsAK2hCbARAAAAMAEAAAAwAo8SJqAQAEaM4PyAAAABBwYXNwAAAAAQAAAAEAAAAQ", + "c3R0cwAAAAAAAAAAAAAAEHN0c2MAAAAAAAAAAAAAABRzdHN6AAAAAAAAAAAAAAAAAAAAEHN0Y28AAAAAAAAAAAAAAChtdmV4", + "AAAAIHRyZXgAAAAAAAAAAQAAAAEAAAAAAAAAAAAAAAAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGly", + "YXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjYyLjEyLjEwMgAAAHhtb29mAAAAEG1m", + "aGQAAAAAAAAAAQAAAGB0cmFmAAAAJHRmaGQAAAA5AAAAAQAAAAAAAAMKAABAAAAAACMBAQAAAAAAFHRmZHQBAAAAAAAAAAAA", + "AAAAAAAgdHJ1bgAAAgUAAAACAAAAgAIAAAAAAAAjAAAACgAAADVtZGF0AAAAH2WIhDoRigACGPHAAED2OAAIeUnJyddddddd", + "dddddeAAAAAGQZogF6CMAAAAQ21mcmEAAAArdGZyYQEAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAMKAQEBAAAAEG1m", + "cm8AAAAAAAAAQw==", +].join(""); function isTransientKimiSearchError(error: unknown): boolean { if (!(error instanceof Error)) { @@ -141,6 +163,75 @@ async function collectDoneMessage( return doneMessage; } +async function proveK3NativeVideoRegion(baseUrl: string, apiKey: string) { + const provider = await registerSingleProviderPlugin(plugin); + const catalog = buildMoonshotProvider(); + const definition = catalog.models.find((model) => model.id === "kimi-k3"); + if (!definition) { + throw new Error("Moonshot catalog does not include kimi-k3"); + } + const model = provider.normalizeResolvedModel?.({ + provider: "moonshot", + modelId: "kimi-k3", + model: { + ...definition, + provider: "moonshot", + api: "openai-completions", + baseUrl, + }, + } as never) as Model<"openai-completions"> | undefined; + const wrapped = provider.wrapStreamFn?.({ + provider: "moonshot", + modelId: "kimi-k3", + thinkingLevel: "max", + streamFn: streamSimple, + } as never); + if (!model?.input.includes("video" as never) || !wrapped) { + throw new Error("registered Moonshot provider did not prepare K3 native video"); + } + const context: ProviderContext = { + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What single color fills this video? Reply with exactly RED, BLUE, or GREEN.", + }, + { type: "video", mimeType: "video/mp4", data: KIMI_K3_LIVE_RED_VIDEO_BASE64 }, + ], + timestamp: Date.now(), + }, + ], + }; + const response = await collectDoneMessage( + (await wrapped(model, context as never, { apiKey, maxTokens: 128 })) as AsyncIterable<{ + type: string; + message?: AssistantMessage; + error?: AssistantMessage; + }>, + ); + const answer = response.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join(" "); + expect(answer).toMatch(/\bRED\b/iu); +} + +describe("moonshot K3 native video live", () => { + itInternationalVideoLive( + "independently understands red video on api.moonshot.ai", + async () => await proveK3NativeVideoRegion("https://api.moonshot.ai/v1", MOONSHOT_API_KEY), + 120_000, + ); + + itChinaVideoLive( + "independently understands red video on api.moonshot.cn", + async () => await proveK3NativeVideoRegion(MOONSHOT_CN_BASE_URL, MOONSHOT_CN_API_KEY), + 120_000, + ); +}); + describeModelLive("moonshot K2.6 replay live", () => { it("accepts a cross-model tool-call replay after backfilling reasoning_content", async () => { const provider = await registerSingleProviderPlugin(plugin); diff --git a/extensions/moonshot/native-video.test.ts b/extensions/moonshot/native-video.test.ts new file mode 100644 index 000000000000..df6173bdae31 --- /dev/null +++ b/extensions/moonshot/native-video.test.ts @@ -0,0 +1,312 @@ +import { createServer } from "node:http"; +import { createOpenAICompletionsTransportStreamFn } from "@openclaw/ai/transports"; +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { attachModelProviderRequestTransport } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + createAssistantMessageEventStream, + type Context, + type Model, + type ProviderContext, +} from "openclaw/plugin-sdk/llm"; +import { registerSingleProviderPlugin } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { describe, expect, it, vi } from "vitest"; +import plugin from "./index.js"; +import { wrapMoonshotStream } from "./native-video.js"; +import { MOONSHOT_BASE_URL } from "./provider-catalog.js"; + +const MP4_A = "data:video/mp4;base64,YWFhYWFhYWFhYWFhYWFhYQ=="; +const MP4_B = "data:video/mp4;base64,YmJiYmJiYmJiYmJiYmJiYg=="; +const WEBM = "data:video/webm;base64,d2VibQ=="; + +function model(overrides: Partial = {}): Model { + return { + id: "kimi-k3", + name: "Kimi K3", + provider: "moonshot", + api: "openai-completions", + baseUrl: MOONSHOT_BASE_URL, + reasoning: true, + input: ["text", "image", "video"] as never, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 1_048_576, + maxTokens: 1_048_576, + ...overrides, + } as Model; +} + +function genericPayload(videoUrls = [MP4_A]) { + return { + model: "kimi-k3", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "before" }, + { type: "image_url", image_url: { url: "data:image/png;base64,aW1hZ2U=" } }, + ...videoUrls.map((url) => ({ type: "video_url", video_url: { url } })), + { type: "text", text: "after" }, + ], + }, + ], + }; +} + +function capturePayloadStream(payload: unknown, capture: (value: unknown) => void): StreamFn { + return async (payloadModel, _context, options) => { + const replacement = await options?.onPayload?.(payload, payloadModel); + capture(replacement === undefined ? payload : replacement); + return createAssistantMessageEventStream(); + }; +} + +function createNativeWrapper(streamFn: StreamFn, requestBytesExclusive = 100_000_000): StreamFn { + return wrapMoonshotStream( + { provider: "moonshot", modelId: "kimi-k3", streamFn } as never, + false, + requestBytesExclusive, + ); +} + +describe("Moonshot native video wrapper", () => { + it("preserves serialized current-user MP4 parts and order", async () => { + const payload = genericPayload(); + let dispatched: unknown; + const caller = vi.fn((value: unknown) => { + expect(JSON.stringify(value)).not.toContain("__openclaw"); + expect(JSON.stringify(value)).not.toContain("/private/"); + expect((value as typeof payload).messages[0]?.content.map((part) => part.type)).toEqual([ + "text", + "image_url", + "video_url", + "text", + ]); + }); + const wrapped = createNativeWrapper( + capturePayloadStream(payload, (value) => (dispatched = value)), + ); + + await wrapped(model(), { messages: [] } as Context, { onPayload: caller }); + + expect(caller).toHaveBeenCalledOnce(); + expect((dispatched as typeof payload).messages[0]?.content[2]).toEqual({ + type: "video_url", + video_url: { url: MP4_A }, + }); + }); + + it("allows valid hook clones and injections while omitting non-MP4 video", async () => { + const payload = genericPayload([MP4_A, WEBM]); + let dispatched: unknown; + const wrapped = createNativeWrapper( + capturePayloadStream(payload, (value) => (dispatched = value)), + ); + + await wrapped(model(), { messages: [] } as Context, { + onPayload(value) { + const content = (value as typeof payload).messages[0]!.content; + const valid = content[2]! as Record; + content.push(structuredClone(valid) as never); + content.push({ type: "video_url", video_url: { url: MP4_B } } as never); + content.push({ type: "image_url", image_url: { url: WEBM } } as never); + }, + }); + + const body = JSON.stringify(dispatched); + expect(body.split(MP4_A)).toHaveLength(3); + expect(body).toContain("YmJiYmJi"); + expect(body).not.toContain("d2VibQ"); + expect(body.match(/video omitted/gu)).toHaveLength(2); + }); + + it("validates a caller replacement after Moonshot thinking post-processing", async () => { + const payload = genericPayload(); + let dispatched: unknown; + const wrapped = createNativeWrapper( + capturePayloadStream(payload, (value) => (dispatched = value)), + ); + + await wrapped(model(), { messages: [] } as Context, { + onPayload(value) { + return structuredClone(value); + }, + }); + + expect(dispatched).toMatchObject({ reasoning_effort: "max" }); + expect((dispatched as typeof payload).messages[0]!.content[2]).toEqual({ + type: "video_url", + video_url: { url: MP4_A }, + }); + }); + + it("evicts later admitted videos in place to satisfy the exclusive final size", async () => { + const payload = genericPayload([MP4_A, MP4_B]); + const projectedWithSecondOmitted = genericPayload([MP4_A]); + projectedWithSecondOmitted.messages[0]!.content.splice(3, 0, { + type: "text", + text: "(video omitted: Moonshot request size limit)", + } as never); + Object.assign(projectedWithSecondOmitted, { reasoning_effort: "max" }); + const ceiling = Buffer.byteLength(JSON.stringify(projectedWithSecondOmitted), "utf8") + 1; + let dispatched: unknown; + const wrapped = createNativeWrapper( + capturePayloadStream(payload, (value) => (dispatched = value)), + ceiling, + ); + + await wrapped(model(), { messages: [] } as Context, {}); + + const content = (dispatched as typeof payload).messages[0]!.content; + expect(content[2]).toEqual({ type: "video_url", video_url: { url: MP4_A } }); + expect(content[3]).toMatchObject({ type: "text", text: expect.stringContaining("size limit") }); + expect(Buffer.byteLength(JSON.stringify(dispatched), "utf8")).toBeLessThan(ceiling); + }); + + it("measures caller replacements and rejects an oversized non-video body", async () => { + const wrapped = createNativeWrapper( + capturePayloadStream(genericPayload(), () => undefined), + 200, + ); + + await expect( + wrapped(model(), { messages: [] } as Context, { + onPayload: () => ({ model: "kimi-k3", messages: [], padding: "x".repeat(300) }), + }), + ).rejects.toThrow("Moonshot request body must be smaller than 200 bytes"); + }); + + it.each(["wrapStreamFn", "wrapSimpleCompletionStreamFn"] as const)( + "keeps thinking outside native video for %s", + async (hookName) => { + const provider = await registerSingleProviderPlugin(plugin); + const payload = genericPayload(); + let dispatched: unknown; + const wrapped = provider[hookName]?.({ + provider: "moonshot", + modelId: "kimi-k3", + thinkingLevel: "off", + streamFn: capturePayloadStream(payload, (value) => (dispatched = value)), + } as never); + if (!wrapped) { + throw new Error(`Moonshot did not register ${hookName}`); + } + + await wrapped(model(), { messages: [] } as Context, { + onPayload(value) { + const record = value as Record; + expect(record.reasoning_effort).toBe("max"); + expect(JSON.stringify(record)).toContain('"type":"video_url"'); + record.reasoning_effort = "low"; + }, + }); + + expect(dispatched).toMatchObject({ reasoning_effort: "max" }); + }, + ); +}); + +describe("Moonshot registered transport boundary", () => { + it("sends ordered video_url content through the real Chat Completions transport", async () => { + let requestBody: Record | undefined; + const server = createServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => (body += chunk)); + request.on("end", () => { + requestBody = JSON.parse(body) as Record; + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end( + `data: {"id":"chatcmpl-test","object":"chat.completion.chunk","model":"kimi-k3","choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n`, + ); + }); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve()); + }); + try { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("missing loopback address"); + } + const provider = await registerSingleProviderPlugin(plugin); + const officialModel = model(); + const normalized = provider.normalizeResolvedModel?.({ + provider: "moonshot", + modelId: "kimi-k3", + model: officialModel, + } as never) as Model | undefined; + expect(normalized?.input).toContain("video"); + const transport = createOpenAICompletionsTransportStreamFn(); + const loopbackTransport: StreamFn = (runtimeModel, context, options) => + transport( + attachModelProviderRequestTransport( + { + ...runtimeModel, + provider: "custom", + baseUrl: `http://127.0.0.1:${address.port}/v1`, + }, + { allowPrivateNetwork: true }, + ), + context, + options, + ); + const wrapped = provider.wrapStreamFn?.({ + provider: "moonshot", + modelId: "kimi-k3", + thinkingLevel: "off", + streamFn: loopbackTransport, + } as never); + if (!wrapped || !normalized) { + throw new Error("Moonshot registered transport unavailable"); + } + const context: ProviderContext = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "before" }, + { type: "image", mimeType: "image/png", data: "aW1hZ2U=" }, + { type: "video", mimeType: "video/mp4", data: "dmlkZW8=" }, + { type: "text", text: "after" }, + ], + timestamp: 1, + }, + ], + }; + let callerPayload: unknown; + const caller = vi.fn((payload: unknown) => (callerPayload = payload)); + const stream = await wrapped(normalized, context as never, { + apiKey: "test-key", + maxRetries: 0, + onPayload: caller, + }); + let streamError: unknown; + for await (const event of stream) { + if (event.type === "error") { + streamError = event.error; + } + } + + expect(caller, JSON.stringify(streamError)).toHaveBeenCalledOnce(); + expect(JSON.stringify(callerPayload)).not.toContain("/private/"); + expect(JSON.stringify(callerPayload)).toContain("data:video/mp4;base64,dmlkZW8="); + expect(requestBody, JSON.stringify(streamError)).toBeDefined(); + const messages = requestBody?.messages as Array<{ content?: Array> }>; + expect(messages[0]?.content).toEqual([ + { type: "text", text: "before" }, + { type: "image_url", image_url: { url: "data:image/png;base64,aW1hZ2U=" } }, + { type: "video_url", video_url: { url: "data:video/mp4;base64,dmlkZW8=" } }, + { type: "text", text: "after" }, + ]); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + } + }); +}); diff --git a/extensions/moonshot/native-video.ts b/extensions/moonshot/native-video.ts new file mode 100644 index 000000000000..ed3a7c646181 --- /dev/null +++ b/extensions/moonshot/native-video.ts @@ -0,0 +1,93 @@ +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { resolveProviderContext, streamSimple } from "openclaw/plugin-sdk/llm"; +import type { ProviderWrapStreamFnContext } from "openclaw/plugin-sdk/plugin-entry"; +import { + createMoonshotThinkingWrapper, + resolveMoonshotThinkingKeep, + resolveMoonshotThinkingType, +} from "openclaw/plugin-sdk/provider-stream-shared"; +import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + isMoonshotAlwaysThinkingModelId, + isMoonshotK3NativeVideoRoute, +} from "./provider-policy-api.js"; + +const VIDEO_PREFIX = "data:video/mp4;base64,"; +const VIDEO_OMISSION = "(video omitted: untrusted or unsupported Moonshot video)"; +const MOONSHOT_REQUEST_BYTES_EXCLUSIVE = 100_000_000; + +function forEachUserContentPart(payload: unknown, visit: (part: Record) => void) { + const messages = isRecord(payload) && Array.isArray(payload.messages) ? payload.messages : []; + for (const message of messages) { + if (!isRecord(message) || message.role !== "user" || !Array.isArray(message.content)) { + continue; + } + for (const part of message.content) { + if (isRecord(part)) { + visit(part); + } + } + } +} + +function partUrl(part: Record, field: "image_url" | "video_url") { + const url = isRecord(part[field]) ? part[field].url : undefined; + return typeof url === "string" ? url : undefined; +} + +function replacePart(part: Record, text: string) { + Object.keys(part).forEach((key) => Reflect.deleteProperty(part, key)); + Object.assign(part, { type: "text", text }); +} + +function finalizePayload( + result: unknown, + payload: Record, + requestBytesExclusive: number, +) { + const admitted: Record[] = []; + forEachUserContentPart(payload, (part) => { + const videoUrl = partUrl(part, "video_url"); + const imageUrl = partUrl(part, "image_url"); + if (part.type === "video_url" && videoUrl?.startsWith(VIDEO_PREFIX)) { + admitted.push(part); + } else if (part.type === "video_url" || imageUrl?.startsWith("data:video/")) { + replacePart(part, VIDEO_OMISSION); + } + }); + const isOversized = () => + Buffer.byteLength(JSON.stringify(payload), "utf8") >= requestBytesExclusive; + while (admitted.length > 0 && isOversized()) { + replacePart(admitted.pop()!, "(video omitted: Moonshot request size limit)"); + } + if (isOversized()) { + throw new Error(`Moonshot request body must be smaller than ${requestBytesExclusive} bytes`); + } + return result; +} + +export function wrapMoonshotStream( + ctx: ProviderWrapStreamFnContext, + simple = false, + requestBytesExclusive = MOONSHOT_REQUEST_BYTES_EXCLUSIVE, +): StreamFn { + const underlying = ctx.streamFn ?? streamSimple; + if (simple && !isMoonshotAlwaysThinkingModelId(ctx.modelId)) { + return underlying; + } + const withVideoContext: StreamFn = (model, context, options) => + isMoonshotK3NativeVideoRoute({ ...model, modelId: model.id }) + ? resolveProviderContext(context, options as never).then((providerContext) => + underlying(model, providerContext as never, options), + ) + : underlying(model, context, options); + return createMoonshotThinkingWrapper( + withVideoContext, + resolveMoonshotThinkingType({ + configuredThinking: ctx.extraParams?.thinking, + thinkingLevel: ctx.thinkingLevel, + }), + resolveMoonshotThinkingKeep({ configuredThinking: ctx.extraParams?.thinking }), + (result, payload) => finalizePayload(result, payload, requestBytesExclusive), + ); +} diff --git a/extensions/moonshot/onboard.ts b/extensions/moonshot/onboard.ts index d3e0a1e02dd4..64348e66c578 100644 --- a/extensions/moonshot/onboard.ts +++ b/extensions/moonshot/onboard.ts @@ -15,20 +15,18 @@ const moonshotPresetAppliers = createDefaultModelPresetAppliers<[string]>({ primaryModelRef: MOONSHOT_DEFAULT_MODEL_REF, resolveParams: (_cfg: OpenClawConfig, baseUrl: string) => { const defaultModel = buildMoonshotProvider().models.find( - (model) => model.id === MOONSHOT_DEFAULT_MODEL_ID, + ({ id }) => id === MOONSHOT_DEFAULT_MODEL_ID, ); - if (!defaultModel) { - return null; - } - - return { - providerId: "moonshot", - api: "openai-completions", - baseUrl, - defaultModel, - defaultModelId: MOONSHOT_DEFAULT_MODEL_ID, - aliases: [{ modelRef: MOONSHOT_DEFAULT_MODEL_REF, alias: "Kimi" }], - }; + return defaultModel + ? { + providerId: "moonshot", + api: "openai-completions", + baseUrl, + defaultModel, + defaultModelId: MOONSHOT_DEFAULT_MODEL_ID, + aliases: [{ modelRef: MOONSHOT_DEFAULT_MODEL_REF, alias: "Kimi" }], + } + : null; }, }); diff --git a/extensions/moonshot/provider-catalog.ts b/extensions/moonshot/provider-catalog.ts index 2744634a6332..ceaa1217e642 100644 --- a/extensions/moonshot/provider-catalog.ts +++ b/extensions/moonshot/provider-catalog.ts @@ -3,7 +3,6 @@ import { applyProviderNativeStreamingUsageCompat, buildManifestModelProviderConfig, readManifestProviderDefaultModelRef, - supportsNativeStreamingUsageCompat, } from "openclaw/plugin-sdk/provider-catalog-shared"; import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared"; import manifest from "./openclaw.plugin.json" with { type: "json" }; @@ -17,10 +16,9 @@ export const MOONSHOT_DEFAULT_MODEL_REF = readManifestProviderDefaultModelRef( export const MOONSHOT_DEFAULT_MODEL_ID = MOONSHOT_DEFAULT_MODEL_REF.slice("moonshot/".length); export function isNativeMoonshotBaseUrl(baseUrl: string | undefined): boolean { - return supportsNativeStreamingUsageCompat({ - providerId: "moonshot", - baseUrl, - }); + return [MOONSHOT_BASE_URL, MOONSHOT_CN_BASE_URL].some( + (official) => baseUrl === official || baseUrl === `${official}/`, + ); } export function applyMoonshotNativeStreamingUsageCompat( diff --git a/extensions/moonshot/provider-contract-api.ts b/extensions/moonshot/provider-contract-api.ts index 90e732ea505f..2dc3139805fb 100644 --- a/extensions/moonshot/provider-contract-api.ts +++ b/extensions/moonshot/provider-contract-api.ts @@ -1,5 +1,6 @@ // Moonshot API module exposes the plugin public contract. import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared"; +import manifest from "./openclaw.plugin.json" with { type: "json" }; const noopAuth = async () => ({ profiles: [] }); @@ -9,27 +10,13 @@ export function createMoonshotProvider(): ProviderPlugin { label: "Moonshot", docsPath: "/providers/moonshot", aliases: ["moonshotai", "moonshot-ai"], - auth: [ - { - id: "api-key", - kind: "api_key", - label: "Kimi API key (.ai)", - hint: "Kimi API models · https://platform.kimi.ai/docs/pricing/chat", - run: noopAuth, - wizard: { - groupLabel: "Moonshot AI (Kimi)", - }, - }, - { - id: "api-key-cn", - kind: "api_key", - label: "Kimi API key (.cn)", - hint: "Kimi API models · https://platform.kimi.ai/docs/pricing/chat", - run: noopAuth, - wizard: { - groupLabel: "Moonshot AI (Kimi)", - }, - }, - ], + auth: manifest.providerAuthChoices.map((choice) => ({ + id: choice.method, + kind: "api_key", + label: choice.choiceLabel, + hint: choice.groupHint, + run: noopAuth, + wizard: { groupLabel: choice.groupLabel }, + })), }; } diff --git a/extensions/moonshot/provider-policy-api.ts b/extensions/moonshot/provider-policy-api.ts index 3cd37fc6fb3f..eefaab2aa297 100644 --- a/extensions/moonshot/provider-policy-api.ts +++ b/extensions/moonshot/provider-policy-api.ts @@ -1,32 +1,41 @@ // Moonshot policy module exposes model-specific thinking controls before runtime registration. import type { ProviderDefaultThinkingPolicyContext } from "openclaw/plugin-sdk/core"; +import { isNativeMoonshotBaseUrl } from "./provider-catalog.js"; export const KIMI_K2_7_CODE_MODEL_ID = "kimi-k2.7-code"; export const KIMI_K2_7_CODE_HIGHSPEED_MODEL_ID = "kimi-k2.7-code-highspeed"; export const KIMI_K3_MODEL_ID = "kimi-k3"; +const ALWAYS_THINKING_PROFILES = { + [KIMI_K3_MODEL_ID]: { id: "max", label: "max" }, + [KIMI_K2_7_CODE_MODEL_ID]: { id: "low", label: "on" }, + [KIMI_K2_7_CODE_HIGHSPEED_MODEL_ID]: { id: "low", label: "on" }, +} as const; + +export function isMoonshotK3NativeVideoRoute(route: { + provider?: string; + modelId?: string; + api?: string; + baseUrl?: string; +}): boolean { + return ( + route.provider === "moonshot" && + route.modelId === KIMI_K3_MODEL_ID && + route.api === "openai-completions" && + isNativeMoonshotBaseUrl(route.baseUrl) + ); +} export function isMoonshotAlwaysThinkingModelId(modelId: string): boolean { - const normalized = modelId.trim().toLowerCase(); - return ( - normalized === KIMI_K2_7_CODE_MODEL_ID || - normalized === KIMI_K2_7_CODE_HIGHSPEED_MODEL_ID || - normalized === KIMI_K3_MODEL_ID - ); + return modelId.trim().toLowerCase() in ALWAYS_THINKING_PROFILES; } export function resolveThinkingProfile(context: ProviderDefaultThinkingPolicyContext) { const modelId = context.modelId.trim().toLowerCase(); - if (modelId === KIMI_K3_MODEL_ID) { + const profile = ALWAYS_THINKING_PROFILES[modelId as keyof typeof ALWAYS_THINKING_PROFILES]; + if (profile) { return { - levels: [{ id: "max" as const, label: "max" }], - defaultLevel: "max" as const, - preserveWhenCatalogReasoningFalse: true, - }; - } - if (modelId === KIMI_K2_7_CODE_MODEL_ID || modelId === KIMI_K2_7_CODE_HIGHSPEED_MODEL_ID) { - return { - levels: [{ id: "low" as const, label: "on" }], - defaultLevel: "low" as const, + levels: [profile], + defaultLevel: profile.id, preserveWhenCatalogReasoningFalse: true, }; } diff --git a/packages/ai/src/openai-completions-messages.test.ts b/packages/ai/src/openai-completions-messages.test.ts index 6f538448e1eb..f3eba605bbe0 100644 --- a/packages/ai/src/openai-completions-messages.test.ts +++ b/packages/ai/src/openai-completions-messages.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { convertMessages } from "./openai-completions-messages.js"; +import type { ProviderContext, ProviderModel } from "./provider-types.js"; import { resolveOpenAICompletionsCompat } from "./transports/openai-completions-compat.js"; import type { AssistantMessage, Context, Model } from "./types.js"; @@ -26,6 +27,40 @@ const emptyUsage = { }; describe("convertMessages assistant text replay", () => { + it("serializes advertised video in ordered Chat Completions user content", () => { + const videoModel = { + ...model, + input: ["text", "image", "video"], + } as ProviderModel<"openai-completions">; + const context: ProviderContext = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "before" }, + { type: "image", mimeType: "image/png", data: "image" }, + { type: "video", mimeType: "video/mp4", data: "video" }, + { type: "text", text: "after" }, + ], + timestamp: 1, + }, + ], + }; + + const converted = convertMessages( + videoModel as Model<"openai-completions">, + context as Context, + resolveOpenAICompletionsCompat(videoModel as Model<"openai-completions">), + ); + + expect(converted[0]?.content).toEqual([ + { type: "text", text: "before" }, + { type: "image_url", image_url: { url: "data:image/png;base64,image" } }, + { type: "video_url", video_url: { url: "data:video/mp4;base64,video" } }, + { type: "text", text: "after" }, + ]); + }); + it("keeps separate assistant text blocks apart", () => { const assistant: AssistantMessage = { role: "assistant", diff --git a/packages/ai/src/openai-completions-messages.ts b/packages/ai/src/openai-completions-messages.ts index 961679ed2bc7..572b74661add 100644 --- a/packages/ai/src/openai-completions-messages.ts +++ b/packages/ai/src/openai-completions-messages.ts @@ -8,6 +8,7 @@ import type { ChatCompletionToolMessageParam, } from "openai/resources/chat/completions.js"; import { transformProviderMessages as transformMessages } from "./provider-transcript-transform.js"; +import type { ProviderMessage } from "./provider-types.js"; import { describeToolResultMediaPlaceholder, extractToolResultText, @@ -19,6 +20,10 @@ import { sanitizeSurrogates } from "./utils/sanitize-unicode.js"; import { stripSystemPromptCacheBoundary } from "./utils/system-prompt-cache-boundary.js"; const EMPTY_TOOL_RESULT_TEXT = "(no output)"; +type ChatCompletionContentPartVideo = { + type: "video_url"; + video_url: { url: string }; +}; function isTextContentBlock(block: { type: string }): block is TextContent { return block.type === "text"; @@ -76,7 +81,7 @@ export function convertMessages( const transformedMessages = transformMessages(context.messages, model, (id) => normalizeToolCallId(id), - ); + ) as ProviderMessage[]; if (context.systemPrompt) { const useDeveloperRole = model.reasoning && compat.supportsDeveloperRole; @@ -114,24 +119,29 @@ export function convertMessages( } params.push(userParam); } else { - const content: ChatCompletionContentPart[] = msg.content.map( - (item): ChatCompletionContentPart => { + const content: Array = + msg.content.map((item) => { if (item.type === "text") { return { type: "text", text: sanitizeSurrogates(item.text), } satisfies ChatCompletionContentPartText; } + if (item.type === "video") { + return { + type: "video_url", + video_url: { url: `data:${item.mimeType};base64,${item.data}` }, + } satisfies ChatCompletionContentPartVideo; + } return { type: "image_url", image_url: { url: `data:${item.mimeType};base64,${item.data}` }, } satisfies ChatCompletionContentPartImage; - }, - ); + }); if (content.length === 0) { continue; } - const userParam: ChatCompletionMessageParam = { role: "user", content }; + const userParam = { role: "user", content } as ChatCompletionMessageParam; if (isRuntimeContextCarrier) { options.cacheOptOutIndexes?.add(params.length); } diff --git a/packages/ai/src/provider-transcript-transform.ts b/packages/ai/src/provider-transcript-transform.ts index f858b8339766..93de187ccd3d 100644 --- a/packages/ai/src/provider-transcript-transform.ts +++ b/packages/ai/src/provider-transcript-transform.ts @@ -4,7 +4,6 @@ import type { ModelInputContent, ProviderMessage, ProviderModel, - VideoContent, } from "./provider-types.js"; import { transformMessages } from "./transcript-transform.js"; import type { Message, Model as CanonicalModel } from "./types.js"; @@ -15,10 +14,15 @@ const VIDEO_OMISSION = "(video omitted: provider does not support video input)"; function projectUserMediaForTransport( content: ModelInputContent[], supportsImages: boolean, -): Exclude[] { - const result: Exclude[] = []; + supportsVideo: boolean, +): ModelInputContent[] { + const result: ModelInputContent[] = []; for (const block of content) { - if (block.type === "text" || (block.type === "image" && supportsImages)) { + const supported = + block.type === "text" || + (block.type === "image" && supportsImages) || + (block.type === "video" && supportsVideo); + if (supported) { result.push(block); continue; } @@ -50,9 +54,13 @@ export function transformProviderMessages( return message as Message; } return Object.assign({}, message, { - content: projectUserMediaForTransport(message.content, model.input.includes("image")), + content: projectUserMediaForTransport( + message.content, + model.input.includes("image"), + model.api === "openai-completions" && model.input.includes("video"), + ), }) as Extract; - }), + }) as Message[], target, normalizeToolCallId, ); diff --git a/packages/ai/src/providers/transform-messages.test.ts b/packages/ai/src/providers/transform-messages.test.ts index e058ae947fe4..ac8ae21d9625 100644 --- a/packages/ai/src/providers/transform-messages.test.ts +++ b/packages/ai/src/providers/transform-messages.test.ts @@ -88,10 +88,21 @@ describe("transformMessages", () => { expect(advertised[0]?.content).toEqual([ { type: "text", text: "before" }, { type: "image", data: "image-one", mimeType: "image/png" }, - { type: "text", text: "(video omitted: provider does not support video input)" }, + { type: "video", data: sentinel, mimeType: "video/mp4" }, { type: "text", text: "after" }, { type: "image", data: "image-two", mimeType: "image/jpeg" }, ]); + + const responsesModel = { + ...advertisedVideoModel, + api: "openai-responses" as const, + } as ProviderModel<"openai-responses">; + const responses = transformProviderMessages(messages, responsesModel); + expect(responses[0]?.content).toContainEqual({ + type: "text", + text: "(video omitted: provider does not support video input)", + }); + expect(JSON.stringify(responses)).not.toContain(sentinel); }); it("preserves structured tool blocks while projecting only real images", () => { diff --git a/src/llm/providers/stream-wrappers/moonshot-thinking.ts b/src/llm/providers/stream-wrappers/moonshot-thinking.ts index ee79d843eb6a..754de054b661 100644 --- a/src/llm/providers/stream-wrappers/moonshot-thinking.ts +++ b/src/llm/providers/stream-wrappers/moonshot-thinking.ts @@ -7,156 +7,114 @@ import { streamSimple } from "../../stream.js"; type MoonshotThinkingType = "enabled" | "disabled"; type MoonshotThinkingKeep = "all"; -const MOONSHOT_THINKING_KEEP_MODEL_ID = "kimi-k2.6"; -const MOONSHOT_PROVIDER_ID = "moonshot"; -const MOONSHOT_K2_7_CODE_MODEL_IDS = ["kimi-k2.7-code", "kimi-k2.7-code-highspeed"] as const; -const MOONSHOT_K3_MODEL_ID = "kimi-k3"; -const MOONSHOT_FIXED_SAMPLING_FIELDS = [ - "temperature", - "top_p", - "n", - "presence_penalty", - "frequency_penalty", -] as const; -type MoonshotK27CodeModel = (typeof MOONSHOT_K2_7_CODE_MODEL_IDS)[number]; -type MoonshotAlwaysThinkingModel = MoonshotK27CodeModel | "kimi-k3"; - -async function loadDefaultStreamFn(): Promise { - return streamSimple; -} +type MoonshotPayloadFinalizer = (result: unknown, payload: Record) => unknown; +const MOONSHOT_ALWAYS_THINKING = { + "kimi-k2.7-code": "low", + "kimi-k2.7-code-highspeed": "low", + "kimi-k3": "max", +} as const; +const FIXED_SAMPLING_FIELDS = "temperature top_p n presence_penalty frequency_penalty".split(" "); +type MoonshotAlwaysThinkingEffort = "low" | "max"; function normalizeMoonshotThinkingType(value: unknown): MoonshotThinkingType | undefined { - if (typeof value === "boolean") { - return value ? "enabled" : "disabled"; + const type = asPayloadRecord(value)?.type ?? value; + if (typeof type === "boolean") { + return type ? "enabled" : "disabled"; } - if (typeof value === "string") { - const normalized = normalizeOptionalLowercaseString(value); - if (!normalized) { - return undefined; - } - if (["enabled", "enable", "on", "true"].includes(normalized)) { - return "enabled"; - } - if (["disabled", "disable", "off", "false"].includes(normalized)) { - return "disabled"; - } - return undefined; + const normalized = normalizeOptionalLowercaseString(type); + if (["enabled", "enable", "on", "true"].includes(normalized ?? "")) { + return "enabled"; } - if (value && typeof value === "object" && !Array.isArray(value)) { - return normalizeMoonshotThinkingType((value as Record).type); + if (["disabled", "disable", "off", "false"].includes(normalized ?? "")) { + return "disabled"; } return undefined; } -function normalizeMoonshotThinkingKeep(value: unknown): MoonshotThinkingKeep | undefined { - if (!value || typeof value !== "object" || Array.isArray(value)) { - return undefined; - } - const keepValue = (value as Record).keep; - if (typeof keepValue !== "string") { - return undefined; - } - return normalizeOptionalLowercaseString(keepValue) === "all" ? "all" : undefined; -} - function isMoonshotToolChoiceCompatible(toolChoice: unknown): boolean { - if (toolChoice == null || toolChoice === "auto" || toolChoice === "none") { - return true; - } - if (typeof toolChoice === "object" && !Array.isArray(toolChoice)) { - const typeValue = (toolChoice as Record).type; - return typeValue === "auto" || typeValue === "none"; - } - return false; + const type = asPayloadRecord(toolChoice)?.type ?? toolChoice; + return type == null || type === "auto" || type === "none"; } -function isPinnedToolChoice(toolChoice: unknown): boolean { - if (!toolChoice || typeof toolChoice !== "object" || Array.isArray(toolChoice)) { - return false; - } - const typeValue = (toolChoice as Record).type; - return typeValue === "tool" || typeValue === "function"; -} - -function ensureMoonshotToolCallReasoningContent(payloadObj: Record): void { - if (!Array.isArray(payloadObj.messages)) { - return; - } - for (const message of payloadObj.messages) { +function ensureMoonshotToolCallReasoningContent(payload: Record): void { + const messages = Array.isArray(payload.messages) ? payload.messages : []; + for (const message of messages) { const record = asPayloadRecord(message); - if ( - record?.role === "assistant" && - Array.isArray(record.tool_calls) && - record.tool_calls.length > 0 && - !("reasoning_content" in record) - ) { + if (record?.role !== "assistant" || !Array.isArray(record.tool_calls)) { + continue; + } + if (record.tool_calls.length > 0 && !("reasoning_content" in record)) { record.reasoning_content = ""; } } } -function sanitizeKimiK27Payload(payloadObj: Record): void { - delete payloadObj.thinking; - delete payloadObj.reasoning_effort; - delete payloadObj.reasoningEffort; - for (const field of MOONSHOT_FIXED_SAMPLING_FIELDS) { - delete payloadObj[field]; - } - if (!isMoonshotToolChoiceCompatible(payloadObj.tool_choice)) { - payloadObj.tool_choice = "auto"; - } -} - -function sanitizeKimiK3Payload(payloadObj: Record): void { - delete payloadObj.thinking; - delete payloadObj.reasoningEffort; - payloadObj.reasoning_effort = "max"; - for (const field of MOONSHOT_FIXED_SAMPLING_FIELDS) { - delete payloadObj[field]; - } +function resolveAlwaysThinkingEffort( + modelId: string, + directMoonshotModel: boolean, +): MoonshotAlwaysThinkingEffort | undefined { + const effort = MOONSHOT_ALWAYS_THINKING[modelId as keyof typeof MOONSHOT_ALWAYS_THINKING]; + return effort && (modelId !== "kimi-k3" || directMoonshotModel) ? effort : undefined; } function sanitizeAlwaysThinkingPayload( - payloadObj: Record, - modelId: MoonshotAlwaysThinkingModel, + payload: Record, + effort: MoonshotAlwaysThinkingEffort, ): void { - if (modelId === MOONSHOT_K3_MODEL_ID) { - sanitizeKimiK3Payload(payloadObj); + delete payload.thinking; + delete payload.reasoningEffort; + FIXED_SAMPLING_FIELDS.forEach((field) => Reflect.deleteProperty(payload, field)); + if (effort === "max") { + payload.reasoning_effort = effort; } else { - sanitizeKimiK27Payload(payloadObj); + delete payload.reasoning_effort; + if (!isMoonshotToolChoiceCompatible(payload.tool_choice)) { + payload.tool_choice = "auto"; + } } } -function sanitizeAlwaysThinkingAfterCaller( - value: unknown, - fallbackPayload: Record, - modelId: MoonshotAlwaysThinkingModel, -): unknown { - const finalPayload = asPayloadRecord(value) ?? fallbackPayload; - sanitizeAlwaysThinkingPayload(finalPayload, modelId); - ensureMoonshotToolCallReasoningContent(finalPayload); - return value; -} - -function resolveAlwaysThinkingModelId( +function prepareThinkingPayload( + payload: Record, modelId: string, directMoonshotModel: boolean, -): MoonshotAlwaysThinkingModel | undefined { - if (MOONSHOT_K2_7_CODE_MODEL_IDS.includes(modelId as MoonshotK27CodeModel)) { - return modelId as MoonshotK27CodeModel; + thinkingType?: MoonshotThinkingType, + thinkingKeep?: MoonshotThinkingKeep, +) { + const payloadModelId = + typeof payload.model === "string" ? payload.model.trim().toLowerCase() : modelId; + let effectiveThinkingType = normalizeMoonshotThinkingType(payload.thinking); + if (thinkingType) { + payload.thinking = { type: thinkingType }; + effectiveThinkingType = thinkingType; } - return directMoonshotModel && modelId === MOONSHOT_K3_MODEL_ID ? MOONSHOT_K3_MODEL_ID : undefined; -} - -function finalizeMoonshotPayloadAfterCaller( - value: unknown, - fallbackPayload: Record, - thinkingEnabled: boolean, -): unknown { - if (thinkingEnabled) { - ensureMoonshotToolCallReasoningContent(asPayloadRecord(value) ?? fallbackPayload); + const effort = resolveAlwaysThinkingEffort(payloadModelId, directMoonshotModel); + if (effort) { + sanitizeAlwaysThinkingPayload(payload, effort); + return (finalPayload: Record) => { + sanitizeAlwaysThinkingPayload(finalPayload, effort); + ensureMoonshotToolCallReasoningContent(finalPayload); + }; } - return value; + if (effectiveThinkingType === "enabled" && !isMoonshotToolChoiceCompatible(payload.tool_choice)) { + const toolChoiceType = asPayloadRecord(payload.tool_choice)?.type; + if (payload.tool_choice === "required") { + payload.tool_choice = "auto"; + } else if (toolChoiceType === "tool" || toolChoiceType === "function") { + payload.thinking = { type: "disabled" }; + effectiveThinkingType = "disabled"; + } + } + const thinking = asPayloadRecord(payload.thinking); + const preserveKeep = + payloadModelId === "kimi-k2.6" && effectiveThinkingType === "enabled" && thinkingKeep === "all"; + if (thinking) { + delete thinking.keep; + Object.assign(thinking, preserveKeep ? { keep: "all" } : {}); + } + return effectiveThinkingType === "enabled" + ? ensureMoonshotToolCallReasoningContent + : () => undefined; } /** @deprecated Moonshot provider-owned stream helper; do not use from third-party plugins. */ @@ -164,21 +122,18 @@ export function resolveMoonshotThinkingType(params: { configuredThinking: unknown; thinkingLevel?: ThinkLevel; }): MoonshotThinkingType | undefined { - const configured = normalizeMoonshotThinkingType(params.configuredThinking); - if (configured) { - return configured; - } - if (!params.thinkingLevel) { - return undefined; - } - return params.thinkingLevel === "off" ? "disabled" : "enabled"; + return ( + normalizeMoonshotThinkingType(params.configuredThinking) ?? + (params.thinkingLevel ? (params.thinkingLevel === "off" ? "disabled" : "enabled") : undefined) + ); } /** @deprecated Moonshot provider-owned stream helper; do not use from third-party plugins. */ export function resolveMoonshotThinkingKeep(params: { configuredThinking: unknown; }): MoonshotThinkingKeep | undefined { - return normalizeMoonshotThinkingKeep(params.configuredThinking); + const keep = normalizeOptionalLowercaseString(asPayloadRecord(params.configuredThinking)?.keep); + return keep === "all" ? "all" : undefined; } /** @deprecated Moonshot provider-owned stream helper; do not use from third-party plugins. */ @@ -186,103 +141,41 @@ export function createMoonshotThinkingWrapper( baseStreamFn: StreamFn | undefined, thinkingType?: MoonshotThinkingType, thinkingKeep?: MoonshotThinkingKeep, + finalizePayload?: MoonshotPayloadFinalizer, ): StreamFn { - const wrap = - (underlying: StreamFn): StreamFn => - (model, context, options) => { - const modelId = model.id.trim().toLowerCase(); - const directMoonshotModel = - normalizeOptionalLowercaseString(model.provider) === MOONSHOT_PROVIDER_ID; - const alwaysThinkingModel = resolveAlwaysThinkingModelId(modelId, directMoonshotModel); - const streamModel = alwaysThinkingModel ? { ...model, reasoning: true } : model; - const streamOptions = alwaysThinkingModel - ? { - ...options, - reasoning: - alwaysThinkingModel === MOONSHOT_K3_MODEL_ID ? ("max" as const) : ("low" as const), - } - : options; - const originalOnPayload = streamOptions?.onPayload; - return underlying(streamModel, context, { - ...streamOptions, - onPayload(payload, payloadModel) { - const payloadObj = asPayloadRecord(payload); - if (!payloadObj) { - return originalOnPayload?.(payload, payloadModel); - } - const payloadModelId = - typeof payloadObj.model === "string" ? payloadObj.model.trim().toLowerCase() : modelId; - let effectiveThinkingType = normalizeMoonshotThinkingType(payloadObj.thinking); - - if (thinkingType) { - payloadObj.thinking = { type: thinkingType }; - effectiveThinkingType = thinkingType; - } - - const payloadAlwaysThinkingModel = resolveAlwaysThinkingModelId( - payloadModelId, - directMoonshotModel, - ); - if (payloadAlwaysThinkingModel) { - // These models fix their reasoning and sampling contract. Reapply it - // after caller hooks so extra_body cannot restore rejected fields. - sanitizeAlwaysThinkingPayload(payloadObj, payloadAlwaysThinkingModel); - const result = originalOnPayload?.(payload, payloadModel); - if (result && typeof (result as Promise).then === "function") { - return Promise.resolve(result).then((resolved) => - sanitizeAlwaysThinkingAfterCaller(resolved, payloadObj, payloadAlwaysThinkingModel), - ); - } - return sanitizeAlwaysThinkingAfterCaller( - result, - payloadObj, - payloadAlwaysThinkingModel, - ); - } - - if ( - effectiveThinkingType === "enabled" && - !isMoonshotToolChoiceCompatible(payloadObj.tool_choice) - ) { - if (payloadObj.tool_choice === "required") { - payloadObj.tool_choice = "auto"; - } else if (isPinnedToolChoice(payloadObj.tool_choice)) { - payloadObj.thinking = { type: "disabled" }; - effectiveThinkingType = "disabled"; - } - } - - // thinking.keep is only valid on kimi-k2.6 when thinking is enabled. Gate - // by the final payload.model and final type so stray config never leaks. - const isKeepCapableModel = payloadModelId === MOONSHOT_THINKING_KEEP_MODEL_ID; - if (payloadObj.thinking && typeof payloadObj.thinking === "object") { - const thinkingObj = payloadObj.thinking as Record; - if ( - isKeepCapableModel && - effectiveThinkingType === "enabled" && - thinkingKeep === "all" - ) { - thinkingObj.keep = "all"; - } else if ("keep" in thinkingObj) { - delete thinkingObj.keep; - } - } - const result = originalOnPayload?.(payload, payloadModel); - const thinkingEnabled = effectiveThinkingType === "enabled"; - if (result && typeof (result as Promise).then === "function") { - return Promise.resolve(result).then((resolved) => - finalizeMoonshotPayloadAfterCaller(resolved, payloadObj, thinkingEnabled), - ); - } - return finalizeMoonshotPayloadAfterCaller(result, payloadObj, thinkingEnabled); - }, - }); - }; - if (baseStreamFn) { - return wrap(baseStreamFn); - } - return async (model, context, options) => { - const underlying = await loadDefaultStreamFn(); - return wrap(underlying)(model, context, options); + const underlying = baseStreamFn ?? streamSimple; + return function moonshotThinkingStream(model, context, options) { + const modelId = model.id.trim().toLowerCase(); + const directMoonshotModel = normalizeOptionalLowercaseString(model.provider) === "moonshot"; + const alwaysThinkingEffort = resolveAlwaysThinkingEffort(modelId, directMoonshotModel); + const streamModel = alwaysThinkingEffort ? { ...model, reasoning: true } : model; + const streamOptions = alwaysThinkingEffort + ? { ...options, reasoning: alwaysThinkingEffort } + : options; + return underlying(streamModel, context, { + ...streamOptions, + onPayload(payload, payloadModel) { + const record = asPayloadRecord(payload); + if (!record) { + return streamOptions?.onPayload?.(payload, payloadModel); + } + const postThinking = prepareThinkingPayload( + record, + modelId, + directMoonshotModel, + thinkingType, + thinkingKeep, + ); + const finish = (result: unknown) => { + const finalPayload = asPayloadRecord(result) ?? record; + postThinking(finalPayload); + return finalizePayload ? finalizePayload(result, finalPayload) : result; + }; + const result = streamOptions?.onPayload?.(payload, payloadModel); + return result && typeof (result as Promise).then === "function" + ? Promise.resolve(result).then(finish) + : finish(result); + }, + }); }; } diff --git a/src/plugin-sdk/provider-stream-shared.ts b/src/plugin-sdk/provider-stream-shared.ts index 50be6e481105..d63fb2c0d851 100644 --- a/src/plugin-sdk/provider-stream-shared.ts +++ b/src/plugin-sdk/provider-stream-shared.ts @@ -700,6 +700,7 @@ export { export { applyAnthropicEphemeralCacheControlMarkers } from "../llm/providers/stream-wrappers/anthropic-cache-control-payload.js"; export { createMoonshotThinkingWrapper, + resolveMoonshotThinkingKeep, resolveMoonshotThinkingType, } from "../llm/providers/stream-wrappers/moonshot-thinking.js"; export { streamWithPayloadPatch };