diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json index 96eb39ae2856..4a1bbdf2fcd0 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json @@ -1 +1 @@ -{"contentHash":"5a9d78a04eff220d33c4d2b7230c25f3d846ccddb4f2165967cf00fb948affa1","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} +{"contentHash":"8c91a4aac45efb318043e133624a8334599c7f0f73c7ba7212dbb661f88c3b17","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json index c37ea0235685..c1db7b6b603f 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json @@ -1 +1 @@ -{"contentHash":"48813fa984919c8790905b995f4033d19fc9178e6eed7f2e5148ea9729c5eba3","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} +{"contentHash":"cea38cb2354e456c9dcb62fcb59d4c5a1850eb7b35cf2c6dc1e6b4052cb239f2","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json index 796dc895d01a..06735feebbbd 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json @@ -1 +1 @@ -{"contentHash":"454a3daca1cb018f303ce3dbcb8ec4ea86dbec180623822c5ca5250953402415","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"} +{"contentHash":"b7c8bb7b8bbe40d1d877fb4c9c410663df59e2ceb236838e1564c04f3bd0e0c5","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-core.json b/docs/.generated/plugin-sdk-api-baseline/channel-core.json index 0699ec800411..2d9266aba832 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-core.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-core.json @@ -1 +1 @@ -{"contentHash":"df88650d813f6efbdffc5570647f183e69fa3015929c748b03e8b22bf0d916e6","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} +{"contentHash":"be5e5a878566af7ed7371bcc8bf6501771b7366c019e2e8132b1bd3d598b7b48","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json index 5afad8c35ad1..b655d7fc6a22 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json @@ -1 +1 @@ -{"contentHash":"d11b2d65878b0886a3dc102c99a74fcd5e9bb80f27809ee810812f26ddde73cf","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} +{"contentHash":"f4a908c9a10596f9aec9e9ebfda956d87e7d38d7a66b98dd36dca655ab0df554","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json index 328e20d36d58..c941b9140c67 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json @@ -1 +1 @@ -{"contentHash":"81f7f67df730331d9fc2de922d1524833548a09ad69b2eb916440113ba91494c","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} +{"contentHash":"57bdf7a00a80ea1e2d06985d3e2989df155f9cd9735b5b190df713abf7184a31","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-message.json b/docs/.generated/plugin-sdk-api-baseline/channel-message.json index e01b7e3d1490..eccdad20a958 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-message.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-message.json @@ -1 +1 @@ -{"contentHash":"3059df01f7a73ba9c511ce97a0ef358fd8109389f524c0a3914984745aa02db5","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} +{"contentHash":"3642828e0c335f3412a575e04ac5179c82226386942fcaecaf6ecf229cefb246","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json index 3244217b6d9e..d0602845248d 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json @@ -1 +1 @@ -{"contentHash":"6fcaaa319bccd859ecbbe26278d3476ced29bdfed4e3ea0f05947c99d0e79bba","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} +{"contentHash":"b976e3217681b57aab8ecab4a94ecf0f8238fe4c68de6c7bb9e2588e9c726622","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json index c57780ebbc7a..fa3243fbeed2 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json @@ -1 +1 @@ -{"contentHash":"2d253351db6cb80191067f6ce1aefba74d35dbbf1816c2741078e91738d608d7","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} +{"contentHash":"b65bc3c8f77d9ae5fcfe03fe596f2cd948ce04a2544fd5df609c172ec3d4d267","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} diff --git a/docs/.generated/plugin-sdk-api-baseline/core.json b/docs/.generated/plugin-sdk-api-baseline/core.json index 598115b6a462..d472fdd43dee 100644 --- a/docs/.generated/plugin-sdk-api-baseline/core.json +++ b/docs/.generated/plugin-sdk-api-baseline/core.json @@ -1 +1 @@ -{"contentHash":"b9d274360639f0589b6036ba6d735d375a66edfe3f25fac3add96f5989c8bf8a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} +{"contentHash":"d22bf2e5ae14ee5ec01ea825c2bc16c908a15f78dd462c3e160393aa871d7cf5","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/discord.json b/docs/.generated/plugin-sdk-api-baseline/discord.json index 309a49da7d53..a04337202e29 100644 --- a/docs/.generated/plugin-sdk-api-baseline/discord.json +++ b/docs/.generated/plugin-sdk-api-baseline/discord.json @@ -1 +1 @@ -{"contentHash":"5fb15ccec47ac420b0a1fb6d0769faeff1e14f3da1905c3c3a29c24887f1aaf1","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} +{"contentHash":"66c63de1253ea9f27ada35301d7f9a53504483c10ca5dd281fe50b645b5bccd2","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} diff --git a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json index c9cc599e555e..94ad586bebc7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json +++ b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json @@ -1 +1 @@ -{"contentHash":"8c220f857b5144582c560810b63948b14ba87524eff0aa7407d39ccb7df7de8f","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} +{"contentHash":"c1226a9266ddddfd3cb3ec3842a3eb43d7b05738ff495e18b9a13654d8bbc3e7","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} diff --git a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json index 66ae476ec9e6..e8bf4f630bd5 100644 --- a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json @@ -1 +1 @@ -{"contentHash":"f7dd2a2df426b3826fb9bf31a062004801675eb89c006508a91cd5f7713a739b","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} +{"contentHash":"538b128a5de8b38c9c3e13cdbb74d0eaa0e6ce7a8066c5852204b77b37536734","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json index e7900f34dce2..48b6af5cce5a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json @@ -1 +1 @@ -{"contentHash":"6c93d74605b93456c5124fc15a3bb99d1a68734b9772a0514af3ded6fb647001","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} +{"contentHash":"9e50d560be0fae99edf2ab5400625033c3515d00b16113b085543d6f218c17cd","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json index 17e3f067c3ce..16700279f529 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json @@ -1 +1 @@ -{"contentHash":"f951f78a1ed81bc8c9eac5d2421e243abf28aac9297d21b2c174e22ce3fac045","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} +{"contentHash":"9748f103ca50929f6a67e77c6708c0fb1a24e5d83ec1ae9e15485a80fdc20398","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json index a2df38de5bd6..5bf9c996e115 100644 --- a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json @@ -1 +1 @@ -{"contentHash":"1291d3c53625156807274d0ddc752b3ae0b01bf937f0fdca504a879698d8194a","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} +{"contentHash":"4c5ae6de0b18a8ad16c4c6f8437bb18c0d2e8f511452ac47ef1442136c7f90a9","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json index 926bfbdb194d..36edd015a6f7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json +++ b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json @@ -1 +1 @@ -{"contentHash":"c4ad78f6f23d67778f2d37cd7492e2c21832bd2622f56fa17d16f9b12fb1e955","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} +{"contentHash":"a204b23b49393b23755ed3f6d5f44c14248abba916063c6a4f363d39a019c800","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} diff --git a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json index 6d34eaea09d0..010c789252ad 100644 --- a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json +++ b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json @@ -1 +1 @@ -{"contentHash":"2d76db26f7ce4188386d49fd13ed7daf5185cb22a6d2ac1fd9ade7288b6c7d0f","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} +{"contentHash":"ed3e64c4f91e222c62209dbb7ae7632ff3ea85ae5658264b559d36997f037b7b","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} diff --git a/extensions/google/google.live.test.ts b/extensions/google/google.live.test.ts index b8925fbe21ed..0a58af762e0a 100644 --- a/extensions/google/google.live.test.ts +++ b/extensions/google/google.live.test.ts @@ -1,6 +1,12 @@ // Google tests cover google plugin behavior. import { toErrorObject as toLintErrorObject } from "openclaw/plugin-sdk/error-runtime"; -import { completeSimple, type Model } from "openclaw/plugin-sdk/llm"; +import { + completeSimple, + type Model, + type ProviderContext, + type ProviderModel, + type ProviderStreamFunction, +} from "openclaw/plugin-sdk/llm"; import { resolveFfmpegBin } from "openclaw/plugin-sdk/media-runtime"; import { createCapturedPluginRegistration, @@ -24,6 +30,25 @@ const GOOGLE_API_KEY = const LIVE = isLiveTestEnabled() && GOOGLE_API_KEY.length > 0; const describeLive = LIVE ? describe : describe.skip; +// Two 64x64 solid-red H.264 frames keep native-video proof bounded and reproducible. +const GOOGLE_LIVE_RED_VIDEO_BASE64 = [ + "AAAAJGZ0eXBpc29tAAACAGlzb21pc282aXNvMmF2YzFtcDQxAAAC5m1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAAA", + "AAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "AAAAAAAAAAIAAAHodHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAA", + "AAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAABAAAAAQAAAAAABhG1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAQAAAAAAA", + "VcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAS9taW5mAAAAFHZtaGQAAAABAAAA", + "AAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAADvc3RibAAAAKNzdHNkAAAAAAAAAAEAAACTYXZj", + "MQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAABAAEAASAAAAEgAAAAAAAAAARVMYXZjNjIuMjguMTAyIGxpYngyNjQAAAAAAAAA", + "AAAAABj//wAAAC1hdmNDAULACv/hABZnQsAK2hCbARAAAAMAEAAAAwAo8SJqAQAEaM4PyAAAABBwYXNwAAAAAQAAAAEAAAAQ", + "c3R0cwAAAAAAAAAAAAAAEHN0c2MAAAAAAAAAAAAAABRzdHN6AAAAAAAAAAAAAAAAAAAAEHN0Y28AAAAAAAAAAAAAAChtdmV4", + "AAAAIHRyZXgAAAAAAAAAAQAAAAEAAAAAAAAAAAAAAAAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGly", + "YXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjYyLjEyLjEwMgAAAHhtb29mAAAAEG1m", + "aGQAAAAAAAAAAQAAAGB0cmFmAAAAJHRmaGQAAAA5AAAAAQAAAAAAAAMKAABAAAAAACMBAQAAAAAAFHRmZHQBAAAAAAAAAAAA", + "AAAAAAAgdHJ1bgAAAgUAAAACAAAAgAIAAAAAAAAjAAAACgAAADVtZGF0AAAAH2WIhDoRigACGPHAAED2OAAIeUnJyddddddd", + "dddddeAAAAAGQZogF6CMAAAAQ21mcmEAAAArdGZyYQEAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAMKAQEBAAAAEG1m", + "cm8AAAAAAAAAQw==", +].join(""); + async function withGoogleApiEnvUnset(fn: () => Promise): Promise { const geminiApiKey = process.env.GEMINI_API_KEY; const googleApiKey = process.env.GOOGLE_API_KEY; @@ -152,6 +177,74 @@ describeLive("google plugin live", () => { 90_000, ); + it("understands native video through the registered Google chat transport", async () => { + const catalog = await buildGoogleLiveCatalogProvider({ + apiKey: "GEMINI_API_KEY", + discoveryApiKey: GOOGLE_API_KEY, + }); + const definition = catalog.models.find((candidate) => candidate.id === "gemini-3.5-flash-lite"); + expect(definition, "gemini-3.5-flash-lite missing from Google models.list").toBeDefined(); + expect(definition?.input).toContain("video"); + + const catalogModel = { + ...definition!, + provider: "google", + baseUrl: catalog.baseUrl, + api: "google-generative-ai", + } as ProviderModel<"google-generative-ai">; + const { providers } = await registerGooglePlugin(); + const provider = requireRegisteredProvider(providers, "google"); + const model = provider.normalizeResolvedModel?.({ + provider: "google", + modelId: catalogModel.id, + model: catalogModel, + } as never) as ProviderModel<"google-generative-ai"> | undefined; + expect(model?.input).toContain("video"); + if (!model) { + throw new Error("registered Google provider did not prepare its native video model"); + } + + const registeredStream = provider.createStreamFn?.({ + provider: "google", + modelId: model.id, + model: model as never, + }); + const streamFn = registeredStream as ProviderStreamFunction<"google-generative-ai"> | undefined; + if (!streamFn) { + throw new Error("registered Google provider did not create its native chat transport"); + } + const context: ProviderContext = { + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What single color fills this video? Reply with exactly RED, BLUE, or GREEN.", + }, + { + type: "video", + mimeType: "video/mp4", + data: GOOGLE_LIVE_RED_VIDEO_BASE64, + }, + ], + timestamp: Date.now(), + }, + ], + }; + + const stream = await Promise.resolve( + streamFn(model, context, { apiKey: GOOGLE_API_KEY, maxTokens: 16 }), + ); + const response = await stream.result(); + expect(response.stopReason, response.errorMessage).not.toBe("error"); + const answer = response.content + .filter((block) => block.type === "text") + .map((block) => block.text) + .join(" "); + expect(answer).toMatch(/\bRED\b/iu); + }, 90_000); + it("synthesizes speech through the registered provider", async () => { const { speechProviders } = await registerGooglePlugin(); const provider = requireRegisteredProvider(speechProviders, "google"); diff --git a/extensions/google/provider-catalog.test.ts b/extensions/google/provider-catalog.test.ts index e3e9c5bca91b..d9f98c624254 100644 --- a/extensions/google/provider-catalog.test.ts +++ b/extensions/google/provider-catalog.test.ts @@ -32,6 +32,7 @@ describe("google provider catalog", () => { contextWindow: 1_048_576, maxTokens: 65_536, reasoning: true, + input: ["text", "image"], }); }); @@ -39,6 +40,7 @@ describe("google provider catalog", () => { expect(buildGoogleVertexStaticCatalogProvider().models.map((model) => model.id)).toEqual( buildGoogleStaticCatalogProvider().models.map((model) => model.id), ); + expect(buildGoogleStaticCatalogProvider().models[0]?.input).toEqual(["text", "image", "video"]); }); it("builds the authenticated text catalog from Google models.list metadata", async () => { @@ -121,7 +123,7 @@ describe("google provider catalog", () => { reasoning: true, contextWindow: 1_048_576, maxTokens: 65_536, - input: ["text", "image"], + input: ["text", "image", "video"], compat: { codeMode: "preferred" }, }), expect.objectContaining({ @@ -130,7 +132,7 @@ describe("google provider catalog", () => { reasoning: true, contextWindow: 1_048_576, maxTokens: 65_536, - input: ["text", "image"], + input: ["text", "image", "video"], compat: { codeMode: "preferred" }, }), expect.objectContaining({ diff --git a/extensions/google/provider-catalog.ts b/extensions/google/provider-catalog.ts index 6ae43afeec40..c93d9a8aefe6 100644 --- a/extensions/google/provider-catalog.ts +++ b/extensions/google/provider-catalog.ts @@ -59,7 +59,10 @@ export function buildGoogleStaticCatalogProvider(): ModelProviderConfig { return { baseUrl: GOOGLE_GEMINI_BASE_URL, api: "google-generative-ai", - models: GOOGLE_GEMINI_TEXT_MODELS, + models: GOOGLE_GEMINI_TEXT_MODELS.map((model) => ({ + ...model, + input: [...model.input, "video"], + })), }; } @@ -73,7 +76,7 @@ function readGoogleLiveModels(body: unknown): readonly unknown[] { function googleLiveModelInput(id: string): ModelDefinitionConfig["input"] { if (!id.startsWith("gemma-")) { - return ["text", "image"]; + return ["text", "image", "video"]; } const isMultimodalGemma = /^gemma-3-(?:4b|12b|27b)(?:-|$)/.test(id) || diff --git a/extensions/google/provider-models.test.ts b/extensions/google/provider-models.test.ts index 0147b135b857..b7d42a18d1b2 100644 --- a/extensions/google/provider-models.test.ts +++ b/extensions/google/provider-models.test.ts @@ -3,6 +3,7 @@ import type { ProviderRuntimeModel } from "openclaw/plugin-sdk/plugin-entry"; import { describe, expect, it } from "vitest"; import { createProviderDynamicModelContext as createContext } from "../test-support/provider-model-test-helpers.js"; import { + isGoogleNativeVideoModelId, isGoogleTextGenerationModelId, isModernGoogleModel, resolveGoogleGeminiForwardCompatModel, @@ -580,4 +581,24 @@ describe("resolveGoogleGeminiForwardCompatModel", () => { ).toBeUndefined(); } }); + + it("classifies only ordinary Gemini generation ids for native video", () => { + for (const modelId of [ + "gemini-2.5-flash", + "google/gemini-3.1-pro-preview", + "models/gemini-flash-latest", + ]) { + expect(isGoogleNativeVideoModelId(modelId), modelId).toBe(true); + } + for (const modelId of [ + "gemma-4-26b-a4b-it", + "tunedModels/gemini-2.5-flash", + "gemini-3.1-flash-image", + "gemini-2.5-computer-use-preview", + "gemini-2.5-flash-tts-preview", + "gemini-2.5-flash-live-preview", + ]) { + expect(isGoogleNativeVideoModelId(modelId), modelId).toBe(false); + } + }); }); diff --git a/extensions/google/provider-models.ts b/extensions/google/provider-models.ts index c757e43fa759..77defb34fd39 100644 --- a/extensions/google/provider-models.ts +++ b/extensions/google/provider-models.ts @@ -49,7 +49,10 @@ function normalizeGeminiProRequestId(id: string): string { } function googleFamilyModelId(id: string): string { - return id.startsWith(GOOGLE_PROVIDER_PREFIX) ? id.slice(GOOGLE_PROVIDER_PREFIX.length) : id; + const unqualified = id.startsWith(GOOGLE_PROVIDER_PREFIX) + ? id.slice(GOOGLE_PROVIDER_PREFIX.length) + : id; + return unqualified.startsWith("models/") ? unqualified.slice("models/".length) : unqualified; } export function isGoogleTextGenerationModelId(id: string): boolean { @@ -71,6 +74,11 @@ export function isGoogleTextGenerationModelId(id: string): boolean { ); } +export function isGoogleNativeVideoModelId(id: string): boolean { + const normalized = normalizeOptionalLowercaseString(googleFamilyModelId(id)) ?? ""; + return normalized.startsWith("gemini-") && isGoogleTextGenerationModelId(normalized); +} + type GoogleForwardCompatFamily = readonly [ googleTemplateIds: readonly string[], cliTemplateIds: readonly string[], diff --git a/extensions/google/provider-policy.ts b/extensions/google/provider-policy.ts index 21a6adfbfd5c..d0bed91ba9b4 100644 --- a/extensions/google/provider-policy.ts +++ b/extensions/google/provider-policy.ts @@ -17,6 +17,7 @@ import { isGoogleGemini3ProModel, isGoogleGemini3ThinkingLevelModel } from "./th export { DEFAULT_GOOGLE_API_BASE_URL, isGoogleGenerativeAiApi, + isOfficialGoogleAiStudioBaseUrl, isGoogleVertexBaseUrl, isGoogleVertexHostname, normalizeGoogleApiBaseUrl, diff --git a/extensions/google/provider-registration.test.ts b/extensions/google/provider-registration.test.ts index 558006de06dc..067f2e3f0f6a 100644 --- a/extensions/google/provider-registration.test.ts +++ b/extensions/google/provider-registration.test.ts @@ -65,4 +65,47 @@ describe("buildGoogleProvider createStreamFn", () => { expect(streamFns.createVertex).not.toHaveBeenCalled(); expect(streamFns.createGenerativeAi).not.toHaveBeenCalled(); }); + + it.each([ + ["gemini-2.5-flash", "https://generativelanguage.googleapis.com", true], + ["google/gemini-3.1-pro-preview", "https://generativelanguage.googleapis.com/v1beta", true], + ["models/gemini-2.5-pro", "https://generativelanguage.googleapis.com/v1beta/", true], + ["gemma-4-26b-a4b-it", "https://generativelanguage.googleapis.com/v1beta", false], + ["gemini-2.5-flash-image", "https://generativelanguage.googleapis.com/v1beta", false], + ["gemini-2.5-flash", "https://proxy.example.test/v1beta", false], + ["gemini-2.5-flash", "https://user@generativelanguage.googleapis.com/v1beta", false], + ["gemini-2.5-flash", "https://generativelanguage.googleapis.com/v1beta?key=x", false], + ["gemini-2.5-flash", "https://generativelanguage.googleapis.com:8443/v1beta", false], + ["gemini-2.5-flash", "https://generativelanguage.googleapis.com/v1beta/openai", false], + ])("normalizes native-video input for exact AI Studio route %s", (modelId, baseUrl, expected) => { + const provider = buildGoogleProvider(); + const normalized = provider.normalizeResolvedModel?.({ + provider: "google", + modelId, + model: model({ + id: modelId, + provider: "google", + api: "google-generative-ai", + baseUrl, + input: ["text", "image", "video"] as never, + }), + } as never); + + expect(((normalized?.input ?? []) as string[]).includes("video")).toBe(expected); + }); + + it("strips inherited video for Vertex and non-Google provider routes", () => { + const provider = buildGoogleProvider(); + for (const [providerId, api] of [ + ["google-vertex", "google-vertex"], + ["custom-google", "google-generative-ai"], + ] as const) { + const normalized = provider.normalizeResolvedModel?.({ + provider: providerId, + modelId: "gemini-2.5-flash", + model: model({ provider: providerId, api, input: ["text", "image", "video"] as never }), + } as never); + expect(normalized?.input).toEqual(["text", "image"]); + } + }); }); diff --git a/extensions/google/provider-registration.ts b/extensions/google/provider-registration.ts index de15ca332aa2..09c39f6b5bc9 100644 --- a/extensions/google/provider-registration.ts +++ b/extensions/google/provider-registration.ts @@ -13,8 +13,13 @@ import { buildGoogleVertexStaticCatalogProvider, } from "./provider-catalog.js"; import { GOOGLE_GEMINI_PROVIDER_HOOKS } from "./provider-hooks.js"; -import { isModernGoogleModel, resolveGoogleGeminiForwardCompatModel } from "./provider-models.js"; import { + isGoogleNativeVideoModelId, + isModernGoogleModel, + resolveGoogleGeminiForwardCompatModel, +} from "./provider-models.js"; +import { + isOfficialGoogleAiStudioBaseUrl, isGoogleVertexBaseUrl, normalizeGoogleProviderConfig, resolveGoogleGenerativeAiTransport, @@ -25,6 +30,18 @@ import { } from "./transport-stream.js"; import { resolveGoogleVertexConfigApiKey } from "./vertex-adc.js"; +function normalizeGoogleVideoInput( + ctx: Parameters>[0], +) { + const input = (ctx.model.input as string[]).filter((type) => type !== "video"); + const supportsVideo = + ctx.provider === "google" && + ctx.model.api === "google-generative-ai" && + isOfficialGoogleAiStudioBaseUrl(ctx.model.baseUrl) && + isGoogleNativeVideoModelId(ctx.modelId); + return { ...ctx.model, input: supportsVideo ? [...input, "video"] : input } as typeof ctx.model; +} + function resolveGoogleReasoningOutputMode( ctx: ProviderReasoningOutputModeContext, ): "native" | "tagged" { @@ -100,6 +117,7 @@ export function buildGoogleProvider(): ProviderPlugin { }, }, normalizeModelId: ({ modelId }) => normalizeGoogleModelId(modelId), + normalizeResolvedModel: normalizeGoogleVideoInput, resolveDynamicModel: (ctx) => resolveGoogleGeminiForwardCompatModel({ providerId: ctx.provider, diff --git a/extensions/google/src/google-api-base-url.ts b/extensions/google/src/google-api-base-url.ts index 761b006d533f..bc2db1e48273 100644 --- a/extensions/google/src/google-api-base-url.ts +++ b/extensions/google/src/google-api-base-url.ts @@ -17,6 +17,19 @@ function isGoogleGenerativeAiUrl(url: URL): boolean { ); } +/** Exact official AI Studio request root eligible for native provider behavior. */ +export function isOfficialGoogleAiStudioBaseUrl(baseUrl?: string | null): boolean { + const raw = normalizeOptionalString(baseUrl) ?? DEFAULT_GOOGLE_API_BASE_URL; + try { + const href = trimTrailingSlashes(new URL(raw).href); + return ( + href === "https://generativelanguage.googleapis.com" || href === DEFAULT_GOOGLE_API_BASE_URL + ); + } catch { + return false; + } +} + function stripUrlUserInfo(url: URL): void { url.username = ""; url.password = ""; diff --git a/extensions/google/transport-stream.test.ts b/extensions/google/transport-stream.test.ts index 4b89dbfe9002..7c6f00eb4d65 100644 --- a/extensions/google/transport-stream.test.ts +++ b/extensions/google/transport-stream.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { gzipSync } from "node:zlib"; import { expectDefined } from "@openclaw/normalization-core"; import { toErrorObject as toLintErrorObject } from "openclaw/plugin-sdk/error-runtime"; -import type { Model } from "openclaw/plugin-sdk/llm"; +import type { Model, ProviderContext } from "openclaw/plugin-sdk/llm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { resetGoogleVertexAdcState } from "./google-oauth.test-support.js"; @@ -133,6 +133,16 @@ async function runGeminiStreamResult( return stream.result(); } +function withProviderContextHandoff( + options: Record, + handoff: () => Promise, +): Record { + return new Proxy(options, { + get: (target, property, receiver) => + typeof property === "symbol" ? handoff : Reflect.get(target, property, receiver), + }); +} + async function runGoogleVertexStreamResult(params: { model?: Model<"google-vertex">; fetch: typeof guardedFetchMock; @@ -469,6 +479,157 @@ describe("google transport stream", () => { vi.resetModules(); }); + it("resolves qualified AI Studio video after payload hooks and preserves part order", async () => { + mockGoogleTextResponse(); + const videoData = "current-video-base64"; + const quicktimeData = "current-quicktime-base64"; + const imageData = "current-image-base64"; + const providerContext: ProviderContext = { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "before" }, + { type: "image", mimeType: "image/png", data: imageData }, + { type: "video", mimeType: "video/mp4", data: videoData }, + { type: "video", mimeType: "video/quicktime", data: quicktimeData }, + { type: "text", text: "after" }, + ], + timestamp: 0, + }, + ], + }; + const handoff = vi.fn(async () => providerContext); + const onPayload = vi.fn((payload: unknown) => { + expect(JSON.stringify(payload)).not.toContain(videoData); + expect(JSON.stringify(payload)).not.toContain(quicktimeData); + expect(JSON.stringify(payload)).toContain("native video slot unavailable"); + return payload; + }); + + const result = await runGeminiStreamResult({ + model: buildGeminiModel({ input: ["text", "image", "video"] as never }), + context: { messages: [{ role: "user", content: "fallback", timestamp: 0 }] }, + options: withProviderContextHandoff({ onPayload }, handoff), + }); + expect(result.errorMessage).toBeUndefined(); + + const body = parseRequestJsonBody( + requireRequestInit(requireMockCall(guardedFetchMock, 0, "guarded fetch"), "guarded fetch"), + ) as { contents: GoogleTestContentTurn[] }; + expect(body.contents[0]?.parts).toEqual([ + { text: "before" }, + { inlineData: { mimeType: "image/png", data: imageData } }, + { inlineData: { mimeType: "video/mp4", data: videoData } }, + { inlineData: { mimeType: "video/quicktime", data: quicktimeData } }, + { text: "after" }, + ]); + expect(handoff).toHaveBeenCalledOnce(); + expect(onPayload).toHaveBeenCalledOnce(); + }); + + it("omits cloned and hook-injected video slots", async () => { + mockGoogleTextResponse(); + const trustedData = "trusted-video-base64"; + const injectedData = "injected-video-base64"; + const providerContext: ProviderContext = { + messages: [ + { + role: "user", + content: [{ type: "video", mimeType: "video/mp4", data: trustedData }], + timestamp: 0, + }, + ], + }; + await runGeminiStreamResult({ + model: buildGeminiModel({ input: ["text", "image", "video"] as never }), + options: withProviderContextHandoff( + { + onPayload: (payload: unknown) => { + const cloned = structuredClone(payload) as { + contents: GoogleTestContentTurn[]; + }; + cloned.contents[0]?.parts.push({ + inlineData: { mimeType: "video/mp4", data: injectedData }, + }); + return cloned; + }, + }, + async () => providerContext, + ), + }); + const bodyText = requireRequestInit( + requireMockCall(guardedFetchMock, 0, "guarded fetch"), + "guarded fetch", + ).body as string; + expect(bodyText).not.toContain(trustedData); + expect(bodyText).not.toContain(injectedData); + expect(bodyText).toContain("native video slot unavailable"); + }); + + it("rejects unsupported Google video MIME after the payload hook", async () => { + mockGoogleTextResponse(); + const videoData = "unsupported-mime-video"; + const onPayload = vi.fn((payload: unknown) => { + expect(JSON.stringify(payload)).not.toContain(videoData); + return payload; + }); + await runGeminiStreamResult({ + model: buildGeminiModel({ input: ["text", "image", "video"] as never }), + options: withProviderContextHandoff({ onPayload }, async () => ({ + messages: [ + { + role: "user", + content: [{ type: "video", mimeType: "video/x-msvideo", data: videoData }], + timestamp: 0, + }, + ], + })), + }); + const body = requireRequestInit( + requireMockCall(guardedFetchMock, 0, "guarded fetch"), + "guarded fetch", + ).body as string; + expect(body).not.toContain(videoData); + expect(body).toContain("unsupported Google video MIME type"); + }); + + it("evicts trusted video until the exact serialized request is below 20MB", async () => { + mockGoogleTextResponse(); + const providerContext: ProviderContext = { + messages: [ + { + role: "user", + content: [{ type: "video", mimeType: "video/mp4", data: "A".repeat(20_000_000) }], + timestamp: 0, + }, + ], + }; + await runGeminiStreamResult({ + model: buildGeminiModel({ input: ["text", "image", "video"] as never }), + options: withProviderContextHandoff({}, async () => providerContext), + }); + const body = requireRequestInit( + requireMockCall(guardedFetchMock, 0, "guarded fetch"), + "guarded fetch", + ).body as string; + expect(new TextEncoder().encode(body).byteLength).toBeLessThan(20_000_000); + expect(body).toContain("native video slot unavailable"); + }); + + it("returns a useful provider error when the non-video request still exceeds 20MB", async () => { + const result = await runGeminiStreamResult({ + context: { + messages: [{ role: "user", content: "A".repeat(20_000_000), timestamp: 0 }], + }, + }); + expect(result).toMatchObject({ + stopReason: "error", + errorMessage: "Google request body must be smaller than 20000000 bytes", + }); + expect(guardedFetchMock).not.toHaveBeenCalled(); + }); + it("uses the guarded fetch transport and parses Gemini SSE output", async () => { guardedFetchMock.mockResolvedValueOnce( buildSseResponse([ @@ -823,6 +984,15 @@ describe("google transport stream", () => { requireRequestInit(requireMockCall(guardedFetchMock, 1, "guarded fetch"), "guarded fetch"), { "x-goog-api-key": "gemini-key-2" }, ); + const firstBody = requireRequestInit( + requireMockCall(guardedFetchMock, 0, "guarded fetch"), + "guarded fetch", + ).body; + const secondBody = requireRequestInit( + requireMockCall(guardedFetchMock, 1, "guarded fetch"), + "guarded fetch", + ).body; + expect(secondBody).toBe(firstBody); }); it.each([ @@ -1249,6 +1419,52 @@ describe("google transport stream", () => { }, ); + it("keeps oversized-video shedding in the Gemini 3 retry payload", async () => { + vi.stubEnv("OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS", "10"); + guardedFetchMock + .mockResolvedValueOnce( + new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }), + ) + .mockResolvedValueOnce( + buildSseResponse([ + { + candidates: [{ content: { parts: [{ text: "recovered" }] }, finishReason: "STOP" }], + }, + ]), + ); + + const result = await runGeminiStreamResult({ + model: buildGeminiModel({ + id: "gemini-3.1-pro-preview", + name: "Gemini 3.1 Pro Preview", + input: ["text", "image", "video"] as never, + }), + options: withProviderContextHandoff({ reasoning: "high" }, async () => ({ + messages: [ + { + role: "user", + content: [{ type: "video", mimeType: "video/mp4", data: "A".repeat(20_000_000) }], + timestamp: 0, + }, + ], + })), + }); + + expect(result.errorMessage).toBeUndefined(); + expect(result.content).toEqual([{ type: "text", text: "recovered" }]); + expect(guardedFetchMock).toHaveBeenCalledTimes(2); + for (const index of [0, 1]) { + const body = requireRequestInit( + requireMockCall(guardedFetchMock, index, "guarded fetch"), + "guarded fetch", + ).body as string; + expect(new TextEncoder().encode(body).byteLength).toBeLessThan(20_000_000); + expect(body).toContain("native video slot unavailable"); + } + }); + it("does not retry a genuinely empty Gemini 3 response", async () => { vi.stubEnv("OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS", "10"); guardedFetchMock.mockResolvedValueOnce(buildRawSseResponse("")); diff --git a/extensions/google/transport-stream.ts b/extensions/google/transport-stream.ts index 31b8f9fa06a1..c1d9a0cb74d7 100644 --- a/extensions/google/transport-stream.ts +++ b/extensions/google/transport-stream.ts @@ -3,10 +3,15 @@ import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; import { calculateCost, getEnvApiKey, + resolveProviderContext, type Context, type Model, + type ProviderCallStreamOptions, + type ProviderContext, + type ProviderModel, type SimpleStreamOptions, type ThinkingLevel, + type VideoContent, } from "openclaw/plugin-sdk/llm"; import { parseStrictNonNegativeInteger } from "openclaw/plugin-sdk/number-runtime"; import { @@ -41,7 +46,8 @@ import { } from "openclaw/plugin-sdk/string-coerce-runtime"; import { parseGeminiAuth } from "./gemini-auth.js"; import { stripGoogleProviderPrefix } from "./model-id.js"; -import { normalizeGoogleApiBaseUrl } from "./provider-policy.js"; +import { isGoogleNativeVideoModelId } from "./provider-models.js"; +import { isOfficialGoogleAiStudioBaseUrl, normalizeGoogleApiBaseUrl } from "./provider-policy.js"; import { isGoogleGemini25ThinkingBudgetModel, isGoogleGemini3FlashModel, @@ -59,30 +65,31 @@ import { type CanonicalGoogleTransportApi = "google-generative-ai" | "google-vertex"; type GoogleTransportApi = CanonicalGoogleTransportApi | "openclaw-google-generative-ai-transport"; -type GoogleTransportModel = Model & { +type GoogleTransportModel = ProviderModel & { headers?: Record; provider: string; }; -type GoogleTransportOptions = SimpleStreamOptions & { - cachedContent?: string; - toolChoice?: - | "auto" - | "none" - | "any" - | "required" - | { - type: "function"; - function: { - name: string; +type GoogleTransportOptions = SimpleStreamOptions & + ProviderCallStreamOptions & { + cachedContent?: string; + toolChoice?: + | "auto" + | "none" + | "any" + | "required" + | { + type: "function"; + function: { + name: string; + }; }; - }; - thinking?: { - enabled: boolean; - budgetTokens?: number; - level?: GoogleThinkingLevel; + thinking?: { + enabled: boolean; + budgetTokens?: number; + level?: GoogleThinkingLevel; + }; }; -}; type GoogleGenerateContentRequest = { cachedContent?: string; @@ -93,6 +100,22 @@ type GoogleGenerateContentRequest = { toolConfig?: Record; }; +const GOOGLE_NATIVE_VIDEO_MIME: ReadonlySet = new Set([ + "video/mp4", + "video/mpeg", + "video/quicktime", + "video/avi", + "video/x-flv", + "video/mpg", + "video/webm", + "video/wmv", + "video/3gpp", +]); +const GOOGLE_VIDEO_SLOT_OMISSION = "(video omitted: native video slot unavailable)"; +const GOOGLE_VIDEO_MIME_OMISSION = "(video omitted: unsupported Google video MIME type)"; +const GOOGLE_REQUEST_BYTES_EXCLUSIVE = 20_000_000; +type GoogleVideoSlots = Map, VideoContent>; + const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_DEFAULT_MS = 45_000; const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_ENV = "OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS"; const GOOGLE_SSE_EVENT_BOUNDARY_RE = /(?:\r\n|\r(?!\n)|\n){2}/u; @@ -282,6 +305,23 @@ function normalizeGoogleTransportModelRoute(model: GoogleTransportModel): Google return api && api !== model.api ? Object.assign({}, model, { api }) : model; } +function canonicalGoogleModel(model: GoogleTransportModel): Model { + return { + ...model, + input: model.input.filter((type) => type !== "video"), + } as Model; +} + +function supportsGoogleNativeVideo(model: GoogleTransportModel): boolean { + return ( + model.provider === "google" && + normalizeGoogleTransportRouteApi(model.api) === "google-generative-ai" && + isOfficialGoogleAiStudioBaseUrl(model.baseUrl) && + isGoogleNativeVideoModelId(model.id) && + model.input.includes("video") + ); +} + function normalizeGoogleTransportMessageRoutes(messages: Context["messages"]): Context["messages"] { return messages.map((msg) => { if (msg.role !== "assistant") { @@ -529,19 +569,23 @@ function normalizeGoogleThinkingConfig( return Object.keys(thinkingConfig).length > 0 ? thinkingConfig : undefined; } -function convertGoogleMessages(model: GoogleTransportModel, context: Context) { +function convertGoogleMessages( + model: GoogleTransportModel, + context: Context | ProviderContext, + videoSlots?: GoogleVideoSlots, +) { const contents: Array> = []; const replayToolCallThoughtSignatures = new Map(); const shouldReplayToolCallThoughtSignature = requiresToolCallThoughtSignature(model.id); const routeModel = normalizeGoogleTransportModelRoute(model); const transformedMessages = transformTransportMessages( - normalizeGoogleTransportMessageRoutes(context.messages), - routeModel, + normalizeGoogleTransportMessageRoutes(context.messages as Context["messages"]), + canonicalGoogleModel(routeModel), (id) => (requiresToolCallId(model.id) ? normalizeToolCallId(id) : id), { preserveCrossModelToolCallThoughtSignature: requiresToolCallThoughtSignature(model.id), }, - ); + ) as ProviderContext["messages"]; // Parallel calls need one immediate function-response turn. Gemini < 3 images cannot // live inside functionResponse, so hold them until the consecutive result run ends. const pendingToolResultImageTurns: Array> = []; @@ -565,16 +609,17 @@ function convertGoogleMessages(model: GoogleTransportModel, context: Context) { continue; } const parts = msg.content - .map((item) => - item.type === "text" - ? { text: sanitizeTransportPayloadText(item.text) || " " } - : { - inlineData: { - mimeType: item.mimeType, - data: item.data, - }, - }, - ) + .map((item) => { + if (item.type === "text") { + return { text: sanitizeTransportPayloadText(item.text) || " " }; + } + if (item.type === "image") { + return { inlineData: { mimeType: item.mimeType, data: item.data } }; + } + const placeholder = { text: GOOGLE_VIDEO_SLOT_OMISSION }; + videoSlots?.set(placeholder, item); + return placeholder; + }) .filter((item) => model.input.includes("image") || !("inlineData" in item)); if (parts.length === 0) { parts.push({ text: " " }); @@ -729,8 +774,9 @@ function convertGoogleTools(tools: NonNullable) { export function buildGoogleGenerativeAiParams( model: GoogleTransportModel, - context: Context, + context: Context | ProviderContext, options?: GoogleTransportOptions, + videoSlots?: GoogleVideoSlots, ): GoogleGenerateContentRequest { const generationConfig: Record = {}; if (typeof options?.temperature === "number") { @@ -748,7 +794,7 @@ export function buildGoogleGenerativeAiParams( } const params: GoogleGenerateContentRequest = { - contents: convertGoogleMessages(model, context), + contents: convertGoogleMessages(model, context, videoSlots), }; const cachedContent = typeof options?.cachedContent === "string" ? options.cachedContent.trim() : ""; @@ -779,6 +825,66 @@ export function buildGoogleGenerativeAiParams( return params; } +function replaceGooglePartWithText(part: Record, text: string): void { + Object.keys(part).forEach((key) => Reflect.deleteProperty(part, key)); + part.text = text; +} + +function materializeGoogleVideoSlots( + request: GoogleGenerateContentRequest, + slots: GoogleVideoSlots, +): Record[] { + const trusted: Record[] = []; + const visit = (value: unknown): void => { + if (Array.isArray(value)) { + value.forEach(visit); + return; + } + if (!isRecord(value)) { + return; + } + const video = slots.get(value); + if (video) { + if (!GOOGLE_NATIVE_VIDEO_MIME.has(video.mimeType)) { + replaceGooglePartWithText(value, GOOGLE_VIDEO_MIME_OMISSION); + return; + } + Object.keys(value).forEach((key) => Reflect.deleteProperty(value, key)); + value.inlineData = { mimeType: video.mimeType, data: video.data }; + trusted.push(value); + return; + } + const inlineData = isRecord(value.inlineData) ? value.inlineData : undefined; + if (normalizeLowercaseStringOrEmpty(inlineData?.mimeType).startsWith("video/")) { + replaceGooglePartWithText(value, GOOGLE_VIDEO_SLOT_OMISSION); + return; + } + Object.values(value).forEach(visit); + }; + visit(request); + return trusted; +} + +function serializeGoogleRequest( + request: GoogleGenerateContentRequest, + videoSlots: Record[], +): string { + let body = JSON.stringify(request); + for (const slot of videoSlots.toReversed()) { + if (Buffer.byteLength(body, "utf8") < GOOGLE_REQUEST_BYTES_EXCLUSIVE) { + break; + } + replaceGooglePartWithText(slot, GOOGLE_VIDEO_SLOT_OMISSION); + body = JSON.stringify(request); + } + if (Buffer.byteLength(body, "utf8") >= GOOGLE_REQUEST_BYTES_EXCLUSIVE) { + throw new Error( + `Google request body must be smaller than ${GOOGLE_REQUEST_BYTES_EXCLUSIVE} bytes`, + ); + } + return body; +} + function buildGoogleHeaders( model: GoogleTransportModel, apiKey: string | undefined, @@ -830,7 +936,7 @@ function collectGoogleTransportApiKeys(params: { }): string[] { if ( params.kind !== "google-generative-ai" || - !isOfficialGoogleGenerativeAiBaseUrl(params.model.baseUrl) || + !isOfficialGoogleAiStudioBaseUrl(params.model.baseUrl) || isGoogleOauthApiKey(params.primaryApiKey) || hasGoogleAuthHeader(params.model.headers) || hasGoogleAuthHeader(params.options?.headers) @@ -878,18 +984,6 @@ function buildGoogleTransportRequestUrl( : buildGoogleGenerativeAiRequestUrl(model); } -function isOfficialGoogleGenerativeAiBaseUrl(baseUrl: string | undefined): boolean { - if (!baseUrl) { - return true; - } - try { - const url = new URL(baseUrl); - return url.protocol === "https:" && url.hostname === "generativelanguage.googleapis.com"; - } catch { - return false; - } -} - function resolveGoogleGemini3FirstResponseRetryMs(env = process.env): number { const raw = env[GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_ENV]; if (raw === undefined || raw.trim() === "") { @@ -905,7 +999,7 @@ function shouldRetryGoogleGemini3FirstResponse(params: { if (params.kind !== "google-generative-ai") { return false; } - if (!isOfficialGoogleGenerativeAiBaseUrl(params.model.baseUrl)) { + if (!isOfficialGoogleAiStudioBaseUrl(params.model.baseUrl)) { return false; } return isGoogleGemini3ProModel(params.model.id) || isGoogleGemini3FlashModel(params.model.id); @@ -1029,6 +1123,7 @@ async function openGoogleSseAttempt(params: { url: string; headers: Record; request: GoogleGenerateContentRequest; + videoSlots: Record[]; parentSignal?: AbortSignal; firstResponseTimeoutMs: number; errorPrefix: string; @@ -1042,7 +1137,7 @@ async function openGoogleSseAttempt(params: { const response = await params.guardedFetch(params.url, { method: "POST", headers: params.headers, - body: JSON.stringify(params.request), + body: serializeGoogleRequest(params.request, params.videoSlots), signal, }); if (!response.ok) { @@ -1080,6 +1175,7 @@ async function openGoogleSseChunks(params: { url: string; headers: Record; request: GoogleGenerateContentRequest; + videoSlots: Record[]; }): Promise> { const errorPrefix = params.kind === "google-vertex" @@ -1089,7 +1185,7 @@ async function openGoogleSseChunks(params: { const response = await params.guardedFetch(params.url, { method: "POST", headers: params.headers, - body: JSON.stringify(params.request), + body: serializeGoogleRequest(params.request, params.videoSlots), signal: params.options?.signal, }); if (!response.ok) { @@ -1102,18 +1198,11 @@ async function openGoogleSseChunks(params: { } const retryMs = resolveGoogleGemini3FirstResponseRetryMs(); - const retryRequest = - retryMs > 0 - ? buildGoogleGemini3FirstResponseRetryParams({ - model: params.model, - request: params.request, - }) - : undefined; - if (!retryRequest) { + if (retryMs <= 0) { const response = await params.guardedFetch(params.url, { method: "POST", headers: params.headers, - body: JSON.stringify(params.request), + body: serializeGoogleRequest(params.request, params.videoSlots), signal: params.options?.signal, }); if (!response.ok) { @@ -1130,6 +1219,7 @@ async function openGoogleSseChunks(params: { url: params.url, headers: params.headers, request: params.request, + videoSlots: params.videoSlots, parentSignal: params.options?.signal, firstResponseTimeoutMs: retryMs, errorPrefix, @@ -1138,11 +1228,18 @@ async function openGoogleSseChunks(params: { return firstAttempt; } + // The first serialization owns video shedding. Clone only after it times out + // so the retry inherits those exact omissions instead of restoring stale bytes. + const retryRequest = buildGoogleGemini3FirstResponseRetryParams({ + model: params.model, + request: params.request, + })!; const retryAttempt = await openGoogleSseAttempt({ guardedFetch: params.guardedFetch, url: params.url, headers: params.headers, request: retryRequest, + videoSlots: params.videoSlots, parentSignal: params.options?.signal, firstResponseTimeoutMs: 0, errorPrefix, @@ -1275,7 +1372,7 @@ function updateUsage( chunk.usageMetadata.totalTokenCount ?? promptTokens + outputTokens + toolUsePromptTokens, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, }; - calculateCost(model, output.usage); + calculateCost(canonicalGoogleModel(model), output.usage); } function pushTextBlockEnd( @@ -1309,6 +1406,7 @@ function pushTextBlockEnd( function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): StreamFn { return (rawModel, context, rawOptions) => { const model = rawModel as GoogleTransportModel; + const canonicalModel = canonicalGoogleModel(model); const options = rawOptions as GoogleTransportOptions | undefined; const { eventStream, stream } = createWritableTransportEventStream(); void (async () => { @@ -1324,12 +1422,17 @@ function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): Strea }; try { const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? undefined; - const guardedFetch = buildGuardedModelFetch(model); - let params = buildGoogleGenerativeAiParams(model, context, options); - const nextParams = await options?.onPayload?.(params, model); + const guardedFetch = buildGuardedModelFetch(canonicalModel); + const providerContext = supportsGoogleNativeVideo(model) + ? await resolveProviderContext(context, options) + : context; + const videoSlots: GoogleVideoSlots = new Map(); + let params = buildGoogleGenerativeAiParams(model, providerContext, options, videoSlots); + const nextParams = await options?.onPayload?.(params, canonicalModel); if (nextParams !== undefined) { params = nextParams as GoogleGenerateContentRequest; } + const trustedVideoSlots = materializeGoogleVideoSlots(params, videoSlots); const requestUrl = buildGoogleTransportRequestUrl(kind, model, options); const fetchImpl = (options as { fetch?: typeof fetch } | undefined)?.fetch; const openSse = async (apiKeyForRequest: string | undefined) => { @@ -1348,6 +1451,7 @@ function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): Strea url: requestUrl, headers: requestHeaders, request: params, + videoSlots: trustedVideoSlots, }); }; const apiKeys = collectGoogleTransportApiKeys({ diff --git a/packages/ai/src/host.ts b/packages/ai/src/host.ts index 0f5aa904c7bd..2ee394c5393c 100644 --- a/packages/ai/src/host.ts +++ b/packages/ai/src/host.ts @@ -182,8 +182,6 @@ export interface AiTransportHost { transformTransportMessages: AiTransformTransportMessages; /** Registers a custom transport API with the host's stream error bridge. */ registerCustomApi(registry: ApiRegistry, api: Api, streamFn: StreamFn): boolean; - /** Prepares the provider-owned Google simple-completion alias when needed. */ - prepareGoogleSimpleCompletionModel(registry: ApiRegistry, model: Model): Model; /** * Emits one transport diagnostic; build runs only when the host logs it and * may return null to suppress the entry (e.g. de-duplication). @@ -263,7 +261,6 @@ const inertAiTransportHost: ActiveAiTransportHost = { transformTransportMessages: (messages, model, normalizeToolCallId) => transformMessages(messages, model, normalizeToolCallId), registerCustomApi: queueCustomApiRegistration, - prepareGoogleSimpleCompletionModel: (_registry, model) => model, logDebug: () => {}, logInfo: () => {}, logWarn: () => {}, diff --git a/packages/ai/src/package.e2e.test.ts b/packages/ai/src/package.e2e.test.ts index 0119fd3ce12d..9ec71e93066a 100644 --- a/packages/ai/src/package.e2e.test.ts +++ b/packages/ai/src/package.e2e.test.ts @@ -18,14 +18,16 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach); const compatibility = { "@openclaw/ai/provider-types": { - values: [], + values: ["PROVIDER_CONTEXT_HANDOFF", "resolveProviderContext"], types: [ "MediaContent", "ModelInputContent", "ProviderContext", + "ProviderContextHandoff", "ProviderMessage", "ProviderModel", "ProviderStreamFunction", + "ProviderStreamOptions", "ProviderUserMessage", "VideoContent", ], diff --git a/packages/ai/src/provider-types.test.ts b/packages/ai/src/provider-types.test.ts index 70c01484c610..7741a6744538 100644 --- a/packages/ai/src/provider-types.test.ts +++ b/packages/ai/src/provider-types.test.ts @@ -8,16 +8,20 @@ import type { ToolResultMessage, UserMessage, } from "@openclaw/llm-core"; -import { describe, expectTypeOf, it } from "vitest"; -import type { - MediaContent, - ModelInputContent, - ProviderContext, - ProviderMessage, - ProviderModel, - ProviderStreamFunction, - ProviderUserMessage, - VideoContent, +import { describe, expect, expectTypeOf, it, vi } from "vitest"; +import { + PROVIDER_CONTEXT_HANDOFF, + resolveProviderContext, + type ProviderContextHandoff, + type ProviderStreamOptions, + type MediaContent, + type ModelInputContent, + type ProviderContext, + type ProviderMessage, + type ProviderModel, + type ProviderStreamFunction, + type ProviderUserMessage, + type VideoContent, } from "./provider-types.js"; describe("provider call types", () => { @@ -33,6 +37,13 @@ describe("provider call types", () => { expectTypeOf().toEqualTypeOf<"text" | "image" | "video">(); expectTypeOf[0]>().toEqualTypeOf(); expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf[2]>().toEqualTypeOf< + ProviderStreamOptions | undefined + >(); + expectTypeOf().toEqualTypeOf< + ProviderContextHandoff | undefined + >(); + expectTypeOf>().toEqualTypeOf<[]>(); expectTypeOf().toEqualTypeOf(); expectTypeOf().toEqualTypeOf(); @@ -42,4 +53,17 @@ describe("provider call types", () => { >(); expectTypeOf().toEqualTypeOf<"text" | "image">(); }); + + it("projects canonical context unless a provider handoff is present", async () => { + const context = { systemPrompt: "system", messages: [], tools: [] }; + await expect(resolveProviderContext(context)).resolves.toBe(context); + + const resolved = { ...context, messages: [] }; + const handoff = vi.fn(async () => resolved); + await expect( + resolveProviderContext(context, { [PROVIDER_CONTEXT_HANDOFF]: handoff }), + ).resolves.toBe(resolved); + expect(handoff).toHaveBeenCalledOnce(); + expect(handoff).toHaveBeenCalledWith(); + }); }); diff --git a/packages/ai/src/provider-types.ts b/packages/ai/src/provider-types.ts index 0887ab5f38da..2fd48c4ed9e5 100644 --- a/packages/ai/src/provider-types.ts +++ b/packages/ai/src/provider-types.ts @@ -1,6 +1,8 @@ import type * as Llm from "@openclaw/llm-core"; export type * from "./types.js"; +export const PROVIDER_CONTEXT_HANDOFF: unique symbol = Symbol("providerContextHandoff"); + export type VideoContent = Omit & { type: "video" }; export type MediaContent = Llm.ImageContent | VideoContent; export type ModelInputContent = Llm.TextContent | MediaContent; @@ -12,14 +14,26 @@ export type ProviderContext = Omit & { messages: Provid export type ProviderModel = Omit, "input"> & { input: ModelInputContent["type"][]; }; +export type ProviderContextHandoff = () => Promise; +export type ProviderStreamOptions = Llm.StreamOptions & { + [PROVIDER_CONTEXT_HANDOFF]?: ProviderContextHandoff; +}; export type ProviderStreamFunction< TApi extends Llm.Api = Llm.Api, - TOptions extends Llm.StreamOptions = Llm.StreamOptions, + TOptions extends Llm.StreamOptions = ProviderStreamOptions, > = ( model: ProviderModel, context: ProviderContext, options?: TOptions, ) => Llm.AssistantMessageEventStreamContract; + +/** Resolves provider-only context without widening the canonical call contract. */ +export async function resolveProviderContext( + context: Llm.Context | ProviderContext, + options?: ProviderStreamOptions, +): Promise { + return options?.[PROVIDER_CONTEXT_HANDOFF]?.() ?? (context as ProviderContext); +} export type { ProviderContext as Context, ProviderMessage as Message, diff --git a/packages/ai/src/transports/simple-completion-transport.test.ts b/packages/ai/src/transports/simple-completion-transport.test.ts index f91ca22cfbc6..911ca038ee69 100644 --- a/packages/ai/src/transports/simple-completion-transport.test.ts +++ b/packages/ai/src/transports/simple-completion-transport.test.ts @@ -15,7 +15,6 @@ const createOpenClawTransportStreamFnForModel = vi.fn(); const createTransportAwareStreamFnForModel = vi.fn(); const prepareTransportAwareSimpleModel = vi.fn(); const resolveTransportAwareSimpleApi = vi.fn(); -const prepareGoogleSimpleCompletionModel = vi.fn((_registry: unknown, model: unknown) => model); const inheritManagedTransport = vi.fn((_source: Model, target: Model) => target); const pluginStreamFn = vi.fn(() => createAssistantMessageEventStream()); const TEST_SECRET = "ollama-provider-secret"; @@ -90,7 +89,6 @@ describe("prepareModelForSimpleCompletion", () => { createTransportAwareStreamFnForModel.mockReset(); prepareTransportAwareSimpleModel.mockReset(); resolveTransportAwareSimpleApi.mockReset(); - prepareGoogleSimpleCompletionModel.mockReset(); inheritManagedTransport.mockClear(); createAnthropicVertexStreamFnForModel.mockReturnValue("vertex-stream"); resolveProviderStreamFn.mockReturnValue(pluginStreamFn); @@ -100,7 +98,6 @@ describe("prepareModelForSimpleCompletion", () => { createTransportAwareStreamFnForModel.mockReturnValue(undefined); prepareTransportAwareSimpleModel.mockImplementation((model) => model); resolveTransportAwareSimpleApi.mockReturnValue(undefined); - prepareGoogleSimpleCompletionModel.mockImplementation((_registry, model) => model); configureAiTransportHost({ plugin: { resolveProviderStream: resolveProviderStreamFn, @@ -110,10 +107,6 @@ describe("prepareModelForSimpleCompletion", () => { }, registerCustomApi: ensureCustomApiRegistered, inheritManagedTransport, - prepareGoogleSimpleCompletionModel: prepareGoogleSimpleCompletionModel as ( - registry: ApiRegistry, - model: Model, - ) => Model, resolveSecretSentinel: (value) => value.replaceAll(TEST_SECRET_SENTINEL, TEST_SECRET), }); }); @@ -438,7 +431,7 @@ describe("prepareModelForSimpleCompletion", () => { }); }); - it("keeps registered Google models on the sanitizer path when the provider owns a stream", () => { + it("routes registered Google simple completions through the provider plugin stream", () => { const model: Model<"google-generative-ai"> = { id: "gemma-4-26b-a4b-it", name: "Gemma 4 26B", @@ -461,24 +454,38 @@ describe("prepareModelForSimpleCompletion", () => { }, SIMPLE_COMPLETION_SOURCE_ID, ); - prepareGoogleSimpleCompletionModel.mockImplementationOnce((_registry: unknown, m: unknown) => ({ - ...(m as Model<"google-generative-ai">), - api: "openclaw-google-generative-ai-simple", - })); - const result = prepareModelForSimpleCompletion({ model }); - expect(resolveProviderStreamFn).not.toHaveBeenCalled(); - expect(prepareTransportAwareSimpleModel).toHaveBeenCalledWith(model, { cfg: undefined }); - expect(prepareGoogleSimpleCompletionModel).toHaveBeenCalledWith(apiRegistry, model); + expect(resolveProviderStreamFn).toHaveBeenCalledOnce(); + expect(prepareTransportAwareSimpleModel).not.toHaveBeenCalled(); expect(buildTransportAwareSimpleStreamFn).not.toHaveBeenCalled(); - expect(result).toEqual({ - ...model, - api: "openclaw-google-generative-ai-simple", - }); + expect(result.api).toMatch(/^openclaw-provider-stream:/); + const registeredStream = ensureCustomApiRegistered.mock.calls.find( + (call) => call[1] === result.api, + )?.[2] as StreamFn | undefined; + void registeredStream?.( + result, + { messages: [] }, + { + reasoning: "high", + apiKey: "google-key", + headers: { "x-test": "value" }, + signal: new AbortController().signal, + }, + ); + expect(pluginStreamFn).toHaveBeenCalledWith( + expect.objectContaining({ id: model.id }), + { messages: [] }, + expect.objectContaining({ + reasoning: "high", + apiKey: "google-key", + headers: { "x-test": "value" }, + signal: expect.any(AbortSignal), + }), + ); }); - it("keeps Google transport-aware models on the transport alias", () => { + it("prefers the Google provider plugin over the duplicate transport-aware path", () => { const model: Model<"google-generative-ai"> = { id: "gemini-flash-latest", name: "Gemini Flash Latest", @@ -493,24 +500,17 @@ describe("prepareModelForSimpleCompletion", () => { headers: {}, }; - const transportModel = { - ...model, - api: "openclaw-google-generative-ai-transport", - }; - buildTransportAwareSimpleStreamFn.mockReturnValueOnce("google-transport-stream"); - prepareTransportAwareSimpleModel.mockReturnValueOnce(transportModel); - const result = prepareModelForSimpleCompletion({ model }); - expect(resolveProviderStreamFn).not.toHaveBeenCalled(); - expect(buildTransportAwareSimpleStreamFn).toHaveBeenCalledWith(model, { cfg: undefined }); + expect(resolveProviderStreamFn).toHaveBeenCalledOnce(); + expect(buildTransportAwareSimpleStreamFn).not.toHaveBeenCalled(); + expect(prepareTransportAwareSimpleModel).not.toHaveBeenCalled(); + expect(result.api).toBe("google-generative-ai"); expect(ensureCustomApiRegistered).toHaveBeenCalledWith( apiRegistry, - "openclaw-google-generative-ai-transport", - "google-transport-stream", + "google-generative-ai", + expect.any(Function), ); - expect(prepareGoogleSimpleCompletionModel).not.toHaveBeenCalled(); - expect(result).toBe(transportModel); }); it.each([ diff --git a/packages/ai/src/transports/simple-completion-transport.ts b/packages/ai/src/transports/simple-completion-transport.ts index bf8c3fc11ced..f7f6071bd730 100644 --- a/packages/ai/src/transports/simple-completion-transport.ts +++ b/packages/ai/src/transports/simple-completion-transport.ts @@ -179,11 +179,6 @@ function prepareProviderStreamModel(params: { cfg?: unknown; apiRegistry: ApiRegistry; }): Model | undefined { - // Google simple completions have managed transport and sanitizer paths below. - // A plugin-native stream here would bypass both and emit unsupported payloads. - if (params.model.api === "google-generative-ai") { - return undefined; - } const pluginModel = resolveModelHeaderSentinels(params.model); const providerStreamFn = getAiTransportHost().plugin.resolveProviderStream({ provider: params.model.provider, @@ -244,15 +239,6 @@ export function prepareModelForSimpleCompletion(params: { } } - if (model.api === "google-generative-ai") { - return applyProviderSimpleCompletionWrapper( - apiRegistry, - getAiTransportHost().prepareGoogleSimpleCompletionModel(apiRegistry, model), - cfg, - model.api, - ); - } - if (model.provider === "anthropic-vertex") { const api = resolveAnthropicVertexSimpleApi(model.baseUrl); const host = getAiTransportHost(); diff --git a/src/agents/ai-transport-runtime-host.ts b/src/agents/ai-transport-runtime-host.ts index 39f87816b61a..a3f19b046d12 100644 --- a/src/agents/ai-transport-runtime-host.ts +++ b/src/agents/ai-transport-runtime-host.ts @@ -15,7 +15,6 @@ import { import { createAnthropicVertexStreamFnForModel } from "./anthropic-vertex-stream.js"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./copilot-dynamic-headers.js"; import { ensureCustomApiRegistered } from "./custom-api-registry.js"; -import { prepareGoogleSimpleCompletionModel } from "./google-simple-completion-stream.js"; import { resolveProviderRequestCapabilities, resolveProviderEndpoint, @@ -101,7 +100,6 @@ export function configureAiTransportRuntimeHost(): void { ), transformTransportMessages, registerCustomApi: ensureCustomApiRegistered, - prepareGoogleSimpleCompletionModel, }); configured = true; } diff --git a/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts b/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts index de1eddd05059..56195f14eeed 100644 --- a/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts +++ b/src/agents/embedded-agent-runner/run.overflow-compaction.loop.test.ts @@ -41,14 +41,6 @@ vi.mock("../subagents/registry/subagent-registry.js", () => ({ settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns, })); -vi.mock("./run/plugin-harness-prompt-images.js", () => ({ - preparePluginHarnessPromptImages: vi.fn(async () => ({ - images: undefined, - imageOrder: undefined, - media: undefined, - })), -})); - vi.mock("./run/skill-workshop-attempt-params.js", () => ({ resolveSkillWorkshopAttemptParams: vi.fn(() => ({})), })); diff --git a/src/agents/embedded-agent-runner/run/attempt-prompt-submit.ts b/src/agents/embedded-agent-runner/run/attempt-prompt-submit.ts index da379b547779..c4f017fd0838 100644 --- a/src/agents/embedded-agent-runner/run/attempt-prompt-submit.ts +++ b/src/agents/embedded-agent-runner/run/attempt-prompt-submit.ts @@ -320,7 +320,13 @@ export async function prepareEmbeddedAttemptPromptExecution(input: { prompt: string; sandbox?: SandboxContext | null; skipPromptSubmission: boolean; -}): Promise { + pluginHarness?: boolean; +}): Promise< + PromptImageResult & { + imageOrder?: PromptExecutionAttempt["imageOrder"]; + media?: PromptExecutionAttempt["media"]; + } +> { if (input.skipPromptSubmission) { return emptyPromptImages(); } @@ -331,7 +337,7 @@ export async function prepareEmbeddedAttemptPromptExecution(input: { (await attempt.userTurnTranscriptRecorder?.resolveMessage()); const persistedMedia = persistedMessage ? (readPersistedMediaFacts(persistedMessage) ?? []) : []; - return await detectAndLoadPromptImages({ + const result = await detectAndLoadPromptImages({ prompt: input.prompt, workspaceDir: input.effectiveWorkspace, model: attempt.model, @@ -349,4 +355,17 @@ export async function prepareEmbeddedAttemptPromptExecution(input: { ? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge } : undefined, }); + if (!input.pluginHarness) { + return result; + } + if (result.failedMediaCount) { + throw new Error( + `failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`, + ); + } + return { + ...result, + imageOrder: result.images.length ? result.images.map(() => "inline" as const) : undefined, + media: undefined, + }; } diff --git a/src/agents/embedded-agent-runner/run/attempt-session-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-session-prepare.ts index 45a10e049922..55d43b1c6cf8 100644 --- a/src/agents/embedded-agent-runner/run/attempt-session-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-session-prepare.ts @@ -4,6 +4,10 @@ */ import type { SessionTranscriptRuntimeTarget } from "../../../config/sessions/session-accessor.js"; import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../../context-engine/host-compat.js"; +import { + attachRuntimePromptMediaFacts, + readPersistedMediaFacts, +} from "../../../media/media-facts.js"; import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js"; import type { PluginMetadataSnapshot } from "../../../plugins/plugin-metadata-snapshot.types.js"; import { createPreparedEmbeddedAgentSettingsManager } from "../../agent-project-settings.js"; @@ -446,12 +450,20 @@ export async function prepareEmbeddedAttemptSessionManager(input: { latestPersistedUserMessage = message; latestRuntimeUserMessage = runtimeMessage; if (runtimeMessage) { + const media = readPersistedMediaFacts(message); + if (media?.length) { + attachRuntimePromptMediaFacts(runtimeMessage, media); + } userTranscriptContextRegistry.record(runtimeMessage, message); } attempt.onUserMessagePersisted?.(message); }, - onUserMessagePersistenceSuppressed: (_message, runtimeMessage) => { + onUserMessagePersistenceSuppressed: (message, runtimeMessage) => { latestRuntimeUserMessage = runtimeMessage; + const media = runtimeMessage ? readPersistedMediaFacts(message) : undefined; + if (runtimeMessage && media?.length) { + attachRuntimePromptMediaFacts(runtimeMessage, media); + } }, onUserMessageBlocked: () => { attempt.userTurnTranscriptRecorder?.markBlocked(); diff --git a/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts index f511c9a46fa9..872e8c89e0c5 100644 --- a/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-session-runtime-prepare.ts @@ -229,6 +229,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: { providerThinkingLevel: input.transport.providerThinkingLevel, sessionAgentId: input.sessionManager.sessionAgentId, workspaceDir: input.effectiveWorkspace, + workspaceOnly: input.effectiveFsWorkspaceOnly, agentDir: input.agentDir, abortSignal: input.transport.abortSignal, getProviderRuntimeHandle: input.transport.getProviderRuntimeHandle, diff --git a/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts b/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts index 09a5a393db9e..a98489a2ff30 100644 --- a/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts +++ b/src/agents/embedded-agent-runner/run/attempt-stream-settle.ts @@ -68,6 +68,8 @@ import { waitForCompactionRetryWithAggregateTimeout, } from "./compaction-retry-aggregate-timeout.js"; import { selectCompactionTimeoutSnapshot } from "./compaction-timeout.js"; +import { materializeProviderContext } from "./images.js"; +import { wrapStreamFnWithMessageTransform } from "./message-transform-stream-wrapper.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js"; /** @@ -441,6 +443,7 @@ export async function prepareEmbeddedAttemptTransport(input: { providerThinkingLevel: ProviderThinkLevel | undefined; sessionAgentId: string; workspaceDir: string; + workspaceOnly: boolean; agentDir: string; abortSignal: AbortSignal; getProviderRuntimeHandle: () => ProviderRuntimePluginHandle; @@ -496,6 +499,23 @@ export async function prepareEmbeddedAttemptTransport(input: { agentDir: input.agentDir, workspaceDir: input.workspaceDir, }); + const directProviderStreamFn = providerStreamFn + ? wrapStreamFnWithMessageTransform( + providerStreamFn, + (messages) => messages, + ({ context, ...provider }) => + materializeProviderContext({ + ...provider, + context, + workspaceDir: input.workspaceDir, + workspaceOnly: input.workspaceOnly, + sandbox: + input.sandbox?.enabled && input.sandbox.fsBridge + ? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge } + : undefined, + }), + ) + : undefined; const transportApiKey = await resolveEmbeddedAgentApiKey({ provider: attempt.model.provider, resolvedApiKey: attempt.resolvedApiKey, @@ -503,13 +523,13 @@ export async function prepareEmbeddedAttemptTransport(input: { }); const streamStrategy = describeEmbeddedAgentStreamStrategy({ currentStreamFn: defaultSessionStreamFn, - providerStreamFn, + providerStreamFn: directProviderStreamFn, model: attempt.model, resolvedApiKey: transportApiKey, }); session.agent.streamFn = resolveEmbeddedAgentStreamFn({ currentStreamFn: defaultSessionStreamFn, - providerStreamFn, + providerStreamFn: directProviderStreamFn, sessionId: attempt.sessionId, promptCacheKey: attempt.promptCacheKey, signal: input.abortSignal, diff --git a/src/agents/embedded-agent-runner/run/images.media-refs.ts b/src/agents/embedded-agent-runner/run/images.media-refs.ts index 2f8a377c26a9..9b57fed161a9 100644 --- a/src/agents/embedded-agent-runner/run/images.media-refs.ts +++ b/src/agents/embedded-agent-runner/run/images.media-refs.ts @@ -4,19 +4,18 @@ import { normalizeMediaFacts, type MediaFact, } from "../../../media/media-facts.js"; -import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js"; import { resolveUserPath } from "../../../utils.js"; const URL_SCHEME_PATTERN = /^[a-z][a-z0-9+.-]*:/i; const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/; -type DetectedImageRef = { +export type MediaFileRef = { raw: string; type: "path" | "media-uri"; resolved: string; }; -export type MediaImageRef = DetectedImageRef & { +export type MediaImageRef = MediaFileRef & { aliases: string[]; detect?: boolean; factIndex: number; @@ -35,6 +34,35 @@ export function isOpenClawCliImageCachePath(filePath: string): boolean { }); } +export function resolveMediaFactLocalRef(fact: MediaFact): MediaFileRef | undefined { + const mediaUri = [fact.url, fact.path].find((value) => value?.startsWith("media://inbound/")); + const identity = mediaUri ?? fact.path ?? fact.url; + if (!identity) { + return undefined; + } + let resolved = mediaUri; + if (!resolved && /^file:/i.test(identity)) { + try { + resolved = safeFileURLToPath(identity); + } catch { + return undefined; + } + } else if ( + !resolved && + (!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity)) + ) { + resolved = identity; + } + if (!resolved) { + return undefined; + } + return { + raw: identity, + type: mediaUri ? "media-uri" : "path", + resolved: resolved.startsWith("~") ? resolveUserPath(resolved) : resolved, + }; +} + function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef | undefined { if (!isImageMediaFact(fact)) { return undefined; @@ -55,25 +83,9 @@ function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef } : undefined; } - let resolved = mediaUri; - if (!resolved && identity && /^file:/i.test(identity)) { - try { - resolved = safeFileURLToPath(identity); - } catch { - resolved = undefined; - } - } else if ( - !resolved && - identity && - (!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity)) - ) { - resolved = identity; - } - if (resolved?.startsWith("~")) { - resolved = resolveUserPath(resolved); - } + const localRef = resolveMediaFactLocalRef(fact); const hydrate = fact.hydrationSuppressed !== true; - if (!resolved || isOpenClawCliImageCachePath(resolved)) { + if (!localRef || isOpenClawCliImageCachePath(localRef.resolved)) { return { aliases: [fact.path, fact.url].filter((value): value is string => Boolean(value)), detect: false, @@ -86,11 +98,11 @@ function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef }; } return { - aliases: [fact.path, fact.url, resolved].filter((value): value is string => Boolean(value)), + ...localRef, + aliases: [fact.path, fact.url, localRef.resolved].filter((value): value is string => + Boolean(value), + ), factIndex, - raw: mediaUri ?? fact.path ?? fact.url ?? resolved, - type: mediaUri ? "media-uri" : "path", - resolved, hydrate, ...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}), }; @@ -104,57 +116,9 @@ export function collectMediaImageRefs( ); } -export function collectIdentitylessMediaImageFactIndexes(media?: readonly MediaFact[]): number[] { - return normalizeMediaFacts(media).flatMap((fact, factIndex) => - isImageMediaFact(fact) && - fact.hydrationSuppressed !== true && - fact.path === undefined && - fact.url === undefined - ? [factIndex] - : [], - ); -} - // Guards for transports that cannot carry attachments (paired-node CLI): only // facts that will actually hydrate an image count; described/remote-only facts // whose hydration is suppressed must not block text-only prompts. export function hasHydratableMediaImages(media?: readonly MediaFact[]): boolean { return collectMediaImageRefs(media).some((ref) => ref?.hydrate === true); } - -export function selectMediaImageRefs(params: { - refs: Array; - existingImageCount: number; - imageOrder?: readonly PromptImageOrderEntry[]; -}): Array { - const { refs } = params; - if (!params.imageOrder?.length) { - // Legacy turns (no layout metadata): identity-less facts are the inline - // images' own slots — pair them positionally so they cannot count as failed - // offloads; identity-bearing refs remain genuine offloaded attachments. - let inlinePairs = params.existingImageCount; - return refs.filter((ref) => { - if (ref === undefined && inlinePairs > 0) { - inlinePairs -= 1; - return false; - } - return true; - }); - } - if (refs.length !== params.imageOrder.length) { - // Partial fact arrays cannot prove positional ownership. Keep every ref as - // an offload so no attachment is silently consumed by an inline slot. - return refs; - } - let remainingExisting = params.existingImageCount; - return params.imageOrder.flatMap((entry, index) => { - if (entry === "offloaded") { - return [refs[index]]; - } - if (remainingExisting > 0) { - remainingExisting -= 1; - return []; - } - return [undefined]; - }); -} diff --git a/src/agents/embedded-agent-runner/run/images.ts b/src/agents/embedded-agent-runner/run/images.ts index b5dbdb502eab..d2487eb55d9e 100644 --- a/src/agents/embedded-agent-runner/run/images.ts +++ b/src/agents/embedded-agent-runner/run/images.ts @@ -1,11 +1,18 @@ import path from "node:path"; +import { MAX_VIDEO_BYTES } from "@openclaw/media-core/constants"; +import { normalizeMimeType } from "@openclaw/media-core/mime"; import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import type { + ModelInputContent, + ProviderContext, +} from "../../../../packages/ai/src/provider-types.js"; import { formatErrorMessage } from "../../../infra/errors.js"; import { assertNoWindowsNetworkPath, safeFileURLToPath } from "../../../infra/local-file-access.js"; -import type { ImageContent } from "../../../llm/types.js"; +import type { Context, ImageContent, TextContent } from "../../../llm/types.js"; import { attachRuntimePromptMediaFacts, isImageMediaFact, + isVideoMediaFact, normalizeMediaFacts, readRuntimePromptImageOrder, readRuntimePromptMediaFacts, @@ -15,7 +22,7 @@ import { import { resolveMediaReferenceLocalPath } from "../../../media/media-reference.js"; import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js"; import { finalizeRuntimePromptImages } from "../../../media/runtime-prompt-image-provenance.js"; -import { loadWebMedia } from "../../../media/web-media.js"; +import { loadWebMedia, type WebMediaResult } from "../../../media/web-media.js"; import { resolveUserPath } from "../../../utils.js"; import type { ImageSanitizationLimits } from "../../image-sanitization.js"; import type { AgentMessage } from "../../runtime/index.js"; @@ -27,19 +34,17 @@ import type { SandboxFsBridge } from "../../sandbox/fs-bridge.js"; import { sanitizeImageBlocks } from "../../tool-images.js"; import { log } from "../logger.js"; import { - collectIdentitylessMediaImageFactIndexes, collectMediaImageRefs, isOpenClawCliImageCachePath, - selectMediaImageRefs, + resolveMediaFactLocalRef, + type MediaFileRef, type MediaImageRef, } from "./images.media-refs.js"; import { type ImageFactIndex, type MediaImageLayout, - countMissingLayoutInlineSlots, readPersistedImageBlockFactIndexes, readPersistedMediaImageLayout, - resolveLayoutInlineFactIndexes, } from "./prompt-image-metadata.js"; export { hasHydratableMediaImages } from "./images.media-refs.js"; @@ -72,12 +77,6 @@ const PATH_PATTERN = new RegExp(PATH_REGEX_SOURCE, "gi"); const LEGACY_ATTACHMENT_MARKER_PATTERN = /\[(?:media attached(?:\s+\d+\/\d+)?:|Image:\s*source:)\s*[^\]]+\]/gi; -interface DetectedImageRef { - raw: string; - type: "path" | "media-uri"; - resolved: string; -} - function isImageExtension(filePath: string): boolean { const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath)); return IMAGE_EXTENSIONS.has(ext); @@ -96,77 +95,6 @@ type PromptImageEntry = { factIndex: ImageFactIndex; }; -function mergePromptAttachmentImages(params: { - imageOrder?: PromptImageOrderEntry[]; - mediaImageLayout?: MediaImageLayout; - existingImages?: ImageContent[]; - existingImageFactIndexes?: readonly ImageFactIndex[]; - offloadedImages?: Array; - promptRefImages?: ImageContent[]; -}): PromptImageEntry[] { - const existingImages = (params.existingImages ?? []).map((image, index) => ({ - image, - factIndex: params.existingImageFactIndexes?.[index] ?? null, - })); - const offloadedImages = params.offloadedImages ?? []; - const promptRefImages = (params.promptRefImages ?? []).map((image) => ({ - image, - factIndex: null, - })); - const slots: MediaImageLayout["slots"] = - params.mediaImageLayout?.slots ?? params.imageOrder?.map((kind) => ({ kind })) ?? []; - if (slots.length === 0) { - const factOwned = [...offloadedImages, ...existingImages] - .filter((entry): entry is PromptImageEntry => entry !== null && entry.factIndex !== null) - .toSorted((left, right) => (left.factIndex ?? 0) - (right.factIndex ?? 0)); - return [ - ...factOwned, - ...existingImages.filter((entry) => entry.factIndex === null), - ...promptRefImages, - ]; - } - - const unusedExisting = [...existingImages]; - const takeExisting = (factIndex: number | null | undefined): PromptImageEntry | undefined => { - const matchIndex = - factIndex === undefined - ? 0 - : unusedExisting.findIndex((entry) => entry.factIndex === factIndex); - if (matchIndex < 0) { - return undefined; - } - return unusedExisting.splice(matchIndex, 1)[0]; - }; - let offloadedIndex = 0; - const ordered = slots.flatMap((slot) => { - const offloaded = slot.kind === "offloaded" ? offloadedImages[offloadedIndex++] : undefined; - const exactExisting = - slot.factIndex !== undefined - ? takeExisting(slot.factIndex) - : slot.kind === "inline" - ? takeExisting(undefined) - : undefined; - const existing = - exactExisting ?? - (slot.kind === "inline" && slot.factIndex !== undefined ? takeExisting(null) : undefined); - if (existing) { - return [existing]; - } - if (slot.kind === "inline") { - return []; - } - return offloaded ? [offloaded] : []; - }); - return [ - ...ordered, - ...unusedExisting, - ...offloadedImages - .slice(offloadedIndex) - .filter((entry): entry is PromptImageEntry => entry !== null), - ...promptRefImages, - ]; -} - async function sanitizeImageEntriesWithLog( entries: PromptImageEntry[], label: string, @@ -193,8 +121,8 @@ async function sanitizeImageEntriesWithLog( } /** Detects explicit local image paths and file URLs in user prompt text. */ -export function detectImageReferences(prompt: string): DetectedImageRef[] { - const refs: DetectedImageRef[] = []; +export function detectImageReferences(prompt: string): MediaFileRef[] { + const refs: MediaFileRef[] = []; const seen = new Set(); const pathPrompt = prompt.replace(LEGACY_ATTACHMENT_MARKER_PATTERN, (marker) => " ".repeat(marker.length), @@ -263,7 +191,7 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] { return refs; } -function refDedupeKey(ref: DetectedImageRef, workspaceDir?: string): string { +function refDedupeKey(ref: MediaFileRef, workspaceDir?: string): string { const resolved = ref.type === "path" && workspaceDir && !path.isAbsolute(ref.resolved) ? path.resolve(workspaceDir, ref.resolved) @@ -279,16 +207,19 @@ function rawAliasDedupeKey(alias: string): string | undefined { : undefined; } -async function loadImageFromRef( - ref: DetectedImageRef, +async function loadMediaFromRef( + ref: MediaFileRef, workspaceDir: string, options?: { + label?: string; maxBytes?: number; + signal?: AbortSignal; workspaceOnly?: boolean; localRoots?: readonly string[]; sandbox?: { root: string; bridge: SandboxFsBridge }; }, -): Promise { +): Promise { + options?.signal?.throwIfAborted(); try { let targetPath = ref.resolved; @@ -310,7 +241,7 @@ async function loadImageFromRef( targetPath = resolved.resolved; } catch (err) { log.debug( - `Native image: sandbox validation failed for ${ref.resolved}: ${formatErrorMessage(err)}`, + `${options?.label ?? "Native media"}: sandbox validation failed: ${formatErrorMessage(err)}`, ); return null; } @@ -331,21 +262,31 @@ async function loadImageFromRef( : options?.maxBytes, ); - if (media.kind !== "image") { - log.debug(`Native image: not an image file: ${targetPath} (got ${media.kind})`); - return null; - } - - const mimeType = media.contentType ?? "image/jpeg"; - const data = media.buffer.toString("base64"); - - return { type: "image", data, mimeType }; + options?.signal?.throwIfAborted(); + return media; } catch (err) { - log.debug(`Native image: failed to load ${ref.resolved}: ${formatErrorMessage(err)}`); + options?.signal?.throwIfAborted(); + log.debug(`${options?.label ?? "Native media"}: failed to load: ${formatErrorMessage(err)}`); return null; } } +async function loadImageFromRef( + ref: MediaFileRef, + workspaceDir: string, + options?: Parameters[2], +): Promise { + const media = await loadMediaFromRef(ref, workspaceDir, { ...options, label: "Native image" }); + if (!media || media.kind !== "image") { + return null; + } + return { + type: "image", + data: media.buffer.toString("base64"), + mimeType: media.contentType ?? "image/jpeg", + }; +} + function modelSupportsImages(model: { input?: string[] }): boolean { return model.input?.includes("image") ?? false; } @@ -367,7 +308,7 @@ export async function detectAndLoadPromptImages(params: { }): Promise<{ images: ImageContent[]; imageFactIndexes: ImageFactIndex[]; - detectedRefs: DetectedImageRef[]; + detectedRefs: MediaFileRef[]; failedMediaCount: number; loadedCount: number; skippedCount: number; @@ -382,97 +323,86 @@ export async function detectAndLoadPromptImages(params: { skippedCount: 0, }; } - - const allMediaRefs = collectMediaImageRefs(params.media); - const suppressedFactIndexes = new Set(params.mediaImageLayout?.suppressedFactIndexes ?? []); - for (const ref of allMediaRefs) { - if (!ref || !suppressedFactIndexes.has(ref.factIndex)) { - continue; + const media = normalizeMediaFacts(params.media); + const suppressed = new Set(params.mediaImageLayout?.suppressedFactIndexes ?? []); + const imageFactIndexes = media.flatMap((fact, factIndex) => + isImageMediaFact(fact) && fact.hydrationSuppressed !== true && !suppressed.has(factIndex) + ? [factIndex] + : [], + ); + const refs = collectMediaImageRefs(media); + const refsByFact = new Map(refs.flatMap((ref) => (ref ? [[ref.factIndex, ref] as const] : []))); + const inferredSlots = (() => { + if (params.imageOrder?.length === imageFactIndexes.length) { + return params.imageOrder.map((kind, index) => ({ + kind, + factIndex: imageFactIndexes[index], + })); } - ref.hydrate = false; - } - const orderRefs = allMediaRefs.filter( - (ref) => !ref || (!suppressedFactIndexes.has(ref.factIndex) && ref.hydrate), - ); - const imageOrder = params.mediaImageLayout?.slots.map((slot) => slot.kind) ?? params.imageOrder; - const refsByFactIndex = new Map( - allMediaRefs.flatMap((ref) => (ref ? [[ref.factIndex, ref] as const] : [])), - ); - // imageOrder describes only images still requiring native delivery; described - // (suppressed) facts must not count against it, or the inference silently - // skips and inline sanitization failures dispatch as success. - const unsuppressedImageFactIndexes = normalizeMediaFacts(params.media).flatMap( - (fact, factIndex) => - isImageMediaFact(fact) && - fact.hydrationSuppressed !== true && - !suppressedFactIndexes.has(factIndex) - ? [factIndex] - : [], - ); - const inferredExistingImageFactIndexes = - imageOrder && unsuppressedImageFactIndexes.length === imageOrder.length - ? imageOrder.flatMap((entry, index) => - entry === "inline" ? [unsuppressedImageFactIndexes[index] ?? null] : [], - ) - : undefined; - const inferredMediaImageLayout = - !params.mediaImageLayout && - imageOrder && - unsuppressedImageFactIndexes.length === imageOrder.length - ? { - slots: imageOrder.map((kind, index) => ({ - kind, - factIndex: unsuppressedImageFactIndexes[index], - })), - suppressedFactIndexes: [], - } - : undefined; - const layoutInlineFactIndexes = resolveLayoutInlineFactIndexes( - params.mediaImageLayout, - params.existingImages?.length ?? 0, - ); - const existingImageFactIndexes = - params.existingImageFactIndexes ?? - layoutInlineFactIndexes ?? - (inferredExistingImageFactIndexes?.length === (params.existingImages?.length ?? 0) - ? inferredExistingImageFactIndexes - : undefined); - const missingInlineMediaCount = countMissingLayoutInlineSlots( - params.mediaImageLayout, - existingImageFactIndexes, - params.existingImages?.length ?? 0, - ); - const attachmentRefs = params.mediaImageLayout - ? params.mediaImageLayout.slots.flatMap((slot) => - slot.kind === "offloaded" - ? [ - { - factIndex: slot.factIndex, - ref: slot.factIndex === undefined ? undefined : refsByFactIndex.get(slot.factIndex), - }, - ] - : [], + if (params.imageOrder?.length) { + const pending = [...imageFactIndexes]; + return [ + ...params.imageOrder.map((kind) => ({ + kind, + ...(kind === "offloaded" && pending.length ? { factIndex: pending.shift() } : {}), + })), + ...pending.map((factIndex) => ({ kind: "offloaded" as const, factIndex })), + ]; + } + return imageFactIndexes.map((factIndex, imageIndex) => ({ + factIndex, + kind: + !media[factIndex]?.path && + !media[factIndex]?.url && + imageIndex < (params.existingImages?.length ?? 0) + ? ("inline" as const) + : ("offloaded" as const), + })); + })(); + const slots = params.mediaImageLayout?.slots.length + ? params.mediaImageLayout.slots.filter( + (slot) => slot.factIndex === undefined || !suppressed.has(slot.factIndex), ) - : selectMediaImageRefs({ - refs: orderRefs, - existingImageCount: params.existingImages?.length ?? 0, - imageOrder, - }).map((ref) => ({ factIndex: ref?.factIndex, ref })); - const materializedFactIndexes = new Set( - (existingImageFactIndexes ?? []).filter((entry): entry is number => typeof entry === "number"), + : inferredSlots; + const layoutInlineIndexes = slots.flatMap((slot) => + slot.kind === "inline" ? [slot.factIndex ?? null] : [], + ); + const existingIndexes = + params.existingImageFactIndexes ?? + (layoutInlineIndexes.length === (params.existingImages?.length ?? 0) + ? layoutInlineIndexes + : params.existingImages?.map(() => null)); + const unusedExisting = (params.existingImages ?? []).map((image, index) => ({ + image, + factIndex: existingIndexes?.[index] ?? null, + })); + const takeExisting = ( + factIndex: number | undefined, + allowUnowned: boolean, + ): PromptImageEntry | undefined => { + const exact = + factIndex === undefined + ? -1 + : unusedExisting.findIndex((entry) => entry.factIndex === factIndex); + const index = + exact >= 0 + ? exact + : allowUnowned + ? unusedExisting.findIndex((entry) => entry.factIndex === null) + : -1; + return index >= 0 ? unusedExisting.splice(index, 1)[0] : undefined; + }; + const availableRefs = refs.filter((ref): ref is MediaImageRef => Boolean(ref)); + const attachmentRefs = slots.flatMap((slot) => + slot.kind === "offloaded" && slot.factIndex !== undefined + ? (refsByFact.get(slot.factIndex) ?? []) + : [], ); - const availableMediaRefs = allMediaRefs.filter((ref): ref is MediaImageRef => ref !== undefined); - const selectedAttachmentRefs = attachmentRefs.flatMap(({ ref }) => (ref ? [ref] : [])); const attachmentKeys = new Set( - selectedAttachmentRefs.map((ref) => refDedupeKey(ref, ref.workspaceDir ?? params.workspaceDir)), + attachmentRefs.map((ref) => refDedupeKey(ref, ref.workspaceDir ?? params.workspaceDir)), ); const attachmentRawKeys = new Set( - selectedAttachmentRefs.flatMap((ref) => - ref.aliases.flatMap((alias) => { - const key = rawAliasDedupeKey(alias); - return key ? [key] : []; - }), - ), + attachmentRefs.flatMap((ref) => ref.aliases.flatMap((alias) => rawAliasDedupeKey(alias) ?? [])), ); const promptRefs = detectImageReferences(params.prompt).filter( (ref) => @@ -480,7 +410,7 @@ export async function detectAndLoadPromptImages(params: { !attachmentKeys.has(refDedupeKey(ref, params.workspaceDir)), ); const detectedRefs = [ - ...availableMediaRefs.flatMap(({ detect, hydrate, raw, type, resolved }) => + ...availableRefs.flatMap(({ detect, hydrate, raw, type, resolved }) => detect !== false && (hydrate || (!resolved.startsWith("http://") && !resolved.startsWith("https://"))) ? [{ raw, type, resolved }] @@ -488,40 +418,10 @@ export async function detectAndLoadPromptImages(params: { ), ...promptRefs, ]; - if (attachmentRefs.length === 0 && promptRefs.length === 0) { - const existingImages = params.existingImages ?? []; - const sanitized = existingImages.length - ? await sanitizeImageEntriesWithLog( - existingImages.map((image, index) => ({ - image, - factIndex: existingImageFactIndexes?.[index] ?? null, - })), - "prompt:images", - { - maxBytes: params.maxBytes, - maxDimensionPx: params.maxDimensionPx, - }, - ) - : { entries: [], failedMediaCount: 0 }; - const finalized = finalizeRuntimePromptImages(sanitized.entries); - return { - ...finalized, - detectedRefs, - failedMediaCount: missingInlineMediaCount + sanitized.failedMediaCount, - loadedCount: 0, - skippedCount: 0, - }; - } - - log.debug( - `Native image: prepared ${attachmentRefs.length} attachment ref(s) and ${promptRefs.length} explicit prompt ref(s)`, - ); let loadedCount = 0; - let failedMediaCount = missingInlineMediaCount; + let failedMediaCount = 0; let skippedCount = 0; - const loadRef = async ( - ref: DetectedImageRef & { workspaceDir?: string }, - ): Promise => { + const loadRef = async (ref: MediaFileRef & { workspaceDir?: string }) => { const image = await loadImageFromRef(ref, ref.workspaceDir ?? params.workspaceDir, { maxBytes: params.maxBytes, workspaceOnly: params.workspaceOnly, @@ -536,42 +436,33 @@ export async function detectAndLoadPromptImages(params: { } return image; }; - const offloadedImages: Array = []; - for (const attachment of attachmentRefs) { - const factIndex = attachment.factIndex; - if (factIndex !== undefined && materializedFactIndexes.has(factIndex)) { - offloadedImages.push(null); + const promptImages: PromptImageEntry[] = []; + for (const slot of slots) { + const existing = takeExisting(slot.factIndex, slot.kind === "inline"); + if (existing) { + promptImages.push(existing); continue; } - const ref = attachment.ref; - if (!ref) { + if (slot.kind === "inline") { failedMediaCount++; - offloadedImages.push(null); continue; } - const image = ref.hydrate ? await loadRef(ref) : null; - if (ref.hydrate && !image) { + const ref = slot.factIndex === undefined ? undefined : refsByFact.get(slot.factIndex); + const image = ref?.hydrate ? await loadRef(ref) : null; + if (ref?.hydrate && !image) { failedMediaCount++; } - offloadedImages.push(image ? { image, factIndex: ref.factIndex } : null); + if (image) { + promptImages.push({ image, factIndex: ref?.factIndex ?? null }); + } } - const promptRefImages: ImageContent[] = []; + promptImages.push(...unusedExisting); for (const ref of promptRefs) { const image = await loadRef(ref); if (image) { - promptRefImages.push(image); + promptImages.push({ image, factIndex: null }); } } - - const promptImages = mergePromptAttachmentImages({ - imageOrder, - mediaImageLayout: params.mediaImageLayout ?? inferredMediaImageLayout, - existingImages: params.existingImages, - existingImageFactIndexes, - offloadedImages, - promptRefImages, - }); - const sanitizedPromptImages = await sanitizeImageEntriesWithLog(promptImages, "prompt:images", { maxBytes: params.maxBytes, maxDimensionPx: params.maxDimensionPx, @@ -587,20 +478,107 @@ export async function detectAndLoadPromptImages(params: { }; } -/** Hydrates non-enumerable facts carried by queued user turns before provider replay. */ -export async function hydratePromptMediaMessages( +type PromptMediaOptions = { + workspaceDir: string; + model: { input?: string[] }; + maxBytes?: number; + maxDimensionPx?: number; + workspaceOnly?: boolean; + localRoots?: readonly string[]; + sandbox?: { root: string; bridge: SandboxFsBridge }; + provider?: boolean; + signal?: AbortSignal; +}; + +const VIDEO_OMISSION = { + unsupported: "(video omitted: provider does not support native video)", + historical: "(video omitted: native historical replay is not available yet)", + unavailable: "(video omitted: source unavailable)", + invalid: "(video omitted: invalid video MIME type)", + limit: "(video omitted: native video byte limit exceeded)", +} as const; + +async function materializeVideoFact( + fact: MediaFact, + budget: { remaining: number }, + options: PromptMediaOptions, +): Promise { + if ((fact.sizeBytes ?? 0) > budget.remaining) { + return { type: "text", text: VIDEO_OMISSION.limit }; + } + const ref = resolveMediaFactLocalRef(fact); + const loaded = ref + ? await loadMediaFromRef(ref, fact.workspaceDir ?? options.workspaceDir, { + label: "Native video", + maxBytes: budget.remaining, + signal: options.signal, + workspaceOnly: options.workspaceOnly, + localRoots: + options.localRoots ?? (options.workspaceOnly ? [options.workspaceDir] : undefined), + sandbox: options.sandbox, + }) + : null; + if (!loaded) { + return { type: "text", text: VIDEO_OMISSION.unavailable }; + } + const mimeType = normalizeMimeType(loaded.contentType); + if (loaded.kind !== "video" || !mimeType?.startsWith("video/")) { + return { type: "text", text: VIDEO_OMISSION.invalid }; + } + if (loaded.buffer.length > budget.remaining) { + return { type: "text", text: VIDEO_OMISSION.limit }; + } + budget.remaining -= loaded.buffer.length; + return { type: "video", data: loaded.buffer.toString("base64"), mimeType }; +} + +async function projectOrderedPromptMedia(params: { + content: Array; + media: MediaFact[]; + images: ImageContent[]; + imageFactIndexes: ImageFactIndex[]; + runtime: boolean; + options: PromptMediaOptions; + budget: { remaining: number }; +}): Promise { + const generatedMarkers = new Set(Object.values(VIDEO_OMISSION)); + const projected: ModelInputContent[] = params.content.filter( + (block): block is TextContent => block.type === "text" && !generatedMarkers.has(block.text), + ); + const imagesByFact = new Map(); + const factlessImages: ImageContent[] = []; + params.images.forEach((image, index) => { + const factIndex = params.imageFactIndexes[index]; + if (factIndex == null) { + factlessImages.push(image); + } else { + imagesByFact.set(factIndex, [...(imagesByFact.get(factIndex) ?? []), image]); + } + }); + for (const [factIndex, fact] of params.media.entries()) { + if (isImageMediaFact(fact)) { + projected.push(...(imagesByFact.get(factIndex) ?? [])); + } else if (isVideoMediaFact(fact)) { + projected.push( + !params.runtime + ? { type: "text", text: VIDEO_OMISSION.historical } + : params.options.provider + ? await materializeVideoFact(fact, params.budget, params.options) + : { type: "text", text: VIDEO_OMISSION.unsupported }, + ); + } + } + projected.push(...factlessImages); + return projected; +} + +/** Hydrates exact-message media facts for canonical replay or one provider call. */ +async function materializePromptMediaMessages( messages: AgentMessage[], - options: { - workspaceDir: string; - model: { input?: string[] }; - maxBytes?: number; - maxDimensionPx?: number; - workspaceOnly?: boolean; - localRoots?: readonly string[]; - sandbox?: { root: string; bridge: SandboxFsBridge }; - }, + options: PromptMediaOptions, ): Promise { let hydrated: AgentMessage[] | undefined; + const videoBudget = { remaining: MAX_VIDEO_BYTES }; for (const [index, message] of messages.entries()) { if (message.role !== "user") { continue; @@ -617,28 +595,13 @@ export async function hydratePromptMediaMessages( ? message.content : [{ type: "text" as const, text: message.content }]; const existingImages = content.filter((block): block is ImageContent => block.type === "image"); - const persistedImageFactIndexes = readPersistedImageBlockFactIndexes(message); - const inlineLayoutFactIndexes = mediaImageLayout?.slots.flatMap((slot) => - slot.kind === "inline" ? [slot.factIndex ?? null] : [], - ); - // Pre-carrier transcripts had no explicit block provenance. Their native - // image blocks were positionally aligned with the first image facts. - const legacyImageFactIndexes = - runtimeMedia === undefined && mediaImageLayout === undefined - ? collectIdentitylessMediaImageFactIndexes(resolvedMedia) - : undefined; - const existingImageFactIndexes = - persistedImageFactIndexes ?? - (inlineLayoutFactIndexes?.length === existingImages.length - ? inlineLayoutFactIndexes - : legacyImageFactIndexes?.slice(0, existingImages.length)); const result = await detectAndLoadPromptImages({ prompt: "", media: resolvedMedia, workspaceDir: options.workspaceDir, model: options.model, existingImages, - existingImageFactIndexes, + existingImageFactIndexes: readPersistedImageBlockFactIndexes(message), imageOrder: runtimeImageOrder, mediaImageLayout, maxBytes: options.maxBytes, @@ -647,6 +610,25 @@ export async function hydratePromptMediaMessages( localRoots: options.localRoots, sandbox: options.sandbox, }); + const projectedContent = await projectOrderedPromptMedia({ + content, + media: resolvedMedia, + images: result.images, + imageFactIndexes: result.imageFactIndexes, + runtime: runtimeMedia !== undefined, + options, + budget: videoBudget, + }); + hydrated ??= messages.slice(); + if (options.provider) { + hydrated[index] = { + role: "user", + content: projectedContent, + timestamp: message.timestamp, + ...(message.runtimeContextCarrier ? { runtimeContextCarrier: true } : {}), + } as ProviderContext["messages"][number] as AgentMessage; + continue; + } const nextMeta = meta && typeof meta === "object" && !Array.isArray(meta) ? { ...(meta as Record) } @@ -656,10 +638,9 @@ export async function hydratePromptMediaMessages( } else { delete nextMeta.mediaImageBlockFactIndexes; } - hydrated ??= messages.slice(); const hydratedMessage = { ...message, - content: [...content.filter((block) => block.type !== "image"), ...result.images], + content: projectedContent, } as AgentMessage; if (Object.keys(nextMeta).length > 0) { (hydratedMessage as unknown as Record)["__openclaw"] = nextMeta; @@ -673,3 +654,35 @@ export async function hydratePromptMediaMessages( } return hydrated ?? messages; } + +/** Hydrates non-enumerable facts carried by queued user turns before canonical replay. */ +export async function hydratePromptMediaMessages( + messages: AgentMessage[], + options: Omit, +): Promise { + return await materializePromptMediaMessages(messages, options); +} + +/** Materializes one transient provider context from exact-message media facts. */ +export async function materializeProviderContext(params: { + context: Context; + signal?: AbortSignal; + workspaceDir: string; + workspaceOnly?: boolean; + localRoots?: readonly string[]; + sandbox?: { root: string; bridge: SandboxFsBridge }; +}): Promise { + const messages = await materializePromptMediaMessages(params.context.messages as AgentMessage[], { + workspaceDir: params.workspaceDir, + model: { input: ["text", "image"] }, + workspaceOnly: params.workspaceOnly, + localRoots: params.localRoots, + sandbox: params.sandbox, + provider: true, + signal: params.signal, + }); + params.signal?.throwIfAborted(); + return messages === params.context.messages + ? (params.context as ProviderContext) + : ({ ...params.context, messages } as ProviderContext); +} diff --git a/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.test.ts b/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.test.ts new file mode 100644 index 000000000000..b95f286535d3 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.test.ts @@ -0,0 +1,240 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createAssistantMessageEventStream } from "@openclaw/llm-core"; +import { MAX_VIDEO_BYTES } from "@openclaw/media-core/constants"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + PROVIDER_CONTEXT_HANDOFF, + resolveProviderContext, + type ProviderContext, + type ProviderStreamOptions, +} from "../../../../packages/ai/src/provider-types.js"; +import { attachRuntimePromptMediaFacts } from "../../../media/media-facts.js"; +import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; +import type { StreamFn } from "../../runtime/index.js"; +import type { SandboxFsBridge } from "../../sandbox/fs-bridge.js"; +import { hydratePromptMediaMessages, materializeProviderContext } from "./images.js"; +import { wrapStreamFnWithMessageTransform } from "./message-transform-stream-wrapper.js"; + +const PNG = { + type: "image" as const, + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==", + mimeType: "image/png", +}; +const MP4 = Buffer.from("0000001c6674797069736f6d0000000069736f6d0000000000000000", "hex"); + +describe("direct provider context handoff", () => { + const tempDirs: string[] = []; + afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })), + ); + }); + + it("keeps canonical omissions while materializing only exact current runtime facts", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-provider-video-")); + tempDirs.push(stateDir); + const env = captureEnv(["OPENCLAW_STATE_DIR"]); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const inbound = path.join(stateDir, "media", "inbound"); + await fs.mkdir(inbound, { recursive: true }); + await fs.writeFile(path.join(inbound, "recent.mp4"), MP4); + await fs.writeFile(path.join(inbound, "steer.mp4"), MP4); + + try { + const historical = { + role: "user" as const, + content: "historical", + timestamp: 1, + __openclaw: { + media: [ + { + kind: "video", + contentType: "video/mp4", + url: "media://inbound/old.mp4", + hydrationSuppressed: true, + }, + ], + }, + }; + const recent = attachRuntimePromptMediaFacts( + { + role: "user" as const, + content: [{ type: "text" as const, text: "recent" }, PNG, { ...PNG }], + timestamp: 2, + __openclaw: { + mediaImageBlockFactIndexes: [0, 2], + mediaImageLayout: { + slots: [ + { kind: "inline", factIndex: 0 }, + { kind: "inline", factIndex: 2 }, + ], + }, + }, + }, + [ + { kind: "image", contentType: "image/png" }, + { + kind: "video", + contentType: "video/mp4", + sizeBytes: MP4.length, + url: "media://inbound/recent.mp4", + hydrationSuppressed: true, + }, + { kind: "image", contentType: "image/png" }, + ], + ); + const steer = attachRuntimePromptMediaFacts( + { + role: "user" as const, + content: [{ type: "text" as const, text: "steer" }, { ...PNG }], + timestamp: 3, + __openclaw: { mediaImageBlockFactIndexes: [1] }, + }, + [ + { + kind: "video", + contentType: "video/mp4", + sizeBytes: MP4.length, + url: "media://inbound/steer.mp4", + }, + { kind: "image", contentType: "image/png" }, + ], + ); + const canonicalMessages = await hydratePromptMediaMessages([historical, recent, steer], { + workspaceDir: stateDir, + model: { input: ["text", "image"] }, + }); + const context = { + systemPrompt: "system", + messages: canonicalMessages, + tools: [], + } as Parameters[1]; + const originalJson = JSON.stringify(context); + const model = { + id: "canonical", + provider: "direct", + api: "test", + input: ["text", "image"], + } as Parameters[0]; + + const firstCandidate = vi.fn((_model, firstContext, options) => { + expect((options as ProviderStreamOptions)[PROVIDER_CONTEXT_HANDOFF]).toBeUndefined(); + expect(JSON.stringify(firstContext)).toContain("provider does not support native video"); + expect(JSON.stringify(firstContext)).toContain("historical replay is not available yet"); + return createAssistantMessageEventStream(); + }); + void firstCandidate(model, context, {}); + + let providerContext: Promise | undefined; + let providerOptions: ProviderStreamOptions | undefined; + const direct = vi.fn((_model, canonical, options) => { + providerOptions = options; + providerContext = resolveProviderContext(canonical, options); + return createAssistantMessageEventStream(); + }); + const wrapped = wrapStreamFnWithMessageTransform( + direct, + (messages) => messages, + (input) => materializeProviderContext({ ...input, workspaceDir: stateDir }), + ); + void wrapped(model, context, {}); + const resolved = await providerContext; + + expect(resolved?.messages[0]?.content).toEqual([ + { type: "text", text: "historical" }, + { type: "text", text: "(video omitted: native historical replay is not available yet)" }, + ]); + expect(resolved?.messages[1]?.content).toEqual([ + { type: "text", text: "recent" }, + PNG, + { type: "video", data: MP4.toString("base64"), mimeType: "video/mp4" }, + { ...PNG }, + ]); + expect(resolved?.messages[2]?.content).toEqual([ + { type: "text", text: "steer" }, + { type: "video", data: MP4.toString("base64"), mimeType: "video/mp4" }, + { ...PNG }, + ]); + for (const message of resolved?.messages.slice(1) ?? []) { + expect(message).not.toHaveProperty("__openclaw"); + expect(Object.getOwnPropertySymbols(message)).toEqual([]); + } + expect(JSON.stringify(resolved)).not.toContain(stateDir); + expect(JSON.stringify(context)).toBe(originalJson); + await expect(providerOptions?.[PROVIDER_CONTEXT_HANDOFF]?.()).rejects.toThrow( + "already consumed", + ); + } finally { + env.restore(); + } + }); + + it("rejects abort after a bounded sandbox read instead of dispatching an omission", async () => { + let finishRead: ((value: Buffer) => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + const bridge = { + resolvePath: ({ filePath }: { filePath: string }) => ({ + containerPath: filePath, + relativePath: filePath.replace(/^\//, ""), + }), + readFile: vi.fn(async () => { + markReadStarted?.(); + return await new Promise((resolve) => { + finishRead = resolve; + }); + }), + } as unknown as SandboxFsBridge; + const controller = new AbortController(); + const current = attachRuntimePromptMediaFacts( + { role: "user" as const, content: "inspect", timestamp: 1 }, + [{ kind: "video", contentType: "video/mp4", path: "/workspace/clip.mp4" }], + ); + const resolving = materializeProviderContext({ + context: { systemPrompt: "system", messages: [current], tools: [] }, + signal: controller.signal, + workspaceDir: "/workspace", + sandbox: { root: "/workspace", bridge }, + }); + await readStarted; + controller.abort(new Error("test abort")); + finishRead?.(MP4); + await expect(resolving).rejects.toThrow("test abort"); + }); + + it("rejects a known over-budget current video before reading it", async () => { + const readFile = vi.fn(); + const bridge = { + resolvePath: ({ filePath }: { filePath: string }) => ({ + containerPath: filePath, + relativePath: filePath.replace(/^\//, ""), + }), + readFile, + } as unknown as SandboxFsBridge; + const current = attachRuntimePromptMediaFacts( + { role: "user" as const, content: "inspect", timestamp: 1 }, + [ + { + kind: "video", + contentType: "video/mp4", + path: "/workspace/clip.mp4", + sizeBytes: MAX_VIDEO_BYTES + 1, + }, + ], + ); + const resolved = await materializeProviderContext({ + context: { systemPrompt: "system", messages: [current], tools: [] }, + workspaceDir: "/workspace", + sandbox: { root: "/workspace", bridge }, + }); + expect(resolved.messages[0]?.content).toContainEqual({ + type: "text", + text: "(video omitted: native video byte limit exceeded)", + }); + expect(readFile).not.toHaveBeenCalled(); + }); +}); diff --git a/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.ts b/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.ts index bf63831f182b..27ba883e9f1e 100644 --- a/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.ts +++ b/src/agents/embedded-agent-runner/run/message-transform-stream-wrapper.ts @@ -1,39 +1,60 @@ +import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; /** * Wraps stream functions with pre-call message transforms. */ -import type { StreamFn } from "openclaw/plugin-sdk/agent-core"; +import { + PROVIDER_CONTEXT_HANDOFF, + type ProviderContext, + type ProviderStreamOptions, +} from "../../../../packages/ai/src/provider-types.js"; import type { AgentMessage } from "../../runtime/index.js"; /** * Stream wrapper for applying message transforms immediately before provider dispatch. */ type MessageTransform = (messages: AgentMessage[], model: unknown) => AgentMessage[]; +type ProviderContextMaterializer = (input: { + context: Parameters[1]; + signal?: AbortSignal; +}) => Promise; /** Wraps a stream function with a conditional message-list transform. */ export function wrapStreamFnWithMessageTransform( streamFn: StreamFn, transform: MessageTransform, + materializeProviderContext?: ProviderContextMaterializer, ): StreamFn { return (model, context, options) => { const messages = (context as unknown as { messages?: unknown })?.messages; - if (!Array.isArray(messages)) { - return streamFn(model, context, options); + const nextMessages = Array.isArray(messages) + ? transform(messages as AgentMessage[], model) + : messages; + const nextContext = + Array.isArray(messages) && nextMessages !== messages + ? ({ + ...(context as unknown as Record), + messages: nextMessages, + } as typeof context) + : context; + if (!materializeProviderContext) { + return streamFn(model, nextContext, options); } - - const nextMessages = transform(messages as AgentMessage[], model); - if (nextMessages === messages) { - return streamFn(model, context, options); - } - - return streamFn( - // Clone the context instead of mutating it so callers can reuse the original assembled - // context for logging, replay, or retry comparisons. - model, - { - ...(context as unknown as Record), - messages: nextMessages, - } as typeof context, - options, - ); + let availableContext: Parameters[1] | undefined = nextContext; + const handoff = async (): Promise => { + const captured = availableContext; + availableContext = undefined; + if (!captured) { + throw new Error("provider context handoff already consumed"); + } + options?.signal?.throwIfAborted(); + return await materializeProviderContext({ + context: captured, + signal: options?.signal, + }); + }; + return streamFn(model, nextContext, { + ...options, + [PROVIDER_CONTEXT_HANDOFF]: handoff, + } as ProviderStreamOptions & NonNullable[2]>); }; } diff --git a/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts b/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts deleted file mode 100644 index 296011fc25bf..000000000000 --- a/src/agents/embedded-agent-runner/run/plugin-harness-prompt-images.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants"; -import { isImageMediaFact, readPersistedMediaFacts } from "../../../media/media-facts.js"; -import { resolveImageSanitizationLimits } from "../../image-sanitization.js"; -import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js"; -import { detectAndLoadPromptImages } from "./images.js"; -import type { RunEmbeddedAgentParams } from "./params.js"; -import { readPersistedMediaImageLayout } from "./prompt-image-metadata.js"; -import type { EmbeddedRunAttemptParams } from "./types.js"; - -function toTypeOnlyImageFact( - fact: NonNullable[number], - hydrationSuppressed: boolean, -): NonNullable[number] { - return { - contentType: fact.contentType, - kind: fact.kind === "sticker" ? "sticker" : "image", - messageId: fact.messageId, - transcribed: fact.transcribed, - ...(fact.hydrationSuppressed === true || hydrationSuppressed - ? { hydrationSuppressed: true } - : {}), - }; -} - -/** Materializes fact-carried images before a plugin harness owns transport. */ -export async function preparePluginHarnessPromptImages(params: { - runParams: RunEmbeddedAgentParams; - runtime: { - sessionId: string; - sessionKey?: string; - workspaceDir: string; - model: EmbeddedRunAttemptParams["model"]; - }; - pluginHarnessOwnsTransport: boolean; -}): Promise<{ - images: RunEmbeddedAgentParams["images"]; - imageOrder: RunEmbeddedAgentParams["imageOrder"]; - media: RunEmbeddedAgentParams["media"]; -}> { - const { runParams, runtime } = params; - if (!params.pluginHarnessOwnsTransport) { - return { - images: runParams.images, - imageOrder: runParams.imageOrder, - media: runParams.media, - }; - } - const persistedMessage = - runParams.userTurnTranscriptRecorder?.message ?? - (await runParams.userTurnTranscriptRecorder?.resolveMessage()); - const persistedMedia = persistedMessage ? (readPersistedMediaFacts(persistedMessage) ?? []) : []; - const hydrationMedia = persistedMedia.length > 0 ? persistedMedia : runParams.media; - if (!hydrationMedia?.some(isImageMediaFact)) { - return { - images: runParams.images, - imageOrder: runParams.imageOrder, - media: runParams.media, - }; - } - - const workspace = await resolveAttemptWorkspaceSandbox({ - ...runParams, - cwd: undefined, - sessionId: runtime.sessionId, - sessionKey: runtime.sessionKey, - workspaceDir: runtime.workspaceDir, - }); - const result = await detectAndLoadPromptImages({ - prompt: "", - media: hydrationMedia, - mediaImageLayout: persistedMessage - ? readPersistedMediaImageLayout(persistedMessage) - : undefined, - workspaceDir: workspace.effectiveWorkspace, - model: runtime.model, - existingImages: runParams.images, - imageOrder: runParams.imageOrder, - maxBytes: MAX_IMAGE_BYTES, - maxDimensionPx: resolveImageSanitizationLimits(runParams.config).maxDimensionPx, - localRoots: workspace.effectiveFsWorkspaceOnly - ? [workspace.effectiveWorkspace, workspace.resolvedWorkspace] - : undefined, - workspaceOnly: workspace.effectiveFsWorkspaceOnly, - sandbox: - workspace.sandbox?.enabled && workspace.sandbox.fsBridge - ? { root: workspace.sandbox.workspaceDir, bridge: workspace.sandbox.fsBridge } - : undefined, - }); - if (result.failedMediaCount > 0) { - throw new Error( - `failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`, - ); - } - const materializedFactIndexes = new Set( - result.imageFactIndexes.filter((index): index is number => index !== null), - ); - const retainedMedia = hydrationMedia?.map((fact, factIndex) => - isImageMediaFact(fact) - ? toTypeOnlyImageFact(fact, !materializedFactIndexes.has(factIndex)) - : fact, - ); - return { - images: result.images, - imageOrder: result.images.length > 0 ? result.images.map(() => "inline" as const) : undefined, - media: retainedMedia?.length ? retainedMedia : undefined, - }; -} diff --git a/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts index 92ac14494320..7ff56606c8ce 100644 --- a/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts +++ b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts @@ -7,48 +7,6 @@ export type MediaImageLayout = { suppressedFactIndexes: number[]; }; -export function resolveLayoutInlineFactIndexes( - layout: MediaImageLayout | undefined, - existingImageCount: number, -): ImageFactIndex[] | undefined { - const factIndexes = layout?.slots.flatMap((slot) => - slot.kind === "inline" ? [slot.factIndex ?? null] : [], - ); - return factIndexes?.length === existingImageCount ? factIndexes : undefined; -} - -export function countMissingLayoutInlineSlots( - layout: MediaImageLayout | undefined, - existingFactIndexes: readonly ImageFactIndex[] | undefined, - existingImageCount: number, -): number { - if (!layout) { - return 0; - } - const available = existingFactIndexes - ? [...existingFactIndexes] - : Array.from({ length: existingImageCount }, () => null); - let missing = 0; - for (const slot of layout.slots) { - if (slot.kind !== "inline") { - continue; - } - const exactIndex = - slot.factIndex === undefined - ? available.length > 0 - ? 0 - : -1 - : available.findIndex((factIndex) => factIndex === slot.factIndex); - const matchIndex = exactIndex >= 0 ? exactIndex : available.indexOf(null); - if (matchIndex >= 0) { - available.splice(matchIndex, 1); - } else { - missing++; - } - } - return missing; -} - export function readPersistedImageBlockFactIndexes( message: AgentMessage, ): ImageFactIndex[] | undefined { diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts index a39c486f137c..0420122c9e55 100644 --- a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.media.test.ts @@ -5,8 +5,33 @@ import { describe, expect, it } from "vitest"; import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js"; import { readRuntimePromptImageFactIndexes } from "../../../media/runtime-prompt-image-provenance.js"; import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js"; -import { detectAndLoadPromptImages } from "./images.js"; -import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js"; +import { prepareEmbeddedAttemptPromptExecution } from "./attempt-prompt-submit.js"; + +async function preparePluginHarnessPromptImages(params: { + runParams: Parameters[0]["attempt"]; + runtime: { + workspaceDir: string; + model: Parameters[0]["attempt"]["model"]; + }; + pluginHarnessOwnsTransport: boolean; +}) { + if (!params.pluginHarnessOwnsTransport) { + return { + images: params.runParams.images, + imageOrder: params.runParams.imageOrder, + media: params.runParams.media, + }; + } + const result = await prepareEmbeddedAttemptPromptExecution({ + attempt: { ...params.runParams, model: params.runtime.model }, + effectiveWorkspace: params.runtime.workspaceDir, + effectiveFsWorkspaceOnly: false, + prompt: "", + skipPromptSubmission: false, + pluginHarness: true, + }); + return { images: result.images, imageOrder: result.imageOrder, media: result.media }; +} const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg=="; @@ -33,7 +58,7 @@ describe("plugin harness prompt media", () => { }, pluginHarnessOwnsTransport: true, } as unknown as Parameters[0]), - ).resolves.toEqual({ images: undefined, imageOrder: undefined, media: undefined }); + ).resolves.toEqual({ images: [], imageOrder: undefined, media: undefined }); }); it.each([ @@ -103,9 +128,8 @@ describe("plugin harness prompt media", () => { expect(result.images ?? []).toHaveLength(testCase.expectedImages); if (testCase.expectedImages > 0) { expect(result.images?.[0]?.mimeType).toBe("image/png"); - } else { - expect(result.media).toEqual(media); } + expect(result.media).toBeUndefined(); } finally { envSnapshot.restore(); await fs.rm(stateDir, { recursive: true, force: true }); @@ -162,24 +186,9 @@ describe("plugin harness prompt media", () => { ]); expect(readRuntimePromptImageFactIndexes(result.images ?? [])).toEqual([0]); expect(result.imageOrder).toEqual(["inline"]); - expect(result.media?.[0]).toMatchObject({ contentType: "image/png", kind: "image" }); - expect(result.media?.[0]).not.toHaveProperty("path"); - expect(result.media?.[0]).not.toHaveProperty("url"); - expect(result.media?.[1]).toMatchObject(documentFact); - - const serialized = JSON.stringify(result); - const restored = JSON.parse(serialized) as typeof result; - const replay = await detectAndLoadPromptImages({ - prompt: "", - media: restored.media, - workspaceDir, - model: { input: ["text", "image"] }, - existingImages: restored.images, - imageOrder: restored.imageOrder, - }); - expect(replay.failedMediaCount).toBe(0); - expect(replay.images).toEqual(result.images); - expect(replay.imageFactIndexes).toEqual([0]); + expect(result.media).toBeUndefined(); + expect(JSON.stringify(result)).not.toContain(imagePath); + expect(structuredClone(result).images).toEqual(result.images); } finally { envSnapshot.restore(); await fs.rm(stateDir, { recursive: true, force: true }); @@ -308,13 +317,7 @@ describe("plugin harness prompt media", () => { } as unknown as Parameters[0]); expect(result.images).toEqual([]); - expect(result.media?.[0]).toMatchObject({ - contentType: "image/png", - kind: "image", - hydrationSuppressed: true, - }); - expect(result.media?.[0]).not.toHaveProperty("path"); - expect(result.media?.[0]).not.toHaveProperty("url"); + expect(result.media).toBeUndefined(); }); it("retains layout-derived suppression after plugin host materialization", async () => { @@ -357,9 +360,7 @@ describe("plugin harness prompt media", () => { expect(result.images).toEqual([inlineImage]); expect(result.imageOrder).toEqual(["inline"]); - expect(result.media?.[0]).toMatchObject({ kind: "image", hydrationSuppressed: true }); - expect(result.media?.[1]).toMatchObject({ kind: "image" }); - expect(result.media?.[1]).not.toHaveProperty("hydrationSuppressed"); + expect(result.media).toBeUndefined(); }); it("keeps unsupported native images as aligned type-only facts", async () => { @@ -383,10 +384,7 @@ describe("plugin harness prompt media", () => { } as unknown as Parameters[0]); expect(result.images).toEqual([]); - expect(result.media?.[0]).toMatchObject({ contentType: "image/png" }); - expect(result.media?.[0]).not.toHaveProperty("path"); - expect(result.media?.[1]).toMatchObject({ kind: "image" }); - expect(result.media?.[1]).not.toHaveProperty("path"); + expect(result.media).toBeUndefined(); }); it("leaves facts untouched when the native harness owns transport", async () => { diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts index 637985e24229..c057755bc1c7 100644 --- a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts @@ -15,14 +15,15 @@ import { } from "../../tools/gateway-caller-context.js"; import type { SystemAgentToolOptions } from "../../tools/system-agent-tool.js"; import { prepareExecApprovalContinuationForAttempt } from "./attempt-exec-approval-continuation.js"; +import { prepareEmbeddedAttemptPromptExecution } from "./attempt-prompt-submit.js"; import { applyResolvedToolPromptFinalizer } from "./attempt-prompt-support.js"; +import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js"; import { runEmbeddedAttemptWithBackend } from "./backend.js"; import { EMBEDDED_RUN_LANE_HEARTBEAT_MS, EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS, } from "./lane-runtime.js"; import type { RunEmbeddedAgentParams } from "./params.js"; -import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js"; import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js"; import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js"; @@ -190,11 +191,26 @@ export async function dispatchEmbeddedRunAttempt(input: { modelMaxTokens: runtime.model.maxTokens, userTurnTranscriptRecorder: params.userTurnTranscriptRecorder, }); - const promptMedia = await preparePluginHarnessPromptImages({ - runParams: params, - runtime, - pluginHarnessOwnsTransport: control.pluginHarnessOwnsTransport, - }); + const promptMedia = control.pluginHarnessOwnsTransport + ? await (async () => { + const workspace = await resolveAttemptWorkspaceSandbox({ + ...params, + cwd: undefined, + sessionId: runtime.sessionId, + sessionKey: runtime.sessionKey, + workspaceDir: runtime.workspaceDir, + }); + return await prepareEmbeddedAttemptPromptExecution({ + attempt: { ...params, model: runtime.model }, + effectiveFsWorkspaceOnly: workspace.effectiveFsWorkspaceOnly, + effectiveWorkspace: workspace.effectiveWorkspace, + prompt: "", + sandbox: workspace.sandbox, + skipPromptSubmission: false, + pluginHarness: true, + }); + })() + : { images: params.images, imageOrder: params.imageOrder, media: params.media }; // Plugin harnesses own their tool materialization, so the host cannot attest // a message tool. Finalize conservatively instead of leaking phantom guidance. const pluginHarnessPrompt = diff --git a/src/agents/google-simple-completion-stream.test.ts b/src/agents/google-simple-completion-stream.test.ts deleted file mode 100644 index c8540f3aa035..000000000000 --- a/src/agents/google-simple-completion-stream.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -import type { ApiRegistry } from "@openclaw/ai"; -// Verifies the Google simple-completion wrapper and thinking-payload sanitizer hook. -import { beforeEach, describe, expect, it, vi } from "vitest"; -import type { Model } from "../llm/types.js"; - -const streamSimple = vi.fn(); -const sanitizeGoogleThinkingPayload = vi.fn(); -const ensureCustomApiRegistered = vi.fn(); -const apiRegistry = { - getApiProvider: vi.fn(() => ({ streamSimple })), -} as unknown as ApiRegistry; - -vi.mock("../llm/providers/stream-wrappers/google-thinking-payload.js", async () => { - const actual = await vi.importActual< - typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js") - >("../llm/providers/stream-wrappers/google-thinking-payload.js"); - return { - ...actual, - sanitizeGoogleThinkingPayload, - }; -}); - -vi.mock("./custom-api-registry.js", () => ({ - ensureCustomApiRegistered, -})); - -const { prepareGoogleSimpleCompletionModel } = await import("./google-simple-completion-stream.js"); - -const GOOGLE_SIMPLE_COMPLETION_API = "openclaw-google-generative-ai-simple"; - -// Mirrors the provider catalog shape closely enough for wrapper registration -// without pulling live Google model discovery into unit tests. -function makeGoogleModel( - id = "gemini-flash-latest", - overrides: Partial> = {}, -): Model<"google-generative-ai"> { - return { - id, - name: id, - api: "google-generative-ai", - provider: "google", - baseUrl: "https://generativelanguage.googleapis.com", - reasoning: true, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 1_000_000, - maxTokens: 8192, - headers: {}, - ...overrides, - }; -} - -describe("prepareGoogleSimpleCompletionModel", () => { - beforeEach(() => { - streamSimple.mockReset(); - sanitizeGoogleThinkingPayload.mockReset(); - ensureCustomApiRegistered.mockReset(); - streamSimple.mockImplementation((_model, _context, options) => { - const payload = { - generationConfig: { - thinkingConfig: { thinkingBudget: -1 }, - }, - }; - options?.onPayload?.(payload, _model); - return { content: [{ type: "text", text: "ok" }], payload }; - }); - }); - - it("returns non-Google models unchanged", () => { - const model = { - ...makeGoogleModel("gpt-5"), - api: "openai-responses", - } as unknown as Model<"openai-responses">; - - const result = prepareGoogleSimpleCompletionModel(apiRegistry, model); - - expect(result).toBe(model); - expect(ensureCustomApiRegistered).not.toHaveBeenCalled(); - }); - - it("registers an OpenClaw-owned Google simple-completion api alias", () => { - const model = makeGoogleModel(); - - const result = prepareGoogleSimpleCompletionModel(apiRegistry, model); - - expect(result).toEqual({ - ...model, - api: GOOGLE_SIMPLE_COMPLETION_API, - }); - expect(ensureCustomApiRegistered).toHaveBeenCalledTimes(1); - expect(ensureCustomApiRegistered.mock.calls[0]?.[0]).toBe(apiRegistry); - expect(ensureCustomApiRegistered.mock.calls[0]?.[1]).toBe(GOOGLE_SIMPLE_COMPLETION_API); - }); - - it.each(["off", "low", "medium", "high", "adaptive"] as const)( - "sanitizes outbound thinking payload for gemini-flash-latest with reasoning=%s", - async (reasoning) => { - const model = makeGoogleModel(); - const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model); - const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as ( - ...args: unknown[] - ) => unknown; - - // The custom alias must unwrap to the real Google API before delegating, - // then sanitize the exact outbound payload produced by streamSimple. - await streamFn(wrapped, { messages: [] }, { apiKey: "key", reasoning }); - - expect(streamSimple).toHaveBeenCalledTimes(1); - expect(streamSimple.mock.calls[0]?.[0]).toEqual({ - ...model, - api: "google-generative-ai", - }); - expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({ - payload: { - generationConfig: { - thinkingConfig: { thinkingBudget: -1 }, - }, - }, - modelId: "gemini-flash-latest", - thinkingLevel: reasoning, - }); - }, - ); - - it("returns the sanitizer-mutated payload shape", async () => { - sanitizeGoogleThinkingPayload.mockImplementationOnce((args: { payload: unknown }) => { - const payload = args.payload as { - generationConfig: { thinkingConfig: Record }; - }; - delete payload.generationConfig.thinkingConfig.thinkingBudget; - payload.generationConfig.thinkingConfig.thinkingLevel = "MINIMAL"; - }); - const model = makeGoogleModel(); - prepareGoogleSimpleCompletionModel(apiRegistry, model); - const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as ( - ...args: unknown[] - ) => unknown; - - const result = await streamFn(model, { messages: [] }, { apiKey: "key", reasoning: "off" }); - - expect(result).toMatchObject({ - payload: { - generationConfig: { - thinkingConfig: { - thinkingLevel: "MINIMAL", - }, - }, - }, - }); - expect( - ( - result as { - payload: { generationConfig: { thinkingConfig: Record } }; - } - ).payload.generationConfig.thinkingConfig, - ).not.toHaveProperty("thinkingBudget"); - }); - - it("removes disabled thinking budget for Gemma 4 when reasoning is omitted", async () => { - const actual = await vi.importActual< - typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js") - >("../llm/providers/stream-wrappers/google-thinking-payload.js"); - sanitizeGoogleThinkingPayload.mockImplementationOnce(actual.sanitizeGoogleThinkingPayload); - streamSimple.mockImplementationOnce((_model, _context, options) => { - const payload = { - generationConfig: { - thinkingConfig: { thinkingBudget: 0 }, - }, - }; - options?.onPayload?.(payload, _model); - return { content: [{ type: "text", text: "ok" }], payload }; - }); - const model = makeGoogleModel("gemma-4-26b-a4b-it"); - const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model); - const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as ( - ...args: unknown[] - ) => unknown; - - const result = await streamFn(wrapped, { messages: [] }, { apiKey: "key" }); - - expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({ - payload: { - generationConfig: {}, - }, - modelId: "gemma-4-26b-a4b-it", - thinkingLevel: undefined, - }); - expect( - ( - result as { - payload: { generationConfig: Record }; - } - ).payload.generationConfig, - ).not.toHaveProperty("thinkingConfig"); - }); - - it.each(["xhigh", "max"] as const)( - "preserves clamped-off intent in the final Gemini 3 payload for reasoning=%s", - async (reasoning) => { - const actual = await vi.importActual< - typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js") - >("../llm/providers/stream-wrappers/google-thinking-payload.js"); - sanitizeGoogleThinkingPayload.mockImplementationOnce(actual.sanitizeGoogleThinkingPayload); - streamSimple.mockImplementationOnce((_model, _context, options) => { - const payload = { - generationConfig: { - thinkingConfig: { thinkingLevel: "MINIMAL" }, - }, - }; - options?.onPayload?.(payload, _model); - return { content: [{ type: "text", text: "ok" }], payload }; - }); - const model = makeGoogleModel("gemini-3-flash-preview", { - thinkingLevelMap: { - minimal: null, - low: null, - medium: null, - high: null, - xhigh: null, - max: null, - }, - }); - const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model); - const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as ( - ...args: unknown[] - ) => unknown; - - const result = await streamFn(wrapped, { messages: [] }, { apiKey: "key", reasoning }); - - expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({ - payload: { - generationConfig: { - thinkingConfig: { thinkingLevel: "MINIMAL" }, - }, - }, - modelId: "gemini-3-flash-preview", - thinkingLevel: "off", - }); - expect(result).toMatchObject({ - payload: { - generationConfig: { - thinkingConfig: { thinkingLevel: "MINIMAL" }, - }, - }, - }); - expect( - ( - result as { - payload: { generationConfig: { thinkingConfig: Record } }; - } - ).payload.generationConfig.thinkingConfig, - ).not.toHaveProperty("includeThoughts"); - }, - ); -}); diff --git a/src/agents/google-simple-completion-stream.ts b/src/agents/google-simple-completion-stream.ts deleted file mode 100644 index 9bbabf5b72b7..000000000000 --- a/src/agents/google-simple-completion-stream.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { ApiRegistry } from "@openclaw/ai"; -/** - * Google simple-completion stream adapter. - * - * This registers a patched Google stream API that keeps the normal Google - * backend but sanitizes unsupported thinking payload options for simple models. - */ -import { clampThinkingLevel } from "@openclaw/ai/internal/runtime"; -import type { StreamFn } from "@openclaw/llm-core"; -import { - sanitizeGoogleThinkingPayload, - type GoogleThinkingInputLevel, -} from "../llm/providers/stream-wrappers/google-thinking-payload.js"; -import { streamWithPayloadPatch } from "../llm/providers/stream-wrappers/stream-payload-utils.js"; -import type { Api, Model, ModelThinkingLevel } from "../llm/types.js"; -import { ensureCustomApiRegistered } from "./custom-api-registry.js"; - -/** Custom API id for the Google simple-completion stream adapter. */ -const GOOGLE_SIMPLE_COMPLETION_API: Api = "openclaw-google-generative-ai-simple"; - -const SOURCE_API: Api = "google-generative-ai"; - -function resolveGoogleSimpleThinkingLevel( - model: Model, - reasoning: unknown, -): GoogleThinkingInputLevel | undefined { - switch (reasoning) { - case "adaptive": - return reasoning; - case "off": - case "minimal": - case "low": - case "medium": - case "high": - case "max": - case "xhigh": - return clampThinkingLevel(model, reasoning as ModelThinkingLevel); - default: - return undefined; - } -} - -function buildGoogleSimpleCompletionStreamFn(registry: ApiRegistry): StreamFn { - return (model, context, options) => { - const googleModel: Model = { ...model, api: SOURCE_API }; - const sourceProvider = registry.getApiProvider(SOURCE_API); - if (!sourceProvider) { - throw new Error(`No API provider registered for api: ${SOURCE_API}`); - } - return streamWithPayloadPatch( - sourceProvider.streamSimple as StreamFn, - googleModel, - context, - options, - (payload) => { - sanitizeGoogleThinkingPayload({ - payload, - modelId: model.id, - thinkingLevel: resolveGoogleSimpleThinkingLevel( - googleModel, - (options as { reasoning?: unknown } | undefined)?.reasoning, - ), - }); - }, - ); - }; -} - -/** Rewrites Google generative-ai models to the simple-completion adapter when needed. */ -export function prepareGoogleSimpleCompletionModel( - registry: ApiRegistry, - model: Model, -): Model { - if (model.api !== SOURCE_API) { - return model; - } - ensureCustomApiRegistered( - registry, - GOOGLE_SIMPLE_COMPLETION_API, - buildGoogleSimpleCompletionStreamFn(registry), - ); - return { ...model, api: GOOGLE_SIMPLE_COMPLETION_API }; -} diff --git a/src/media/media-facts.ts b/src/media/media-facts.ts index c7e0db19921f..9b560e4cc129 100644 --- a/src/media/media-facts.ts +++ b/src/media/media-facts.ts @@ -292,30 +292,43 @@ export function isGenericBinaryMediaContentType(contentType?: string | null): bo ); } -/** Returns whether a fact can produce native image input. */ -export function isImageMediaFact(fact: MediaFactInput): boolean { +function classifyMediaFact(fact: MediaFactInput): MediaKind | undefined { if (fact.kind && fact.kind !== "unknown") { - return fact.kind === "image" || fact.kind === "sticker"; + return fact.kind; } const normalizedContentType = normalizeMimeType(fact.contentType); if (normalizedContentType && !isGenericBinaryMediaContentType(normalizedContentType)) { const mimeKind = kindFromMime(normalizedContentType); if (mimeKind) { - return mimeKind === "image"; + return mimeKind; } // Legacy channel-mode projections persist bare image or sticker kind as MediaType. - return normalizedContentType === "image" || normalizedContentType === "sticker"; + return LEGACY_MEDIA_KINDS.has(normalizedContentType as MediaKind) + ? (normalizedContentType as MediaKind) + : undefined; } const pathValue = normalizeOptionalString(fact.path) ?? normalizeOptionalString(fact.url); const inferredMime = mimeTypeFromFilePath(pathValue); if (inferredMime === "image/svg+xml") { - return false; + return undefined; } - if (kindFromMime(inferredMime) === "image") { - return true; + const inferredKind = kindFromMime(inferredMime); + if (inferredKind) { + return inferredKind; } const extension = getFileExtension(pathValue); - return extension === ".tif" || extension === ".tiff"; + return extension === ".tif" || extension === ".tiff" ? "image" : undefined; +} + +/** Returns whether a fact can produce native image input. */ +export function isImageMediaFact(fact: MediaFactInput): boolean { + const kind = classifyMediaFact(fact); + return kind === "image" || kind === "sticker"; +} + +/** Returns whether a fact can produce native video input. */ +export function isVideoMediaFact(fact: MediaFactInput): boolean { + return classifyMediaFact(fact) === "video"; } type MediaFactDefaults = { diff --git a/src/plugin-sdk/llm.ts b/src/plugin-sdk/llm.ts index e3fee06e3c2f..e034a2846656 100644 --- a/src/plugin-sdk/llm.ts +++ b/src/plugin-sdk/llm.ts @@ -2,6 +2,14 @@ * Public SDK subpath for LLM streaming, model utils, and validation. */ export type { ApiProvider } from "@openclaw/ai"; +export { resolveProviderContext } from "../../packages/ai/src/provider-types.js"; +export type { + ProviderContext, + ProviderModel, + ProviderStreamFunction, + ProviderStreamOptions as ProviderCallStreamOptions, + VideoContent, +} from "../../packages/ai/src/provider-types.js"; export { calculateCost, clampThinkingLevel, diff --git a/tsdown.ai.config.ts b/tsdown.ai.config.ts index c7aa5ee8e9a7..7f053b2001bc 100644 --- a/tsdown.ai.config.ts +++ b/tsdown.ai.config.ts @@ -20,6 +20,7 @@ const config = { diagnostics: "packages/ai/src/utils/diagnostics.ts", "event-stream": "packages/ai/src/utils/event-stream.ts", types: "packages/ai/src/types.ts", + "provider-types": "packages/ai/src/provider-types.ts", validation: "packages/ai/src/validation.ts", "internal/anthropic": "packages/ai/src/internal/anthropic.ts", "internal/openai": "packages/ai/src/internal/openai.ts",