mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(google): add current-turn native video input (#122074)
* feat(agents): add current-turn Gemini video handoff * test(google): add live native video regression * build(ai): emit provider types entrypoint * fix(google): preserve video shedding on retry
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
01cd9cc2a8
commit
7eed2c3f21
@@ -1 +1 @@
|
||||
{"contentHash":"5a9d78a04eff220d33c4d2b7230c25f3d846ccddb4f2165967cf00fb948affa1","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
{"contentHash":"8c91a4aac45efb318043e133624a8334599c7f0f73c7ba7212dbb661f88c3b17","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"48813fa984919c8790905b995f4033d19fc9178e6eed7f2e5148ea9729c5eba3","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
{"contentHash":"cea38cb2354e456c9dcb62fcb59d4c5a1850eb7b35cf2c6dc1e6b4052cb239f2","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"454a3daca1cb018f303ce3dbcb8ec4ea86dbec180623822c5ca5250953402415","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"}
|
||||
{"contentHash":"b7c8bb7b8bbe40d1d877fb4c9c410663df59e2ceb236838e1564c04f3bd0e0c5","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"df88650d813f6efbdffc5570647f183e69fa3015929c748b03e8b22bf0d916e6","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
{"contentHash":"be5e5a878566af7ed7371bcc8bf6501771b7366c019e2e8132b1bd3d598b7b48","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"d11b2d65878b0886a3dc102c99a74fcd5e9bb80f27809ee810812f26ddde73cf","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
{"contentHash":"f4a908c9a10596f9aec9e9ebfda956d87e7d38d7a66b98dd36dca655ab0df554","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"81f7f67df730331d9fc2de922d1524833548a09ad69b2eb916440113ba91494c","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
{"contentHash":"57bdf7a00a80ea1e2d06985d3e2989df155f9cd9735b5b190df713abf7184a31","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"3059df01f7a73ba9c511ce97a0ef358fd8109389f524c0a3914984745aa02db5","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
{"contentHash":"3642828e0c335f3412a575e04ac5179c82226386942fcaecaf6ecf229cefb246","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"6fcaaa319bccd859ecbbe26278d3476ced29bdfed4e3ea0f05947c99d0e79bba","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
{"contentHash":"b976e3217681b57aab8ecab4a94ecf0f8238fe4c68de6c7bb9e2588e9c726622","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"2d253351db6cb80191067f6ce1aefba74d35dbbf1816c2741078e91738d608d7","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
{"contentHash":"b65bc3c8f77d9ae5fcfe03fe596f2cd948ce04a2544fd5df609c172ec3d4d267","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"b9d274360639f0589b6036ba6d735d375a66edfe3f25fac3add96f5989c8bf8a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
{"contentHash":"d22bf2e5ae14ee5ec01ea825c2bc16c908a15f78dd462c3e160393aa871d7cf5","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"5fb15ccec47ac420b0a1fb6d0769faeff1e14f3da1905c3c3a29c24887f1aaf1","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
{"contentHash":"66c63de1253ea9f27ada35301d7f9a53504483c10ca5dd281fe50b645b5bccd2","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"8c220f857b5144582c560810b63948b14ba87524eff0aa7407d39ccb7df7de8f","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
{"contentHash":"c1226a9266ddddfd3cb3ec3842a3eb43d7b05738ff495e18b9a13654d8bbc3e7","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"f7dd2a2df426b3826fb9bf31a062004801675eb89c006508a91cd5f7713a739b","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
{"contentHash":"538b128a5de8b38c9c3e13cdbb74d0eaa0e6ce7a8066c5852204b77b37536734","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"6c93d74605b93456c5124fc15a3bb99d1a68734b9772a0514af3ded6fb647001","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
{"contentHash":"9e50d560be0fae99edf2ab5400625033c3515d00b16113b085543d6f218c17cd","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"f951f78a1ed81bc8c9eac5d2421e243abf28aac9297d21b2c174e22ce3fac045","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
{"contentHash":"9748f103ca50929f6a67e77c6708c0fb1a24e5d83ec1ae9e15485a80fdc20398","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"1291d3c53625156807274d0ddc752b3ae0b01bf937f0fdca504a879698d8194a","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
{"contentHash":"4c5ae6de0b18a8ad16c4c6f8437bb18c0d2e8f511452ac47ef1442136c7f90a9","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"}
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"contentHash":"c4ad78f6f23d67778f2d37cd7492e2c21832bd2622f56fa17d16f9b12fb1e955","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
{"contentHash":"a204b23b49393b23755ed3f6d5f44c14248abba916063c6a4f363d39a019c800","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"contentHash":"2d76db26f7ce4188386d49fd13ed7daf5185cb22a6d2ac1fd9ade7288b6c7d0f","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
{"contentHash":"ed3e64c4f91e222c62209dbb7ae7632ff3ea85ae5658264b559d36997f037b7b","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"}
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
// Google tests cover google plugin behavior.
|
||||
import { toErrorObject as toLintErrorObject } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { completeSimple, type Model } from "openclaw/plugin-sdk/llm";
|
||||
import {
|
||||
completeSimple,
|
||||
type Model,
|
||||
type ProviderContext,
|
||||
type ProviderModel,
|
||||
type ProviderStreamFunction,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
import { resolveFfmpegBin } from "openclaw/plugin-sdk/media-runtime";
|
||||
import {
|
||||
createCapturedPluginRegistration,
|
||||
@@ -24,6 +30,25 @@ const GOOGLE_API_KEY =
|
||||
const LIVE = isLiveTestEnabled() && GOOGLE_API_KEY.length > 0;
|
||||
const describeLive = LIVE ? describe : describe.skip;
|
||||
|
||||
// Two 64x64 solid-red H.264 frames keep native-video proof bounded and reproducible.
|
||||
const GOOGLE_LIVE_RED_VIDEO_BASE64 = [
|
||||
"AAAAJGZ0eXBpc29tAAACAGlzb21pc282aXNvMmF2YzFtcDQxAAAC5m1vb3YAAABsbXZoZAAAAAAAAAAAAAAAAAAAA+gAAAAA",
|
||||
"AAEAAAEAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
"AAAAAAAAAAIAAAHodHJhawAAAFx0a2hkAAAAAwAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAA",
|
||||
"AAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAQAAAAABAAAAAQAAAAAABhG1kaWEAAAAgbWRoZAAAAAAAAAAAAAAAAAAAQAAAAAAA",
|
||||
"VcQAAAAAAC1oZGxyAAAAAAAAAAB2aWRlAAAAAAAAAAAAAAAAVmlkZW9IYW5kbGVyAAAAAS9taW5mAAAAFHZtaGQAAAABAAAA",
|
||||
"AAAAAAAAAAAkZGluZgAAABxkcmVmAAAAAAAAAAEAAAAMdXJsIAAAAAEAAADvc3RibAAAAKNzdHNkAAAAAAAAAAEAAACTYXZj",
|
||||
"MQAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAABAAEAASAAAAEgAAAAAAAAAARVMYXZjNjIuMjguMTAyIGxpYngyNjQAAAAAAAAA",
|
||||
"AAAAABj//wAAAC1hdmNDAULACv/hABZnQsAK2hCbARAAAAMAEAAAAwAo8SJqAQAEaM4PyAAAABBwYXNwAAAAAQAAAAEAAAAQ",
|
||||
"c3R0cwAAAAAAAAAAAAAAEHN0c2MAAAAAAAAAAAAAABRzdHN6AAAAAAAAAAAAAAAAAAAAEHN0Y28AAAAAAAAAAAAAAChtdmV4",
|
||||
"AAAAIHRyZXgAAAAAAAAAAQAAAAEAAAAAAAAAAAAAAAAAAABidWR0YQAAAFptZXRhAAAAAAAAACFoZGxyAAAAAAAAAABtZGly",
|
||||
"YXBwbAAAAAAAAAAAAAAAAC1pbHN0AAAAJal0b28AAAAdZGF0YQAAAAEAAAAATGF2ZjYyLjEyLjEwMgAAAHhtb29mAAAAEG1m",
|
||||
"aGQAAAAAAAAAAQAAAGB0cmFmAAAAJHRmaGQAAAA5AAAAAQAAAAAAAAMKAABAAAAAACMBAQAAAAAAFHRmZHQBAAAAAAAAAAAA",
|
||||
"AAAAAAAgdHJ1bgAAAgUAAAACAAAAgAIAAAAAAAAjAAAACgAAADVtZGF0AAAAH2WIhDoRigACGPHAAED2OAAIeUnJyddddddd",
|
||||
"dddddeAAAAAGQZogF6CMAAAAQ21mcmEAAAArdGZyYQEAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAMKAQEBAAAAEG1m",
|
||||
"cm8AAAAAAAAAQw==",
|
||||
].join("");
|
||||
|
||||
async function withGoogleApiEnvUnset<T>(fn: () => Promise<T>): Promise<T> {
|
||||
const geminiApiKey = process.env.GEMINI_API_KEY;
|
||||
const googleApiKey = process.env.GOOGLE_API_KEY;
|
||||
@@ -152,6 +177,74 @@ describeLive("google plugin live", () => {
|
||||
90_000,
|
||||
);
|
||||
|
||||
it("understands native video through the registered Google chat transport", async () => {
|
||||
const catalog = await buildGoogleLiveCatalogProvider({
|
||||
apiKey: "GEMINI_API_KEY",
|
||||
discoveryApiKey: GOOGLE_API_KEY,
|
||||
});
|
||||
const definition = catalog.models.find((candidate) => candidate.id === "gemini-3.5-flash-lite");
|
||||
expect(definition, "gemini-3.5-flash-lite missing from Google models.list").toBeDefined();
|
||||
expect(definition?.input).toContain("video");
|
||||
|
||||
const catalogModel = {
|
||||
...definition!,
|
||||
provider: "google",
|
||||
baseUrl: catalog.baseUrl,
|
||||
api: "google-generative-ai",
|
||||
} as ProviderModel<"google-generative-ai">;
|
||||
const { providers } = await registerGooglePlugin();
|
||||
const provider = requireRegisteredProvider(providers, "google");
|
||||
const model = provider.normalizeResolvedModel?.({
|
||||
provider: "google",
|
||||
modelId: catalogModel.id,
|
||||
model: catalogModel,
|
||||
} as never) as ProviderModel<"google-generative-ai"> | undefined;
|
||||
expect(model?.input).toContain("video");
|
||||
if (!model) {
|
||||
throw new Error("registered Google provider did not prepare its native video model");
|
||||
}
|
||||
|
||||
const registeredStream = provider.createStreamFn?.({
|
||||
provider: "google",
|
||||
modelId: model.id,
|
||||
model: model as never,
|
||||
});
|
||||
const streamFn = registeredStream as ProviderStreamFunction<"google-generative-ai"> | undefined;
|
||||
if (!streamFn) {
|
||||
throw new Error("registered Google provider did not create its native chat transport");
|
||||
}
|
||||
const context: ProviderContext = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "What single color fills this video? Reply with exactly RED, BLUE, or GREEN.",
|
||||
},
|
||||
{
|
||||
type: "video",
|
||||
mimeType: "video/mp4",
|
||||
data: GOOGLE_LIVE_RED_VIDEO_BASE64,
|
||||
},
|
||||
],
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const stream = await Promise.resolve(
|
||||
streamFn(model, context, { apiKey: GOOGLE_API_KEY, maxTokens: 16 }),
|
||||
);
|
||||
const response = await stream.result();
|
||||
expect(response.stopReason, response.errorMessage).not.toBe("error");
|
||||
const answer = response.content
|
||||
.filter((block) => block.type === "text")
|
||||
.map((block) => block.text)
|
||||
.join(" ");
|
||||
expect(answer).toMatch(/\bRED\b/iu);
|
||||
}, 90_000);
|
||||
|
||||
it("synthesizes speech through the registered provider", async () => {
|
||||
const { speechProviders } = await registerGooglePlugin();
|
||||
const provider = requireRegisteredProvider(speechProviders, "google");
|
||||
|
||||
@@ -32,6 +32,7 @@ describe("google provider catalog", () => {
|
||||
contextWindow: 1_048_576,
|
||||
maxTokens: 65_536,
|
||||
reasoning: true,
|
||||
input: ["text", "image"],
|
||||
});
|
||||
});
|
||||
|
||||
@@ -39,6 +40,7 @@ describe("google provider catalog", () => {
|
||||
expect(buildGoogleVertexStaticCatalogProvider().models.map((model) => model.id)).toEqual(
|
||||
buildGoogleStaticCatalogProvider().models.map((model) => model.id),
|
||||
);
|
||||
expect(buildGoogleStaticCatalogProvider().models[0]?.input).toEqual(["text", "image", "video"]);
|
||||
});
|
||||
|
||||
it("builds the authenticated text catalog from Google models.list metadata", async () => {
|
||||
@@ -121,7 +123,7 @@ describe("google provider catalog", () => {
|
||||
reasoning: true,
|
||||
contextWindow: 1_048_576,
|
||||
maxTokens: 65_536,
|
||||
input: ["text", "image"],
|
||||
input: ["text", "image", "video"],
|
||||
compat: { codeMode: "preferred" },
|
||||
}),
|
||||
expect.objectContaining({
|
||||
@@ -130,7 +132,7 @@ describe("google provider catalog", () => {
|
||||
reasoning: true,
|
||||
contextWindow: 1_048_576,
|
||||
maxTokens: 65_536,
|
||||
input: ["text", "image"],
|
||||
input: ["text", "image", "video"],
|
||||
compat: { codeMode: "preferred" },
|
||||
}),
|
||||
expect.objectContaining({
|
||||
|
||||
@@ -59,7 +59,10 @@ export function buildGoogleStaticCatalogProvider(): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl: GOOGLE_GEMINI_BASE_URL,
|
||||
api: "google-generative-ai",
|
||||
models: GOOGLE_GEMINI_TEXT_MODELS,
|
||||
models: GOOGLE_GEMINI_TEXT_MODELS.map((model) => ({
|
||||
...model,
|
||||
input: [...model.input, "video"],
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -73,7 +76,7 @@ function readGoogleLiveModels(body: unknown): readonly unknown[] {
|
||||
|
||||
function googleLiveModelInput(id: string): ModelDefinitionConfig["input"] {
|
||||
if (!id.startsWith("gemma-")) {
|
||||
return ["text", "image"];
|
||||
return ["text", "image", "video"];
|
||||
}
|
||||
const isMultimodalGemma =
|
||||
/^gemma-3-(?:4b|12b|27b)(?:-|$)/.test(id) ||
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { ProviderRuntimeModel } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createProviderDynamicModelContext as createContext } from "../test-support/provider-model-test-helpers.js";
|
||||
import {
|
||||
isGoogleNativeVideoModelId,
|
||||
isGoogleTextGenerationModelId,
|
||||
isModernGoogleModel,
|
||||
resolveGoogleGeminiForwardCompatModel,
|
||||
@@ -580,4 +581,24 @@ describe("resolveGoogleGeminiForwardCompatModel", () => {
|
||||
).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("classifies only ordinary Gemini generation ids for native video", () => {
|
||||
for (const modelId of [
|
||||
"gemini-2.5-flash",
|
||||
"google/gemini-3.1-pro-preview",
|
||||
"models/gemini-flash-latest",
|
||||
]) {
|
||||
expect(isGoogleNativeVideoModelId(modelId), modelId).toBe(true);
|
||||
}
|
||||
for (const modelId of [
|
||||
"gemma-4-26b-a4b-it",
|
||||
"tunedModels/gemini-2.5-flash",
|
||||
"gemini-3.1-flash-image",
|
||||
"gemini-2.5-computer-use-preview",
|
||||
"gemini-2.5-flash-tts-preview",
|
||||
"gemini-2.5-flash-live-preview",
|
||||
]) {
|
||||
expect(isGoogleNativeVideoModelId(modelId), modelId).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -49,7 +49,10 @@ function normalizeGeminiProRequestId(id: string): string {
|
||||
}
|
||||
|
||||
function googleFamilyModelId(id: string): string {
|
||||
return id.startsWith(GOOGLE_PROVIDER_PREFIX) ? id.slice(GOOGLE_PROVIDER_PREFIX.length) : id;
|
||||
const unqualified = id.startsWith(GOOGLE_PROVIDER_PREFIX)
|
||||
? id.slice(GOOGLE_PROVIDER_PREFIX.length)
|
||||
: id;
|
||||
return unqualified.startsWith("models/") ? unqualified.slice("models/".length) : unqualified;
|
||||
}
|
||||
|
||||
export function isGoogleTextGenerationModelId(id: string): boolean {
|
||||
@@ -71,6 +74,11 @@ export function isGoogleTextGenerationModelId(id: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
export function isGoogleNativeVideoModelId(id: string): boolean {
|
||||
const normalized = normalizeOptionalLowercaseString(googleFamilyModelId(id)) ?? "";
|
||||
return normalized.startsWith("gemini-") && isGoogleTextGenerationModelId(normalized);
|
||||
}
|
||||
|
||||
type GoogleForwardCompatFamily = readonly [
|
||||
googleTemplateIds: readonly string[],
|
||||
cliTemplateIds: readonly string[],
|
||||
|
||||
@@ -17,6 +17,7 @@ import { isGoogleGemini3ProModel, isGoogleGemini3ThinkingLevelModel } from "./th
|
||||
export {
|
||||
DEFAULT_GOOGLE_API_BASE_URL,
|
||||
isGoogleGenerativeAiApi,
|
||||
isOfficialGoogleAiStudioBaseUrl,
|
||||
isGoogleVertexBaseUrl,
|
||||
isGoogleVertexHostname,
|
||||
normalizeGoogleApiBaseUrl,
|
||||
|
||||
@@ -65,4 +65,47 @@ describe("buildGoogleProvider createStreamFn", () => {
|
||||
expect(streamFns.createVertex).not.toHaveBeenCalled();
|
||||
expect(streamFns.createGenerativeAi).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["gemini-2.5-flash", "https://generativelanguage.googleapis.com", true],
|
||||
["google/gemini-3.1-pro-preview", "https://generativelanguage.googleapis.com/v1beta", true],
|
||||
["models/gemini-2.5-pro", "https://generativelanguage.googleapis.com/v1beta/", true],
|
||||
["gemma-4-26b-a4b-it", "https://generativelanguage.googleapis.com/v1beta", false],
|
||||
["gemini-2.5-flash-image", "https://generativelanguage.googleapis.com/v1beta", false],
|
||||
["gemini-2.5-flash", "https://proxy.example.test/v1beta", false],
|
||||
["gemini-2.5-flash", "https://user@generativelanguage.googleapis.com/v1beta", false],
|
||||
["gemini-2.5-flash", "https://generativelanguage.googleapis.com/v1beta?key=x", false],
|
||||
["gemini-2.5-flash", "https://generativelanguage.googleapis.com:8443/v1beta", false],
|
||||
["gemini-2.5-flash", "https://generativelanguage.googleapis.com/v1beta/openai", false],
|
||||
])("normalizes native-video input for exact AI Studio route %s", (modelId, baseUrl, expected) => {
|
||||
const provider = buildGoogleProvider();
|
||||
const normalized = provider.normalizeResolvedModel?.({
|
||||
provider: "google",
|
||||
modelId,
|
||||
model: model({
|
||||
id: modelId,
|
||||
provider: "google",
|
||||
api: "google-generative-ai",
|
||||
baseUrl,
|
||||
input: ["text", "image", "video"] as never,
|
||||
}),
|
||||
} as never);
|
||||
|
||||
expect(((normalized?.input ?? []) as string[]).includes("video")).toBe(expected);
|
||||
});
|
||||
|
||||
it("strips inherited video for Vertex and non-Google provider routes", () => {
|
||||
const provider = buildGoogleProvider();
|
||||
for (const [providerId, api] of [
|
||||
["google-vertex", "google-vertex"],
|
||||
["custom-google", "google-generative-ai"],
|
||||
] as const) {
|
||||
const normalized = provider.normalizeResolvedModel?.({
|
||||
provider: providerId,
|
||||
modelId: "gemini-2.5-flash",
|
||||
model: model({ provider: providerId, api, input: ["text", "image", "video"] as never }),
|
||||
} as never);
|
||||
expect(normalized?.input).toEqual(["text", "image"]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,8 +13,13 @@ import {
|
||||
buildGoogleVertexStaticCatalogProvider,
|
||||
} from "./provider-catalog.js";
|
||||
import { GOOGLE_GEMINI_PROVIDER_HOOKS } from "./provider-hooks.js";
|
||||
import { isModernGoogleModel, resolveGoogleGeminiForwardCompatModel } from "./provider-models.js";
|
||||
import {
|
||||
isGoogleNativeVideoModelId,
|
||||
isModernGoogleModel,
|
||||
resolveGoogleGeminiForwardCompatModel,
|
||||
} from "./provider-models.js";
|
||||
import {
|
||||
isOfficialGoogleAiStudioBaseUrl,
|
||||
isGoogleVertexBaseUrl,
|
||||
normalizeGoogleProviderConfig,
|
||||
resolveGoogleGenerativeAiTransport,
|
||||
@@ -25,6 +30,18 @@ import {
|
||||
} from "./transport-stream.js";
|
||||
import { resolveGoogleVertexConfigApiKey } from "./vertex-adc.js";
|
||||
|
||||
function normalizeGoogleVideoInput(
|
||||
ctx: Parameters<NonNullable<ProviderPlugin["normalizeResolvedModel"]>>[0],
|
||||
) {
|
||||
const input = (ctx.model.input as string[]).filter((type) => type !== "video");
|
||||
const supportsVideo =
|
||||
ctx.provider === "google" &&
|
||||
ctx.model.api === "google-generative-ai" &&
|
||||
isOfficialGoogleAiStudioBaseUrl(ctx.model.baseUrl) &&
|
||||
isGoogleNativeVideoModelId(ctx.modelId);
|
||||
return { ...ctx.model, input: supportsVideo ? [...input, "video"] : input } as typeof ctx.model;
|
||||
}
|
||||
|
||||
function resolveGoogleReasoningOutputMode(
|
||||
ctx: ProviderReasoningOutputModeContext,
|
||||
): "native" | "tagged" {
|
||||
@@ -100,6 +117,7 @@ export function buildGoogleProvider(): ProviderPlugin {
|
||||
},
|
||||
},
|
||||
normalizeModelId: ({ modelId }) => normalizeGoogleModelId(modelId),
|
||||
normalizeResolvedModel: normalizeGoogleVideoInput,
|
||||
resolveDynamicModel: (ctx) =>
|
||||
resolveGoogleGeminiForwardCompatModel({
|
||||
providerId: ctx.provider,
|
||||
|
||||
@@ -17,6 +17,19 @@ function isGoogleGenerativeAiUrl(url: URL): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/** Exact official AI Studio request root eligible for native provider behavior. */
|
||||
export function isOfficialGoogleAiStudioBaseUrl(baseUrl?: string | null): boolean {
|
||||
const raw = normalizeOptionalString(baseUrl) ?? DEFAULT_GOOGLE_API_BASE_URL;
|
||||
try {
|
||||
const href = trimTrailingSlashes(new URL(raw).href);
|
||||
return (
|
||||
href === "https://generativelanguage.googleapis.com" || href === DEFAULT_GOOGLE_API_BASE_URL
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function stripUrlUserInfo(url: URL): void {
|
||||
url.username = "";
|
||||
url.password = "";
|
||||
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { toErrorObject as toLintErrorObject } from "openclaw/plugin-sdk/error-runtime";
|
||||
import type { Model } from "openclaw/plugin-sdk/llm";
|
||||
import type { Model, ProviderContext } from "openclaw/plugin-sdk/llm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resetGoogleVertexAdcState } from "./google-oauth.test-support.js";
|
||||
|
||||
@@ -133,6 +133,16 @@ async function runGeminiStreamResult(
|
||||
return stream.result();
|
||||
}
|
||||
|
||||
function withProviderContextHandoff(
|
||||
options: Record<string, unknown>,
|
||||
handoff: () => Promise<ProviderContext>,
|
||||
): Record<string, unknown> {
|
||||
return new Proxy(options, {
|
||||
get: (target, property, receiver) =>
|
||||
typeof property === "symbol" ? handoff : Reflect.get(target, property, receiver),
|
||||
});
|
||||
}
|
||||
|
||||
async function runGoogleVertexStreamResult(params: {
|
||||
model?: Model<"google-vertex">;
|
||||
fetch: typeof guardedFetchMock;
|
||||
@@ -469,6 +479,157 @@ describe("google transport stream", () => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("resolves qualified AI Studio video after payload hooks and preserves part order", async () => {
|
||||
mockGoogleTextResponse();
|
||||
const videoData = "current-video-base64";
|
||||
const quicktimeData = "current-quicktime-base64";
|
||||
const imageData = "current-image-base64";
|
||||
const providerContext: ProviderContext = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: "before" },
|
||||
{ type: "image", mimeType: "image/png", data: imageData },
|
||||
{ type: "video", mimeType: "video/mp4", data: videoData },
|
||||
{ type: "video", mimeType: "video/quicktime", data: quicktimeData },
|
||||
{ type: "text", text: "after" },
|
||||
],
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
const handoff = vi.fn(async () => providerContext);
|
||||
const onPayload = vi.fn((payload: unknown) => {
|
||||
expect(JSON.stringify(payload)).not.toContain(videoData);
|
||||
expect(JSON.stringify(payload)).not.toContain(quicktimeData);
|
||||
expect(JSON.stringify(payload)).toContain("native video slot unavailable");
|
||||
return payload;
|
||||
});
|
||||
|
||||
const result = await runGeminiStreamResult({
|
||||
model: buildGeminiModel({ input: ["text", "image", "video"] as never }),
|
||||
context: { messages: [{ role: "user", content: "fallback", timestamp: 0 }] },
|
||||
options: withProviderContextHandoff({ onPayload }, handoff),
|
||||
});
|
||||
expect(result.errorMessage).toBeUndefined();
|
||||
|
||||
const body = parseRequestJsonBody(
|
||||
requireRequestInit(requireMockCall(guardedFetchMock, 0, "guarded fetch"), "guarded fetch"),
|
||||
) as { contents: GoogleTestContentTurn[] };
|
||||
expect(body.contents[0]?.parts).toEqual([
|
||||
{ text: "before" },
|
||||
{ inlineData: { mimeType: "image/png", data: imageData } },
|
||||
{ inlineData: { mimeType: "video/mp4", data: videoData } },
|
||||
{ inlineData: { mimeType: "video/quicktime", data: quicktimeData } },
|
||||
{ text: "after" },
|
||||
]);
|
||||
expect(handoff).toHaveBeenCalledOnce();
|
||||
expect(onPayload).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("omits cloned and hook-injected video slots", async () => {
|
||||
mockGoogleTextResponse();
|
||||
const trustedData = "trusted-video-base64";
|
||||
const injectedData = "injected-video-base64";
|
||||
const providerContext: ProviderContext = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "video", mimeType: "video/mp4", data: trustedData }],
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
await runGeminiStreamResult({
|
||||
model: buildGeminiModel({ input: ["text", "image", "video"] as never }),
|
||||
options: withProviderContextHandoff(
|
||||
{
|
||||
onPayload: (payload: unknown) => {
|
||||
const cloned = structuredClone(payload) as {
|
||||
contents: GoogleTestContentTurn[];
|
||||
};
|
||||
cloned.contents[0]?.parts.push({
|
||||
inlineData: { mimeType: "video/mp4", data: injectedData },
|
||||
});
|
||||
return cloned;
|
||||
},
|
||||
},
|
||||
async () => providerContext,
|
||||
),
|
||||
});
|
||||
const bodyText = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, 0, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body as string;
|
||||
expect(bodyText).not.toContain(trustedData);
|
||||
expect(bodyText).not.toContain(injectedData);
|
||||
expect(bodyText).toContain("native video slot unavailable");
|
||||
});
|
||||
|
||||
it("rejects unsupported Google video MIME after the payload hook", async () => {
|
||||
mockGoogleTextResponse();
|
||||
const videoData = "unsupported-mime-video";
|
||||
const onPayload = vi.fn((payload: unknown) => {
|
||||
expect(JSON.stringify(payload)).not.toContain(videoData);
|
||||
return payload;
|
||||
});
|
||||
await runGeminiStreamResult({
|
||||
model: buildGeminiModel({ input: ["text", "image", "video"] as never }),
|
||||
options: withProviderContextHandoff({ onPayload }, async () => ({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "video", mimeType: "video/x-msvideo", data: videoData }],
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
})),
|
||||
});
|
||||
const body = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, 0, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body as string;
|
||||
expect(body).not.toContain(videoData);
|
||||
expect(body).toContain("unsupported Google video MIME type");
|
||||
});
|
||||
|
||||
it("evicts trusted video until the exact serialized request is below 20MB", async () => {
|
||||
mockGoogleTextResponse();
|
||||
const providerContext: ProviderContext = {
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "video", mimeType: "video/mp4", data: "A".repeat(20_000_000) }],
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
await runGeminiStreamResult({
|
||||
model: buildGeminiModel({ input: ["text", "image", "video"] as never }),
|
||||
options: withProviderContextHandoff({}, async () => providerContext),
|
||||
});
|
||||
const body = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, 0, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body as string;
|
||||
expect(new TextEncoder().encode(body).byteLength).toBeLessThan(20_000_000);
|
||||
expect(body).toContain("native video slot unavailable");
|
||||
});
|
||||
|
||||
it("returns a useful provider error when the non-video request still exceeds 20MB", async () => {
|
||||
const result = await runGeminiStreamResult({
|
||||
context: {
|
||||
messages: [{ role: "user", content: "A".repeat(20_000_000), timestamp: 0 }],
|
||||
},
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
stopReason: "error",
|
||||
errorMessage: "Google request body must be smaller than 20000000 bytes",
|
||||
});
|
||||
expect(guardedFetchMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the guarded fetch transport and parses Gemini SSE output", async () => {
|
||||
guardedFetchMock.mockResolvedValueOnce(
|
||||
buildSseResponse([
|
||||
@@ -823,6 +984,15 @@ describe("google transport stream", () => {
|
||||
requireRequestInit(requireMockCall(guardedFetchMock, 1, "guarded fetch"), "guarded fetch"),
|
||||
{ "x-goog-api-key": "gemini-key-2" },
|
||||
);
|
||||
const firstBody = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, 0, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body;
|
||||
const secondBody = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, 1, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body;
|
||||
expect(secondBody).toBe(firstBody);
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -1249,6 +1419,52 @@ describe("google transport stream", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps oversized-video shedding in the Gemini 3 retry payload", async () => {
|
||||
vi.stubEnv("OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS", "10");
|
||||
guardedFetchMock
|
||||
.mockResolvedValueOnce(
|
||||
new Response(new ReadableStream<Uint8Array>(), {
|
||||
headers: { "content-type": "text/event-stream" },
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
buildSseResponse([
|
||||
{
|
||||
candidates: [{ content: { parts: [{ text: "recovered" }] }, finishReason: "STOP" }],
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const result = await runGeminiStreamResult({
|
||||
model: buildGeminiModel({
|
||||
id: "gemini-3.1-pro-preview",
|
||||
name: "Gemini 3.1 Pro Preview",
|
||||
input: ["text", "image", "video"] as never,
|
||||
}),
|
||||
options: withProviderContextHandoff({ reasoning: "high" }, async () => ({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "video", mimeType: "video/mp4", data: "A".repeat(20_000_000) }],
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
})),
|
||||
});
|
||||
|
||||
expect(result.errorMessage).toBeUndefined();
|
||||
expect(result.content).toEqual([{ type: "text", text: "recovered" }]);
|
||||
expect(guardedFetchMock).toHaveBeenCalledTimes(2);
|
||||
for (const index of [0, 1]) {
|
||||
const body = requireRequestInit(
|
||||
requireMockCall(guardedFetchMock, index, "guarded fetch"),
|
||||
"guarded fetch",
|
||||
).body as string;
|
||||
expect(new TextEncoder().encode(body).byteLength).toBeLessThan(20_000_000);
|
||||
expect(body).toContain("native video slot unavailable");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry a genuinely empty Gemini 3 response", async () => {
|
||||
vi.stubEnv("OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS", "10");
|
||||
guardedFetchMock.mockResolvedValueOnce(buildRawSseResponse(""));
|
||||
|
||||
@@ -3,10 +3,15 @@ import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import {
|
||||
calculateCost,
|
||||
getEnvApiKey,
|
||||
resolveProviderContext,
|
||||
type Context,
|
||||
type Model,
|
||||
type ProviderCallStreamOptions,
|
||||
type ProviderContext,
|
||||
type ProviderModel,
|
||||
type SimpleStreamOptions,
|
||||
type ThinkingLevel,
|
||||
type VideoContent,
|
||||
} from "openclaw/plugin-sdk/llm";
|
||||
import { parseStrictNonNegativeInteger } from "openclaw/plugin-sdk/number-runtime";
|
||||
import {
|
||||
@@ -41,7 +46,8 @@ import {
|
||||
} from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { parseGeminiAuth } from "./gemini-auth.js";
|
||||
import { stripGoogleProviderPrefix } from "./model-id.js";
|
||||
import { normalizeGoogleApiBaseUrl } from "./provider-policy.js";
|
||||
import { isGoogleNativeVideoModelId } from "./provider-models.js";
|
||||
import { isOfficialGoogleAiStudioBaseUrl, normalizeGoogleApiBaseUrl } from "./provider-policy.js";
|
||||
import {
|
||||
isGoogleGemini25ThinkingBudgetModel,
|
||||
isGoogleGemini3FlashModel,
|
||||
@@ -59,30 +65,31 @@ import {
|
||||
type CanonicalGoogleTransportApi = "google-generative-ai" | "google-vertex";
|
||||
type GoogleTransportApi = CanonicalGoogleTransportApi | "openclaw-google-generative-ai-transport";
|
||||
|
||||
type GoogleTransportModel = Model<GoogleTransportApi> & {
|
||||
type GoogleTransportModel = ProviderModel<GoogleTransportApi> & {
|
||||
headers?: Record<string, string>;
|
||||
provider: string;
|
||||
};
|
||||
|
||||
type GoogleTransportOptions = SimpleStreamOptions & {
|
||||
cachedContent?: string;
|
||||
toolChoice?:
|
||||
| "auto"
|
||||
| "none"
|
||||
| "any"
|
||||
| "required"
|
||||
| {
|
||||
type: "function";
|
||||
function: {
|
||||
name: string;
|
||||
type GoogleTransportOptions = SimpleStreamOptions &
|
||||
ProviderCallStreamOptions & {
|
||||
cachedContent?: string;
|
||||
toolChoice?:
|
||||
| "auto"
|
||||
| "none"
|
||||
| "any"
|
||||
| "required"
|
||||
| {
|
||||
type: "function";
|
||||
function: {
|
||||
name: string;
|
||||
};
|
||||
};
|
||||
};
|
||||
thinking?: {
|
||||
enabled: boolean;
|
||||
budgetTokens?: number;
|
||||
level?: GoogleThinkingLevel;
|
||||
thinking?: {
|
||||
enabled: boolean;
|
||||
budgetTokens?: number;
|
||||
level?: GoogleThinkingLevel;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
type GoogleGenerateContentRequest = {
|
||||
cachedContent?: string;
|
||||
@@ -93,6 +100,22 @@ type GoogleGenerateContentRequest = {
|
||||
toolConfig?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
const GOOGLE_NATIVE_VIDEO_MIME: ReadonlySet<string> = new Set([
|
||||
"video/mp4",
|
||||
"video/mpeg",
|
||||
"video/quicktime",
|
||||
"video/avi",
|
||||
"video/x-flv",
|
||||
"video/mpg",
|
||||
"video/webm",
|
||||
"video/wmv",
|
||||
"video/3gpp",
|
||||
]);
|
||||
const GOOGLE_VIDEO_SLOT_OMISSION = "(video omitted: native video slot unavailable)";
|
||||
const GOOGLE_VIDEO_MIME_OMISSION = "(video omitted: unsupported Google video MIME type)";
|
||||
const GOOGLE_REQUEST_BYTES_EXCLUSIVE = 20_000_000;
|
||||
type GoogleVideoSlots = Map<Record<string, unknown>, VideoContent>;
|
||||
|
||||
const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_DEFAULT_MS = 45_000;
|
||||
const GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_ENV = "OPENCLAW_GOOGLE_GEMINI_FIRST_RESPONSE_RETRY_MS";
|
||||
const GOOGLE_SSE_EVENT_BOUNDARY_RE = /(?:\r\n|\r(?!\n)|\n){2}/u;
|
||||
@@ -282,6 +305,23 @@ function normalizeGoogleTransportModelRoute(model: GoogleTransportModel): Google
|
||||
return api && api !== model.api ? Object.assign({}, model, { api }) : model;
|
||||
}
|
||||
|
||||
function canonicalGoogleModel(model: GoogleTransportModel): Model<GoogleTransportApi> {
|
||||
return {
|
||||
...model,
|
||||
input: model.input.filter((type) => type !== "video"),
|
||||
} as Model<GoogleTransportApi>;
|
||||
}
|
||||
|
||||
function supportsGoogleNativeVideo(model: GoogleTransportModel): boolean {
|
||||
return (
|
||||
model.provider === "google" &&
|
||||
normalizeGoogleTransportRouteApi(model.api) === "google-generative-ai" &&
|
||||
isOfficialGoogleAiStudioBaseUrl(model.baseUrl) &&
|
||||
isGoogleNativeVideoModelId(model.id) &&
|
||||
model.input.includes("video")
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeGoogleTransportMessageRoutes(messages: Context["messages"]): Context["messages"] {
|
||||
return messages.map((msg) => {
|
||||
if (msg.role !== "assistant") {
|
||||
@@ -529,19 +569,23 @@ function normalizeGoogleThinkingConfig(
|
||||
return Object.keys(thinkingConfig).length > 0 ? thinkingConfig : undefined;
|
||||
}
|
||||
|
||||
function convertGoogleMessages(model: GoogleTransportModel, context: Context) {
|
||||
function convertGoogleMessages(
|
||||
model: GoogleTransportModel,
|
||||
context: Context | ProviderContext,
|
||||
videoSlots?: GoogleVideoSlots,
|
||||
) {
|
||||
const contents: Array<Record<string, unknown>> = [];
|
||||
const replayToolCallThoughtSignatures = new Map<string, string>();
|
||||
const shouldReplayToolCallThoughtSignature = requiresToolCallThoughtSignature(model.id);
|
||||
const routeModel = normalizeGoogleTransportModelRoute(model);
|
||||
const transformedMessages = transformTransportMessages(
|
||||
normalizeGoogleTransportMessageRoutes(context.messages),
|
||||
routeModel,
|
||||
normalizeGoogleTransportMessageRoutes(context.messages as Context["messages"]),
|
||||
canonicalGoogleModel(routeModel),
|
||||
(id) => (requiresToolCallId(model.id) ? normalizeToolCallId(id) : id),
|
||||
{
|
||||
preserveCrossModelToolCallThoughtSignature: requiresToolCallThoughtSignature(model.id),
|
||||
},
|
||||
);
|
||||
) as ProviderContext["messages"];
|
||||
// Parallel calls need one immediate function-response turn. Gemini < 3 images cannot
|
||||
// live inside functionResponse, so hold them until the consecutive result run ends.
|
||||
const pendingToolResultImageTurns: Array<Record<string, unknown>> = [];
|
||||
@@ -565,16 +609,17 @@ function convertGoogleMessages(model: GoogleTransportModel, context: Context) {
|
||||
continue;
|
||||
}
|
||||
const parts = msg.content
|
||||
.map((item) =>
|
||||
item.type === "text"
|
||||
? { text: sanitizeTransportPayloadText(item.text) || " " }
|
||||
: {
|
||||
inlineData: {
|
||||
mimeType: item.mimeType,
|
||||
data: item.data,
|
||||
},
|
||||
},
|
||||
)
|
||||
.map((item) => {
|
||||
if (item.type === "text") {
|
||||
return { text: sanitizeTransportPayloadText(item.text) || " " };
|
||||
}
|
||||
if (item.type === "image") {
|
||||
return { inlineData: { mimeType: item.mimeType, data: item.data } };
|
||||
}
|
||||
const placeholder = { text: GOOGLE_VIDEO_SLOT_OMISSION };
|
||||
videoSlots?.set(placeholder, item);
|
||||
return placeholder;
|
||||
})
|
||||
.filter((item) => model.input.includes("image") || !("inlineData" in item));
|
||||
if (parts.length === 0) {
|
||||
parts.push({ text: " " });
|
||||
@@ -729,8 +774,9 @@ function convertGoogleTools(tools: NonNullable<Context["tools"]>) {
|
||||
|
||||
export function buildGoogleGenerativeAiParams(
|
||||
model: GoogleTransportModel,
|
||||
context: Context,
|
||||
context: Context | ProviderContext,
|
||||
options?: GoogleTransportOptions,
|
||||
videoSlots?: GoogleVideoSlots,
|
||||
): GoogleGenerateContentRequest {
|
||||
const generationConfig: Record<string, unknown> = {};
|
||||
if (typeof options?.temperature === "number") {
|
||||
@@ -748,7 +794,7 @@ export function buildGoogleGenerativeAiParams(
|
||||
}
|
||||
|
||||
const params: GoogleGenerateContentRequest = {
|
||||
contents: convertGoogleMessages(model, context),
|
||||
contents: convertGoogleMessages(model, context, videoSlots),
|
||||
};
|
||||
const cachedContent =
|
||||
typeof options?.cachedContent === "string" ? options.cachedContent.trim() : "";
|
||||
@@ -779,6 +825,66 @@ export function buildGoogleGenerativeAiParams(
|
||||
return params;
|
||||
}
|
||||
|
||||
function replaceGooglePartWithText(part: Record<string, unknown>, text: string): void {
|
||||
Object.keys(part).forEach((key) => Reflect.deleteProperty(part, key));
|
||||
part.text = text;
|
||||
}
|
||||
|
||||
function materializeGoogleVideoSlots(
|
||||
request: GoogleGenerateContentRequest,
|
||||
slots: GoogleVideoSlots,
|
||||
): Record<string, unknown>[] {
|
||||
const trusted: Record<string, unknown>[] = [];
|
||||
const visit = (value: unknown): void => {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach(visit);
|
||||
return;
|
||||
}
|
||||
if (!isRecord(value)) {
|
||||
return;
|
||||
}
|
||||
const video = slots.get(value);
|
||||
if (video) {
|
||||
if (!GOOGLE_NATIVE_VIDEO_MIME.has(video.mimeType)) {
|
||||
replaceGooglePartWithText(value, GOOGLE_VIDEO_MIME_OMISSION);
|
||||
return;
|
||||
}
|
||||
Object.keys(value).forEach((key) => Reflect.deleteProperty(value, key));
|
||||
value.inlineData = { mimeType: video.mimeType, data: video.data };
|
||||
trusted.push(value);
|
||||
return;
|
||||
}
|
||||
const inlineData = isRecord(value.inlineData) ? value.inlineData : undefined;
|
||||
if (normalizeLowercaseStringOrEmpty(inlineData?.mimeType).startsWith("video/")) {
|
||||
replaceGooglePartWithText(value, GOOGLE_VIDEO_SLOT_OMISSION);
|
||||
return;
|
||||
}
|
||||
Object.values(value).forEach(visit);
|
||||
};
|
||||
visit(request);
|
||||
return trusted;
|
||||
}
|
||||
|
||||
function serializeGoogleRequest(
|
||||
request: GoogleGenerateContentRequest,
|
||||
videoSlots: Record<string, unknown>[],
|
||||
): string {
|
||||
let body = JSON.stringify(request);
|
||||
for (const slot of videoSlots.toReversed()) {
|
||||
if (Buffer.byteLength(body, "utf8") < GOOGLE_REQUEST_BYTES_EXCLUSIVE) {
|
||||
break;
|
||||
}
|
||||
replaceGooglePartWithText(slot, GOOGLE_VIDEO_SLOT_OMISSION);
|
||||
body = JSON.stringify(request);
|
||||
}
|
||||
if (Buffer.byteLength(body, "utf8") >= GOOGLE_REQUEST_BYTES_EXCLUSIVE) {
|
||||
throw new Error(
|
||||
`Google request body must be smaller than ${GOOGLE_REQUEST_BYTES_EXCLUSIVE} bytes`,
|
||||
);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
function buildGoogleHeaders(
|
||||
model: GoogleTransportModel,
|
||||
apiKey: string | undefined,
|
||||
@@ -830,7 +936,7 @@ function collectGoogleTransportApiKeys(params: {
|
||||
}): string[] {
|
||||
if (
|
||||
params.kind !== "google-generative-ai" ||
|
||||
!isOfficialGoogleGenerativeAiBaseUrl(params.model.baseUrl) ||
|
||||
!isOfficialGoogleAiStudioBaseUrl(params.model.baseUrl) ||
|
||||
isGoogleOauthApiKey(params.primaryApiKey) ||
|
||||
hasGoogleAuthHeader(params.model.headers) ||
|
||||
hasGoogleAuthHeader(params.options?.headers)
|
||||
@@ -878,18 +984,6 @@ function buildGoogleTransportRequestUrl(
|
||||
: buildGoogleGenerativeAiRequestUrl(model);
|
||||
}
|
||||
|
||||
function isOfficialGoogleGenerativeAiBaseUrl(baseUrl: string | undefined): boolean {
|
||||
if (!baseUrl) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
const url = new URL(baseUrl);
|
||||
return url.protocol === "https:" && url.hostname === "generativelanguage.googleapis.com";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveGoogleGemini3FirstResponseRetryMs(env = process.env): number {
|
||||
const raw = env[GOOGLE_GEMINI3_FIRST_RESPONSE_RETRY_ENV];
|
||||
if (raw === undefined || raw.trim() === "") {
|
||||
@@ -905,7 +999,7 @@ function shouldRetryGoogleGemini3FirstResponse(params: {
|
||||
if (params.kind !== "google-generative-ai") {
|
||||
return false;
|
||||
}
|
||||
if (!isOfficialGoogleGenerativeAiBaseUrl(params.model.baseUrl)) {
|
||||
if (!isOfficialGoogleAiStudioBaseUrl(params.model.baseUrl)) {
|
||||
return false;
|
||||
}
|
||||
return isGoogleGemini3ProModel(params.model.id) || isGoogleGemini3FlashModel(params.model.id);
|
||||
@@ -1029,6 +1123,7 @@ async function openGoogleSseAttempt(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
request: GoogleGenerateContentRequest;
|
||||
videoSlots: Record<string, unknown>[];
|
||||
parentSignal?: AbortSignal;
|
||||
firstResponseTimeoutMs: number;
|
||||
errorPrefix: string;
|
||||
@@ -1042,7 +1137,7 @@ async function openGoogleSseAttempt(params: {
|
||||
const response = await params.guardedFetch(params.url, {
|
||||
method: "POST",
|
||||
headers: params.headers,
|
||||
body: JSON.stringify(params.request),
|
||||
body: serializeGoogleRequest(params.request, params.videoSlots),
|
||||
signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
@@ -1080,6 +1175,7 @@ async function openGoogleSseChunks(params: {
|
||||
url: string;
|
||||
headers: Record<string, string>;
|
||||
request: GoogleGenerateContentRequest;
|
||||
videoSlots: Record<string, unknown>[];
|
||||
}): Promise<Extract<GoogleSseAttempt, { type: "ready" }>> {
|
||||
const errorPrefix =
|
||||
params.kind === "google-vertex"
|
||||
@@ -1089,7 +1185,7 @@ async function openGoogleSseChunks(params: {
|
||||
const response = await params.guardedFetch(params.url, {
|
||||
method: "POST",
|
||||
headers: params.headers,
|
||||
body: JSON.stringify(params.request),
|
||||
body: serializeGoogleRequest(params.request, params.videoSlots),
|
||||
signal: params.options?.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
@@ -1102,18 +1198,11 @@ async function openGoogleSseChunks(params: {
|
||||
}
|
||||
|
||||
const retryMs = resolveGoogleGemini3FirstResponseRetryMs();
|
||||
const retryRequest =
|
||||
retryMs > 0
|
||||
? buildGoogleGemini3FirstResponseRetryParams({
|
||||
model: params.model,
|
||||
request: params.request,
|
||||
})
|
||||
: undefined;
|
||||
if (!retryRequest) {
|
||||
if (retryMs <= 0) {
|
||||
const response = await params.guardedFetch(params.url, {
|
||||
method: "POST",
|
||||
headers: params.headers,
|
||||
body: JSON.stringify(params.request),
|
||||
body: serializeGoogleRequest(params.request, params.videoSlots),
|
||||
signal: params.options?.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
@@ -1130,6 +1219,7 @@ async function openGoogleSseChunks(params: {
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
request: params.request,
|
||||
videoSlots: params.videoSlots,
|
||||
parentSignal: params.options?.signal,
|
||||
firstResponseTimeoutMs: retryMs,
|
||||
errorPrefix,
|
||||
@@ -1138,11 +1228,18 @@ async function openGoogleSseChunks(params: {
|
||||
return firstAttempt;
|
||||
}
|
||||
|
||||
// The first serialization owns video shedding. Clone only after it times out
|
||||
// so the retry inherits those exact omissions instead of restoring stale bytes.
|
||||
const retryRequest = buildGoogleGemini3FirstResponseRetryParams({
|
||||
model: params.model,
|
||||
request: params.request,
|
||||
})!;
|
||||
const retryAttempt = await openGoogleSseAttempt({
|
||||
guardedFetch: params.guardedFetch,
|
||||
url: params.url,
|
||||
headers: params.headers,
|
||||
request: retryRequest,
|
||||
videoSlots: params.videoSlots,
|
||||
parentSignal: params.options?.signal,
|
||||
firstResponseTimeoutMs: 0,
|
||||
errorPrefix,
|
||||
@@ -1275,7 +1372,7 @@ function updateUsage(
|
||||
chunk.usageMetadata.totalTokenCount ?? promptTokens + outputTokens + toolUsePromptTokens,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
calculateCost(model, output.usage);
|
||||
calculateCost(canonicalGoogleModel(model), output.usage);
|
||||
}
|
||||
|
||||
function pushTextBlockEnd(
|
||||
@@ -1309,6 +1406,7 @@ function pushTextBlockEnd(
|
||||
function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): StreamFn {
|
||||
return (rawModel, context, rawOptions) => {
|
||||
const model = rawModel as GoogleTransportModel;
|
||||
const canonicalModel = canonicalGoogleModel(model);
|
||||
const options = rawOptions as GoogleTransportOptions | undefined;
|
||||
const { eventStream, stream } = createWritableTransportEventStream();
|
||||
void (async () => {
|
||||
@@ -1324,12 +1422,17 @@ function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): Strea
|
||||
};
|
||||
try {
|
||||
const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? undefined;
|
||||
const guardedFetch = buildGuardedModelFetch(model);
|
||||
let params = buildGoogleGenerativeAiParams(model, context, options);
|
||||
const nextParams = await options?.onPayload?.(params, model);
|
||||
const guardedFetch = buildGuardedModelFetch(canonicalModel);
|
||||
const providerContext = supportsGoogleNativeVideo(model)
|
||||
? await resolveProviderContext(context, options)
|
||||
: context;
|
||||
const videoSlots: GoogleVideoSlots = new Map();
|
||||
let params = buildGoogleGenerativeAiParams(model, providerContext, options, videoSlots);
|
||||
const nextParams = await options?.onPayload?.(params, canonicalModel);
|
||||
if (nextParams !== undefined) {
|
||||
params = nextParams as GoogleGenerateContentRequest;
|
||||
}
|
||||
const trustedVideoSlots = materializeGoogleVideoSlots(params, videoSlots);
|
||||
const requestUrl = buildGoogleTransportRequestUrl(kind, model, options);
|
||||
const fetchImpl = (options as { fetch?: typeof fetch } | undefined)?.fetch;
|
||||
const openSse = async (apiKeyForRequest: string | undefined) => {
|
||||
@@ -1348,6 +1451,7 @@ function createGoogleTransportStreamFn(kind: CanonicalGoogleTransportApi): Strea
|
||||
url: requestUrl,
|
||||
headers: requestHeaders,
|
||||
request: params,
|
||||
videoSlots: trustedVideoSlots,
|
||||
});
|
||||
};
|
||||
const apiKeys = collectGoogleTransportApiKeys({
|
||||
|
||||
@@ -182,8 +182,6 @@ export interface AiTransportHost {
|
||||
transformTransportMessages: AiTransformTransportMessages;
|
||||
/** Registers a custom transport API with the host's stream error bridge. */
|
||||
registerCustomApi(registry: ApiRegistry, api: Api, streamFn: StreamFn): boolean;
|
||||
/** Prepares the provider-owned Google simple-completion alias when needed. */
|
||||
prepareGoogleSimpleCompletionModel(registry: ApiRegistry, model: Model): Model;
|
||||
/**
|
||||
* Emits one transport diagnostic; build runs only when the host logs it and
|
||||
* may return null to suppress the entry (e.g. de-duplication).
|
||||
@@ -263,7 +261,6 @@ const inertAiTransportHost: ActiveAiTransportHost = {
|
||||
transformTransportMessages: (messages, model, normalizeToolCallId) =>
|
||||
transformMessages(messages, model, normalizeToolCallId),
|
||||
registerCustomApi: queueCustomApiRegistration,
|
||||
prepareGoogleSimpleCompletionModel: (_registry, model) => model,
|
||||
logDebug: () => {},
|
||||
logInfo: () => {},
|
||||
logWarn: () => {},
|
||||
|
||||
@@ -18,14 +18,16 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
const compatibility = {
|
||||
"@openclaw/ai/provider-types": {
|
||||
values: [],
|
||||
values: ["PROVIDER_CONTEXT_HANDOFF", "resolveProviderContext"],
|
||||
types: [
|
||||
"MediaContent",
|
||||
"ModelInputContent",
|
||||
"ProviderContext",
|
||||
"ProviderContextHandoff",
|
||||
"ProviderMessage",
|
||||
"ProviderModel",
|
||||
"ProviderStreamFunction",
|
||||
"ProviderStreamOptions",
|
||||
"ProviderUserMessage",
|
||||
"VideoContent",
|
||||
],
|
||||
|
||||
@@ -8,16 +8,20 @@ import type {
|
||||
ToolResultMessage,
|
||||
UserMessage,
|
||||
} from "@openclaw/llm-core";
|
||||
import { describe, expectTypeOf, it } from "vitest";
|
||||
import type {
|
||||
MediaContent,
|
||||
ModelInputContent,
|
||||
ProviderContext,
|
||||
ProviderMessage,
|
||||
ProviderModel,
|
||||
ProviderStreamFunction,
|
||||
ProviderUserMessage,
|
||||
VideoContent,
|
||||
import { describe, expect, expectTypeOf, it, vi } from "vitest";
|
||||
import {
|
||||
PROVIDER_CONTEXT_HANDOFF,
|
||||
resolveProviderContext,
|
||||
type ProviderContextHandoff,
|
||||
type ProviderStreamOptions,
|
||||
type MediaContent,
|
||||
type ModelInputContent,
|
||||
type ProviderContext,
|
||||
type ProviderMessage,
|
||||
type ProviderModel,
|
||||
type ProviderStreamFunction,
|
||||
type ProviderUserMessage,
|
||||
type VideoContent,
|
||||
} from "./provider-types.js";
|
||||
|
||||
describe("provider call types", () => {
|
||||
@@ -33,6 +37,13 @@ describe("provider call types", () => {
|
||||
expectTypeOf<ProviderModel["input"][number]>().toEqualTypeOf<"text" | "image" | "video">();
|
||||
expectTypeOf<Parameters<ProviderStreamFunction>[0]>().toEqualTypeOf<ProviderModel>();
|
||||
expectTypeOf<Parameters<ProviderStreamFunction>[1]>().toEqualTypeOf<ProviderContext>();
|
||||
expectTypeOf<Parameters<ProviderStreamFunction>[2]>().toEqualTypeOf<
|
||||
ProviderStreamOptions | undefined
|
||||
>();
|
||||
expectTypeOf<ProviderStreamOptions[typeof PROVIDER_CONTEXT_HANDOFF]>().toEqualTypeOf<
|
||||
ProviderContextHandoff | undefined
|
||||
>();
|
||||
expectTypeOf<Parameters<ProviderContextHandoff>>().toEqualTypeOf<[]>();
|
||||
|
||||
expectTypeOf<UserMessage["content"]>().toEqualTypeOf<string | (TextContent | ImageContent)[]>();
|
||||
expectTypeOf<Context["messages"][number]>().toEqualTypeOf<Message>();
|
||||
@@ -42,4 +53,17 @@ describe("provider call types", () => {
|
||||
>();
|
||||
expectTypeOf<ImagesModel["input"][number]>().toEqualTypeOf<"text" | "image">();
|
||||
});
|
||||
|
||||
it("projects canonical context unless a provider handoff is present", async () => {
|
||||
const context = { systemPrompt: "system", messages: [], tools: [] };
|
||||
await expect(resolveProviderContext(context)).resolves.toBe(context);
|
||||
|
||||
const resolved = { ...context, messages: [] };
|
||||
const handoff = vi.fn(async () => resolved);
|
||||
await expect(
|
||||
resolveProviderContext(context, { [PROVIDER_CONTEXT_HANDOFF]: handoff }),
|
||||
).resolves.toBe(resolved);
|
||||
expect(handoff).toHaveBeenCalledOnce();
|
||||
expect(handoff).toHaveBeenCalledWith();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import type * as Llm from "@openclaw/llm-core";
|
||||
export type * from "./types.js";
|
||||
|
||||
export const PROVIDER_CONTEXT_HANDOFF: unique symbol = Symbol("providerContextHandoff");
|
||||
|
||||
export type VideoContent = Omit<Llm.ImageContent, "type"> & { type: "video" };
|
||||
export type MediaContent = Llm.ImageContent | VideoContent;
|
||||
export type ModelInputContent = Llm.TextContent | MediaContent;
|
||||
@@ -12,14 +14,26 @@ export type ProviderContext = Omit<Llm.Context, "messages"> & { messages: Provid
|
||||
export type ProviderModel<TApi extends Llm.Api = Llm.Api> = Omit<Llm.Model<TApi>, "input"> & {
|
||||
input: ModelInputContent["type"][];
|
||||
};
|
||||
export type ProviderContextHandoff = () => Promise<ProviderContext>;
|
||||
export type ProviderStreamOptions = Llm.StreamOptions & {
|
||||
[PROVIDER_CONTEXT_HANDOFF]?: ProviderContextHandoff;
|
||||
};
|
||||
export type ProviderStreamFunction<
|
||||
TApi extends Llm.Api = Llm.Api,
|
||||
TOptions extends Llm.StreamOptions = Llm.StreamOptions,
|
||||
TOptions extends Llm.StreamOptions = ProviderStreamOptions,
|
||||
> = (
|
||||
model: ProviderModel<TApi>,
|
||||
context: ProviderContext,
|
||||
options?: TOptions,
|
||||
) => Llm.AssistantMessageEventStreamContract;
|
||||
|
||||
/** Resolves provider-only context without widening the canonical call contract. */
|
||||
export async function resolveProviderContext(
|
||||
context: Llm.Context | ProviderContext,
|
||||
options?: ProviderStreamOptions,
|
||||
): Promise<ProviderContext> {
|
||||
return options?.[PROVIDER_CONTEXT_HANDOFF]?.() ?? (context as ProviderContext);
|
||||
}
|
||||
export type {
|
||||
ProviderContext as Context,
|
||||
ProviderMessage as Message,
|
||||
|
||||
@@ -15,7 +15,6 @@ const createOpenClawTransportStreamFnForModel = vi.fn();
|
||||
const createTransportAwareStreamFnForModel = vi.fn();
|
||||
const prepareTransportAwareSimpleModel = vi.fn();
|
||||
const resolveTransportAwareSimpleApi = vi.fn();
|
||||
const prepareGoogleSimpleCompletionModel = vi.fn((_registry: unknown, model: unknown) => model);
|
||||
const inheritManagedTransport = vi.fn((_source: Model, target: Model) => target);
|
||||
const pluginStreamFn = vi.fn(() => createAssistantMessageEventStream());
|
||||
const TEST_SECRET = "ollama-provider-secret";
|
||||
@@ -90,7 +89,6 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
createTransportAwareStreamFnForModel.mockReset();
|
||||
prepareTransportAwareSimpleModel.mockReset();
|
||||
resolveTransportAwareSimpleApi.mockReset();
|
||||
prepareGoogleSimpleCompletionModel.mockReset();
|
||||
inheritManagedTransport.mockClear();
|
||||
createAnthropicVertexStreamFnForModel.mockReturnValue("vertex-stream");
|
||||
resolveProviderStreamFn.mockReturnValue(pluginStreamFn);
|
||||
@@ -100,7 +98,6 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
createTransportAwareStreamFnForModel.mockReturnValue(undefined);
|
||||
prepareTransportAwareSimpleModel.mockImplementation((model) => model);
|
||||
resolveTransportAwareSimpleApi.mockReturnValue(undefined);
|
||||
prepareGoogleSimpleCompletionModel.mockImplementation((_registry, model) => model);
|
||||
configureAiTransportHost({
|
||||
plugin: {
|
||||
resolveProviderStream: resolveProviderStreamFn,
|
||||
@@ -110,10 +107,6 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
},
|
||||
registerCustomApi: ensureCustomApiRegistered,
|
||||
inheritManagedTransport,
|
||||
prepareGoogleSimpleCompletionModel: prepareGoogleSimpleCompletionModel as (
|
||||
registry: ApiRegistry,
|
||||
model: Model,
|
||||
) => Model,
|
||||
resolveSecretSentinel: (value) => value.replaceAll(TEST_SECRET_SENTINEL, TEST_SECRET),
|
||||
});
|
||||
});
|
||||
@@ -438,7 +431,7 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps registered Google models on the sanitizer path when the provider owns a stream", () => {
|
||||
it("routes registered Google simple completions through the provider plugin stream", () => {
|
||||
const model: Model<"google-generative-ai"> = {
|
||||
id: "gemma-4-26b-a4b-it",
|
||||
name: "Gemma 4 26B",
|
||||
@@ -461,24 +454,38 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
},
|
||||
SIMPLE_COMPLETION_SOURCE_ID,
|
||||
);
|
||||
prepareGoogleSimpleCompletionModel.mockImplementationOnce((_registry: unknown, m: unknown) => ({
|
||||
...(m as Model<"google-generative-ai">),
|
||||
api: "openclaw-google-generative-ai-simple",
|
||||
}));
|
||||
|
||||
const result = prepareModelForSimpleCompletion({ model });
|
||||
|
||||
expect(resolveProviderStreamFn).not.toHaveBeenCalled();
|
||||
expect(prepareTransportAwareSimpleModel).toHaveBeenCalledWith(model, { cfg: undefined });
|
||||
expect(prepareGoogleSimpleCompletionModel).toHaveBeenCalledWith(apiRegistry, model);
|
||||
expect(resolveProviderStreamFn).toHaveBeenCalledOnce();
|
||||
expect(prepareTransportAwareSimpleModel).not.toHaveBeenCalled();
|
||||
expect(buildTransportAwareSimpleStreamFn).not.toHaveBeenCalled();
|
||||
expect(result).toEqual({
|
||||
...model,
|
||||
api: "openclaw-google-generative-ai-simple",
|
||||
});
|
||||
expect(result.api).toMatch(/^openclaw-provider-stream:/);
|
||||
const registeredStream = ensureCustomApiRegistered.mock.calls.find(
|
||||
(call) => call[1] === result.api,
|
||||
)?.[2] as StreamFn | undefined;
|
||||
void registeredStream?.(
|
||||
result,
|
||||
{ messages: [] },
|
||||
{
|
||||
reasoning: "high",
|
||||
apiKey: "google-key",
|
||||
headers: { "x-test": "value" },
|
||||
signal: new AbortController().signal,
|
||||
},
|
||||
);
|
||||
expect(pluginStreamFn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: model.id }),
|
||||
{ messages: [] },
|
||||
expect.objectContaining({
|
||||
reasoning: "high",
|
||||
apiKey: "google-key",
|
||||
headers: { "x-test": "value" },
|
||||
signal: expect.any(AbortSignal),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps Google transport-aware models on the transport alias", () => {
|
||||
it("prefers the Google provider plugin over the duplicate transport-aware path", () => {
|
||||
const model: Model<"google-generative-ai"> = {
|
||||
id: "gemini-flash-latest",
|
||||
name: "Gemini Flash Latest",
|
||||
@@ -493,24 +500,17 @@ describe("prepareModelForSimpleCompletion", () => {
|
||||
headers: {},
|
||||
};
|
||||
|
||||
const transportModel = {
|
||||
...model,
|
||||
api: "openclaw-google-generative-ai-transport",
|
||||
};
|
||||
buildTransportAwareSimpleStreamFn.mockReturnValueOnce("google-transport-stream");
|
||||
prepareTransportAwareSimpleModel.mockReturnValueOnce(transportModel);
|
||||
|
||||
const result = prepareModelForSimpleCompletion({ model });
|
||||
|
||||
expect(resolveProviderStreamFn).not.toHaveBeenCalled();
|
||||
expect(buildTransportAwareSimpleStreamFn).toHaveBeenCalledWith(model, { cfg: undefined });
|
||||
expect(resolveProviderStreamFn).toHaveBeenCalledOnce();
|
||||
expect(buildTransportAwareSimpleStreamFn).not.toHaveBeenCalled();
|
||||
expect(prepareTransportAwareSimpleModel).not.toHaveBeenCalled();
|
||||
expect(result.api).toBe("google-generative-ai");
|
||||
expect(ensureCustomApiRegistered).toHaveBeenCalledWith(
|
||||
apiRegistry,
|
||||
"openclaw-google-generative-ai-transport",
|
||||
"google-transport-stream",
|
||||
"google-generative-ai",
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(prepareGoogleSimpleCompletionModel).not.toHaveBeenCalled();
|
||||
expect(result).toBe(transportModel);
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -179,11 +179,6 @@ function prepareProviderStreamModel<TApi extends Api>(params: {
|
||||
cfg?: unknown;
|
||||
apiRegistry: ApiRegistry;
|
||||
}): Model | undefined {
|
||||
// Google simple completions have managed transport and sanitizer paths below.
|
||||
// A plugin-native stream here would bypass both and emit unsupported payloads.
|
||||
if (params.model.api === "google-generative-ai") {
|
||||
return undefined;
|
||||
}
|
||||
const pluginModel = resolveModelHeaderSentinels(params.model);
|
||||
const providerStreamFn = getAiTransportHost().plugin.resolveProviderStream({
|
||||
provider: params.model.provider,
|
||||
@@ -244,15 +239,6 @@ export function prepareModelForSimpleCompletion<TApi extends Api>(params: {
|
||||
}
|
||||
}
|
||||
|
||||
if (model.api === "google-generative-ai") {
|
||||
return applyProviderSimpleCompletionWrapper(
|
||||
apiRegistry,
|
||||
getAiTransportHost().prepareGoogleSimpleCompletionModel(apiRegistry, model),
|
||||
cfg,
|
||||
model.api,
|
||||
);
|
||||
}
|
||||
|
||||
if (model.provider === "anthropic-vertex") {
|
||||
const api = resolveAnthropicVertexSimpleApi(model.baseUrl);
|
||||
const host = getAiTransportHost();
|
||||
|
||||
@@ -15,7 +15,6 @@ import {
|
||||
import { createAnthropicVertexStreamFnForModel } from "./anthropic-vertex-stream.js";
|
||||
import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./copilot-dynamic-headers.js";
|
||||
import { ensureCustomApiRegistered } from "./custom-api-registry.js";
|
||||
import { prepareGoogleSimpleCompletionModel } from "./google-simple-completion-stream.js";
|
||||
import {
|
||||
resolveProviderRequestCapabilities,
|
||||
resolveProviderEndpoint,
|
||||
@@ -101,7 +100,6 @@ export function configureAiTransportRuntimeHost(): void {
|
||||
),
|
||||
transformTransportMessages,
|
||||
registerCustomApi: ensureCustomApiRegistered,
|
||||
prepareGoogleSimpleCompletionModel,
|
||||
});
|
||||
configured = true;
|
||||
}
|
||||
|
||||
@@ -41,14 +41,6 @@ vi.mock("../subagents/registry/subagent-registry.js", () => ({
|
||||
settleRequesterAfterSessionSpawns: mocks.settleRequesterAfterSessionSpawns,
|
||||
}));
|
||||
|
||||
vi.mock("./run/plugin-harness-prompt-images.js", () => ({
|
||||
preparePluginHarnessPromptImages: vi.fn(async () => ({
|
||||
images: undefined,
|
||||
imageOrder: undefined,
|
||||
media: undefined,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("./run/skill-workshop-attempt-params.js", () => ({
|
||||
resolveSkillWorkshopAttemptParams: vi.fn(() => ({})),
|
||||
}));
|
||||
|
||||
@@ -320,7 +320,13 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
prompt: string;
|
||||
sandbox?: SandboxContext | null;
|
||||
skipPromptSubmission: boolean;
|
||||
}): Promise<PromptImageResult> {
|
||||
pluginHarness?: boolean;
|
||||
}): Promise<
|
||||
PromptImageResult & {
|
||||
imageOrder?: PromptExecutionAttempt["imageOrder"];
|
||||
media?: PromptExecutionAttempt["media"];
|
||||
}
|
||||
> {
|
||||
if (input.skipPromptSubmission) {
|
||||
return emptyPromptImages();
|
||||
}
|
||||
@@ -331,7 +337,7 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
(await attempt.userTurnTranscriptRecorder?.resolveMessage());
|
||||
const persistedMedia = persistedMessage ? (readPersistedMediaFacts(persistedMessage) ?? []) : [];
|
||||
|
||||
return await detectAndLoadPromptImages({
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: input.prompt,
|
||||
workspaceDir: input.effectiveWorkspace,
|
||||
model: attempt.model,
|
||||
@@ -349,4 +355,17 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge }
|
||||
: undefined,
|
||||
});
|
||||
if (!input.pluginHarness) {
|
||||
return result;
|
||||
}
|
||||
if (result.failedMediaCount) {
|
||||
throw new Error(
|
||||
`failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
imageOrder: result.images.length ? result.images.map(() => "inline" as const) : undefined,
|
||||
media: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
*/
|
||||
import type { SessionTranscriptRuntimeTarget } from "../../../config/sessions/session-accessor.js";
|
||||
import { OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST } from "../../../context-engine/host-compat.js";
|
||||
import {
|
||||
attachRuntimePromptMediaFacts,
|
||||
readPersistedMediaFacts,
|
||||
} from "../../../media/media-facts.js";
|
||||
import { getGlobalHookRunner } from "../../../plugins/hook-runner-global.js";
|
||||
import type { PluginMetadataSnapshot } from "../../../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { createPreparedEmbeddedAgentSettingsManager } from "../../agent-project-settings.js";
|
||||
@@ -446,12 +450,20 @@ export async function prepareEmbeddedAttemptSessionManager(input: {
|
||||
latestPersistedUserMessage = message;
|
||||
latestRuntimeUserMessage = runtimeMessage;
|
||||
if (runtimeMessage) {
|
||||
const media = readPersistedMediaFacts(message);
|
||||
if (media?.length) {
|
||||
attachRuntimePromptMediaFacts(runtimeMessage, media);
|
||||
}
|
||||
userTranscriptContextRegistry.record(runtimeMessage, message);
|
||||
}
|
||||
attempt.onUserMessagePersisted?.(message);
|
||||
},
|
||||
onUserMessagePersistenceSuppressed: (_message, runtimeMessage) => {
|
||||
onUserMessagePersistenceSuppressed: (message, runtimeMessage) => {
|
||||
latestRuntimeUserMessage = runtimeMessage;
|
||||
const media = runtimeMessage ? readPersistedMediaFacts(message) : undefined;
|
||||
if (runtimeMessage && media?.length) {
|
||||
attachRuntimePromptMediaFacts(runtimeMessage, media);
|
||||
}
|
||||
},
|
||||
onUserMessageBlocked: () => {
|
||||
attempt.userTurnTranscriptRecorder?.markBlocked();
|
||||
|
||||
@@ -229,6 +229,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
providerThinkingLevel: input.transport.providerThinkingLevel,
|
||||
sessionAgentId: input.sessionManager.sessionAgentId,
|
||||
workspaceDir: input.effectiveWorkspace,
|
||||
workspaceOnly: input.effectiveFsWorkspaceOnly,
|
||||
agentDir: input.agentDir,
|
||||
abortSignal: input.transport.abortSignal,
|
||||
getProviderRuntimeHandle: input.transport.getProviderRuntimeHandle,
|
||||
|
||||
@@ -68,6 +68,8 @@ import {
|
||||
waitForCompactionRetryWithAggregateTimeout,
|
||||
} from "./compaction-retry-aggregate-timeout.js";
|
||||
import { selectCompactionTimeoutSnapshot } from "./compaction-timeout.js";
|
||||
import { materializeProviderContext } from "./images.js";
|
||||
import { wrapStreamFnWithMessageTransform } from "./message-transform-stream-wrapper.js";
|
||||
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
/**
|
||||
@@ -441,6 +443,7 @@ export async function prepareEmbeddedAttemptTransport(input: {
|
||||
providerThinkingLevel: ProviderThinkLevel | undefined;
|
||||
sessionAgentId: string;
|
||||
workspaceDir: string;
|
||||
workspaceOnly: boolean;
|
||||
agentDir: string;
|
||||
abortSignal: AbortSignal;
|
||||
getProviderRuntimeHandle: () => ProviderRuntimePluginHandle;
|
||||
@@ -496,6 +499,23 @@ export async function prepareEmbeddedAttemptTransport(input: {
|
||||
agentDir: input.agentDir,
|
||||
workspaceDir: input.workspaceDir,
|
||||
});
|
||||
const directProviderStreamFn = providerStreamFn
|
||||
? wrapStreamFnWithMessageTransform(
|
||||
providerStreamFn,
|
||||
(messages) => messages,
|
||||
({ context, ...provider }) =>
|
||||
materializeProviderContext({
|
||||
...provider,
|
||||
context,
|
||||
workspaceDir: input.workspaceDir,
|
||||
workspaceOnly: input.workspaceOnly,
|
||||
sandbox:
|
||||
input.sandbox?.enabled && input.sandbox.fsBridge
|
||||
? { root: input.sandbox.workspaceDir, bridge: input.sandbox.fsBridge }
|
||||
: undefined,
|
||||
}),
|
||||
)
|
||||
: undefined;
|
||||
const transportApiKey = await resolveEmbeddedAgentApiKey({
|
||||
provider: attempt.model.provider,
|
||||
resolvedApiKey: attempt.resolvedApiKey,
|
||||
@@ -503,13 +523,13 @@ export async function prepareEmbeddedAttemptTransport(input: {
|
||||
});
|
||||
const streamStrategy = describeEmbeddedAgentStreamStrategy({
|
||||
currentStreamFn: defaultSessionStreamFn,
|
||||
providerStreamFn,
|
||||
providerStreamFn: directProviderStreamFn,
|
||||
model: attempt.model,
|
||||
resolvedApiKey: transportApiKey,
|
||||
});
|
||||
session.agent.streamFn = resolveEmbeddedAgentStreamFn({
|
||||
currentStreamFn: defaultSessionStreamFn,
|
||||
providerStreamFn,
|
||||
providerStreamFn: directProviderStreamFn,
|
||||
sessionId: attempt.sessionId,
|
||||
promptCacheKey: attempt.promptCacheKey,
|
||||
signal: input.abortSignal,
|
||||
|
||||
@@ -4,19 +4,18 @@ import {
|
||||
normalizeMediaFacts,
|
||||
type MediaFact,
|
||||
} from "../../../media/media-facts.js";
|
||||
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
|
||||
import { resolveUserPath } from "../../../utils.js";
|
||||
|
||||
const URL_SCHEME_PATTERN = /^[a-z][a-z0-9+.-]*:/i;
|
||||
const WINDOWS_DRIVE_PATH_PATTERN = /^[A-Za-z]:[\\/]/;
|
||||
|
||||
type DetectedImageRef = {
|
||||
export type MediaFileRef = {
|
||||
raw: string;
|
||||
type: "path" | "media-uri";
|
||||
resolved: string;
|
||||
};
|
||||
|
||||
export type MediaImageRef = DetectedImageRef & {
|
||||
export type MediaImageRef = MediaFileRef & {
|
||||
aliases: string[];
|
||||
detect?: boolean;
|
||||
factIndex: number;
|
||||
@@ -35,6 +34,35 @@ export function isOpenClawCliImageCachePath(filePath: string): boolean {
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveMediaFactLocalRef(fact: MediaFact): MediaFileRef | undefined {
|
||||
const mediaUri = [fact.url, fact.path].find((value) => value?.startsWith("media://inbound/"));
|
||||
const identity = mediaUri ?? fact.path ?? fact.url;
|
||||
if (!identity) {
|
||||
return undefined;
|
||||
}
|
||||
let resolved = mediaUri;
|
||||
if (!resolved && /^file:/i.test(identity)) {
|
||||
try {
|
||||
resolved = safeFileURLToPath(identity);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
} else if (
|
||||
!resolved &&
|
||||
(!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity))
|
||||
) {
|
||||
resolved = identity;
|
||||
}
|
||||
if (!resolved) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
raw: identity,
|
||||
type: mediaUri ? "media-uri" : "path",
|
||||
resolved: resolved.startsWith("~") ? resolveUserPath(resolved) : resolved,
|
||||
};
|
||||
}
|
||||
|
||||
function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef | undefined {
|
||||
if (!isImageMediaFact(fact)) {
|
||||
return undefined;
|
||||
@@ -55,25 +83,9 @@ function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
let resolved = mediaUri;
|
||||
if (!resolved && identity && /^file:/i.test(identity)) {
|
||||
try {
|
||||
resolved = safeFileURLToPath(identity);
|
||||
} catch {
|
||||
resolved = undefined;
|
||||
}
|
||||
} else if (
|
||||
!resolved &&
|
||||
identity &&
|
||||
(!URL_SCHEME_PATTERN.test(identity) || WINDOWS_DRIVE_PATH_PATTERN.test(identity))
|
||||
) {
|
||||
resolved = identity;
|
||||
}
|
||||
if (resolved?.startsWith("~")) {
|
||||
resolved = resolveUserPath(resolved);
|
||||
}
|
||||
const localRef = resolveMediaFactLocalRef(fact);
|
||||
const hydrate = fact.hydrationSuppressed !== true;
|
||||
if (!resolved || isOpenClawCliImageCachePath(resolved)) {
|
||||
if (!localRef || isOpenClawCliImageCachePath(localRef.resolved)) {
|
||||
return {
|
||||
aliases: [fact.path, fact.url].filter((value): value is string => Boolean(value)),
|
||||
detect: false,
|
||||
@@ -86,11 +98,11 @@ function mediaFactToImageRef(fact: MediaFact, factIndex: number): MediaImageRef
|
||||
};
|
||||
}
|
||||
return {
|
||||
aliases: [fact.path, fact.url, resolved].filter((value): value is string => Boolean(value)),
|
||||
...localRef,
|
||||
aliases: [fact.path, fact.url, localRef.resolved].filter((value): value is string =>
|
||||
Boolean(value),
|
||||
),
|
||||
factIndex,
|
||||
raw: mediaUri ?? fact.path ?? fact.url ?? resolved,
|
||||
type: mediaUri ? "media-uri" : "path",
|
||||
resolved,
|
||||
hydrate,
|
||||
...(fact.workspaceDir ? { workspaceDir: fact.workspaceDir } : {}),
|
||||
};
|
||||
@@ -104,57 +116,9 @@ export function collectMediaImageRefs(
|
||||
);
|
||||
}
|
||||
|
||||
export function collectIdentitylessMediaImageFactIndexes(media?: readonly MediaFact[]): number[] {
|
||||
return normalizeMediaFacts(media).flatMap((fact, factIndex) =>
|
||||
isImageMediaFact(fact) &&
|
||||
fact.hydrationSuppressed !== true &&
|
||||
fact.path === undefined &&
|
||||
fact.url === undefined
|
||||
? [factIndex]
|
||||
: [],
|
||||
);
|
||||
}
|
||||
|
||||
// Guards for transports that cannot carry attachments (paired-node CLI): only
|
||||
// facts that will actually hydrate an image count; described/remote-only facts
|
||||
// whose hydration is suppressed must not block text-only prompts.
|
||||
export function hasHydratableMediaImages(media?: readonly MediaFact[]): boolean {
|
||||
return collectMediaImageRefs(media).some((ref) => ref?.hydrate === true);
|
||||
}
|
||||
|
||||
export function selectMediaImageRefs(params: {
|
||||
refs: Array<MediaImageRef | undefined>;
|
||||
existingImageCount: number;
|
||||
imageOrder?: readonly PromptImageOrderEntry[];
|
||||
}): Array<MediaImageRef | undefined> {
|
||||
const { refs } = params;
|
||||
if (!params.imageOrder?.length) {
|
||||
// Legacy turns (no layout metadata): identity-less facts are the inline
|
||||
// images' own slots — pair them positionally so they cannot count as failed
|
||||
// offloads; identity-bearing refs remain genuine offloaded attachments.
|
||||
let inlinePairs = params.existingImageCount;
|
||||
return refs.filter((ref) => {
|
||||
if (ref === undefined && inlinePairs > 0) {
|
||||
inlinePairs -= 1;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}
|
||||
if (refs.length !== params.imageOrder.length) {
|
||||
// Partial fact arrays cannot prove positional ownership. Keep every ref as
|
||||
// an offload so no attachment is silently consumed by an inline slot.
|
||||
return refs;
|
||||
}
|
||||
let remainingExisting = params.existingImageCount;
|
||||
return params.imageOrder.flatMap((entry, index) => {
|
||||
if (entry === "offloaded") {
|
||||
return [refs[index]];
|
||||
}
|
||||
if (remainingExisting > 0) {
|
||||
remainingExisting -= 1;
|
||||
return [];
|
||||
}
|
||||
return [undefined];
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
import path from "node:path";
|
||||
import { MAX_VIDEO_BYTES } from "@openclaw/media-core/constants";
|
||||
import { normalizeMimeType } from "@openclaw/media-core/mime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import type {
|
||||
ModelInputContent,
|
||||
ProviderContext,
|
||||
} from "../../../../packages/ai/src/provider-types.js";
|
||||
import { formatErrorMessage } from "../../../infra/errors.js";
|
||||
import { assertNoWindowsNetworkPath, safeFileURLToPath } from "../../../infra/local-file-access.js";
|
||||
import type { ImageContent } from "../../../llm/types.js";
|
||||
import type { Context, ImageContent, TextContent } from "../../../llm/types.js";
|
||||
import {
|
||||
attachRuntimePromptMediaFacts,
|
||||
isImageMediaFact,
|
||||
isVideoMediaFact,
|
||||
normalizeMediaFacts,
|
||||
readRuntimePromptImageOrder,
|
||||
readRuntimePromptMediaFacts,
|
||||
@@ -15,7 +22,7 @@ import {
|
||||
import { resolveMediaReferenceLocalPath } from "../../../media/media-reference.js";
|
||||
import type { PromptImageOrderEntry } from "../../../media/prompt-image-order.js";
|
||||
import { finalizeRuntimePromptImages } from "../../../media/runtime-prompt-image-provenance.js";
|
||||
import { loadWebMedia } from "../../../media/web-media.js";
|
||||
import { loadWebMedia, type WebMediaResult } from "../../../media/web-media.js";
|
||||
import { resolveUserPath } from "../../../utils.js";
|
||||
import type { ImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
@@ -27,19 +34,17 @@ import type { SandboxFsBridge } from "../../sandbox/fs-bridge.js";
|
||||
import { sanitizeImageBlocks } from "../../tool-images.js";
|
||||
import { log } from "../logger.js";
|
||||
import {
|
||||
collectIdentitylessMediaImageFactIndexes,
|
||||
collectMediaImageRefs,
|
||||
isOpenClawCliImageCachePath,
|
||||
selectMediaImageRefs,
|
||||
resolveMediaFactLocalRef,
|
||||
type MediaFileRef,
|
||||
type MediaImageRef,
|
||||
} from "./images.media-refs.js";
|
||||
import {
|
||||
type ImageFactIndex,
|
||||
type MediaImageLayout,
|
||||
countMissingLayoutInlineSlots,
|
||||
readPersistedImageBlockFactIndexes,
|
||||
readPersistedMediaImageLayout,
|
||||
resolveLayoutInlineFactIndexes,
|
||||
} from "./prompt-image-metadata.js";
|
||||
|
||||
export { hasHydratableMediaImages } from "./images.media-refs.js";
|
||||
@@ -72,12 +77,6 @@ const PATH_PATTERN = new RegExp(PATH_REGEX_SOURCE, "gi");
|
||||
const LEGACY_ATTACHMENT_MARKER_PATTERN =
|
||||
/\[(?:media attached(?:\s+\d+\/\d+)?:|Image:\s*source:)\s*[^\]]+\]/gi;
|
||||
|
||||
interface DetectedImageRef {
|
||||
raw: string;
|
||||
type: "path" | "media-uri";
|
||||
resolved: string;
|
||||
}
|
||||
|
||||
function isImageExtension(filePath: string): boolean {
|
||||
const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath));
|
||||
return IMAGE_EXTENSIONS.has(ext);
|
||||
@@ -96,77 +95,6 @@ type PromptImageEntry = {
|
||||
factIndex: ImageFactIndex;
|
||||
};
|
||||
|
||||
function mergePromptAttachmentImages(params: {
|
||||
imageOrder?: PromptImageOrderEntry[];
|
||||
mediaImageLayout?: MediaImageLayout;
|
||||
existingImages?: ImageContent[];
|
||||
existingImageFactIndexes?: readonly ImageFactIndex[];
|
||||
offloadedImages?: Array<PromptImageEntry | null>;
|
||||
promptRefImages?: ImageContent[];
|
||||
}): PromptImageEntry[] {
|
||||
const existingImages = (params.existingImages ?? []).map((image, index) => ({
|
||||
image,
|
||||
factIndex: params.existingImageFactIndexes?.[index] ?? null,
|
||||
}));
|
||||
const offloadedImages = params.offloadedImages ?? [];
|
||||
const promptRefImages = (params.promptRefImages ?? []).map((image) => ({
|
||||
image,
|
||||
factIndex: null,
|
||||
}));
|
||||
const slots: MediaImageLayout["slots"] =
|
||||
params.mediaImageLayout?.slots ?? params.imageOrder?.map((kind) => ({ kind })) ?? [];
|
||||
if (slots.length === 0) {
|
||||
const factOwned = [...offloadedImages, ...existingImages]
|
||||
.filter((entry): entry is PromptImageEntry => entry !== null && entry.factIndex !== null)
|
||||
.toSorted((left, right) => (left.factIndex ?? 0) - (right.factIndex ?? 0));
|
||||
return [
|
||||
...factOwned,
|
||||
...existingImages.filter((entry) => entry.factIndex === null),
|
||||
...promptRefImages,
|
||||
];
|
||||
}
|
||||
|
||||
const unusedExisting = [...existingImages];
|
||||
const takeExisting = (factIndex: number | null | undefined): PromptImageEntry | undefined => {
|
||||
const matchIndex =
|
||||
factIndex === undefined
|
||||
? 0
|
||||
: unusedExisting.findIndex((entry) => entry.factIndex === factIndex);
|
||||
if (matchIndex < 0) {
|
||||
return undefined;
|
||||
}
|
||||
return unusedExisting.splice(matchIndex, 1)[0];
|
||||
};
|
||||
let offloadedIndex = 0;
|
||||
const ordered = slots.flatMap((slot) => {
|
||||
const offloaded = slot.kind === "offloaded" ? offloadedImages[offloadedIndex++] : undefined;
|
||||
const exactExisting =
|
||||
slot.factIndex !== undefined
|
||||
? takeExisting(slot.factIndex)
|
||||
: slot.kind === "inline"
|
||||
? takeExisting(undefined)
|
||||
: undefined;
|
||||
const existing =
|
||||
exactExisting ??
|
||||
(slot.kind === "inline" && slot.factIndex !== undefined ? takeExisting(null) : undefined);
|
||||
if (existing) {
|
||||
return [existing];
|
||||
}
|
||||
if (slot.kind === "inline") {
|
||||
return [];
|
||||
}
|
||||
return offloaded ? [offloaded] : [];
|
||||
});
|
||||
return [
|
||||
...ordered,
|
||||
...unusedExisting,
|
||||
...offloadedImages
|
||||
.slice(offloadedIndex)
|
||||
.filter((entry): entry is PromptImageEntry => entry !== null),
|
||||
...promptRefImages,
|
||||
];
|
||||
}
|
||||
|
||||
async function sanitizeImageEntriesWithLog(
|
||||
entries: PromptImageEntry[],
|
||||
label: string,
|
||||
@@ -193,8 +121,8 @@ async function sanitizeImageEntriesWithLog(
|
||||
}
|
||||
|
||||
/** Detects explicit local image paths and file URLs in user prompt text. */
|
||||
export function detectImageReferences(prompt: string): DetectedImageRef[] {
|
||||
const refs: DetectedImageRef[] = [];
|
||||
export function detectImageReferences(prompt: string): MediaFileRef[] {
|
||||
const refs: MediaFileRef[] = [];
|
||||
const seen = new Set<string>();
|
||||
const pathPrompt = prompt.replace(LEGACY_ATTACHMENT_MARKER_PATTERN, (marker) =>
|
||||
" ".repeat(marker.length),
|
||||
@@ -263,7 +191,7 @@ export function detectImageReferences(prompt: string): DetectedImageRef[] {
|
||||
return refs;
|
||||
}
|
||||
|
||||
function refDedupeKey(ref: DetectedImageRef, workspaceDir?: string): string {
|
||||
function refDedupeKey(ref: MediaFileRef, workspaceDir?: string): string {
|
||||
const resolved =
|
||||
ref.type === "path" && workspaceDir && !path.isAbsolute(ref.resolved)
|
||||
? path.resolve(workspaceDir, ref.resolved)
|
||||
@@ -279,16 +207,19 @@ function rawAliasDedupeKey(alias: string): string | undefined {
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async function loadImageFromRef(
|
||||
ref: DetectedImageRef,
|
||||
async function loadMediaFromRef(
|
||||
ref: MediaFileRef,
|
||||
workspaceDir: string,
|
||||
options?: {
|
||||
label?: string;
|
||||
maxBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
workspaceOnly?: boolean;
|
||||
localRoots?: readonly string[];
|
||||
sandbox?: { root: string; bridge: SandboxFsBridge };
|
||||
},
|
||||
): Promise<ImageContent | null> {
|
||||
): Promise<WebMediaResult | null> {
|
||||
options?.signal?.throwIfAborted();
|
||||
try {
|
||||
let targetPath = ref.resolved;
|
||||
|
||||
@@ -310,7 +241,7 @@ async function loadImageFromRef(
|
||||
targetPath = resolved.resolved;
|
||||
} catch (err) {
|
||||
log.debug(
|
||||
`Native image: sandbox validation failed for ${ref.resolved}: ${formatErrorMessage(err)}`,
|
||||
`${options?.label ?? "Native media"}: sandbox validation failed: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
@@ -331,21 +262,31 @@ async function loadImageFromRef(
|
||||
: options?.maxBytes,
|
||||
);
|
||||
|
||||
if (media.kind !== "image") {
|
||||
log.debug(`Native image: not an image file: ${targetPath} (got ${media.kind})`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const mimeType = media.contentType ?? "image/jpeg";
|
||||
const data = media.buffer.toString("base64");
|
||||
|
||||
return { type: "image", data, mimeType };
|
||||
options?.signal?.throwIfAborted();
|
||||
return media;
|
||||
} catch (err) {
|
||||
log.debug(`Native image: failed to load ${ref.resolved}: ${formatErrorMessage(err)}`);
|
||||
options?.signal?.throwIfAborted();
|
||||
log.debug(`${options?.label ?? "Native media"}: failed to load: ${formatErrorMessage(err)}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadImageFromRef(
|
||||
ref: MediaFileRef,
|
||||
workspaceDir: string,
|
||||
options?: Parameters<typeof loadMediaFromRef>[2],
|
||||
): Promise<ImageContent | null> {
|
||||
const media = await loadMediaFromRef(ref, workspaceDir, { ...options, label: "Native image" });
|
||||
if (!media || media.kind !== "image") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
type: "image",
|
||||
data: media.buffer.toString("base64"),
|
||||
mimeType: media.contentType ?? "image/jpeg",
|
||||
};
|
||||
}
|
||||
|
||||
function modelSupportsImages(model: { input?: string[] }): boolean {
|
||||
return model.input?.includes("image") ?? false;
|
||||
}
|
||||
@@ -367,7 +308,7 @@ export async function detectAndLoadPromptImages(params: {
|
||||
}): Promise<{
|
||||
images: ImageContent[];
|
||||
imageFactIndexes: ImageFactIndex[];
|
||||
detectedRefs: DetectedImageRef[];
|
||||
detectedRefs: MediaFileRef[];
|
||||
failedMediaCount: number;
|
||||
loadedCount: number;
|
||||
skippedCount: number;
|
||||
@@ -382,97 +323,86 @@ export async function detectAndLoadPromptImages(params: {
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const allMediaRefs = collectMediaImageRefs(params.media);
|
||||
const suppressedFactIndexes = new Set(params.mediaImageLayout?.suppressedFactIndexes ?? []);
|
||||
for (const ref of allMediaRefs) {
|
||||
if (!ref || !suppressedFactIndexes.has(ref.factIndex)) {
|
||||
continue;
|
||||
const media = normalizeMediaFacts(params.media);
|
||||
const suppressed = new Set(params.mediaImageLayout?.suppressedFactIndexes ?? []);
|
||||
const imageFactIndexes = media.flatMap((fact, factIndex) =>
|
||||
isImageMediaFact(fact) && fact.hydrationSuppressed !== true && !suppressed.has(factIndex)
|
||||
? [factIndex]
|
||||
: [],
|
||||
);
|
||||
const refs = collectMediaImageRefs(media);
|
||||
const refsByFact = new Map(refs.flatMap((ref) => (ref ? [[ref.factIndex, ref] as const] : [])));
|
||||
const inferredSlots = (() => {
|
||||
if (params.imageOrder?.length === imageFactIndexes.length) {
|
||||
return params.imageOrder.map((kind, index) => ({
|
||||
kind,
|
||||
factIndex: imageFactIndexes[index],
|
||||
}));
|
||||
}
|
||||
ref.hydrate = false;
|
||||
}
|
||||
const orderRefs = allMediaRefs.filter(
|
||||
(ref) => !ref || (!suppressedFactIndexes.has(ref.factIndex) && ref.hydrate),
|
||||
);
|
||||
const imageOrder = params.mediaImageLayout?.slots.map((slot) => slot.kind) ?? params.imageOrder;
|
||||
const refsByFactIndex = new Map(
|
||||
allMediaRefs.flatMap((ref) => (ref ? [[ref.factIndex, ref] as const] : [])),
|
||||
);
|
||||
// imageOrder describes only images still requiring native delivery; described
|
||||
// (suppressed) facts must not count against it, or the inference silently
|
||||
// skips and inline sanitization failures dispatch as success.
|
||||
const unsuppressedImageFactIndexes = normalizeMediaFacts(params.media).flatMap(
|
||||
(fact, factIndex) =>
|
||||
isImageMediaFact(fact) &&
|
||||
fact.hydrationSuppressed !== true &&
|
||||
!suppressedFactIndexes.has(factIndex)
|
||||
? [factIndex]
|
||||
: [],
|
||||
);
|
||||
const inferredExistingImageFactIndexes =
|
||||
imageOrder && unsuppressedImageFactIndexes.length === imageOrder.length
|
||||
? imageOrder.flatMap((entry, index) =>
|
||||
entry === "inline" ? [unsuppressedImageFactIndexes[index] ?? null] : [],
|
||||
)
|
||||
: undefined;
|
||||
const inferredMediaImageLayout =
|
||||
!params.mediaImageLayout &&
|
||||
imageOrder &&
|
||||
unsuppressedImageFactIndexes.length === imageOrder.length
|
||||
? {
|
||||
slots: imageOrder.map((kind, index) => ({
|
||||
kind,
|
||||
factIndex: unsuppressedImageFactIndexes[index],
|
||||
})),
|
||||
suppressedFactIndexes: [],
|
||||
}
|
||||
: undefined;
|
||||
const layoutInlineFactIndexes = resolveLayoutInlineFactIndexes(
|
||||
params.mediaImageLayout,
|
||||
params.existingImages?.length ?? 0,
|
||||
);
|
||||
const existingImageFactIndexes =
|
||||
params.existingImageFactIndexes ??
|
||||
layoutInlineFactIndexes ??
|
||||
(inferredExistingImageFactIndexes?.length === (params.existingImages?.length ?? 0)
|
||||
? inferredExistingImageFactIndexes
|
||||
: undefined);
|
||||
const missingInlineMediaCount = countMissingLayoutInlineSlots(
|
||||
params.mediaImageLayout,
|
||||
existingImageFactIndexes,
|
||||
params.existingImages?.length ?? 0,
|
||||
);
|
||||
const attachmentRefs = params.mediaImageLayout
|
||||
? params.mediaImageLayout.slots.flatMap((slot) =>
|
||||
slot.kind === "offloaded"
|
||||
? [
|
||||
{
|
||||
factIndex: slot.factIndex,
|
||||
ref: slot.factIndex === undefined ? undefined : refsByFactIndex.get(slot.factIndex),
|
||||
},
|
||||
]
|
||||
: [],
|
||||
if (params.imageOrder?.length) {
|
||||
const pending = [...imageFactIndexes];
|
||||
return [
|
||||
...params.imageOrder.map((kind) => ({
|
||||
kind,
|
||||
...(kind === "offloaded" && pending.length ? { factIndex: pending.shift() } : {}),
|
||||
})),
|
||||
...pending.map((factIndex) => ({ kind: "offloaded" as const, factIndex })),
|
||||
];
|
||||
}
|
||||
return imageFactIndexes.map((factIndex, imageIndex) => ({
|
||||
factIndex,
|
||||
kind:
|
||||
!media[factIndex]?.path &&
|
||||
!media[factIndex]?.url &&
|
||||
imageIndex < (params.existingImages?.length ?? 0)
|
||||
? ("inline" as const)
|
||||
: ("offloaded" as const),
|
||||
}));
|
||||
})();
|
||||
const slots = params.mediaImageLayout?.slots.length
|
||||
? params.mediaImageLayout.slots.filter(
|
||||
(slot) => slot.factIndex === undefined || !suppressed.has(slot.factIndex),
|
||||
)
|
||||
: selectMediaImageRefs({
|
||||
refs: orderRefs,
|
||||
existingImageCount: params.existingImages?.length ?? 0,
|
||||
imageOrder,
|
||||
}).map((ref) => ({ factIndex: ref?.factIndex, ref }));
|
||||
const materializedFactIndexes = new Set(
|
||||
(existingImageFactIndexes ?? []).filter((entry): entry is number => typeof entry === "number"),
|
||||
: inferredSlots;
|
||||
const layoutInlineIndexes = slots.flatMap((slot) =>
|
||||
slot.kind === "inline" ? [slot.factIndex ?? null] : [],
|
||||
);
|
||||
const existingIndexes =
|
||||
params.existingImageFactIndexes ??
|
||||
(layoutInlineIndexes.length === (params.existingImages?.length ?? 0)
|
||||
? layoutInlineIndexes
|
||||
: params.existingImages?.map(() => null));
|
||||
const unusedExisting = (params.existingImages ?? []).map((image, index) => ({
|
||||
image,
|
||||
factIndex: existingIndexes?.[index] ?? null,
|
||||
}));
|
||||
const takeExisting = (
|
||||
factIndex: number | undefined,
|
||||
allowUnowned: boolean,
|
||||
): PromptImageEntry | undefined => {
|
||||
const exact =
|
||||
factIndex === undefined
|
||||
? -1
|
||||
: unusedExisting.findIndex((entry) => entry.factIndex === factIndex);
|
||||
const index =
|
||||
exact >= 0
|
||||
? exact
|
||||
: allowUnowned
|
||||
? unusedExisting.findIndex((entry) => entry.factIndex === null)
|
||||
: -1;
|
||||
return index >= 0 ? unusedExisting.splice(index, 1)[0] : undefined;
|
||||
};
|
||||
const availableRefs = refs.filter((ref): ref is MediaImageRef => Boolean(ref));
|
||||
const attachmentRefs = slots.flatMap((slot) =>
|
||||
slot.kind === "offloaded" && slot.factIndex !== undefined
|
||||
? (refsByFact.get(slot.factIndex) ?? [])
|
||||
: [],
|
||||
);
|
||||
const availableMediaRefs = allMediaRefs.filter((ref): ref is MediaImageRef => ref !== undefined);
|
||||
const selectedAttachmentRefs = attachmentRefs.flatMap(({ ref }) => (ref ? [ref] : []));
|
||||
const attachmentKeys = new Set(
|
||||
selectedAttachmentRefs.map((ref) => refDedupeKey(ref, ref.workspaceDir ?? params.workspaceDir)),
|
||||
attachmentRefs.map((ref) => refDedupeKey(ref, ref.workspaceDir ?? params.workspaceDir)),
|
||||
);
|
||||
const attachmentRawKeys = new Set(
|
||||
selectedAttachmentRefs.flatMap((ref) =>
|
||||
ref.aliases.flatMap((alias) => {
|
||||
const key = rawAliasDedupeKey(alias);
|
||||
return key ? [key] : [];
|
||||
}),
|
||||
),
|
||||
attachmentRefs.flatMap((ref) => ref.aliases.flatMap((alias) => rawAliasDedupeKey(alias) ?? [])),
|
||||
);
|
||||
const promptRefs = detectImageReferences(params.prompt).filter(
|
||||
(ref) =>
|
||||
@@ -480,7 +410,7 @@ export async function detectAndLoadPromptImages(params: {
|
||||
!attachmentKeys.has(refDedupeKey(ref, params.workspaceDir)),
|
||||
);
|
||||
const detectedRefs = [
|
||||
...availableMediaRefs.flatMap(({ detect, hydrate, raw, type, resolved }) =>
|
||||
...availableRefs.flatMap(({ detect, hydrate, raw, type, resolved }) =>
|
||||
detect !== false &&
|
||||
(hydrate || (!resolved.startsWith("http://") && !resolved.startsWith("https://")))
|
||||
? [{ raw, type, resolved }]
|
||||
@@ -488,40 +418,10 @@ export async function detectAndLoadPromptImages(params: {
|
||||
),
|
||||
...promptRefs,
|
||||
];
|
||||
if (attachmentRefs.length === 0 && promptRefs.length === 0) {
|
||||
const existingImages = params.existingImages ?? [];
|
||||
const sanitized = existingImages.length
|
||||
? await sanitizeImageEntriesWithLog(
|
||||
existingImages.map((image, index) => ({
|
||||
image,
|
||||
factIndex: existingImageFactIndexes?.[index] ?? null,
|
||||
})),
|
||||
"prompt:images",
|
||||
{
|
||||
maxBytes: params.maxBytes,
|
||||
maxDimensionPx: params.maxDimensionPx,
|
||||
},
|
||||
)
|
||||
: { entries: [], failedMediaCount: 0 };
|
||||
const finalized = finalizeRuntimePromptImages(sanitized.entries);
|
||||
return {
|
||||
...finalized,
|
||||
detectedRefs,
|
||||
failedMediaCount: missingInlineMediaCount + sanitized.failedMediaCount,
|
||||
loadedCount: 0,
|
||||
skippedCount: 0,
|
||||
};
|
||||
}
|
||||
|
||||
log.debug(
|
||||
`Native image: prepared ${attachmentRefs.length} attachment ref(s) and ${promptRefs.length} explicit prompt ref(s)`,
|
||||
);
|
||||
let loadedCount = 0;
|
||||
let failedMediaCount = missingInlineMediaCount;
|
||||
let failedMediaCount = 0;
|
||||
let skippedCount = 0;
|
||||
const loadRef = async (
|
||||
ref: DetectedImageRef & { workspaceDir?: string },
|
||||
): Promise<ImageContent | null> => {
|
||||
const loadRef = async (ref: MediaFileRef & { workspaceDir?: string }) => {
|
||||
const image = await loadImageFromRef(ref, ref.workspaceDir ?? params.workspaceDir, {
|
||||
maxBytes: params.maxBytes,
|
||||
workspaceOnly: params.workspaceOnly,
|
||||
@@ -536,42 +436,33 @@ export async function detectAndLoadPromptImages(params: {
|
||||
}
|
||||
return image;
|
||||
};
|
||||
const offloadedImages: Array<PromptImageEntry | null> = [];
|
||||
for (const attachment of attachmentRefs) {
|
||||
const factIndex = attachment.factIndex;
|
||||
if (factIndex !== undefined && materializedFactIndexes.has(factIndex)) {
|
||||
offloadedImages.push(null);
|
||||
const promptImages: PromptImageEntry[] = [];
|
||||
for (const slot of slots) {
|
||||
const existing = takeExisting(slot.factIndex, slot.kind === "inline");
|
||||
if (existing) {
|
||||
promptImages.push(existing);
|
||||
continue;
|
||||
}
|
||||
const ref = attachment.ref;
|
||||
if (!ref) {
|
||||
if (slot.kind === "inline") {
|
||||
failedMediaCount++;
|
||||
offloadedImages.push(null);
|
||||
continue;
|
||||
}
|
||||
const image = ref.hydrate ? await loadRef(ref) : null;
|
||||
if (ref.hydrate && !image) {
|
||||
const ref = slot.factIndex === undefined ? undefined : refsByFact.get(slot.factIndex);
|
||||
const image = ref?.hydrate ? await loadRef(ref) : null;
|
||||
if (ref?.hydrate && !image) {
|
||||
failedMediaCount++;
|
||||
}
|
||||
offloadedImages.push(image ? { image, factIndex: ref.factIndex } : null);
|
||||
if (image) {
|
||||
promptImages.push({ image, factIndex: ref?.factIndex ?? null });
|
||||
}
|
||||
}
|
||||
const promptRefImages: ImageContent[] = [];
|
||||
promptImages.push(...unusedExisting);
|
||||
for (const ref of promptRefs) {
|
||||
const image = await loadRef(ref);
|
||||
if (image) {
|
||||
promptRefImages.push(image);
|
||||
promptImages.push({ image, factIndex: null });
|
||||
}
|
||||
}
|
||||
|
||||
const promptImages = mergePromptAttachmentImages({
|
||||
imageOrder,
|
||||
mediaImageLayout: params.mediaImageLayout ?? inferredMediaImageLayout,
|
||||
existingImages: params.existingImages,
|
||||
existingImageFactIndexes,
|
||||
offloadedImages,
|
||||
promptRefImages,
|
||||
});
|
||||
|
||||
const sanitizedPromptImages = await sanitizeImageEntriesWithLog(promptImages, "prompt:images", {
|
||||
maxBytes: params.maxBytes,
|
||||
maxDimensionPx: params.maxDimensionPx,
|
||||
@@ -587,20 +478,107 @@ export async function detectAndLoadPromptImages(params: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Hydrates non-enumerable facts carried by queued user turns before provider replay. */
|
||||
export async function hydratePromptMediaMessages(
|
||||
type PromptMediaOptions = {
|
||||
workspaceDir: string;
|
||||
model: { input?: string[] };
|
||||
maxBytes?: number;
|
||||
maxDimensionPx?: number;
|
||||
workspaceOnly?: boolean;
|
||||
localRoots?: readonly string[];
|
||||
sandbox?: { root: string; bridge: SandboxFsBridge };
|
||||
provider?: boolean;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
const VIDEO_OMISSION = {
|
||||
unsupported: "(video omitted: provider does not support native video)",
|
||||
historical: "(video omitted: native historical replay is not available yet)",
|
||||
unavailable: "(video omitted: source unavailable)",
|
||||
invalid: "(video omitted: invalid video MIME type)",
|
||||
limit: "(video omitted: native video byte limit exceeded)",
|
||||
} as const;
|
||||
|
||||
async function materializeVideoFact(
|
||||
fact: MediaFact,
|
||||
budget: { remaining: number },
|
||||
options: PromptMediaOptions,
|
||||
): Promise<ModelInputContent> {
|
||||
if ((fact.sizeBytes ?? 0) > budget.remaining) {
|
||||
return { type: "text", text: VIDEO_OMISSION.limit };
|
||||
}
|
||||
const ref = resolveMediaFactLocalRef(fact);
|
||||
const loaded = ref
|
||||
? await loadMediaFromRef(ref, fact.workspaceDir ?? options.workspaceDir, {
|
||||
label: "Native video",
|
||||
maxBytes: budget.remaining,
|
||||
signal: options.signal,
|
||||
workspaceOnly: options.workspaceOnly,
|
||||
localRoots:
|
||||
options.localRoots ?? (options.workspaceOnly ? [options.workspaceDir] : undefined),
|
||||
sandbox: options.sandbox,
|
||||
})
|
||||
: null;
|
||||
if (!loaded) {
|
||||
return { type: "text", text: VIDEO_OMISSION.unavailable };
|
||||
}
|
||||
const mimeType = normalizeMimeType(loaded.contentType);
|
||||
if (loaded.kind !== "video" || !mimeType?.startsWith("video/")) {
|
||||
return { type: "text", text: VIDEO_OMISSION.invalid };
|
||||
}
|
||||
if (loaded.buffer.length > budget.remaining) {
|
||||
return { type: "text", text: VIDEO_OMISSION.limit };
|
||||
}
|
||||
budget.remaining -= loaded.buffer.length;
|
||||
return { type: "video", data: loaded.buffer.toString("base64"), mimeType };
|
||||
}
|
||||
|
||||
async function projectOrderedPromptMedia(params: {
|
||||
content: Array<TextContent | ImageContent>;
|
||||
media: MediaFact[];
|
||||
images: ImageContent[];
|
||||
imageFactIndexes: ImageFactIndex[];
|
||||
runtime: boolean;
|
||||
options: PromptMediaOptions;
|
||||
budget: { remaining: number };
|
||||
}): Promise<ModelInputContent[]> {
|
||||
const generatedMarkers = new Set<string>(Object.values(VIDEO_OMISSION));
|
||||
const projected: ModelInputContent[] = params.content.filter(
|
||||
(block): block is TextContent => block.type === "text" && !generatedMarkers.has(block.text),
|
||||
);
|
||||
const imagesByFact = new Map<number, ImageContent[]>();
|
||||
const factlessImages: ImageContent[] = [];
|
||||
params.images.forEach((image, index) => {
|
||||
const factIndex = params.imageFactIndexes[index];
|
||||
if (factIndex == null) {
|
||||
factlessImages.push(image);
|
||||
} else {
|
||||
imagesByFact.set(factIndex, [...(imagesByFact.get(factIndex) ?? []), image]);
|
||||
}
|
||||
});
|
||||
for (const [factIndex, fact] of params.media.entries()) {
|
||||
if (isImageMediaFact(fact)) {
|
||||
projected.push(...(imagesByFact.get(factIndex) ?? []));
|
||||
} else if (isVideoMediaFact(fact)) {
|
||||
projected.push(
|
||||
!params.runtime
|
||||
? { type: "text", text: VIDEO_OMISSION.historical }
|
||||
: params.options.provider
|
||||
? await materializeVideoFact(fact, params.budget, params.options)
|
||||
: { type: "text", text: VIDEO_OMISSION.unsupported },
|
||||
);
|
||||
}
|
||||
}
|
||||
projected.push(...factlessImages);
|
||||
return projected;
|
||||
}
|
||||
|
||||
/** Hydrates exact-message media facts for canonical replay or one provider call. */
|
||||
async function materializePromptMediaMessages(
|
||||
messages: AgentMessage[],
|
||||
options: {
|
||||
workspaceDir: string;
|
||||
model: { input?: string[] };
|
||||
maxBytes?: number;
|
||||
maxDimensionPx?: number;
|
||||
workspaceOnly?: boolean;
|
||||
localRoots?: readonly string[];
|
||||
sandbox?: { root: string; bridge: SandboxFsBridge };
|
||||
},
|
||||
options: PromptMediaOptions,
|
||||
): Promise<AgentMessage[]> {
|
||||
let hydrated: AgentMessage[] | undefined;
|
||||
const videoBudget = { remaining: MAX_VIDEO_BYTES };
|
||||
for (const [index, message] of messages.entries()) {
|
||||
if (message.role !== "user") {
|
||||
continue;
|
||||
@@ -617,28 +595,13 @@ export async function hydratePromptMediaMessages(
|
||||
? message.content
|
||||
: [{ type: "text" as const, text: message.content }];
|
||||
const existingImages = content.filter((block): block is ImageContent => block.type === "image");
|
||||
const persistedImageFactIndexes = readPersistedImageBlockFactIndexes(message);
|
||||
const inlineLayoutFactIndexes = mediaImageLayout?.slots.flatMap((slot) =>
|
||||
slot.kind === "inline" ? [slot.factIndex ?? null] : [],
|
||||
);
|
||||
// Pre-carrier transcripts had no explicit block provenance. Their native
|
||||
// image blocks were positionally aligned with the first image facts.
|
||||
const legacyImageFactIndexes =
|
||||
runtimeMedia === undefined && mediaImageLayout === undefined
|
||||
? collectIdentitylessMediaImageFactIndexes(resolvedMedia)
|
||||
: undefined;
|
||||
const existingImageFactIndexes =
|
||||
persistedImageFactIndexes ??
|
||||
(inlineLayoutFactIndexes?.length === existingImages.length
|
||||
? inlineLayoutFactIndexes
|
||||
: legacyImageFactIndexes?.slice(0, existingImages.length));
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: resolvedMedia,
|
||||
workspaceDir: options.workspaceDir,
|
||||
model: options.model,
|
||||
existingImages,
|
||||
existingImageFactIndexes,
|
||||
existingImageFactIndexes: readPersistedImageBlockFactIndexes(message),
|
||||
imageOrder: runtimeImageOrder,
|
||||
mediaImageLayout,
|
||||
maxBytes: options.maxBytes,
|
||||
@@ -647,6 +610,25 @@ export async function hydratePromptMediaMessages(
|
||||
localRoots: options.localRoots,
|
||||
sandbox: options.sandbox,
|
||||
});
|
||||
const projectedContent = await projectOrderedPromptMedia({
|
||||
content,
|
||||
media: resolvedMedia,
|
||||
images: result.images,
|
||||
imageFactIndexes: result.imageFactIndexes,
|
||||
runtime: runtimeMedia !== undefined,
|
||||
options,
|
||||
budget: videoBudget,
|
||||
});
|
||||
hydrated ??= messages.slice();
|
||||
if (options.provider) {
|
||||
hydrated[index] = {
|
||||
role: "user",
|
||||
content: projectedContent,
|
||||
timestamp: message.timestamp,
|
||||
...(message.runtimeContextCarrier ? { runtimeContextCarrier: true } : {}),
|
||||
} as ProviderContext["messages"][number] as AgentMessage;
|
||||
continue;
|
||||
}
|
||||
const nextMeta =
|
||||
meta && typeof meta === "object" && !Array.isArray(meta)
|
||||
? { ...(meta as Record<string, unknown>) }
|
||||
@@ -656,10 +638,9 @@ export async function hydratePromptMediaMessages(
|
||||
} else {
|
||||
delete nextMeta.mediaImageBlockFactIndexes;
|
||||
}
|
||||
hydrated ??= messages.slice();
|
||||
const hydratedMessage = {
|
||||
...message,
|
||||
content: [...content.filter((block) => block.type !== "image"), ...result.images],
|
||||
content: projectedContent,
|
||||
} as AgentMessage;
|
||||
if (Object.keys(nextMeta).length > 0) {
|
||||
(hydratedMessage as unknown as Record<string, unknown>)["__openclaw"] = nextMeta;
|
||||
@@ -673,3 +654,35 @@ export async function hydratePromptMediaMessages(
|
||||
}
|
||||
return hydrated ?? messages;
|
||||
}
|
||||
|
||||
/** Hydrates non-enumerable facts carried by queued user turns before canonical replay. */
|
||||
export async function hydratePromptMediaMessages(
|
||||
messages: AgentMessage[],
|
||||
options: Omit<PromptMediaOptions, "provider">,
|
||||
): Promise<AgentMessage[]> {
|
||||
return await materializePromptMediaMessages(messages, options);
|
||||
}
|
||||
|
||||
/** Materializes one transient provider context from exact-message media facts. */
|
||||
export async function materializeProviderContext(params: {
|
||||
context: Context;
|
||||
signal?: AbortSignal;
|
||||
workspaceDir: string;
|
||||
workspaceOnly?: boolean;
|
||||
localRoots?: readonly string[];
|
||||
sandbox?: { root: string; bridge: SandboxFsBridge };
|
||||
}): Promise<ProviderContext> {
|
||||
const messages = await materializePromptMediaMessages(params.context.messages as AgentMessage[], {
|
||||
workspaceDir: params.workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
workspaceOnly: params.workspaceOnly,
|
||||
localRoots: params.localRoots,
|
||||
sandbox: params.sandbox,
|
||||
provider: true,
|
||||
signal: params.signal,
|
||||
});
|
||||
params.signal?.throwIfAborted();
|
||||
return messages === params.context.messages
|
||||
? (params.context as ProviderContext)
|
||||
: ({ ...params.context, messages } as ProviderContext);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createAssistantMessageEventStream } from "@openclaw/llm-core";
|
||||
import { MAX_VIDEO_BYTES } from "@openclaw/media-core/constants";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
PROVIDER_CONTEXT_HANDOFF,
|
||||
resolveProviderContext,
|
||||
type ProviderContext,
|
||||
type ProviderStreamOptions,
|
||||
} from "../../../../packages/ai/src/provider-types.js";
|
||||
import { attachRuntimePromptMediaFacts } from "../../../media/media-facts.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import type { StreamFn } from "../../runtime/index.js";
|
||||
import type { SandboxFsBridge } from "../../sandbox/fs-bridge.js";
|
||||
import { hydratePromptMediaMessages, materializeProviderContext } from "./images.js";
|
||||
import { wrapStreamFnWithMessageTransform } from "./message-transform-stream-wrapper.js";
|
||||
|
||||
const PNG = {
|
||||
type: "image" as const,
|
||||
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==",
|
||||
mimeType: "image/png",
|
||||
};
|
||||
const MP4 = Buffer.from("0000001c6674797069736f6d0000000069736f6d0000000000000000", "hex");
|
||||
|
||||
describe("direct provider context handoff", () => {
|
||||
const tempDirs: string[] = [];
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps canonical omissions while materializing only exact current runtime facts", async () => {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-provider-video-"));
|
||||
tempDirs.push(stateDir);
|
||||
const env = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", stateDir);
|
||||
const inbound = path.join(stateDir, "media", "inbound");
|
||||
await fs.mkdir(inbound, { recursive: true });
|
||||
await fs.writeFile(path.join(inbound, "recent.mp4"), MP4);
|
||||
await fs.writeFile(path.join(inbound, "steer.mp4"), MP4);
|
||||
|
||||
try {
|
||||
const historical = {
|
||||
role: "user" as const,
|
||||
content: "historical",
|
||||
timestamp: 1,
|
||||
__openclaw: {
|
||||
media: [
|
||||
{
|
||||
kind: "video",
|
||||
contentType: "video/mp4",
|
||||
url: "media://inbound/old.mp4",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const recent = attachRuntimePromptMediaFacts(
|
||||
{
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "recent" }, PNG, { ...PNG }],
|
||||
timestamp: 2,
|
||||
__openclaw: {
|
||||
mediaImageBlockFactIndexes: [0, 2],
|
||||
mediaImageLayout: {
|
||||
slots: [
|
||||
{ kind: "inline", factIndex: 0 },
|
||||
{ kind: "inline", factIndex: 2 },
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
[
|
||||
{ kind: "image", contentType: "image/png" },
|
||||
{
|
||||
kind: "video",
|
||||
contentType: "video/mp4",
|
||||
sizeBytes: MP4.length,
|
||||
url: "media://inbound/recent.mp4",
|
||||
hydrationSuppressed: true,
|
||||
},
|
||||
{ kind: "image", contentType: "image/png" },
|
||||
],
|
||||
);
|
||||
const steer = attachRuntimePromptMediaFacts(
|
||||
{
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "steer" }, { ...PNG }],
|
||||
timestamp: 3,
|
||||
__openclaw: { mediaImageBlockFactIndexes: [1] },
|
||||
},
|
||||
[
|
||||
{
|
||||
kind: "video",
|
||||
contentType: "video/mp4",
|
||||
sizeBytes: MP4.length,
|
||||
url: "media://inbound/steer.mp4",
|
||||
},
|
||||
{ kind: "image", contentType: "image/png" },
|
||||
],
|
||||
);
|
||||
const canonicalMessages = await hydratePromptMediaMessages([historical, recent, steer], {
|
||||
workspaceDir: stateDir,
|
||||
model: { input: ["text", "image"] },
|
||||
});
|
||||
const context = {
|
||||
systemPrompt: "system",
|
||||
messages: canonicalMessages,
|
||||
tools: [],
|
||||
} as Parameters<StreamFn>[1];
|
||||
const originalJson = JSON.stringify(context);
|
||||
const model = {
|
||||
id: "canonical",
|
||||
provider: "direct",
|
||||
api: "test",
|
||||
input: ["text", "image"],
|
||||
} as Parameters<StreamFn>[0];
|
||||
|
||||
const firstCandidate = vi.fn<StreamFn>((_model, firstContext, options) => {
|
||||
expect((options as ProviderStreamOptions)[PROVIDER_CONTEXT_HANDOFF]).toBeUndefined();
|
||||
expect(JSON.stringify(firstContext)).toContain("provider does not support native video");
|
||||
expect(JSON.stringify(firstContext)).toContain("historical replay is not available yet");
|
||||
return createAssistantMessageEventStream();
|
||||
});
|
||||
void firstCandidate(model, context, {});
|
||||
|
||||
let providerContext: Promise<ProviderContext> | undefined;
|
||||
let providerOptions: ProviderStreamOptions | undefined;
|
||||
const direct = vi.fn<StreamFn>((_model, canonical, options) => {
|
||||
providerOptions = options;
|
||||
providerContext = resolveProviderContext(canonical, options);
|
||||
return createAssistantMessageEventStream();
|
||||
});
|
||||
const wrapped = wrapStreamFnWithMessageTransform(
|
||||
direct,
|
||||
(messages) => messages,
|
||||
(input) => materializeProviderContext({ ...input, workspaceDir: stateDir }),
|
||||
);
|
||||
void wrapped(model, context, {});
|
||||
const resolved = await providerContext;
|
||||
|
||||
expect(resolved?.messages[0]?.content).toEqual([
|
||||
{ type: "text", text: "historical" },
|
||||
{ type: "text", text: "(video omitted: native historical replay is not available yet)" },
|
||||
]);
|
||||
expect(resolved?.messages[1]?.content).toEqual([
|
||||
{ type: "text", text: "recent" },
|
||||
PNG,
|
||||
{ type: "video", data: MP4.toString("base64"), mimeType: "video/mp4" },
|
||||
{ ...PNG },
|
||||
]);
|
||||
expect(resolved?.messages[2]?.content).toEqual([
|
||||
{ type: "text", text: "steer" },
|
||||
{ type: "video", data: MP4.toString("base64"), mimeType: "video/mp4" },
|
||||
{ ...PNG },
|
||||
]);
|
||||
for (const message of resolved?.messages.slice(1) ?? []) {
|
||||
expect(message).not.toHaveProperty("__openclaw");
|
||||
expect(Object.getOwnPropertySymbols(message)).toEqual([]);
|
||||
}
|
||||
expect(JSON.stringify(resolved)).not.toContain(stateDir);
|
||||
expect(JSON.stringify(context)).toBe(originalJson);
|
||||
await expect(providerOptions?.[PROVIDER_CONTEXT_HANDOFF]?.()).rejects.toThrow(
|
||||
"already consumed",
|
||||
);
|
||||
} finally {
|
||||
env.restore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects abort after a bounded sandbox read instead of dispatching an omission", async () => {
|
||||
let finishRead: ((value: Buffer) => void) | undefined;
|
||||
let markReadStarted: (() => void) | undefined;
|
||||
const readStarted = new Promise<void>((resolve) => {
|
||||
markReadStarted = resolve;
|
||||
});
|
||||
const bridge = {
|
||||
resolvePath: ({ filePath }: { filePath: string }) => ({
|
||||
containerPath: filePath,
|
||||
relativePath: filePath.replace(/^\//, ""),
|
||||
}),
|
||||
readFile: vi.fn(async () => {
|
||||
markReadStarted?.();
|
||||
return await new Promise<Buffer>((resolve) => {
|
||||
finishRead = resolve;
|
||||
});
|
||||
}),
|
||||
} as unknown as SandboxFsBridge;
|
||||
const controller = new AbortController();
|
||||
const current = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: "inspect", timestamp: 1 },
|
||||
[{ kind: "video", contentType: "video/mp4", path: "/workspace/clip.mp4" }],
|
||||
);
|
||||
const resolving = materializeProviderContext({
|
||||
context: { systemPrompt: "system", messages: [current], tools: [] },
|
||||
signal: controller.signal,
|
||||
workspaceDir: "/workspace",
|
||||
sandbox: { root: "/workspace", bridge },
|
||||
});
|
||||
await readStarted;
|
||||
controller.abort(new Error("test abort"));
|
||||
finishRead?.(MP4);
|
||||
await expect(resolving).rejects.toThrow("test abort");
|
||||
});
|
||||
|
||||
it("rejects a known over-budget current video before reading it", async () => {
|
||||
const readFile = vi.fn();
|
||||
const bridge = {
|
||||
resolvePath: ({ filePath }: { filePath: string }) => ({
|
||||
containerPath: filePath,
|
||||
relativePath: filePath.replace(/^\//, ""),
|
||||
}),
|
||||
readFile,
|
||||
} as unknown as SandboxFsBridge;
|
||||
const current = attachRuntimePromptMediaFacts(
|
||||
{ role: "user" as const, content: "inspect", timestamp: 1 },
|
||||
[
|
||||
{
|
||||
kind: "video",
|
||||
contentType: "video/mp4",
|
||||
path: "/workspace/clip.mp4",
|
||||
sizeBytes: MAX_VIDEO_BYTES + 1,
|
||||
},
|
||||
],
|
||||
);
|
||||
const resolved = await materializeProviderContext({
|
||||
context: { systemPrompt: "system", messages: [current], tools: [] },
|
||||
workspaceDir: "/workspace",
|
||||
sandbox: { root: "/workspace", bridge },
|
||||
});
|
||||
expect(resolved.messages[0]?.content).toContainEqual({
|
||||
type: "text",
|
||||
text: "(video omitted: native video byte limit exceeded)",
|
||||
});
|
||||
expect(readFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,39 +1,60 @@
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
/**
|
||||
* Wraps stream functions with pre-call message transforms.
|
||||
*/
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import {
|
||||
PROVIDER_CONTEXT_HANDOFF,
|
||||
type ProviderContext,
|
||||
type ProviderStreamOptions,
|
||||
} from "../../../../packages/ai/src/provider-types.js";
|
||||
import type { AgentMessage } from "../../runtime/index.js";
|
||||
|
||||
/**
|
||||
* Stream wrapper for applying message transforms immediately before provider dispatch.
|
||||
*/
|
||||
type MessageTransform = (messages: AgentMessage[], model: unknown) => AgentMessage[];
|
||||
type ProviderContextMaterializer = (input: {
|
||||
context: Parameters<StreamFn>[1];
|
||||
signal?: AbortSignal;
|
||||
}) => Promise<ProviderContext>;
|
||||
|
||||
/** Wraps a stream function with a conditional message-list transform. */
|
||||
export function wrapStreamFnWithMessageTransform(
|
||||
streamFn: StreamFn,
|
||||
transform: MessageTransform,
|
||||
materializeProviderContext?: ProviderContextMaterializer,
|
||||
): StreamFn {
|
||||
return (model, context, options) => {
|
||||
const messages = (context as unknown as { messages?: unknown })?.messages;
|
||||
if (!Array.isArray(messages)) {
|
||||
return streamFn(model, context, options);
|
||||
const nextMessages = Array.isArray(messages)
|
||||
? transform(messages as AgentMessage[], model)
|
||||
: messages;
|
||||
const nextContext =
|
||||
Array.isArray(messages) && nextMessages !== messages
|
||||
? ({
|
||||
...(context as unknown as Record<string, unknown>),
|
||||
messages: nextMessages,
|
||||
} as typeof context)
|
||||
: context;
|
||||
if (!materializeProviderContext) {
|
||||
return streamFn(model, nextContext, options);
|
||||
}
|
||||
|
||||
const nextMessages = transform(messages as AgentMessage[], model);
|
||||
if (nextMessages === messages) {
|
||||
return streamFn(model, context, options);
|
||||
}
|
||||
|
||||
return streamFn(
|
||||
// Clone the context instead of mutating it so callers can reuse the original assembled
|
||||
// context for logging, replay, or retry comparisons.
|
||||
model,
|
||||
{
|
||||
...(context as unknown as Record<string, unknown>),
|
||||
messages: nextMessages,
|
||||
} as typeof context,
|
||||
options,
|
||||
);
|
||||
let availableContext: Parameters<StreamFn>[1] | undefined = nextContext;
|
||||
const handoff = async (): Promise<ProviderContext> => {
|
||||
const captured = availableContext;
|
||||
availableContext = undefined;
|
||||
if (!captured) {
|
||||
throw new Error("provider context handoff already consumed");
|
||||
}
|
||||
options?.signal?.throwIfAborted();
|
||||
return await materializeProviderContext({
|
||||
context: captured,
|
||||
signal: options?.signal,
|
||||
});
|
||||
};
|
||||
return streamFn(model, nextContext, {
|
||||
...options,
|
||||
[PROVIDER_CONTEXT_HANDOFF]: handoff,
|
||||
} as ProviderStreamOptions & NonNullable<Parameters<StreamFn>[2]>);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import { isImageMediaFact, readPersistedMediaFacts } from "../../../media/media-facts.js";
|
||||
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import { readPersistedMediaImageLayout } from "./prompt-image-metadata.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
function toTypeOnlyImageFact(
|
||||
fact: NonNullable<RunEmbeddedAgentParams["media"]>[number],
|
||||
hydrationSuppressed: boolean,
|
||||
): NonNullable<RunEmbeddedAgentParams["media"]>[number] {
|
||||
return {
|
||||
contentType: fact.contentType,
|
||||
kind: fact.kind === "sticker" ? "sticker" : "image",
|
||||
messageId: fact.messageId,
|
||||
transcribed: fact.transcribed,
|
||||
...(fact.hydrationSuppressed === true || hydrationSuppressed
|
||||
? { hydrationSuppressed: true }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Materializes fact-carried images before a plugin harness owns transport. */
|
||||
export async function preparePluginHarnessPromptImages(params: {
|
||||
runParams: RunEmbeddedAgentParams;
|
||||
runtime: {
|
||||
sessionId: string;
|
||||
sessionKey?: string;
|
||||
workspaceDir: string;
|
||||
model: EmbeddedRunAttemptParams["model"];
|
||||
};
|
||||
pluginHarnessOwnsTransport: boolean;
|
||||
}): Promise<{
|
||||
images: RunEmbeddedAgentParams["images"];
|
||||
imageOrder: RunEmbeddedAgentParams["imageOrder"];
|
||||
media: RunEmbeddedAgentParams["media"];
|
||||
}> {
|
||||
const { runParams, runtime } = params;
|
||||
if (!params.pluginHarnessOwnsTransport) {
|
||||
return {
|
||||
images: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
media: runParams.media,
|
||||
};
|
||||
}
|
||||
const persistedMessage =
|
||||
runParams.userTurnTranscriptRecorder?.message ??
|
||||
(await runParams.userTurnTranscriptRecorder?.resolveMessage());
|
||||
const persistedMedia = persistedMessage ? (readPersistedMediaFacts(persistedMessage) ?? []) : [];
|
||||
const hydrationMedia = persistedMedia.length > 0 ? persistedMedia : runParams.media;
|
||||
if (!hydrationMedia?.some(isImageMediaFact)) {
|
||||
return {
|
||||
images: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
media: runParams.media,
|
||||
};
|
||||
}
|
||||
|
||||
const workspace = await resolveAttemptWorkspaceSandbox({
|
||||
...runParams,
|
||||
cwd: undefined,
|
||||
sessionId: runtime.sessionId,
|
||||
sessionKey: runtime.sessionKey,
|
||||
workspaceDir: runtime.workspaceDir,
|
||||
});
|
||||
const result = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: hydrationMedia,
|
||||
mediaImageLayout: persistedMessage
|
||||
? readPersistedMediaImageLayout(persistedMessage)
|
||||
: undefined,
|
||||
workspaceDir: workspace.effectiveWorkspace,
|
||||
model: runtime.model,
|
||||
existingImages: runParams.images,
|
||||
imageOrder: runParams.imageOrder,
|
||||
maxBytes: MAX_IMAGE_BYTES,
|
||||
maxDimensionPx: resolveImageSanitizationLimits(runParams.config).maxDimensionPx,
|
||||
localRoots: workspace.effectiveFsWorkspaceOnly
|
||||
? [workspace.effectiveWorkspace, workspace.resolvedWorkspace]
|
||||
: undefined,
|
||||
workspaceOnly: workspace.effectiveFsWorkspaceOnly,
|
||||
sandbox:
|
||||
workspace.sandbox?.enabled && workspace.sandbox.fsBridge
|
||||
? { root: workspace.sandbox.workspaceDir, bridge: workspace.sandbox.fsBridge }
|
||||
: undefined,
|
||||
});
|
||||
if (result.failedMediaCount > 0) {
|
||||
throw new Error(
|
||||
`failed to hydrate ${result.failedMediaCount} structured image attachment(s) for plugin harness input`,
|
||||
);
|
||||
}
|
||||
const materializedFactIndexes = new Set(
|
||||
result.imageFactIndexes.filter((index): index is number => index !== null),
|
||||
);
|
||||
const retainedMedia = hydrationMedia?.map((fact, factIndex) =>
|
||||
isImageMediaFact(fact)
|
||||
? toTypeOnlyImageFact(fact, !materializedFactIndexes.has(factIndex))
|
||||
: fact,
|
||||
);
|
||||
return {
|
||||
images: result.images,
|
||||
imageOrder: result.images.length > 0 ? result.images.map(() => "inline" as const) : undefined,
|
||||
media: retainedMedia?.length ? retainedMedia : undefined,
|
||||
};
|
||||
}
|
||||
@@ -7,48 +7,6 @@ export type MediaImageLayout = {
|
||||
suppressedFactIndexes: number[];
|
||||
};
|
||||
|
||||
export function resolveLayoutInlineFactIndexes(
|
||||
layout: MediaImageLayout | undefined,
|
||||
existingImageCount: number,
|
||||
): ImageFactIndex[] | undefined {
|
||||
const factIndexes = layout?.slots.flatMap((slot) =>
|
||||
slot.kind === "inline" ? [slot.factIndex ?? null] : [],
|
||||
);
|
||||
return factIndexes?.length === existingImageCount ? factIndexes : undefined;
|
||||
}
|
||||
|
||||
export function countMissingLayoutInlineSlots(
|
||||
layout: MediaImageLayout | undefined,
|
||||
existingFactIndexes: readonly ImageFactIndex[] | undefined,
|
||||
existingImageCount: number,
|
||||
): number {
|
||||
if (!layout) {
|
||||
return 0;
|
||||
}
|
||||
const available = existingFactIndexes
|
||||
? [...existingFactIndexes]
|
||||
: Array.from({ length: existingImageCount }, () => null);
|
||||
let missing = 0;
|
||||
for (const slot of layout.slots) {
|
||||
if (slot.kind !== "inline") {
|
||||
continue;
|
||||
}
|
||||
const exactIndex =
|
||||
slot.factIndex === undefined
|
||||
? available.length > 0
|
||||
? 0
|
||||
: -1
|
||||
: available.findIndex((factIndex) => factIndex === slot.factIndex);
|
||||
const matchIndex = exactIndex >= 0 ? exactIndex : available.indexOf(null);
|
||||
if (matchIndex >= 0) {
|
||||
available.splice(matchIndex, 1);
|
||||
} else {
|
||||
missing++;
|
||||
}
|
||||
}
|
||||
return missing;
|
||||
}
|
||||
|
||||
export function readPersistedImageBlockFactIndexes(
|
||||
message: AgentMessage,
|
||||
): ImageFactIndex[] | undefined {
|
||||
|
||||
@@ -5,8 +5,33 @@ import { describe, expect, it } from "vitest";
|
||||
import { buildInboundMediaNoteProjection } from "../../../auto-reply/media-note.js";
|
||||
import { readRuntimePromptImageFactIndexes } from "../../../media/runtime-prompt-image-provenance.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
|
||||
import { prepareEmbeddedAttemptPromptExecution } from "./attempt-prompt-submit.js";
|
||||
|
||||
async function preparePluginHarnessPromptImages(params: {
|
||||
runParams: Parameters<typeof prepareEmbeddedAttemptPromptExecution>[0]["attempt"];
|
||||
runtime: {
|
||||
workspaceDir: string;
|
||||
model: Parameters<typeof prepareEmbeddedAttemptPromptExecution>[0]["attempt"]["model"];
|
||||
};
|
||||
pluginHarnessOwnsTransport: boolean;
|
||||
}) {
|
||||
if (!params.pluginHarnessOwnsTransport) {
|
||||
return {
|
||||
images: params.runParams.images,
|
||||
imageOrder: params.runParams.imageOrder,
|
||||
media: params.runParams.media,
|
||||
};
|
||||
}
|
||||
const result = await prepareEmbeddedAttemptPromptExecution({
|
||||
attempt: { ...params.runParams, model: params.runtime.model },
|
||||
effectiveWorkspace: params.runtime.workspaceDir,
|
||||
effectiveFsWorkspaceOnly: false,
|
||||
prompt: "",
|
||||
skipPromptSubmission: false,
|
||||
pluginHarness: true,
|
||||
});
|
||||
return { images: result.images, imageOrder: result.imageOrder, media: result.media };
|
||||
}
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACXBIWXMAAAsTAAALEwEAmpwYAAAADUlEQVR4nGP4////KwAJ5gPoxLp9owAAAABJRU5ErkJggg==";
|
||||
@@ -33,7 +58,7 @@ describe("plugin harness prompt media", () => {
|
||||
},
|
||||
pluginHarnessOwnsTransport: true,
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]),
|
||||
).resolves.toEqual({ images: undefined, imageOrder: undefined, media: undefined });
|
||||
).resolves.toEqual({ images: [], imageOrder: undefined, media: undefined });
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -103,9 +128,8 @@ describe("plugin harness prompt media", () => {
|
||||
expect(result.images ?? []).toHaveLength(testCase.expectedImages);
|
||||
if (testCase.expectedImages > 0) {
|
||||
expect(result.images?.[0]?.mimeType).toBe("image/png");
|
||||
} else {
|
||||
expect(result.media).toEqual(media);
|
||||
}
|
||||
expect(result.media).toBeUndefined();
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
@@ -162,24 +186,9 @@ describe("plugin harness prompt media", () => {
|
||||
]);
|
||||
expect(readRuntimePromptImageFactIndexes(result.images ?? [])).toEqual([0]);
|
||||
expect(result.imageOrder).toEqual(["inline"]);
|
||||
expect(result.media?.[0]).toMatchObject({ contentType: "image/png", kind: "image" });
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[0]).not.toHaveProperty("url");
|
||||
expect(result.media?.[1]).toMatchObject(documentFact);
|
||||
|
||||
const serialized = JSON.stringify(result);
|
||||
const restored = JSON.parse(serialized) as typeof result;
|
||||
const replay = await detectAndLoadPromptImages({
|
||||
prompt: "",
|
||||
media: restored.media,
|
||||
workspaceDir,
|
||||
model: { input: ["text", "image"] },
|
||||
existingImages: restored.images,
|
||||
imageOrder: restored.imageOrder,
|
||||
});
|
||||
expect(replay.failedMediaCount).toBe(0);
|
||||
expect(replay.images).toEqual(result.images);
|
||||
expect(replay.imageFactIndexes).toEqual([0]);
|
||||
expect(result.media).toBeUndefined();
|
||||
expect(JSON.stringify(result)).not.toContain(imagePath);
|
||||
expect(structuredClone(result).images).toEqual(result.images);
|
||||
} finally {
|
||||
envSnapshot.restore();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
@@ -308,13 +317,7 @@ describe("plugin harness prompt media", () => {
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result.images).toEqual([]);
|
||||
expect(result.media?.[0]).toMatchObject({
|
||||
contentType: "image/png",
|
||||
kind: "image",
|
||||
hydrationSuppressed: true,
|
||||
});
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[0]).not.toHaveProperty("url");
|
||||
expect(result.media).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains layout-derived suppression after plugin host materialization", async () => {
|
||||
@@ -357,9 +360,7 @@ describe("plugin harness prompt media", () => {
|
||||
|
||||
expect(result.images).toEqual([inlineImage]);
|
||||
expect(result.imageOrder).toEqual(["inline"]);
|
||||
expect(result.media?.[0]).toMatchObject({ kind: "image", hydrationSuppressed: true });
|
||||
expect(result.media?.[1]).toMatchObject({ kind: "image" });
|
||||
expect(result.media?.[1]).not.toHaveProperty("hydrationSuppressed");
|
||||
expect(result.media).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps unsupported native images as aligned type-only facts", async () => {
|
||||
@@ -383,10 +384,7 @@ describe("plugin harness prompt media", () => {
|
||||
} as unknown as Parameters<typeof preparePluginHarnessPromptImages>[0]);
|
||||
|
||||
expect(result.images).toEqual([]);
|
||||
expect(result.media?.[0]).toMatchObject({ contentType: "image/png" });
|
||||
expect(result.media?.[0]).not.toHaveProperty("path");
|
||||
expect(result.media?.[1]).toMatchObject({ kind: "image" });
|
||||
expect(result.media?.[1]).not.toHaveProperty("path");
|
||||
expect(result.media).toBeUndefined();
|
||||
});
|
||||
|
||||
it("leaves facts untouched when the native harness owns transport", async () => {
|
||||
|
||||
@@ -15,14 +15,15 @@ import {
|
||||
} from "../../tools/gateway-caller-context.js";
|
||||
import type { SystemAgentToolOptions } from "../../tools/system-agent-tool.js";
|
||||
import { prepareExecApprovalContinuationForAttempt } from "./attempt-exec-approval-continuation.js";
|
||||
import { prepareEmbeddedAttemptPromptExecution } from "./attempt-prompt-submit.js";
|
||||
import { applyResolvedToolPromptFinalizer } from "./attempt-prompt-support.js";
|
||||
import { resolveAttemptWorkspaceSandbox } from "./attempt-setup.js";
|
||||
import { runEmbeddedAttemptWithBackend } from "./backend.js";
|
||||
import {
|
||||
EMBEDDED_RUN_LANE_HEARTBEAT_MS,
|
||||
EMBEDDED_RUN_LANE_TIMEOUT_GRACE_MS,
|
||||
} from "./lane-runtime.js";
|
||||
import type { RunEmbeddedAgentParams } from "./params.js";
|
||||
import { preparePluginHarnessPromptImages } from "./plugin-harness-prompt-images.js";
|
||||
import { resolveSkillWorkshopAttemptParams } from "./skill-workshop-attempt-params.js";
|
||||
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptTrajectoryRecorder } from "./types.js";
|
||||
|
||||
@@ -190,11 +191,26 @@ export async function dispatchEmbeddedRunAttempt(input: {
|
||||
modelMaxTokens: runtime.model.maxTokens,
|
||||
userTurnTranscriptRecorder: params.userTurnTranscriptRecorder,
|
||||
});
|
||||
const promptMedia = await preparePluginHarnessPromptImages({
|
||||
runParams: params,
|
||||
runtime,
|
||||
pluginHarnessOwnsTransport: control.pluginHarnessOwnsTransport,
|
||||
});
|
||||
const promptMedia = control.pluginHarnessOwnsTransport
|
||||
? await (async () => {
|
||||
const workspace = await resolveAttemptWorkspaceSandbox({
|
||||
...params,
|
||||
cwd: undefined,
|
||||
sessionId: runtime.sessionId,
|
||||
sessionKey: runtime.sessionKey,
|
||||
workspaceDir: runtime.workspaceDir,
|
||||
});
|
||||
return await prepareEmbeddedAttemptPromptExecution({
|
||||
attempt: { ...params, model: runtime.model },
|
||||
effectiveFsWorkspaceOnly: workspace.effectiveFsWorkspaceOnly,
|
||||
effectiveWorkspace: workspace.effectiveWorkspace,
|
||||
prompt: "",
|
||||
sandbox: workspace.sandbox,
|
||||
skipPromptSubmission: false,
|
||||
pluginHarness: true,
|
||||
});
|
||||
})()
|
||||
: { images: params.images, imageOrder: params.imageOrder, media: params.media };
|
||||
// Plugin harnesses own their tool materialization, so the host cannot attest
|
||||
// a message tool. Finalize conservatively instead of leaking phantom guidance.
|
||||
const pluginHarnessPrompt =
|
||||
|
||||
@@ -1,255 +0,0 @@
|
||||
import type { ApiRegistry } from "@openclaw/ai";
|
||||
// Verifies the Google simple-completion wrapper and thinking-payload sanitizer hook.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { Model } from "../llm/types.js";
|
||||
|
||||
const streamSimple = vi.fn();
|
||||
const sanitizeGoogleThinkingPayload = vi.fn();
|
||||
const ensureCustomApiRegistered = vi.fn();
|
||||
const apiRegistry = {
|
||||
getApiProvider: vi.fn(() => ({ streamSimple })),
|
||||
} as unknown as ApiRegistry;
|
||||
|
||||
vi.mock("../llm/providers/stream-wrappers/google-thinking-payload.js", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js")
|
||||
>("../llm/providers/stream-wrappers/google-thinking-payload.js");
|
||||
return {
|
||||
...actual,
|
||||
sanitizeGoogleThinkingPayload,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./custom-api-registry.js", () => ({
|
||||
ensureCustomApiRegistered,
|
||||
}));
|
||||
|
||||
const { prepareGoogleSimpleCompletionModel } = await import("./google-simple-completion-stream.js");
|
||||
|
||||
const GOOGLE_SIMPLE_COMPLETION_API = "openclaw-google-generative-ai-simple";
|
||||
|
||||
// Mirrors the provider catalog shape closely enough for wrapper registration
|
||||
// without pulling live Google model discovery into unit tests.
|
||||
function makeGoogleModel(
|
||||
id = "gemini-flash-latest",
|
||||
overrides: Partial<Model<"google-generative-ai">> = {},
|
||||
): Model<"google-generative-ai"> {
|
||||
return {
|
||||
id,
|
||||
name: id,
|
||||
api: "google-generative-ai",
|
||||
provider: "google",
|
||||
baseUrl: "https://generativelanguage.googleapis.com",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 1_000_000,
|
||||
maxTokens: 8192,
|
||||
headers: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("prepareGoogleSimpleCompletionModel", () => {
|
||||
beforeEach(() => {
|
||||
streamSimple.mockReset();
|
||||
sanitizeGoogleThinkingPayload.mockReset();
|
||||
ensureCustomApiRegistered.mockReset();
|
||||
streamSimple.mockImplementation((_model, _context, options) => {
|
||||
const payload = {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingBudget: -1 },
|
||||
},
|
||||
};
|
||||
options?.onPayload?.(payload, _model);
|
||||
return { content: [{ type: "text", text: "ok" }], payload };
|
||||
});
|
||||
});
|
||||
|
||||
it("returns non-Google models unchanged", () => {
|
||||
const model = {
|
||||
...makeGoogleModel("gpt-5"),
|
||||
api: "openai-responses",
|
||||
} as unknown as Model<"openai-responses">;
|
||||
|
||||
const result = prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
|
||||
expect(result).toBe(model);
|
||||
expect(ensureCustomApiRegistered).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("registers an OpenClaw-owned Google simple-completion api alias", () => {
|
||||
const model = makeGoogleModel();
|
||||
|
||||
const result = prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
|
||||
expect(result).toEqual({
|
||||
...model,
|
||||
api: GOOGLE_SIMPLE_COMPLETION_API,
|
||||
});
|
||||
expect(ensureCustomApiRegistered).toHaveBeenCalledTimes(1);
|
||||
expect(ensureCustomApiRegistered.mock.calls[0]?.[0]).toBe(apiRegistry);
|
||||
expect(ensureCustomApiRegistered.mock.calls[0]?.[1]).toBe(GOOGLE_SIMPLE_COMPLETION_API);
|
||||
});
|
||||
|
||||
it.each(["off", "low", "medium", "high", "adaptive"] as const)(
|
||||
"sanitizes outbound thinking payload for gemini-flash-latest with reasoning=%s",
|
||||
async (reasoning) => {
|
||||
const model = makeGoogleModel();
|
||||
const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as (
|
||||
...args: unknown[]
|
||||
) => unknown;
|
||||
|
||||
// The custom alias must unwrap to the real Google API before delegating,
|
||||
// then sanitize the exact outbound payload produced by streamSimple.
|
||||
await streamFn(wrapped, { messages: [] }, { apiKey: "key", reasoning });
|
||||
|
||||
expect(streamSimple).toHaveBeenCalledTimes(1);
|
||||
expect(streamSimple.mock.calls[0]?.[0]).toEqual({
|
||||
...model,
|
||||
api: "google-generative-ai",
|
||||
});
|
||||
expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({
|
||||
payload: {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingBudget: -1 },
|
||||
},
|
||||
},
|
||||
modelId: "gemini-flash-latest",
|
||||
thinkingLevel: reasoning,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("returns the sanitizer-mutated payload shape", async () => {
|
||||
sanitizeGoogleThinkingPayload.mockImplementationOnce((args: { payload: unknown }) => {
|
||||
const payload = args.payload as {
|
||||
generationConfig: { thinkingConfig: Record<string, unknown> };
|
||||
};
|
||||
delete payload.generationConfig.thinkingConfig.thinkingBudget;
|
||||
payload.generationConfig.thinkingConfig.thinkingLevel = "MINIMAL";
|
||||
});
|
||||
const model = makeGoogleModel();
|
||||
prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as (
|
||||
...args: unknown[]
|
||||
) => unknown;
|
||||
|
||||
const result = await streamFn(model, { messages: [] }, { apiKey: "key", reasoning: "off" });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
payload: {
|
||||
generationConfig: {
|
||||
thinkingConfig: {
|
||||
thinkingLevel: "MINIMAL",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(
|
||||
(
|
||||
result as {
|
||||
payload: { generationConfig: { thinkingConfig: Record<string, unknown> } };
|
||||
}
|
||||
).payload.generationConfig.thinkingConfig,
|
||||
).not.toHaveProperty("thinkingBudget");
|
||||
});
|
||||
|
||||
it("removes disabled thinking budget for Gemma 4 when reasoning is omitted", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js")
|
||||
>("../llm/providers/stream-wrappers/google-thinking-payload.js");
|
||||
sanitizeGoogleThinkingPayload.mockImplementationOnce(actual.sanitizeGoogleThinkingPayload);
|
||||
streamSimple.mockImplementationOnce((_model, _context, options) => {
|
||||
const payload = {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingBudget: 0 },
|
||||
},
|
||||
};
|
||||
options?.onPayload?.(payload, _model);
|
||||
return { content: [{ type: "text", text: "ok" }], payload };
|
||||
});
|
||||
const model = makeGoogleModel("gemma-4-26b-a4b-it");
|
||||
const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as (
|
||||
...args: unknown[]
|
||||
) => unknown;
|
||||
|
||||
const result = await streamFn(wrapped, { messages: [] }, { apiKey: "key" });
|
||||
|
||||
expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({
|
||||
payload: {
|
||||
generationConfig: {},
|
||||
},
|
||||
modelId: "gemma-4-26b-a4b-it",
|
||||
thinkingLevel: undefined,
|
||||
});
|
||||
expect(
|
||||
(
|
||||
result as {
|
||||
payload: { generationConfig: Record<string, unknown> };
|
||||
}
|
||||
).payload.generationConfig,
|
||||
).not.toHaveProperty("thinkingConfig");
|
||||
});
|
||||
|
||||
it.each(["xhigh", "max"] as const)(
|
||||
"preserves clamped-off intent in the final Gemini 3 payload for reasoning=%s",
|
||||
async (reasoning) => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("../llm/providers/stream-wrappers/google-thinking-payload.js")
|
||||
>("../llm/providers/stream-wrappers/google-thinking-payload.js");
|
||||
sanitizeGoogleThinkingPayload.mockImplementationOnce(actual.sanitizeGoogleThinkingPayload);
|
||||
streamSimple.mockImplementationOnce((_model, _context, options) => {
|
||||
const payload = {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingLevel: "MINIMAL" },
|
||||
},
|
||||
};
|
||||
options?.onPayload?.(payload, _model);
|
||||
return { content: [{ type: "text", text: "ok" }], payload };
|
||||
});
|
||||
const model = makeGoogleModel("gemini-3-flash-preview", {
|
||||
thinkingLevelMap: {
|
||||
minimal: null,
|
||||
low: null,
|
||||
medium: null,
|
||||
high: null,
|
||||
xhigh: null,
|
||||
max: null,
|
||||
},
|
||||
});
|
||||
const wrapped = prepareGoogleSimpleCompletionModel(apiRegistry, model);
|
||||
const streamFn = ensureCustomApiRegistered.mock.calls[0]?.[2] as (
|
||||
...args: unknown[]
|
||||
) => unknown;
|
||||
|
||||
const result = await streamFn(wrapped, { messages: [] }, { apiKey: "key", reasoning });
|
||||
|
||||
expect(sanitizeGoogleThinkingPayload).toHaveBeenCalledWith({
|
||||
payload: {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingLevel: "MINIMAL" },
|
||||
},
|
||||
},
|
||||
modelId: "gemini-3-flash-preview",
|
||||
thinkingLevel: "off",
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
payload: {
|
||||
generationConfig: {
|
||||
thinkingConfig: { thinkingLevel: "MINIMAL" },
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(
|
||||
(
|
||||
result as {
|
||||
payload: { generationConfig: { thinkingConfig: Record<string, unknown> } };
|
||||
}
|
||||
).payload.generationConfig.thinkingConfig,
|
||||
).not.toHaveProperty("includeThoughts");
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,83 +0,0 @@
|
||||
import type { ApiRegistry } from "@openclaw/ai";
|
||||
/**
|
||||
* Google simple-completion stream adapter.
|
||||
*
|
||||
* This registers a patched Google stream API that keeps the normal Google
|
||||
* backend but sanitizes unsupported thinking payload options for simple models.
|
||||
*/
|
||||
import { clampThinkingLevel } from "@openclaw/ai/internal/runtime";
|
||||
import type { StreamFn } from "@openclaw/llm-core";
|
||||
import {
|
||||
sanitizeGoogleThinkingPayload,
|
||||
type GoogleThinkingInputLevel,
|
||||
} from "../llm/providers/stream-wrappers/google-thinking-payload.js";
|
||||
import { streamWithPayloadPatch } from "../llm/providers/stream-wrappers/stream-payload-utils.js";
|
||||
import type { Api, Model, ModelThinkingLevel } from "../llm/types.js";
|
||||
import { ensureCustomApiRegistered } from "./custom-api-registry.js";
|
||||
|
||||
/** Custom API id for the Google simple-completion stream adapter. */
|
||||
const GOOGLE_SIMPLE_COMPLETION_API: Api = "openclaw-google-generative-ai-simple";
|
||||
|
||||
const SOURCE_API: Api = "google-generative-ai";
|
||||
|
||||
function resolveGoogleSimpleThinkingLevel(
|
||||
model: Model,
|
||||
reasoning: unknown,
|
||||
): GoogleThinkingInputLevel | undefined {
|
||||
switch (reasoning) {
|
||||
case "adaptive":
|
||||
return reasoning;
|
||||
case "off":
|
||||
case "minimal":
|
||||
case "low":
|
||||
case "medium":
|
||||
case "high":
|
||||
case "max":
|
||||
case "xhigh":
|
||||
return clampThinkingLevel(model, reasoning as ModelThinkingLevel);
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function buildGoogleSimpleCompletionStreamFn(registry: ApiRegistry): StreamFn {
|
||||
return (model, context, options) => {
|
||||
const googleModel: Model = { ...model, api: SOURCE_API };
|
||||
const sourceProvider = registry.getApiProvider(SOURCE_API);
|
||||
if (!sourceProvider) {
|
||||
throw new Error(`No API provider registered for api: ${SOURCE_API}`);
|
||||
}
|
||||
return streamWithPayloadPatch(
|
||||
sourceProvider.streamSimple as StreamFn,
|
||||
googleModel,
|
||||
context,
|
||||
options,
|
||||
(payload) => {
|
||||
sanitizeGoogleThinkingPayload({
|
||||
payload,
|
||||
modelId: model.id,
|
||||
thinkingLevel: resolveGoogleSimpleThinkingLevel(
|
||||
googleModel,
|
||||
(options as { reasoning?: unknown } | undefined)?.reasoning,
|
||||
),
|
||||
});
|
||||
},
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
/** Rewrites Google generative-ai models to the simple-completion adapter when needed. */
|
||||
export function prepareGoogleSimpleCompletionModel<TApi extends Api>(
|
||||
registry: ApiRegistry,
|
||||
model: Model<TApi>,
|
||||
): Model {
|
||||
if (model.api !== SOURCE_API) {
|
||||
return model;
|
||||
}
|
||||
ensureCustomApiRegistered(
|
||||
registry,
|
||||
GOOGLE_SIMPLE_COMPLETION_API,
|
||||
buildGoogleSimpleCompletionStreamFn(registry),
|
||||
);
|
||||
return { ...model, api: GOOGLE_SIMPLE_COMPLETION_API };
|
||||
}
|
||||
@@ -292,30 +292,43 @@ export function isGenericBinaryMediaContentType(contentType?: string | null): bo
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns whether a fact can produce native image input. */
|
||||
export function isImageMediaFact(fact: MediaFactInput): boolean {
|
||||
function classifyMediaFact(fact: MediaFactInput): MediaKind | undefined {
|
||||
if (fact.kind && fact.kind !== "unknown") {
|
||||
return fact.kind === "image" || fact.kind === "sticker";
|
||||
return fact.kind;
|
||||
}
|
||||
const normalizedContentType = normalizeMimeType(fact.contentType);
|
||||
if (normalizedContentType && !isGenericBinaryMediaContentType(normalizedContentType)) {
|
||||
const mimeKind = kindFromMime(normalizedContentType);
|
||||
if (mimeKind) {
|
||||
return mimeKind === "image";
|
||||
return mimeKind;
|
||||
}
|
||||
// Legacy channel-mode projections persist bare image or sticker kind as MediaType.
|
||||
return normalizedContentType === "image" || normalizedContentType === "sticker";
|
||||
return LEGACY_MEDIA_KINDS.has(normalizedContentType as MediaKind)
|
||||
? (normalizedContentType as MediaKind)
|
||||
: undefined;
|
||||
}
|
||||
const pathValue = normalizeOptionalString(fact.path) ?? normalizeOptionalString(fact.url);
|
||||
const inferredMime = mimeTypeFromFilePath(pathValue);
|
||||
if (inferredMime === "image/svg+xml") {
|
||||
return false;
|
||||
return undefined;
|
||||
}
|
||||
if (kindFromMime(inferredMime) === "image") {
|
||||
return true;
|
||||
const inferredKind = kindFromMime(inferredMime);
|
||||
if (inferredKind) {
|
||||
return inferredKind;
|
||||
}
|
||||
const extension = getFileExtension(pathValue);
|
||||
return extension === ".tif" || extension === ".tiff";
|
||||
return extension === ".tif" || extension === ".tiff" ? "image" : undefined;
|
||||
}
|
||||
|
||||
/** Returns whether a fact can produce native image input. */
|
||||
export function isImageMediaFact(fact: MediaFactInput): boolean {
|
||||
const kind = classifyMediaFact(fact);
|
||||
return kind === "image" || kind === "sticker";
|
||||
}
|
||||
|
||||
/** Returns whether a fact can produce native video input. */
|
||||
export function isVideoMediaFact(fact: MediaFactInput): boolean {
|
||||
return classifyMediaFact(fact) === "video";
|
||||
}
|
||||
|
||||
type MediaFactDefaults<TInput extends MediaFactInput = MediaFactInput> = {
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
* Public SDK subpath for LLM streaming, model utils, and validation.
|
||||
*/
|
||||
export type { ApiProvider } from "@openclaw/ai";
|
||||
export { resolveProviderContext } from "../../packages/ai/src/provider-types.js";
|
||||
export type {
|
||||
ProviderContext,
|
||||
ProviderModel,
|
||||
ProviderStreamFunction,
|
||||
ProviderStreamOptions as ProviderCallStreamOptions,
|
||||
VideoContent,
|
||||
} from "../../packages/ai/src/provider-types.js";
|
||||
export {
|
||||
calculateCost,
|
||||
clampThinkingLevel,
|
||||
|
||||
@@ -20,6 +20,7 @@ const config = {
|
||||
diagnostics: "packages/ai/src/utils/diagnostics.ts",
|
||||
"event-stream": "packages/ai/src/utils/event-stream.ts",
|
||||
types: "packages/ai/src/types.ts",
|
||||
"provider-types": "packages/ai/src/provider-types.ts",
|
||||
validation: "packages/ai/src/validation.ts",
|
||||
"internal/anthropic": "packages/ai/src/internal/anthropic.ts",
|
||||
"internal/openai": "packages/ai/src/internal/openai.ts",
|
||||
|
||||
Reference in New Issue
Block a user