From 49302fcb7df52cf709e22b355f249659e1ad7aa8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 5 Jul 2026 02:53:51 -0700 Subject: [PATCH] fix: DashScope and Moonshot endpoints misclassified as custom when the provider plugin is not installed (#100125) Externalized provider plugins (qwen, moonshot, zai, deepseek, groq, cerebras, chutes) are excluded from dist packaging, so their manifest-declared endpoint classes were invisible to installed gateways and built source checkouts: DashScope/Moonshot-class base URLs classified as custom, breaking image prompt placement, streaming-usage and developer-role compat defaults, and deterministically failing image.test.ts / model-compat DashScope cases after pnpm build. The official external provider catalog now mirrors each plugin's providerEndpoints and provider attribution appends catalog metadata after installed/bundled manifests (first match wins; repo-bundled catalog only, hosted feeds never influence classification). Includes a catalog-manifest mirror contract test and a dist-simulation regression test. --- docs/plugins/manifest.md | 6 + .../official-external-provider-catalog.json | 82 +++++++++++ ...ider-attribution.catalog-endpoints.test.ts | 80 ++++++++++ src/agents/provider-attribution.ts | 9 ++ .../official-external-plugin-catalog.ts | 6 + ...ficial-external-provider-endpoints.test.ts | 137 ++++++++++++++++++ .../official-external-provider-endpoints.ts | 40 +++++ 7 files changed, 360 insertions(+) create mode 100644 src/agents/provider-attribution.catalog-endpoints.test.ts create mode 100644 src/plugins/official-external-provider-endpoints.test.ts create mode 100644 src/plugins/official-external-provider-endpoints.ts diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index bd7c3bcae8e3..9ec6bd23e3e3 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -940,6 +940,12 @@ Provider fields: Use `providerEndpoints` for endpoint classification that generic request policy must know before provider runtime loads. Core still owns the meaning of each `endpointClass`; plugin manifests own the host and base URL metadata. +Officially externalized provider plugins are excluded from the core dist, so +their manifests are invisible until installed. Their `providerEndpoints` must +also be mirrored in `scripts/lib/official-external-provider-catalog.json` so +endpoint classification keeps working without the plugin; a contract test +enforces the mirror. + Endpoint fields: | Field | Type | What it means | diff --git a/scripts/lib/official-external-provider-catalog.json b/scripts/lib/official-external-provider-catalog.json index 1dbb912d0652..8f0fb94e0de2 100644 --- a/scripts/lib/official-external-provider-catalog.json +++ b/scripts/lib/official-external-provider-catalog.json @@ -151,6 +151,14 @@ "id": "cerebras", "label": "Cerebras" }, + "providerEndpoints": [ + { + "endpointClass": "cerebras-native", + "hosts": [ + "api.cerebras.ai" + ] + } + ], "providers": [ { "id": "cerebras", @@ -200,6 +208,14 @@ "id": "chutes", "label": "Chutes" }, + "providerEndpoints": [ + { + "endpointClass": "chutes-native", + "hosts": [ + "llm.chutes.ai" + ] + } + ], "providers": [ { "id": "chutes", @@ -400,6 +416,14 @@ "id": "deepinfra", "label": "DeepInfra" }, + "providerEndpoints": [ + { + "endpointClass": "deepinfra-native", + "hosts": [ + "api.deepinfra.com" + ] + } + ], "providers": [ { "id": "deepinfra", @@ -467,6 +491,14 @@ "id": "deepseek", "label": "DeepSeek" }, + "providerEndpoints": [ + { + "endpointClass": "deepseek-native", + "hosts": [ + "api.deepseek.com" + ] + } + ], "providers": [ { "id": "deepseek", @@ -516,6 +548,14 @@ "id": "gmi", "label": "GMI Cloud" }, + "providerEndpoints": [ + { + "endpointClass": "gmi-native", + "hosts": [ + "api.gmi-serving.com" + ] + } + ], "providers": [ { "id": "gmi", @@ -559,6 +599,14 @@ "id": "groq", "label": "Groq" }, + "providerEndpoints": [ + { + "endpointClass": "groq-native", + "hosts": [ + "api.groq.com" + ] + } + ], "providers": [ { "id": "groq", @@ -806,6 +854,23 @@ "id": "qwen", "label": "Qwen Cloud" }, + "providerEndpoints": [ + { + "endpointClass": "modelstudio-native", + "baseUrls": [ + "https://coding-intl.dashscope.aliyuncs.com/v1", + "https://coding.dashscope.aliyuncs.com/v1", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" + ] + }, + { + "endpointClass": "qwen-portal-native", + "baseUrls": [ + "https://portal.qwen.ai/v1" + ] + } + ], "providers": [ { "id": "qwen", @@ -1018,6 +1083,15 @@ "id": "moonshot", "label": "Moonshot" }, + "providerEndpoints": [ + { + "endpointClass": "moonshot-native", + "baseUrls": [ + "https://api.moonshot.ai/v1", + "https://api.moonshot.cn/v1" + ] + } + ], "providers": [ { "id": "moonshot", @@ -1267,6 +1341,14 @@ "id": "zai", "label": "Z.AI" }, + "providerEndpoints": [ + { + "endpointClass": "zai-native", + "hosts": [ + "api.z.ai" + ] + } + ], "providers": [ { "id": "zai", diff --git a/src/agents/provider-attribution.catalog-endpoints.test.ts b/src/agents/provider-attribution.catalog-endpoints.test.ts new file mode 100644 index 000000000000..9287243d6f4d --- /dev/null +++ b/src/agents/provider-attribution.catalog-endpoints.test.ts @@ -0,0 +1,80 @@ +// Verifies catalog-backed endpoint classification for externalized official providers. +import { describe, expect, it, vi } from "vitest"; + +// Simulates a built dist tree: externalized provider plugins (qwen, moonshot, +// zai, ...) are excluded from dist packaging, so no plugin manifest supplies +// their endpoint metadata. Classification must come from the bundled catalog. +// The single conflicting manifest entry proves installed manifests stay +// authoritative over catalog metadata (first match wins). +vi.mock("../plugins/manifest-metadata-scan.js", () => ({ + listOpenClawPluginManifestMetadata: () => [ + { + pluginDir: "installed-conflict-fixture", + manifest: { + providerEndpoints: [ + { endpointClass: "openai-public", hosts: ["coding.dashscope.aliyuncs.com"] }, + ], + }, + origin: "installed", + }, + ], +})); + +import { + resolveProviderEndpoint, + resolveProviderRequestCapabilities, +} from "./provider-attribution.js"; + +describe("catalog-backed provider endpoint classification", () => { + it.each([ + ["https://dashscope.aliyuncs.com/compatible-mode/v1", "modelstudio-native"], + ["https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "modelstudio-native"], + ["https://coding-intl.dashscope.aliyuncs.com/v1", "modelstudio-native"], + ["https://api.moonshot.ai/v1", "moonshot-native"], + ["https://api.moonshot.cn/v1", "moonshot-native"], + ["https://api.z.ai/api/coding/paas/v4", "zai-native"], + ["https://api.deepseek.com", "deepseek-native"], + ["https://api.groq.com/openai/v1", "groq-native"], + ["https://api.cerebras.ai/v1", "cerebras-native"], + ["https://llm.chutes.ai/v1", "chutes-native"], + ])("classifies %s as %s without an installed plugin manifest", (baseUrl, endpointClass) => { + expect(resolveProviderEndpoint(baseUrl).endpointClass).toBe(endpointClass); + }); + + it("resolves DashScope request capabilities from catalog metadata", () => { + // Image describe placement and streaming-usage compat both key off this + // classification; see shouldPlaceImagePromptInUserContent and + // normalizeModelCompat regressions when qwen was externalized. + const capabilities = resolveProviderRequestCapabilities({ + provider: "qwen", + api: "openai-completions", + baseUrl: "https://dashscope.aliyuncs.com/compatible-mode/v1", + capability: "image", + transport: "media-understanding", + }); + expect(capabilities.endpointClass).toBe("modelstudio-native"); + expect(capabilities.supportsNativeStreamingUsageCompat).toBe(true); + expect(capabilities.isKnownNativeEndpoint).toBe(true); + }); + + it("prefers installed plugin manifest endpoints over catalog metadata", () => { + expect(resolveProviderEndpoint("https://coding.dashscope.aliyuncs.com/v1").endpointClass).toBe( + "openai-public", + ); + }); + + it("keeps unknown hosts classified as custom", () => { + expect(resolveProviderEndpoint("https://proxy.example.com/v1").endpointClass).toBe("custom"); + }); + + it("drops catalog endpoint classes core does not recognize", () => { + // qwen-portal-native, deepinfra-native, and gmi-native are mirrored + // faithfully from their manifests but are not core ProviderEndpointClass + // members, so they must stay inert (same filtering as installed manifests). + expect(resolveProviderEndpoint("https://portal.qwen.ai/v1").endpointClass).toBe("custom"); + expect(resolveProviderEndpoint("https://api.deepinfra.com/v1/openai").endpointClass).toBe( + "custom", + ); + expect(resolveProviderEndpoint("https://api.gmi-serving.com/v1").endpointClass).toBe("custom"); + }); +}); diff --git a/src/agents/provider-attribution.ts b/src/agents/provider-attribution.ts index 1635f22bc9d4..e2fa76a9a4dc 100644 --- a/src/agents/provider-attribution.ts +++ b/src/agents/provider-attribution.ts @@ -12,6 +12,7 @@ import { } from "@openclaw/normalization-core/string-coerce"; import { normalizeTrimmedStringList } from "@openclaw/normalization-core/string-normalization"; import { listOpenClawPluginManifestMetadata } from "../plugins/manifest-metadata-scan.js"; +import { listOfficialExternalProviderEndpointManifests } from "../plugins/official-external-provider-endpoints.js"; import { asBoolean } from "../utils/boolean.js"; import type { RuntimeVersionEnv } from "../version.js"; import { resolveRuntimeServiceVersion } from "../version.js"; @@ -331,6 +332,14 @@ function collectManifestProviderEndpoints(): ManifestProviderEndpointCacheEntry[ for (const { manifest } of listOpenClawPluginManifestMetadata()) { entries.push(...readManifestProviderEndpoints(manifest)); } + // Externalized official provider plugins are excluded from dist builds, so + // their manifests are invisible unless installed. The bundled catalog keeps + // their endpoint classes resolvable: users can point a generic provider key + // at DashScope/Moonshot/... and still need native request policy. Matching + // is first-wins, so installed/bundled manifests stay authoritative. + for (const manifest of listOfficialExternalProviderEndpointManifests()) { + entries.push(...readManifestProviderEndpoints(manifest)); + } return entries; } diff --git a/src/plugins/official-external-plugin-catalog.ts b/src/plugins/official-external-plugin-catalog.ts index b815bb7be167..cb9118f9c114 100644 --- a/src/plugins/official-external-plugin-catalog.ts +++ b/src/plugins/official-external-plugin-catalog.ts @@ -11,6 +11,7 @@ import { isRecord } from "../utils.js"; import type { PluginManifestChannelConfig, PluginManifestContracts, + PluginManifestProviderEndpoint, PluginPackageInstall, } from "./manifest.js"; @@ -81,6 +82,11 @@ export type OfficialExternalPluginCatalogManifest = { envVars?: readonly string[]; }; providers?: readonly OfficialExternalProviderCatalogProvider[]; + /** + * Mirrors the plugin manifest's providerEndpoints so endpoint classification + * keeps working when the plugin is not installed (dist excludes it). + */ + providerEndpoints?: readonly PluginManifestProviderEndpoint[]; webSearchProviders?: readonly OfficialExternalWebSearchProvider[]; install?: PluginPackageInstall & { sourceRef?: string }; contracts?: PluginManifestContracts; diff --git a/src/plugins/official-external-provider-endpoints.test.ts b/src/plugins/official-external-provider-endpoints.test.ts new file mode 100644 index 000000000000..963aab19ec5d --- /dev/null +++ b/src/plugins/official-external-provider-endpoints.test.ts @@ -0,0 +1,137 @@ +// Guards the catalog mirror of externalized provider plugin endpoint metadata. +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { describe, expect, it } from "vitest"; +import rootPackageJson from "../../package.json" with { type: "json" }; +import officialExternalProviderCatalog from "../../scripts/lib/official-external-provider-catalog.json" with { type: "json" }; +import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; +import { listOfficialExternalProviderEndpointManifests } from "./official-external-provider-endpoints.js"; + +const repoRoot = fileURLToPath(new URL("../..", import.meta.url)); + +type ExtensionManifestRecord = { + dirName: string; + manifest: Record; +}; + +function listExtensionManifests(): ExtensionManifestRecord[] { + const extensionsDir = path.join(repoRoot, "extensions"); + const records: ExtensionManifestRecord[] = []; + for (const entry of fs.readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const manifestPath = path.join(extensionsDir, entry.name, "openclaw.plugin.json"); + if (!fs.existsSync(manifestPath)) { + continue; + } + const manifest = parseJsonWithJson5Fallback(fs.readFileSync(manifestPath, "utf8")); + if (isRecord(manifest)) { + records.push({ dirName: entry.name, manifest }); + } + } + return records; +} + +// Dist packaging is what makes these plugins invisible to bundled discovery, +// so the package excludes are the source of truth for which manifests must be +// mirrored into the catalog. +const distExcludedExtensionDirs = new Set( + (rootPackageJson.files ?? []).flatMap((entry) => { + const match = /^!dist\/extensions\/([^/*]+)\/\*\*$/.exec(entry); + return match?.[1] ? [match[1]] : []; + }), +); + +function listCatalogManifestsByPluginId(): Map> { + const byPluginId = new Map>(); + for (const entry of officialExternalProviderCatalog.entries) { + if (!isRecord(entry)) { + continue; + } + const manifest = entry.openclaw; + if (!isRecord(manifest) || !isRecord(manifest.plugin)) { + continue; + } + const pluginId = manifest.plugin.id; + if (typeof pluginId === "string" && pluginId.trim()) { + byPluginId.set(pluginId, manifest); + } + } + return byPluginId; +} + +describe("official external provider endpoint catalog mirror", () => { + const extensionManifests = listExtensionManifests(); + const catalogManifestsByPluginId = listCatalogManifestsByPluginId(); + + it("mirrors providerEndpoints for every dist-excluded plugin manifest that declares them", () => { + const checkedPluginIds: string[] = []; + for (const { dirName, manifest } of extensionManifests) { + if (!Array.isArray(manifest.providerEndpoints)) { + continue; + } + if (!distExcludedExtensionDirs.has(dirName)) { + continue; + } + const pluginId = typeof manifest.id === "string" ? manifest.id : undefined; + const catalogManifest = pluginId ? catalogManifestsByPluginId.get(pluginId) : undefined; + expect( + catalogManifest, + `extensions/${dirName} is excluded from dist and declares providerEndpoints; ` + + `official-external-provider-catalog.json needs an entry for plugin "${pluginId}" ` + + `mirroring them, or endpoint classification breaks when the plugin is not installed`, + ).toBeDefined(); + expect( + catalogManifest?.providerEndpoints, + `catalog providerEndpoints for plugin "${pluginId}" must mirror extensions/${dirName}/openclaw.plugin.json`, + ).toEqual(manifest.providerEndpoints); + if (pluginId) { + checkedPluginIds.push(pluginId); + } + } + // The mirror set going empty means the scan above stopped covering the + // externalized providers this contract exists for. + expect(checkedPluginIds).toContain("qwen"); + expect(checkedPluginIds).toContain("moonshot"); + }); + + it("keeps catalog providerEndpoints in sync with local plugin manifests", () => { + const extensionManifestsById = new Map( + extensionManifests + .filter((record) => typeof record.manifest.id === "string") + .map((record) => [record.manifest.id as string, record]), + ); + for (const [pluginId, catalogManifest] of catalogManifestsByPluginId) { + if (catalogManifest.providerEndpoints === undefined) { + continue; + } + const local = extensionManifestsById.get(pluginId); + if (!local) { + // Catalog-only plugins have no in-repo manifest to compare against. + continue; + } + expect( + catalogManifest.providerEndpoints, + `catalog providerEndpoints for plugin "${pluginId}" must mirror extensions/${local.dirName}/openclaw.plugin.json`, + ).toEqual(local.manifest.providerEndpoints); + } + }); + + it("exposes endpoint metadata for externalized providers", () => { + const endpointClasses = listOfficialExternalProviderEndpointManifests().flatMap((manifest) => + Array.isArray(manifest.providerEndpoints) + ? manifest.providerEndpoints.flatMap((endpoint) => + isRecord(endpoint) && typeof endpoint.endpointClass === "string" + ? [endpoint.endpointClass] + : [], + ) + : [], + ); + expect(endpointClasses).toContain("modelstudio-native"); + expect(endpointClasses).toContain("moonshot-native"); + expect(endpointClasses).toContain("zai-native"); + }); +}); diff --git a/src/plugins/official-external-provider-endpoints.ts b/src/plugins/official-external-provider-endpoints.ts new file mode 100644 index 000000000000..739af8c9c03a --- /dev/null +++ b/src/plugins/official-external-provider-endpoints.ts @@ -0,0 +1,40 @@ +/** + * Provider endpoint metadata for officially externalized provider plugins. + * + * Endpoint classification (SSRF, attribution, payload-compat policy) keys off + * base URLs and must keep working when the owning plugin is not installed: + * dist packages exclude externalized plugins, so their manifests are invisible + * to bundled discovery. Only the repo-bundled catalog JSON feeds this table; + * hosted marketplace feeds must never influence endpoint classification. + * Kept separate from official-external-plugin-catalog.ts so provider + * transports do not pull the ClawHub install/marketplace module graph. + */ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import officialExternalProviderCatalog from "../../scripts/lib/official-external-provider-catalog.json" with { type: "json" }; +import { MANIFEST_KEY } from "../compat/legacy-names.js"; + +/** + * Lists manifest-shaped catalog metadata blocks that declare provider endpoints. + * + * The catalog mirrors manifests faithfully, including endpoint classes core + * does not (yet) recognize (e.g. deepinfra-native, gmi-native). The endpoint + * reader filters unknown classes exactly as it does for installed manifests, + * so they stay inert instead of complicating the mirror contract. + */ +export function listOfficialExternalProviderEndpointManifests(): Record[] { + const entries = (officialExternalProviderCatalog as { entries?: unknown }).entries; + if (!Array.isArray(entries)) { + return []; + } + const manifests: Record[] = []; + for (const entry of entries) { + if (!isRecord(entry)) { + continue; + } + const manifest = entry[MANIFEST_KEY]; + if (isRecord(manifest) && Array.isArray(manifest.providerEndpoints)) { + manifests.push(manifest); + } + } + return manifests; +}