refactor: move plugin model catalogs into plugin state

This commit is contained in:
Peter Steinberger
2026-05-29 02:39:46 +01:00
parent 94db48d028
commit 51b5f75b92
24 changed files with 1276 additions and 69 deletions
+1 -1
View File
@@ -340,7 +340,7 @@ When live probes run in a TTY, you can select fallbacks interactively. In non-in
## Models registry (`models.json`)
Custom providers in `models.providers` are written into `models.json` under the agent directory (default `~/.openclaw/agents/<agentId>/agent/models.json`). This file is merged by default unless `models.mode` is set to `replace`.
Custom providers in `models.providers` are written into `models.json` under the agent directory (default `~/.openclaw/agents/<agentId>/agent/models.json`). Provider-plugin catalogs are stored as generated plugin-owned catalog shards under the agent's plugin state and loaded automatically. This file is merged by default unless `models.mode` is set to `replace`.
<AccordionGroup>
<Accordion title="Merge mode precedence">
+2 -1
View File
@@ -488,7 +488,8 @@ Configuring a custom/local provider `baseUrl` is also the narrow network trust d
- Empty or missing agent `apiKey`/`baseUrl` fall back to `models.providers` in config.
- Matching model `contextWindow`/`maxTokens` use the higher value between explicit config and implicit catalog values.
- Matching model `contextTokens` preserves an explicit runtime cap when present; use it to limit effective context without changing native model metadata.
- Use `models.mode: "replace"` when you want config to fully rewrite `models.json`.
- Provider-plugin catalogs are stored as generated plugin-owned catalog shards under the agent's plugin state.
- Use `models.mode: "replace"` when you want config to fully rewrite `models.json` and active plugin catalog shards.
- Marker persistence is source-authoritative: markers are written from the active source config snapshot (pre-resolution), not from resolved runtime secret values.
</Accordion>
+9 -1
View File
@@ -1,5 +1,6 @@
import path from "node:path";
import type { Model } from "../llm/types.js";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js";
import { normalizeModelCompat } from "../plugins/provider-model-compat.js";
import {
applyProviderResolvedTransportWithPlugin,
@@ -31,6 +32,8 @@ type DiscoveredProviderRuntimeModelLike = Omit<ProviderRuntimeModelLike, "api">
type DiscoverModelsOptions = {
providerFilter?: string;
pluginMetadataSnapshot?: Pick<PluginMetadataSnapshot, "owners">;
workspaceDir?: string;
normalizeModels?: boolean;
};
@@ -84,7 +87,12 @@ function createOpenClawModelRegistry(
agentDir: string,
options?: DiscoverModelsOptions,
): AgentModelRegistry {
const registry = ModelRegistry.create(authStorage, modelsJsonPath);
const registry = ModelRegistry.create(authStorage, modelsJsonPath, {
...(options?.pluginMetadataSnapshot
? { pluginMetadataSnapshot: options.pluginMetadataSnapshot }
: {}),
...(options?.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
});
const getAll = registry.getAll.bind(registry);
const getAvailable = registry.getAvailable.bind(registry);
const find = registry.find.bind(registry);
+3
View File
@@ -547,6 +547,9 @@ describe("runBtwSideQuestion", () => {
const ensureArgs = mockCall(ensureOpenClawModelsJsonMock);
expect(ensureArgs?.[1]).toBe(DEFAULT_AGENT_DIR);
expect(ensureArgs?.[2]).toEqual({ workspaceDir: "/tmp/workspace" });
expect(discoverModelsMock).toHaveBeenCalledWith(undefined, DEFAULT_AGENT_DIR, {
workspaceDir: "/tmp/workspace",
});
});
it("routes Codex-selected BTW questions through the harness side-question hook", async () => {
+1 -1
View File
@@ -252,7 +252,7 @@ async function resolveRuntimeModel(params: {
const modelsOptions = params.workspaceDir ? { workspaceDir: params.workspaceDir } : undefined;
await ensureOpenClawModelsJson(params.cfg, params.agentDir, modelsOptions);
const authStorage = discoverAuthStorage(params.agentDir);
const modelRegistry = discoverModels(authStorage, params.agentDir);
const modelRegistry = discoverModels(authStorage, params.agentDir, modelsOptions);
const model = resolveModelWithRegistry({
provider: params.provider,
modelId: params.model,
+4 -1
View File
@@ -271,7 +271,10 @@ describe("lookupContextTokens", () => {
expect(
path.normalize(discoverAgentDir).endsWith(path.join(".openclaw", "agents", "main", "agent")),
).toBe(true);
expect(discoverCall[2]).toEqual({ normalizeModels: false });
expect(discoverCall[2]).toEqual({
normalizeModels: false,
workspaceDir: expect.any(String),
});
expect(lookupContextTokens("anthropic/claude-opus-4.7-20260219")).toBe(1_048_576);
});
+13 -3
View File
@@ -6,7 +6,11 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { computeBackoff, type BackoffPolicy } from "../infra/backoff.js";
import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js";
import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import {
resolveAgentWorkspaceDir,
resolveDefaultAgentDir,
resolveDefaultAgentId,
} from "./agent-scope.js";
import { lookupCachedContextTokens, MODEL_CONTEXT_TOKEN_CACHE } from "./context-cache.js";
import { CONTEXT_WINDOW_RUNTIME_STATE } from "./context-runtime-state.js";
import { normalizeProviderId } from "./model-selection.js";
@@ -158,17 +162,23 @@ export function ensureContextWindowCacheLoaded(): Promise<void> {
}
CONTEXT_WINDOW_RUNTIME_STATE.loadPromise = (async () => {
const agentDir = resolveDefaultAgentDir(cfg);
const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
try {
await (await loadModelsConfigRuntime()).ensureOpenClawModelsJson(cfg);
await (
await loadModelsConfigRuntime()
).ensureOpenClawModelsJson(cfg, agentDir, {
workspaceDir,
});
} catch {
// Continue with best-effort discovery/overrides.
}
try {
const agentDir = resolveDefaultAgentDir(cfg);
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir, {
normalizeModels: false,
workspaceDir,
}) as unknown as ModelRegistryLike;
const models =
typeof modelRegistry.getAvailable === "function"
@@ -1,5 +1,9 @@
import { statSync } from "node:fs";
import path from "node:path";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js";
import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js";
import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js";
import {
resolveRuntimeExternalAuthProviderRefs,
resolveRuntimeSyntheticAuthProviderRefs,
@@ -7,6 +11,7 @@ import {
import { discoverAuthStorage, discoverModels } from "../agent-model-discovery.js";
import { resolveDefaultAgentDir } from "../agent-scope.js";
import { hasAnyRuntimeAuthProfileStoreSource } from "../auth-profiles/runtime-snapshots.js";
import { listPluginModelCatalogPaths } from "../plugin-model-catalog.js";
import type { AuthStorage, ModelRegistry } from "../sessions/index.js";
type DiscoveryStores = {
@@ -16,7 +21,9 @@ type DiscoveryStores = {
type DiscoverCachedAgentStoresOptions = {
agentDir: string;
config?: OpenClawConfig;
inheritedAuthDir?: string;
workspaceDir?: string;
};
type CacheEntry = DiscoveryStores & {
@@ -47,7 +54,20 @@ function authFingerprint(agentDir: string): object {
};
}
function discoveryFingerprint(params: DiscoverCachedAgentStoresOptions): string {
function pluginModelCatalogFingerprint(
agentDir: string,
): Array<[string, ReturnType<typeof fileFingerprint>]> {
return listPluginModelCatalogPaths(agentDir).map((catalogPath) => [
path.relative(agentDir, catalogPath),
fileFingerprint(catalogPath),
]);
}
function discoveryFingerprint(
params: DiscoverCachedAgentStoresOptions & {
pluginMetadataSnapshot?: PluginMetadataSnapshot;
},
): string {
const inheritedAuthDir =
params.inheritedAuthDir && params.inheritedAuthDir !== params.agentDir
? params.inheritedAuthDir
@@ -58,6 +78,8 @@ function discoveryFingerprint(params: DiscoverCachedAgentStoresOptions): string
localAuth: authFingerprint(params.agentDir),
inheritedAuth: inheritedAuthDir ? authFingerprint(inheritedAuthDir) : undefined,
modelsJson: fileFingerprint(path.join(params.agentDir, "models.json")),
pluginMetadata: pluginMetadataFingerprint(params.pluginMetadataSnapshot),
pluginModelCatalogs: pluginModelCatalogFingerprint(params.agentDir),
});
}
@@ -82,9 +104,46 @@ function pruneDiscoveryStoreCache(): void {
}
}
function discoverFreshAgentStores(agentDir: string): DiscoveryStores {
function resolvePluginMetadataSnapshotForDiscovery(
options: DiscoverCachedAgentStoresOptions,
): PluginMetadataSnapshot | undefined {
try {
return (
getCurrentPluginMetadataSnapshot({
allowWorkspaceScopedSnapshot: true,
config: options.config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}) ??
resolvePluginMetadataSnapshot({
config: options.config ?? {},
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
})
);
} catch {
return undefined;
}
}
function pluginMetadataFingerprint(snapshot: PluginMetadataSnapshot | undefined): object {
return {
configFingerprint: snapshot?.configFingerprint,
policyHash: snapshot?.policyHash,
workspaceDir: snapshot?.workspaceDir,
};
}
function discoverFreshAgentStores(
agentDir: string,
options: Pick<DiscoverCachedAgentStoresOptions, "workspaceDir">,
pluginMetadataSnapshot: PluginMetadataSnapshot | undefined,
): DiscoveryStores {
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir);
const modelRegistry = discoverModels(authStorage, agentDir, {
...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
});
return { authStorage, modelRegistry };
}
@@ -96,11 +155,16 @@ export function discoverCachedAgentStores(
options.inheritedAuthDir ?? resolveDefaultAgentDir({}),
);
if (hasAnyRuntimeAuthProfileStoreSource(agentDir) || hasRuntimePluginAuthSources()) {
return discoverFreshAgentStores(agentDir);
return discoverFreshAgentStores(
agentDir,
options,
resolvePluginMetadataSnapshotForDiscovery(options),
);
}
const pluginMetadataSnapshot = resolvePluginMetadataSnapshotForDiscovery(options);
const cacheKey = JSON.stringify({ agentDir, inheritedAuthDir });
const fingerprint = discoveryFingerprint({ agentDir, inheritedAuthDir });
const fingerprint = discoveryFingerprint({ agentDir, inheritedAuthDir, pluginMetadataSnapshot });
const cached = DISCOVERY_STORE_CACHE.get(cacheKey);
if (cached?.fingerprint === fingerprint) {
cached.lastUsedAt = Date.now();
@@ -110,7 +174,7 @@ export function discoverCachedAgentStores(
};
}
const stores = discoverFreshAgentStores(agentDir);
const stores = discoverFreshAgentStores(agentDir, options, pluginMetadataSnapshot);
DISCOVERY_STORE_CACHE.set(cacheKey, {
authStorage: stores.authStorage,
fingerprint,
@@ -7,6 +7,10 @@ import {
clearRuntimeAuthProfileStoreSnapshots,
replaceRuntimeAuthProfileStoreSnapshots,
} from "../auth-profiles.js";
import {
PLUGIN_MODEL_CATALOG_FILE,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
} from "../plugin-model-catalog.js";
import { resetModelDiscoveryCacheForTest } from "./model-discovery-cache.js";
import { createProviderRuntimeTestMock } from "./model.provider-runtime.test-support.js";
@@ -296,6 +300,40 @@ describe("resolveModel", () => {
expect(discoverModels).toHaveBeenCalledTimes(1);
});
it("invalidates agent discovery stores when generated plugin catalogs change", async () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-cache-plugin-"));
const agentDir = path.join(rootDir, "agent");
fs.mkdirSync(agentDir, { recursive: true });
mockDiscoveredModel(discoverModels, {
provider: "zai",
modelId: "glm-5.1",
templateModel: {
provider: "zai",
...makeModel("glm-5.1"),
},
});
const first = await resolveModelAsync("zai", "glm-5.1", agentDir, undefined, {
runtimeHooks: createRuntimeHooks(),
});
const catalogPath = path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
fs.mkdirSync(path.dirname(catalogPath), { recursive: true });
fs.writeFileSync(
catalogPath,
JSON.stringify({
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {},
}),
);
const second = await resolveModelAsync("zai", "glm-5.1", agentDir, undefined, {
runtimeHooks: createRuntimeHooks(),
});
expectResolvedModel(first);
expectResolvedModel(second);
expect(discoverModels).toHaveBeenCalledTimes(2);
});
it("invalidates agent discovery stores when inherited default auth changes", async () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-cache-"));
const agentDir = path.join(rootDir, "agent");
@@ -336,6 +374,37 @@ describe("resolveModel", () => {
expect(discoverModels).toHaveBeenCalledTimes(2);
});
it("uses the resolved default agent workspace for cached model discovery", () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-workspace-"));
const agentDir = path.join(rootDir, "agent");
const workspaceDir = path.join(rootDir, "workspace");
fs.mkdirSync(agentDir, { recursive: true });
mockDiscoveredModel(discoverModels, {
provider: "openai",
modelId: "gpt-5.5",
templateModel: {
provider: "openai",
...makeModel("gpt-5.5"),
},
});
const cfg = {
agents: {
list: [{ id: "workspace-agent", default: true, agentDir, workspace: workspaceDir }],
},
} as unknown as OpenClawConfig;
const result = resolveModel("openai", "gpt-5.5", agentDir, cfg, {
runtimeHooks: createRuntimeHooks(),
});
expectResolvedModel(result);
expect(discoverModels).toHaveBeenCalledWith(
expect.anything(),
agentDir,
expect.objectContaining({ workspaceDir }),
);
});
it("invalidates agent discovery stores when implicit main auth changes without config", async () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-model-cache-state-"));
vi.stubEnv("OPENCLAW_STATE_DIR", rootDir);
+22 -5
View File
@@ -13,7 +13,11 @@ import {
shouldPreferProviderRuntimeResolvedModel,
} from "../../plugins/provider-runtime.js";
import { discoverAuthStorage, discoverModels } from "../agent-model-discovery.js";
import { resolveDefaultAgentDir } from "../agent-scope.js";
import {
resolveAgentWorkspaceDir,
resolveDefaultAgentDir,
resolveDefaultAgentId,
} from "../agent-scope.js";
import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js";
import { buildModelAliasLines } from "../model-alias-lines.js";
import { modelKey, normalizeStaticProviderModelId } from "../model-ref-shared.js";
@@ -136,16 +140,29 @@ function resolveRuntimeHooks(params?: {
function discoverCachedAgentStoresForAgent(
resolvedAgentDir: string,
cfg: OpenClawConfig | undefined,
workspaceDir: string | undefined,
): {
authStorage: AuthStorage;
modelRegistry: ModelRegistry;
} {
return discoverCachedAgentStores({
agentDir: resolvedAgentDir,
...(cfg ? { config: cfg } : {}),
inheritedAuthDir: resolveDefaultAgentDir(cfg ?? {}),
...(workspaceDir ? { workspaceDir } : {}),
});
}
function resolveModelWorkspaceDir(
cfg: OpenClawConfig | undefined,
explicitWorkspaceDir: string | undefined,
): string | undefined {
if (explicitWorkspaceDir !== undefined || !cfg) {
return explicitWorkspaceDir;
}
return resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
}
function canonicalizeLegacyResolvedModel(params: { provider: string; model: Model }): Model {
if (
normalizeProviderId(params.provider) !== "openai-codex" ||
@@ -1199,12 +1216,12 @@ export function resolveModel(
authStorage: AuthStorage;
modelRegistry: ModelRegistry;
} {
const workspaceDir = options?.workspaceDir ?? cfg?.agents?.defaults?.workspace;
const workspaceDir = resolveModelWorkspaceDir(cfg, options?.workspaceDir);
const normalizedRef = normalizeProviderModelRef({ provider, modelId, cfg, workspaceDir });
const resolvedAgentDir = agentDir ?? resolveDefaultAgentDir(cfg ?? {});
const cachedStores =
!options?.authStorage && !options?.modelRegistry
? discoverCachedAgentStoresForAgent(resolvedAgentDir, cfg)
? discoverCachedAgentStoresForAgent(resolvedAgentDir, cfg, workspaceDir)
: undefined;
const authStorage =
options?.authStorage ?? cachedStores?.authStorage ?? discoverAuthStorage(resolvedAgentDir);
@@ -1261,7 +1278,7 @@ export async function resolveModelAsync(
authStorage: AuthStorage;
modelRegistry: ModelRegistry;
}> {
const workspaceDir = options?.workspaceDir ?? cfg?.agents?.defaults?.workspace;
const workspaceDir = resolveModelWorkspaceDir(cfg, options?.workspaceDir);
const normalizedRef = normalizeProviderModelRef({ provider, modelId, cfg, workspaceDir });
const resolvedAgentDir = agentDir ?? resolveDefaultAgentDir(cfg ?? {});
const emptyDiscoveryStores =
@@ -1270,7 +1287,7 @@ export async function resolveModelAsync(
: undefined;
const cachedStores =
!emptyDiscoveryStores && !options?.authStorage && !options?.modelRegistry
? discoverCachedAgentStoresForAgent(resolvedAgentDir, cfg)
? discoverCachedAgentStoresForAgent(resolvedAgentDir, cfg, workspaceDir)
: undefined;
const authStorage =
options?.authStorage ??
+125
View File
@@ -1,6 +1,8 @@
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { resetLogger, setLoggerOverride } from "../logging/logger.js";
import { PLUGIN_MODEL_CATALOG_GENERATED_BY } from "./plugin-model-catalog.js";
type AgentModelDiscoveryModule = typeof import("./agent-model-discovery.js");
@@ -94,6 +96,16 @@ function emptyPluginMetadataSnapshot() {
policyHash: "test-policy",
plugins: [],
},
owners: {
channels: new Map(),
channelConfigs: new Map(),
providers: new Map(),
modelCatalogProviders: new Map(),
cliBackends: new Map(),
setupProviders: new Map(),
commandAliases: new Map(),
contracts: new Map(),
},
plugins: [],
};
}
@@ -229,7 +241,13 @@ describe("loadModelCatalog", () => {
ensureOpenClawModelsJson: ensureOpenClawModelsJsonMock,
}));
vi.doMock("./agent-scope.js", () => ({
resolveAgentWorkspaceDir: (cfg: OpenClawConfig, agentId: string) => {
const entry = cfg.agents?.list?.find((entry) => entry.id === agentId);
return entry?.workspace ?? cfg.agents?.defaults?.workspace ?? "/tmp/openclaw-workspace";
},
resolveDefaultAgentDir: () => "/tmp/openclaw",
resolveDefaultAgentId: (cfg: OpenClawConfig) =>
cfg.agents?.list?.find((entry) => entry.default)?.id ?? cfg.agents?.list?.[0]?.id ?? "main",
}));
vi.doMock("../plugins/provider-runtime.runtime.js", () => ({
augmentModelCatalogWithProviderPlugins: vi.fn().mockResolvedValue([]),
@@ -305,6 +323,40 @@ describe("loadModelCatalog", () => {
}
});
it("uses the resolved default agent workspace for registry discovery", async () => {
const discoverModels = vi.fn(() => ({
getAll() {
return [];
},
}));
setModelCatalogImportForTest(
async () =>
({
discoverAuthStorage: () => ({}),
AuthStorage: function AuthStorage() {},
discoverModels,
ModelRegistry: class {
getAll() {
return [];
}
},
}) as unknown as AgentModelDiscoveryModule,
);
const config = {
agents: {
list: [{ id: "workspace-agent", default: true, workspace: "/tmp/workspace-agent" }],
},
} as OpenClawConfig;
await loadModelCatalog({ config });
expect(discoverModels).toHaveBeenCalledWith(
expect.anything(),
"/tmp/openclaw",
expect.objectContaining({ workspaceDir: "/tmp/workspace-agent" }),
);
});
it("reloads dynamic registry entries after clearing the cache", async () => {
const models = [{ id: "existing", name: "Existing", provider: "ollama" }];
mockAgentDiscoveryModels(models);
@@ -490,6 +542,79 @@ describe("loadModelCatalog", () => {
expect(augmentCatalogMock).not.toHaveBeenCalled();
});
it("loads generated plugin catalog rows in read-only mode", async () => {
const catalogPath = "/tmp/openclaw/plugins/read-only-shard/catalog.json";
mkdirSync("/tmp/openclaw/plugins/read-only-shard", { recursive: true });
writeFileSync(catalogPath, "{}");
try {
readFileMock.mockImplementation(async (pathname: string) => {
if (pathname.endsWith("models.json")) {
return JSON.stringify({ providers: {} });
}
if (pathname === catalogPath) {
return JSON.stringify({
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
zai: {
models: [
{
id: "glm-5.1",
name: "GLM 5.1",
reasoning: true,
contextWindow: 131072,
input: ["text"],
},
],
},
},
});
}
throw Object.assign(new Error("not found"), { code: "ENOENT" });
});
loadPluginMetadataSnapshotMock.mockReturnValueOnce({
...emptyPluginMetadataSnapshot(),
index: {
policyHash: "test-policy",
plugins: [{ pluginId: "read-only-shard", enabled: true }],
},
normalizePluginId: (id: string) => id,
owners: {
providers: new Map([["zai", ["read-only-shard"]]]),
modelCatalogProviders: new Map([["zai", ["read-only-shard"]]]),
setupProviders: new Map(),
},
});
const result = await loadModelCatalog({
config: {
agents: {
list: [{ id: "workspace-agent", default: true, workspace: "/tmp/read-only-workspace" }],
},
} as OpenClawConfig,
readOnly: true,
});
expect(requireCatalogEntry(result, "zai", "glm-5.1")).toMatchObject({
provider: "zai",
id: "glm-5.1",
name: "GLM 5.1",
reasoning: true,
contextWindow: 131072,
});
expect(
loadPluginMetadataSnapshotMock.mock.calls.some(([call]) => {
return (
typeof call === "object" &&
call !== null &&
(call as { workspaceDir?: string }).workspaceDir === "/tmp/read-only-workspace"
);
}),
).toBe(true);
} finally {
rmSync("/tmp/openclaw/plugins/read-only-shard", { recursive: true, force: true });
}
});
it("falls back to manifest catalog rows when persisted read-only catalog has no model rows", async () => {
readFileMock.mockResolvedValueOnce(
JSON.stringify({
+78 -15
View File
@@ -1,5 +1,5 @@
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { join, relative } from "node:path";
import { getRuntimeConfig } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
@@ -17,7 +17,11 @@ import {
normalizeLowercaseStringOrEmpty,
normalizeOptionalString,
} from "../shared/string-coerce.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import {
resolveAgentWorkspaceDir,
resolveDefaultAgentDir,
resolveDefaultAgentId,
} from "./agent-scope.js";
import { ensureAuthProfileStoreWithoutExternalProfiles } from "./auth-profiles.js";
import { modelSupportsInput as modelCatalogEntrySupportsInput } from "./model-catalog-lookup.js";
import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js";
@@ -31,6 +35,12 @@ import {
hasConfiguredProviderModelRows,
} from "./model-selection-shared.js";
import { ensureOpenClawModelsJson } from "./models-config.js";
import {
decodePluginModelCatalogRelativePathPluginId,
isGeneratedPluginModelCatalog,
listPluginModelCatalogPaths,
resolvePluginModelCatalogOwnerPluginId,
} from "./plugin-model-catalog.js";
import { normalizeProviderId } from "./provider-id.js";
const log = createSubsystemLogger("model-catalog");
@@ -306,31 +316,79 @@ function normalizePersistedModelCatalogEntry(
};
}
function readProviderCatalogRows(parsed: unknown): Record<string, Record<string, unknown>> {
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return {};
}
const providers = (parsed as { providers?: unknown }).providers;
return providers && typeof providers === "object" && !Array.isArray(providers)
? (providers as Record<string, Record<string, unknown>>)
: {};
}
async function loadReadOnlyPersistedProviderRows(
agentDir: string,
getPluginMetadataSnapshot: () => Pick<PluginMetadataSnapshot, "owners">,
): Promise<Record<string, Record<string, unknown>>> {
const raw = await readFile(join(agentDir, "models.json"), "utf8");
const providers = { ...readProviderCatalogRows(JSON.parse(raw) as unknown) };
for (const catalogPath of listPluginModelCatalogPaths(agentDir)) {
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(
relative(agentDir, catalogPath),
);
if (!catalogPluginId) {
continue;
}
const catalogRaw = await readFile(catalogPath, "utf8").catch(() => undefined);
if (!catalogRaw) {
continue;
}
let parsed: unknown;
try {
parsed = JSON.parse(catalogRaw) as unknown;
} catch {
continue;
}
if (isGeneratedPluginModelCatalog(parsed)) {
for (const [providerId, provider] of Object.entries(readProviderCatalogRows(parsed))) {
const ownerPluginId = resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: getPluginMetadataSnapshot(),
});
if (ownerPluginId === catalogPluginId) {
providers[providerId] = provider;
}
}
}
}
return providers;
}
async function loadReadOnlyPersistedModelCatalog(params?: {
config?: OpenClawConfig;
metadataSnapshot?: PluginMetadataSnapshot;
}): Promise<ModelCatalogEntry[]> {
const cfg = params?.config ?? getRuntimeConfig();
const agentDir = resolveDefaultAgentDir(cfg);
const raw = await readFile(join(agentDir, "models.json"), "utf8");
const parsed = JSON.parse(raw) as Record<string, unknown>;
const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
const models: ModelCatalogEntry[] = [];
const { buildShouldSuppressBuiltInModel } = await loadModelSuppression();
const shouldSuppressBuiltInModel = buildShouldSuppressBuiltInModel({ config: cfg });
let metadataSnapshot: PluginMetadataSnapshot | undefined = params?.metadataSnapshot;
const getMetadataSnapshot = () => {
metadataSnapshot ??= loadManifestMetadataSnapshot({
config: cfg,
env: process.env,
workspaceDir,
});
return metadataSnapshot;
};
let manifestPlugins: ProviderModelIdNormalizationOptions["manifestPlugins"];
const getManifestPlugins = () => {
manifestPlugins ??=
params?.metadataSnapshot?.plugins ??
loadManifestMetadataSnapshot({
config: cfg,
env: process.env,
}).plugins;
manifestPlugins ??= getMetadataSnapshot().plugins;
return manifestPlugins;
};
const providers =
parsed?.providers && typeof parsed.providers === "object"
? (parsed.providers as Record<string, Record<string, unknown>>)
: {};
const providers = await loadReadOnlyPersistedProviderRows(agentDir, getMetadataSnapshot);
for (const [providerRaw, providerConfig] of Object.entries(providers)) {
if (!Array.isArray(providerConfig?.models)) {
continue;
@@ -461,6 +519,7 @@ export async function loadModelCatalog(params?: {
const sortModels = sortModelCatalogEntries;
try {
const cfg = params?.config ?? getRuntimeConfig();
const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
let manifestMetadataSnapshot: PluginMetadataSnapshot | undefined;
let manifestPlugins: ProviderModelIdNormalizationOptions["manifestPlugins"];
const getManifestMetadataSnapshot = () => {
@@ -469,6 +528,7 @@ export async function loadModelCatalog(params?: {
loadManifestMetadataSnapshot({
config: cfg,
env: process.env,
workspaceDir,
});
return manifestMetadataSnapshot;
};
@@ -492,7 +552,10 @@ export async function loadModelCatalog(params?: {
readOnly ? { readOnly: true } : undefined,
);
logStage("auth-storage-ready");
const registry = agentDiscovery.discoverModels(authStorage, agentDir);
const registry = agentDiscovery.discoverModels(authStorage, agentDir, {
pluginMetadataSnapshot: getManifestMetadataSnapshot(),
workspaceDir,
});
logStage("registry-ready");
const entries = registry.getAll() as DiscoveredModel[];
logStage("registry-read", `entries=${entries.length}`);
+7
View File
@@ -1,4 +1,5 @@
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import type { ModelRegistry } from "./sessions/index.js";
@@ -23,7 +24,13 @@ export function loadAgentModelRegistry(
workspaceDir: options.workspaceDir,
});
const registry = discoverModels(authStorage, agentDir, {
pluginMetadataSnapshot: resolvePluginMetadataSnapshot({
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}),
providerFilter: options.providerFilter,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
normalizeModels: options.normalizeModels,
});
return { agentDir, registry };
@@ -8,6 +8,7 @@ import {
resolveProvidersForModelsJsonWithDeps,
} from "./models-config.plan.js";
import type { ProviderConfig } from "./models-config.providers.secrets.js";
import { encodePluginModelCatalogRelativePath } from "./plugin-model-catalog.js";
const TEST_ENV_VAR = "OPENCLAW_MODELS_CONFIG_TEST_ENV";
@@ -97,7 +98,8 @@ async function resolveProvidersAndCaptureDiscoveryEnv(cfg: OpenClawConfig) {
describe("models-config", () => {
it("threads plugin metadata snapshots into implicit provider discovery", async () => {
const pluginMetadataSnapshot = {
index: { plugins: [] },
index: { plugins: [{ pluginId: "zai", enabled: true }] },
normalizePluginId: (pluginId: string) => pluginId,
manifestRegistry: { plugins: [], diagnostics: [] },
owners: { providers: new Map() },
} as unknown as Pick<PluginMetadataSnapshot, "index" | "manifestRegistry" | "owners">;
@@ -179,7 +181,8 @@ describe("models-config", () => {
it("threads plugin metadata snapshots through models.json planning", async () => {
const pluginMetadataSnapshot = {
index: { plugins: [] },
index: { plugins: [{ pluginId: "zai", enabled: true }] },
normalizePluginId: (pluginId: string) => pluginId,
manifestRegistry: { plugins: [], diagnostics: [] },
owners: { providers: new Map() },
} as unknown as Pick<PluginMetadataSnapshot, "index" | "manifestRegistry" | "owners">;
@@ -237,6 +240,78 @@ describe("models-config", () => {
expect(parsed.providers?.["auth-only"]).toBeDefined();
});
it("treats empty replace-mode provider sets as authoritative", async () => {
const plan = await planOpenClawModelsJsonWithDeps(
{
cfg: { models: { mode: "replace", providers: {} } },
agentDir: "/tmp/openclaw-models-config-env-vars-test",
env: {},
existingRaw: `${JSON.stringify({ providers: { stale: {} } }, null, 2)}\n`,
existingParsed: { providers: { stale: {} } },
},
{
resolveImplicitProviders: async () => ({}),
},
);
expect(plan.action).toBe("write");
if (plan.action !== "write") {
throw new Error("Expected models.json write plan");
}
expect(JSON.parse(plan.contents)).toEqual({ providers: {} });
expect(plan.pluginCatalogWrites).toEqual({});
});
it("moves plugin-owned provider catalogs into plugin-scoped files", async () => {
const pluginMetadataSnapshot = {
index: { plugins: [{ pluginId: "zai", enabled: true }] },
normalizePluginId: (pluginId: string) => pluginId,
manifestRegistry: { plugins: [], diagnostics: [] },
owners: {
providers: new Map([["zai", ["zai"]]]),
modelCatalogProviders: new Map([["zai", ["zai"]]]),
setupProviders: new Map(),
},
} as unknown as Pick<PluginMetadataSnapshot, "index" | "manifestRegistry" | "owners">;
const plan = await planOpenClawModelsJsonWithDeps(
{
cfg: { models: { providers: {} } },
agentDir: "/tmp/openclaw-models-config-env-vars-test",
env: { ZAI_API_KEY: "sk-test" } as NodeJS.ProcessEnv,
existingRaw: "",
existingParsed: null,
pluginMetadataSnapshot,
},
{
resolveImplicitProviders: async () => ({
zai: createImplicitOpenAiProvider({
baseUrl: "https://api.z.ai/api/paas/v4",
apiKey: "ZAI_API_KEY",
}),
custom: createImplicitOpenAiProvider({
baseUrl: "https://custom.example/v1",
apiKey: "CUSTOM_API_KEY",
}),
}),
},
);
expect(plan.action).toBe("write");
if (plan.action !== "write") {
throw new Error("Expected models.json write plan");
}
const root = JSON.parse(plan.contents) as {
providers?: Record<string, unknown>;
};
expect(Object.keys(root.providers ?? {})).toEqual(["custom"]);
expect(root).not.toHaveProperty("pluginCatalogs");
const zaiCatalogPath = encodePluginModelCatalogRelativePath("zai");
const zaiCatalog = JSON.parse(plan.pluginCatalogWrites?.[zaiCatalogPath] ?? "{}") as {
providers?: Record<string, unknown>;
};
expect(Object.keys(zaiCatalog.providers ?? {})).toEqual(["zai"]);
});
it("falls back to canonical env markers when provider runtime has no api-key policy", async () => {
const plan = await planOpenClawModelsJsonWithDeps(
{
+65 -3
View File
@@ -14,6 +14,11 @@ import {
resolveImplicitProviders,
type ProviderConfig,
} from "./models-config.providers.js";
import {
encodePluginModelCatalogRelativePath,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
resolvePluginModelCatalogOwnerPluginId,
} from "./plugin-model-catalog.js";
type ModelsConfig = NonNullable<OpenClawConfig["models"]>;
export type ResolveImplicitProvidersForModelsJson = (params: {
@@ -31,15 +36,53 @@ export type ResolveImplicitProvidersForModelsJson = (params: {
export type ModelsJsonPlan =
| {
action: "skip";
pluginCatalogWrites?: Record<string, string>;
}
| {
action: "noop";
pluginCatalogWrites?: Record<string, string>;
}
| {
action: "write";
contents: string;
pluginCatalogWrites?: Record<string, string>;
};
function splitProvidersByPluginOwner(params: {
providers: Record<string, ProviderConfig>;
pluginMetadataSnapshot?: Pick<PluginMetadataSnapshot, "owners">;
}): {
rootProviders: Record<string, ProviderConfig>;
pluginProviders: Record<string, Record<string, ProviderConfig>>;
} {
const rootProviders: Record<string, ProviderConfig> = {};
const pluginProviders: Record<string, Record<string, ProviderConfig>> = {};
for (const [providerId, provider] of Object.entries(params.providers)) {
const pluginId = resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
});
if (!pluginId) {
rootProviders[providerId] = provider;
continue;
}
const pluginCatalog = (pluginProviders[pluginId] ??= {});
pluginCatalog[providerId] = provider;
}
return { rootProviders, pluginProviders };
}
function buildPluginCatalogWrites(
pluginProviders: Record<string, Record<string, ProviderConfig>>,
): Record<string, string> {
return Object.fromEntries(
Object.entries(pluginProviders).map(([pluginId, providers]) => [
encodePluginModelCatalogRelativePath(pluginId),
`${JSON.stringify({ generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, providers }, null, 2)}\n`,
]),
);
}
export async function resolveProvidersForModelsJsonWithDeps(
params: {
cfg: OpenClawConfig;
@@ -163,6 +206,13 @@ export async function planOpenClawModelsJsonWithDeps(
);
if (Object.keys(providers).length === 0) {
if (params.cfg.models?.mode === "replace") {
return {
action: "write",
contents: `${JSON.stringify({ providers: {} }, null, 2)}\n`,
pluginCatalogWrites: {},
};
}
return { action: "skip" };
}
@@ -200,15 +250,27 @@ export async function planOpenClawModelsJsonWithDeps(
const finalProviders = applyNativeStreamingUsageCompat(
filterWritableProviders(secretEnforcedProviders),
);
const nextContents = `${JSON.stringify({ providers: finalProviders }, null, 2)}\n`;
const splitProviders = splitProvidersByPluginOwner({
providers: finalProviders,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
});
const pluginCatalogWrites = buildPluginCatalogWrites(splitProviders.pluginProviders);
const nextContents = `${JSON.stringify(
{
providers: splitProviders.rootProviders,
},
null,
2,
)}\n`;
if (params.existingRaw === nextContents) {
return { action: "noop" };
if (params.existingRaw === nextContents && Object.keys(pluginCatalogWrites).length === 0) {
return { action: "noop", pluginCatalogWrites };
}
return {
action: "write",
contents: nextContents,
pluginCatalogWrites,
};
}
@@ -1,6 +1,7 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import {
CUSTOM_PROXY_MODELS_CONFIG,
@@ -11,6 +12,10 @@ import {
withModelsTempHome as withTempHome,
} from "./models-config.e2e-harness.js";
import type { ProviderConfig as ModelsProviderConfig } from "./models-config.providers.secrets.js";
import {
PLUGIN_MODEL_CATALOG_FILE,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
} from "./plugin-model-catalog.js";
vi.mock("./auth-profiles/external-cli-sync.js", () => ({
resolveExternalCliAuthProfiles: () => [],
@@ -98,6 +103,39 @@ type ParsedProviderConfig = {
models?: Array<{ id: string }>;
};
async function readGeneratedProviders(
agentDir: string,
): Promise<Record<string, ParsedProviderConfig>> {
const raw = await fs.readFile(path.join(agentDir, "models.json"), "utf8");
const parsed = JSON.parse(raw) as { providers?: Record<string, ParsedProviderConfig> };
const providers = { ...parsed.providers };
const pluginsDir = path.join(agentDir, "plugins");
let pluginDirs: Array<import("node:fs").Dirent>;
try {
pluginDirs = await fs.readdir(pluginsDir, { withFileTypes: true });
} catch {
return providers;
}
for (const entry of pluginDirs) {
if (!entry.isDirectory()) {
continue;
}
const catalogPath = path.join(pluginsDir, entry.name, PLUGIN_MODEL_CATALOG_FILE);
const catalogRaw = await fs.readFile(catalogPath, "utf8").catch(() => undefined);
if (!catalogRaw) {
continue;
}
const catalog = JSON.parse(catalogRaw) as {
generatedBy?: string;
providers?: Record<string, ParsedProviderConfig>;
};
if (catalog.generatedBy === PLUGIN_MODEL_CATALOG_GENERATED_BY) {
Object.assign(providers, catalog.providers);
}
}
return providers;
}
async function runEnvProviderCase(params: {
envVar: "MINIMAX_API_KEY" | "SYNTHETIC_API_KEY";
envValue: string;
@@ -109,10 +147,7 @@ async function runEnvProviderCase(params: {
try {
await ensureOpenClawModelsJson({});
const modelPath = path.join(resolveDefaultAgentDir({}), "models.json");
const raw = await fs.readFile(modelPath, "utf8");
const parsed = JSON.parse(raw) as { providers: Record<string, ParsedProviderConfig> };
const provider = parsed.providers[params.providerKey];
const provider = (await readGeneratedProviders(resolveDefaultAgentDir({})))[params.providerKey];
expect(provider?.apiKey).toBe(params.expectedApiKeyRef);
} finally {
if (previousValue === undefined) {
@@ -162,19 +197,18 @@ describe("models-config", () => {
agentDir,
);
const raw = await fs.readFile(path.join(agentDir, "models.json"), "utf8");
const parsed = JSON.parse(raw) as { providers: Record<string, ParsedProviderConfig> };
const providers = await readGeneratedProviders(agentDir);
expect(result.wrote).toBe(true);
expect(Object.keys(parsed.providers)).toStrictEqual([
expect(Object.keys(providers).toSorted()).toStrictEqual([
"chutes",
"deepseek",
"mistral",
"xai",
]);
expect(parsed.providers["openai"]).toBeUndefined();
expect(parsed.providers["minimax"]).toBeUndefined();
expect(parsed.providers["synthetic"]).toBeUndefined();
expect(providers["openai"]).toBeUndefined();
expect(providers["minimax"]).toBeUndefined();
expect(providers["synthetic"]).toBeUndefined();
});
});
});
@@ -205,6 +239,54 @@ describe("models-config", () => {
});
});
it("preserves existing generated plugin catalog secrets in merge mode", async () => {
await withTempHome(async (home) => {
const agentDir = path.join(home, "agent-plugin-merge");
const catalogPath = path.join(agentDir, "plugins", "deepseek", PLUGIN_MODEL_CATALOG_FILE);
await fs.mkdir(path.dirname(catalogPath), { recursive: true });
await fs.writeFile(path.join(agentDir, "models.json"), JSON.stringify({ providers: {} }));
await fs.writeFile(
catalogPath,
JSON.stringify(
{
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
deepseek: {
baseUrl: "https://persisted.example/v1",
api: "openai-completions",
apiKey: "persisted-key",
models: [{ id: "test-model" }],
},
},
},
null,
2,
),
);
const pluginMetadataSnapshot = {
index: { plugins: [{ pluginId: "deepseek", enabled: true }] },
normalizePluginId: (pluginId: string) => pluginId,
manifestRegistry: { plugins: [], diagnostics: [] },
owners: {
providers: new Map([["deepseek", ["deepseek"]]]),
modelCatalogProviders: new Map([["deepseek", ["deepseek"]]]),
setupProviders: new Map(),
},
} as unknown as Pick<PluginMetadataSnapshot, "index" | "manifestRegistry" | "owners">;
await ensureOpenClawModelsJson({ models: { providers: {} } }, agentDir, {
pluginMetadataSnapshot,
});
const raw = await fs.readFile(catalogPath, "utf8");
const parsed = JSON.parse(raw) as {
providers: Record<string, ParsedProviderConfig>;
};
expect(parsed.providers.deepseek?.baseUrl).toBe("https://persisted.example/v1");
expect(parsed.providers.deepseek).toBeDefined();
});
});
it("adds minimax provider when MINIMAX_API_KEY is set", async () => {
await withTempHome(async () => {
await runEnvProviderCase({
+152 -5
View File
@@ -20,6 +20,13 @@ import {
} from "./agent-scope.js";
import { MODELS_JSON_STATE } from "./models-config-state.js";
import { planOpenClawModelsJson } from "./models-config.plan.js";
import {
decodePluginModelCatalogRelativePathPluginId,
isGeneratedPluginModelCatalog,
isPluginModelCatalogRelativePath,
listPluginModelCatalogRelativePaths,
resolvePluginModelCatalogOwnerPluginId,
} from "./plugin-model-catalog.js";
import { stableStringify } from "./stable-stringify.js";
export { resetModelsJsonReadyCacheForTest } from "./models-config-state.js";
@@ -33,6 +40,18 @@ async function readFileMtimeMs(pathname: string): Promise<number | null> {
}
}
async function readPluginCatalogMtimes(agentDir: string): Promise<Array<[string, number | null]>> {
const entries = await Promise.all(
listPluginModelCatalogRelativePaths(agentDir).map(async (relativePath) => {
return [relativePath, await readFileMtimeMs(path.join(agentDir, relativePath))] satisfies [
string,
number | null,
];
}),
);
return entries.toSorted(([left], [right]) => left.localeCompare(right));
}
async function buildModelsJsonFingerprint(params: {
config: OpenClawConfig;
sourceConfigForSecrets: OpenClawConfig;
@@ -47,6 +66,7 @@ async function buildModelsJsonFingerprint(params: {
path.join(params.agentDir, "auth-profiles.json"),
);
const modelsFileMtimeMs = await readFileMtimeMs(path.join(params.agentDir, "models.json"));
const pluginCatalogMtimes = await readPluginCatalogMtimes(params.agentDir);
const envShape = createConfigRuntimeEnv(params.config, {});
const pluginMetadataSnapshotIndexFingerprint = params.pluginMetadataSnapshot
? resolveInstalledManifestRegistryIndexFingerprint(params.pluginMetadataSnapshot.index)
@@ -57,6 +77,7 @@ async function buildModelsJsonFingerprint(params: {
envShape,
authProfilesMtimeMs,
modelsFileMtimeMs,
pluginCatalogMtimes,
workspaceDir: params.workspaceDir,
pluginMetadataSnapshotIndexFingerprint,
providerDiscoveryProviderIds: params.providerDiscoveryProviderIds,
@@ -108,6 +129,111 @@ export async function writeModelsFileAtomicForModelsJson(
await privateFileStore(path.dirname(targetPath)).writeText(path.basename(targetPath), contents);
}
async function isGeneratedPluginCatalogFile(targetPath: string): Promise<boolean> {
return (await readGeneratedPluginCatalog(targetPath)) !== undefined;
}
async function readGeneratedPluginCatalog(targetPath: string): Promise<unknown> {
const existing = await readExistingModelsFile(targetPath);
const parsed = existing.parsed;
return isGeneratedPluginModelCatalog(parsed) ? parsed : undefined;
}
function isRecordLike(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: {
agentDir: string;
existingParsed: unknown;
pluginMetadataSnapshot?: Pick<PluginMetadataSnapshot, "owners">;
}): Promise<unknown> {
const root = isRecordLike(params.existingParsed) ? params.existingParsed : {};
const providers = isRecordLike(root.providers) ? { ...root.providers } : {};
let changed = false;
for (const relativePath of listPluginModelCatalogRelativePaths(params.agentDir)) {
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
if (!catalogPluginId) {
continue;
}
const catalog = await readGeneratedPluginCatalog(path.join(params.agentDir, relativePath));
if (!isRecordLike(catalog) || !isRecordLike(catalog.providers)) {
continue;
}
for (const [providerId, provider] of Object.entries(catalog.providers)) {
const currentOwnerPluginId = resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
});
if (currentOwnerPluginId !== catalogPluginId) {
continue;
}
providers[providerId] = provider;
changed = true;
}
}
if (!changed) {
return params.existingParsed;
}
return { ...root, providers };
}
async function removeStalePluginCatalogs(params: {
agentDir: string;
activeRelativePaths: ReadonlySet<string>;
}): Promise<boolean> {
let wrote = false;
for (const relativePath of listPluginModelCatalogRelativePaths(params.agentDir)) {
if (params.activeRelativePaths.has(path.normalize(relativePath))) {
continue;
}
const targetPath = path.join(params.agentDir, relativePath);
if (!(await isGeneratedPluginCatalogFile(targetPath))) {
continue;
}
await fs.unlink(targetPath).catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return;
}
throw error;
});
wrote = true;
}
return wrote;
}
async function writePluginCatalogsForModelsJson(params: {
agentDir: string;
pluginCatalogWrites?: Record<string, string>;
}): Promise<boolean> {
if (!params.pluginCatalogWrites) {
return false;
}
let wrote = false;
const activeRelativePaths = new Set<string>();
for (const [relativePath, contents] of Object.entries(params.pluginCatalogWrites)) {
if (!isPluginModelCatalogRelativePath(relativePath)) {
continue;
}
activeRelativePaths.add(path.normalize(relativePath));
const targetPath = path.join(params.agentDir, relativePath);
const existing = await readExistingModelsFile(targetPath);
if (existing.raw === contents) {
await ensureModelsFileModeForModelsJson(targetPath);
continue;
}
await fs.mkdir(path.dirname(targetPath), { recursive: true, mode: 0o700 });
await writeModelsFileAtomicForModelsJson(targetPath, contents);
await ensureModelsFileModeForModelsJson(targetPath);
wrote = true;
}
const removedStale = await removeStalePluginCatalogs({
agentDir: params.agentDir,
activeRelativePaths,
});
return wrote || removedStale;
}
function resolveModelsConfigInput(config?: OpenClawConfig): {
config: OpenClawConfig;
sourceConfigForSecrets: OpenClawConfig;
@@ -212,6 +338,11 @@ export async function ensureOpenClawModelsJson(
// are available to provider discovery without mutating process.env.
const env = createConfigRuntimeEnv(cfg);
const existingModelsFile = await readExistingModelsFile(targetPath);
const existingParsedForMerge = await mergeGeneratedPluginCatalogProvidersIntoExistingParsed({
agentDir,
existingParsed: existingModelsFile.parsed,
...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
});
const plan = await planOpenClawModelsJson({
cfg,
sourceConfigForSecrets: resolved.sourceConfigForSecrets,
@@ -219,7 +350,7 @@ export async function ensureOpenClawModelsJson(
env,
...(workspaceDir ? { workspaceDir } : {}),
existingRaw: existingModelsFile.raw,
existingParsed: existingModelsFile.parsed,
existingParsed: existingParsedForMerge,
...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
...(options.providerDiscoveryProviderIds
? { providerDiscoveryProviderIds: options.providerDiscoveryProviderIds }
@@ -233,18 +364,34 @@ export async function ensureOpenClawModelsJson(
});
if (plan.action === "skip") {
return { fingerprint, result: { agentDir, wrote: false } };
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
agentDir,
pluginCatalogWrites: plan.pluginCatalogWrites,
});
return { fingerprint, result: { agentDir, wrote: wrotePluginCatalog } };
}
if (plan.action === "noop") {
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
agentDir,
pluginCatalogWrites: plan.pluginCatalogWrites,
});
await ensureModelsFileModeForModelsJson(targetPath);
return { fingerprint, result: { agentDir, wrote: false } };
return { fingerprint, result: { agentDir, wrote: wrotePluginCatalog } };
}
await fs.mkdir(agentDir, { recursive: true, mode: 0o700 });
await writeModelsFileAtomicForModelsJson(targetPath, plan.contents);
const existingRoot = existingModelsFile.raw;
const wroteRoot = existingRoot !== plan.contents;
if (wroteRoot) {
await writeModelsFileAtomicForModelsJson(targetPath, plan.contents);
}
await ensureModelsFileModeForModelsJson(targetPath);
return { fingerprint, result: { agentDir, wrote: true } };
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
agentDir,
pluginCatalogWrites: plan.pluginCatalogWrites,
});
return { fingerprint, result: { agentDir, wrote: wroteRoot || wrotePluginCatalog } };
});
MODELS_JSON_STATE.readyCache.set(cacheKey, pending);
try {
@@ -10,6 +10,11 @@ import {
withModelsTempHome,
} from "./models-config.e2e-harness.js";
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
import {
encodePluginModelCatalogRelativePath,
PLUGIN_MODEL_CATALOG_FILE,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
} from "./plugin-model-catalog.js";
const planOpenClawModelsJsonMock = vi.fn();
const writePrivateStoreTextWriteMock = vi.fn();
@@ -211,6 +216,102 @@ describe("models-config write serialization", () => {
});
});
it("writes plugin-owned model catalogs beside the agent plugin state", async () => {
await withModelsTempHome(async (home) => {
const agentDir = path.join(home, "agent");
planOpenClawModelsJsonMock.mockImplementation(async () => ({
action: "write",
contents: `${JSON.stringify({ providers: {} }, null, 2)}\n`,
pluginCatalogWrites: {
[encodePluginModelCatalogRelativePath("zai")]: `${JSON.stringify(
{
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
zai: {
baseUrl: "https://api.z.ai/api/paas/v4",
api: "openai-completions",
apiKey: "ZAI_API_KEY",
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
},
},
},
null,
2,
)}\n`,
},
}));
await ensureOpenClawModelsJson({}, agentDir);
const root = JSON.parse(await fs.readFile(path.join(agentDir, "models.json"), "utf8")) as {
providers?: Record<string, unknown>;
};
const catalog = JSON.parse(
await fs.readFile(path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE), "utf8"),
) as { providers?: Record<string, unknown> };
expect(root.providers).toEqual({});
expect(root).not.toHaveProperty("pluginCatalogs");
expect(Object.keys(catalog.providers ?? {})).toEqual(["zai"]);
});
});
it("removes stale plugin-owned model catalogs", async () => {
await withModelsTempHome(async (home) => {
const agentDir = path.join(home, "agent");
const staleCatalog = path.join(
agentDir,
"plugins",
"old-provider",
PLUGIN_MODEL_CATALOG_FILE,
);
await fs.mkdir(path.dirname(staleCatalog), { recursive: true });
await fs.writeFile(
staleCatalog,
`${JSON.stringify(
{ generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, providers: {} },
null,
2,
)}\n`,
);
planOpenClawModelsJsonMock.mockImplementation(async () => ({
action: "noop",
pluginCatalogWrites: {},
}));
await fs.mkdir(agentDir, { recursive: true });
await fs.writeFile(
path.join(agentDir, "models.json"),
`${JSON.stringify({ providers: {} })}\n`,
);
const result = await ensureOpenClawModelsJson({}, agentDir);
expect(result.wrote).toBe(true);
await expect(fs.access(staleCatalog)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("keeps generated plugin catalogs on non-authoritative skip plans", async () => {
await withModelsTempHome(async (home) => {
const agentDir = path.join(home, "agent");
const catalogPath = path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
await fs.mkdir(path.dirname(catalogPath), { recursive: true });
await fs.writeFile(
catalogPath,
`${JSON.stringify(
{ generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, providers: {} },
null,
2,
)}\n`,
);
planOpenClawModelsJsonMock.mockImplementation(async () => ({ action: "skip" }));
const result = await ensureOpenClawModelsJson({}, agentDir);
expect(result.wrote).toBe(false);
await expect(fs.access(catalogPath)).resolves.toBeUndefined();
});
});
it("does not reuse scoped startup discovery cache for a different provider scope", async () => {
await withModelsTempHome(async (home) => {
planOpenClawModelsJsonMock.mockImplementation(async () => ({ action: "skip" }));
+108
View File
@@ -0,0 +1,108 @@
import { existsSync, readdirSync } from "node:fs";
import path from "node:path";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js";
import { normalizeProviderId } from "./provider-id.js";
export const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
export const PLUGIN_MODEL_CATALOG_GENERATED_BY = "openclaw-plugin-model-catalog-v1";
export type PluginModelCatalogMetadataSnapshot = Pick<PluginMetadataSnapshot, "owners"> & {
index?: {
plugins: ReadonlyArray<{
enabled: boolean;
pluginId: string;
}>;
};
normalizePluginId?: (pluginId: string) => string;
};
export function encodePluginModelCatalogRelativePath(pluginId: string): string {
return `plugins/${encodeURIComponent(pluginId)}/${PLUGIN_MODEL_CATALOG_FILE}`;
}
export function isPluginModelCatalogRelativePath(relativePath: string): boolean {
const parts = relativePath.split(/[\\/]/);
return (
!path.isAbsolute(relativePath) &&
parts.length === 3 &&
parts[0] === "plugins" &&
parts[1] !== "" &&
parts[1] !== "." &&
parts[1] !== ".." &&
parts[2] === PLUGIN_MODEL_CATALOG_FILE
);
}
export function decodePluginModelCatalogRelativePathPluginId(
relativePath: string,
): string | undefined {
if (!isPluginModelCatalogRelativePath(relativePath)) {
return undefined;
}
const encodedPluginId = relativePath.split(/[\\/]/)[1];
try {
return decodeURIComponent(encodedPluginId);
} catch {
return undefined;
}
}
export function listPluginModelCatalogRelativePaths(agentDir: string): string[] {
const pluginsDir = path.join(agentDir, "plugins");
let pluginDirs: Array<import("node:fs").Dirent>;
try {
pluginDirs = readdirSync(pluginsDir, { withFileTypes: true });
} catch {
return [];
}
return pluginDirs
.filter((entry) => entry.isDirectory())
.map((entry) => path.join("plugins", entry.name, PLUGIN_MODEL_CATALOG_FILE))
.filter(isPluginModelCatalogRelativePath)
.toSorted((left, right) => left.localeCompare(right));
}
export function listPluginModelCatalogPaths(agentDir: string): string[] {
return listPluginModelCatalogRelativePaths(agentDir)
.map((relativePath) => path.join(agentDir, relativePath))
.filter((catalogPath) => existsSync(catalogPath));
}
export function isGeneratedPluginModelCatalog(value: unknown): boolean {
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
(value as { generatedBy?: unknown }).generatedBy === PLUGIN_MODEL_CATALOG_GENERATED_BY
);
}
export function resolvePluginModelCatalogOwnerPluginId(params: {
providerId: string;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
}): string | undefined {
const snapshot = params.pluginMetadataSnapshot;
const owners = snapshot?.owners;
if (!owners) {
return undefined;
}
const providerId = normalizeProviderId(params.providerId);
const candidates = [
owners.modelCatalogProviders.get(providerId),
owners.providers.get(providerId),
owners.setupProviders.get(providerId),
].find((entry): entry is readonly string[] => Array.isArray(entry) && entry.length > 0);
const pluginId = candidates?.length === 1 ? candidates[0] : undefined;
if (!pluginId) {
return undefined;
}
if (!snapshot?.index) {
return pluginId;
}
const normalizedPluginId = snapshot.normalizePluginId?.(pluginId) ?? pluginId;
return snapshot.index.plugins.some(
(plugin) => plugin.pluginId === normalizedPluginId && plugin.enabled,
)
? normalizedPluginId
: undefined;
}
+146 -2
View File
@@ -1,7 +1,11 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { dirname, join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
PLUGIN_MODEL_CATALOG_FILE,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
} from "../plugin-model-catalog.js";
import { AuthStorage } from "./auth-storage.js";
import { ModelRegistry } from "./model-registry.js";
@@ -15,6 +19,40 @@ function writeModelsJson(contents: unknown): string {
return file;
}
function writeModelsJsonWithPluginCatalog(params: {
root: unknown;
pluginRelativePath: string;
pluginCatalog: unknown;
}): string {
const dir = mkdtempSync(join(tmpdir(), "openclaw-model-registry-"));
tempDirs.push(dir);
const file = join(dir, "models.json");
const pluginFile = join(dir, params.pluginRelativePath);
mkdirSync(dirname(pluginFile), { recursive: true });
writeFileSync(file, JSON.stringify(params.root, null, 2), "utf-8");
writeFileSync(pluginFile, JSON.stringify(params.pluginCatalog, null, 2), "utf-8");
return file;
}
function pluginOwnerSnapshot(providerId: string, pluginId: string, enabled = true) {
return {
index: {
plugins: [{ pluginId, enabled }],
},
normalizePluginId: (id: string) => id,
owners: {
channels: new Map(),
channelConfigs: new Map(),
providers: new Map([[providerId, [pluginId]]]),
modelCatalogProviders: new Map([[providerId, [pluginId]]]),
cliBackends: new Map(),
setupProviders: new Map(),
commandAliases: new Map(),
contracts: new Map(),
},
};
}
afterEach(() => {
for (const dir of tempDirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
@@ -73,4 +111,110 @@ describe("ModelRegistry models.json auth", () => {
expect(registry.getError()).toContain('Provider custom: "apiKey" is required');
expect(registry.find("custom", "example-model")).toBeUndefined();
});
it("loads provider models from generated plugin catalog shards", () => {
const modelsPath = writeModelsJsonWithPluginCatalog({
root: { providers: {} },
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
pluginCatalog: {
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
zai: {
baseUrl: "https://api.z.ai/api/paas/v4",
api: "openai-completions",
apiKey: "ZAI_API_KEY",
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
},
},
},
});
const registry = ModelRegistry.create(
AuthStorage.inMemory({ zai: { type: "api_key", key: "sk-test" } }),
modelsPath,
{ pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai") },
);
expect(registry.getError()).toBeUndefined();
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
});
it("ignores non-generated plugin catalog files", () => {
const modelsPath = writeModelsJsonWithPluginCatalog({
root: { providers: {} },
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
pluginCatalog: {
providers: {
zai: {
baseUrl: "https://api.z.ai/api/paas/v4",
api: "openai-completions",
apiKey: "ZAI_API_KEY",
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
},
},
},
});
const registry = ModelRegistry.create(
AuthStorage.inMemory({ zai: { type: "api_key", key: "sk-test" } }),
modelsPath,
);
expect(registry.getError()).toBeUndefined();
expect(registry.find("zai", "glm-5.1")).toBeUndefined();
});
it("ignores generated plugin catalog providers without current ownership", () => {
const modelsPath = writeModelsJsonWithPluginCatalog({
root: { providers: {} },
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
pluginCatalog: {
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
zai: {
baseUrl: "https://api.z.ai/api/paas/v4",
api: "openai-completions",
apiKey: "ZAI_API_KEY",
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
},
},
},
});
const registry = ModelRegistry.create(
AuthStorage.inMemory({ zai: { type: "api_key", key: "sk-test" } }),
modelsPath,
{ pluginMetadataSnapshot: pluginOwnerSnapshot("other", "other") },
);
expect(registry.getError()).toBeUndefined();
expect(registry.find("zai", "glm-5.1")).toBeUndefined();
});
it("ignores generated plugin catalog providers owned by disabled plugins", () => {
const modelsPath = writeModelsJsonWithPluginCatalog({
root: { providers: {} },
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
pluginCatalog: {
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: {
zai: {
baseUrl: "https://api.z.ai/api/paas/v4",
api: "openai-completions",
apiKey: "ZAI_API_KEY",
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
},
},
},
});
const registry = ModelRegistry.create(
AuthStorage.inMemory({ zai: { type: "api_key", key: "sk-test" } }),
modelsPath,
{ pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai", false) },
);
expect(registry.getError()).toBeUndefined();
expect(registry.find("zai", "glm-5.1")).toBeUndefined();
});
});
+122 -8
View File
@@ -3,10 +3,11 @@
*/
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { dirname, join, relative } from "node:path";
import { type Static, Type } from "typebox";
import { Compile } from "typebox/compile";
import type { TLocalizedValidationError } from "typebox/error";
import { getRuntimeConfig } from "../../config/config.js";
import { registerApiProvider } from "../../llm/api-registry.js";
import { resetApiProviders } from "../../llm/providers/register-builtins.js";
import {
@@ -21,7 +22,16 @@ import {
} from "../../llm/types.js";
import { registerOAuthProvider, resetOAuthProviders } from "../../llm/utils/oauth/index.js";
import type { OAuthProviderInterface } from "../../llm/utils/oauth/types.js";
import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js";
import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js";
import { getAgentDir } from "../config.js";
import {
decodePluginModelCatalogRelativePathPluginId,
isGeneratedPluginModelCatalog,
listPluginModelCatalogPaths,
type PluginModelCatalogMetadataSnapshot,
resolvePluginModelCatalogOwnerPluginId,
} from "../plugin-model-catalog.js";
import type { AuthStatus, AuthStorage } from "./auth-storage.js";
import { BUILT_IN_PROVIDER_DISPLAY_NAMES } from "./provider-display-names.js";
import {
@@ -179,6 +189,7 @@ const ProviderConfigSchema = Type.Object({
});
const ModelsConfigSchema = Type.Object({
generatedBy: Type.Optional(Type.String()),
providers: Type.Record(Type.String(), ProviderConfigSchema),
});
@@ -239,6 +250,54 @@ function emptyCustomModelsResult(error?: string): CustomModelsResult {
return { models: [], error };
}
type ModelRegistryOptions = {
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
workspaceDir?: string;
};
function resolvePluginMetadataSnapshotForModelRegistry(
options: Pick<ModelRegistryOptions, "workspaceDir"> = {},
): PluginModelCatalogMetadataSnapshot | undefined {
try {
const config = getRuntimeConfig();
return (
getCurrentPluginMetadataSnapshot({
allowWorkspaceScopedSnapshot: true,
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}) ??
loadPluginMetadataSnapshot({
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
})
);
} catch {
return undefined;
}
}
function filterGeneratedPluginCatalogProviders(params: {
catalogPluginId?: string;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
providers: ModelsConfig["providers"];
}): ModelsConfig["providers"] {
if (!params.catalogPluginId || !params.pluginMetadataSnapshot) {
return {};
}
return Object.fromEntries(
Object.entries(params.providers).filter(([providerId]) => {
return (
resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
}) === params.catalogPluginId
);
}),
);
}
function mergeCompat(
baseCompat: Model["compat"],
overrideCompat: Model["compat"],
@@ -289,18 +348,26 @@ export class ModelRegistry {
private loadError: string | undefined = undefined;
readonly authStorage: AuthStorage;
private modelsJsonPath: string | undefined;
private pluginMetadataSnapshot: PluginModelCatalogMetadataSnapshot | undefined;
private constructor(authStorage: AuthStorage, modelsJsonPath: string | undefined) {
private constructor(
authStorage: AuthStorage,
modelsJsonPath: string | undefined,
options: ModelRegistryOptions = {},
) {
this.authStorage = authStorage;
this.modelsJsonPath = modelsJsonPath;
this.pluginMetadataSnapshot =
options.pluginMetadataSnapshot ?? resolvePluginMetadataSnapshotForModelRegistry(options);
this.loadModels();
}
static create(
authStorage: AuthStorage,
modelsJsonPath: string = join(getAgentDir(), "models.json"),
options: ModelRegistryOptions = {},
): ModelRegistry {
return new ModelRegistry(authStorage, modelsJsonPath);
return new ModelRegistry(authStorage, modelsJsonPath, options);
}
static inMemory(authStorage: AuthStorage): ModelRegistry {
@@ -334,7 +401,8 @@ export class ModelRegistry {
}
private loadModels(): void {
// Load configured models and request settings from models.json
// Load configured models and request settings from models.json plus
// generated plugin-owned catalog shards under the agent plugin state.
const { models: customModels, error } = this.modelsJsonPath
? this.loadCustomModels(this.modelsJsonPath)
: emptyCustomModelsResult();
@@ -357,7 +425,16 @@ export class ModelRegistry {
this.models = combined;
}
private loadCustomModels(modelsJsonPath: string): CustomModelsResult {
private loadCustomModels(
modelsJsonPath: string,
options: {
catalogPluginId?: string;
includePluginCatalogs?: boolean;
requireGeneratedCatalog?: boolean;
} = {
includePluginCatalogs: true,
},
): CustomModelsResult {
if (!existsSync(modelsJsonPath)) {
return emptyCustomModelsResult();
}
@@ -365,6 +442,9 @@ export class ModelRegistry {
try {
const content = readFileSync(modelsJsonPath, "utf-8");
const parsed = JSON.parse(stripJsonComments(content)) as unknown;
if (options.requireGeneratedCatalog === true && !isGeneratedPluginModelCatalog(parsed)) {
return emptyCustomModelsResult();
}
if (!validateModelsConfig.Check(parsed)) {
const errors =
@@ -378,19 +458,53 @@ export class ModelRegistry {
}
const config = parsed;
const providers =
options.requireGeneratedCatalog === true
? filterGeneratedPluginCatalogProviders({
catalogPluginId: options.catalogPluginId,
pluginMetadataSnapshot: this.pluginMetadataSnapshot,
providers: config.providers,
})
: config.providers;
const configForUse = { ...config, providers };
if (options.requireGeneratedCatalog === true && Object.keys(providers).length === 0) {
return emptyCustomModelsResult();
}
// Additional validation
this.validateConfig(config);
this.validateConfig(configForUse);
for (const [providerName, providerConfig] of Object.entries(config.providers)) {
for (const [providerName, providerConfig] of Object.entries(configForUse.providers)) {
if ((providerConfig.models ?? []).length > 0) {
this.storeProviderRequestConfig(providerName, providerConfig);
}
}
return { models: this.parseModels(config), error: undefined };
const models = this.parseModels(configForUse);
if (options.includePluginCatalogs !== false) {
const agentDir = dirname(modelsJsonPath);
for (const pluginCatalogPath of listPluginModelCatalogPaths(dirname(modelsJsonPath))) {
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(
relative(agentDir, pluginCatalogPath),
);
const pluginResult = this.loadCustomModels(pluginCatalogPath, {
catalogPluginId,
includePluginCatalogs: false,
requireGeneratedCatalog: true,
});
if (pluginResult.error) {
return pluginResult;
}
models.push(...pluginResult.models);
}
}
return { models, error: undefined };
} catch (error) {
if (error instanceof SyntaxError) {
if (options.requireGeneratedCatalog === true) {
return emptyCustomModelsResult();
}
return emptyCustomModelsResult(
`Failed to parse models.json: ${error.message}\n\nFile: ${modelsJsonPath}`,
);
+3
View File
@@ -511,6 +511,9 @@ describe("createPdfTool", () => {
);
expect(modelsAgentDir).toBe(agentDir);
expect(modelsOptions).toEqual({ workspaceDir });
expect(modelDiscovery.discoverModels).toHaveBeenCalledWith(expect.anything(), agentDir, {
workspaceDir,
});
expect(extractSpy).not.toHaveBeenCalled();
expect(result.content).toEqual([{ type: "text", text: "native summary" }]);
expectFields(result.details, {
+1 -1
View File
@@ -161,7 +161,7 @@ async function runPdfPrompt(params: {
const modelsOptions = params.workspaceDir ? { workspaceDir: params.workspaceDir } : undefined;
await ensureOpenClawModelsJson(effectiveCfg, params.agentDir, modelsOptions);
const authStorage = discoverAuthStorage(params.agentDir);
const modelRegistry = discoverModels(authStorage, params.agentDir);
const modelRegistry = discoverModels(authStorage, params.agentDir, modelsOptions);
let extractionCache: PdfExtractedContent[] | null = null;
const getExtractions = async (): Promise<PdfExtractedContent[]> => {
+5 -4
View File
@@ -154,10 +154,11 @@ function createIndex(
function createPersistableIndex(pluginId: string): InstalledPluginIndex {
const index = createIndex(pluginId);
for (const plugin of index.plugins) {
plugin.enabled = false;
}
return index;
const plugins = index.plugins.map((plugin) => Object.assign({}, plugin, { enabled: false }));
return {
...index,
plugins,
};
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {