fix(models): migrate catalog credentials into SQLite (#87166)

* fix(models): migrate catalog credentials into SQLite

* refactor(models): share catalog JSON parser
This commit is contained in:
Peter Steinberger
2026-08-10 02:47:51 -07:00
committed by GitHub
parent 1ff50d7e7d
commit fe9bd7583e
6 changed files with 590 additions and 8 deletions
+10
View File
@@ -0,0 +1,10 @@
/** Parses the JSON-with-comments syntax accepted by root model catalogs. */
export function parseModelCatalogJson(input: string): unknown {
const json = input
.replace(/"(?:\\.|[^"\\])*"|\/\/[^\n]*/g, (match) => (match[0] === '"' ? match : ""))
.replace(
/"(?:\\.|[^"\\])*"|,(\s*[}\]])/g,
(match, tail) => tail ?? (match[0] === '"' ? match : ""),
);
return JSON.parse(json) as unknown;
}
+2 -8
View File
@@ -20,6 +20,7 @@ import type {
import type { OAuthProviderInterface } from "../../llm/utils/oauth/types.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { getAgentDir } from "../config.js";
import { parseModelCatalogJson } from "../model-catalog-json.js";
import { resolveModelPluginMetadataSnapshot } from "../model-discovery-context.js";
import {
filterGeneratedPluginModelCatalogProviders,
@@ -229,13 +230,6 @@ function formatValidationPath(error: TLocalizedValidationError): string {
return path || "root";
}
/** Strip `//` line comments and trailing commas from JSON, leaving string literals untouched. */
function stripJsonComments(input: string): string {
return input
.replace(/"(?:\\.|[^"\\])*"|\/\/[^\n]*/g, (m) => (m[0] === '"' ? m : ""))
.replace(/"(?:\\.|[^"\\])*"|,(\s*[}\]])/g, (m, tail) => tail ?? (m[0] === '"' ? m : ""));
}
interface ProviderRequestConfig {
apiKey?: string;
auth?: ProviderAuthMode;
@@ -539,7 +533,7 @@ export class ModelRegistry {
try {
const content = options.contents ?? readFileSync(modelsJsonPath, "utf-8");
const parsed = JSON.parse(stripJsonComments(content)) as unknown;
const parsed = parseModelCatalogJson(content);
if (options.requireGeneratedCatalog === true && !isGeneratedPluginModelCatalog(parsed)) {
return emptyCustomModelsResult();
}
@@ -0,0 +1,235 @@
// Doctor migrates model credentials before removing plaintext from generated catalogs.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js";
import {
readPersistedAuthProfileStoreRaw,
writePersistedAuthProfileStoreRaw,
} from "../agents/auth-profiles/sqlite.js";
import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
import {
encodePluginModelCatalogRelativePath,
loadPersistedPluginModelCatalogsReadOnly,
PLUGIN_MODEL_CATALOG_GENERATED_BY,
replacePersistedPluginModelCatalogs,
} from "../agents/plugin-model-catalog.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { RuntimeEnv } from "../runtime.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { maybeMigrateModelCatalogCredentials } from "./doctor-model-catalog-credentials.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
const note = vi.hoisted(() => vi.fn());
vi.mock("../../packages/terminal-core/src/note.js", () => ({ note }));
const tempDirs: string[] = [];
function createState(): { agentDir: string; env: NodeJS.ProcessEnv; stateDir: string } {
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-catalog-credentials-"));
tempDirs.push(stateDir);
const agentDir = path.join(stateDir, "agents", "main", "agent");
fs.mkdirSync(agentDir, { recursive: true });
return {
agentDir,
stateDir,
env: { ...process.env, HOME: stateDir, OPENCLAW_STATE_DIR: stateDir },
};
}
function provider(apiKey: string) {
return {
api: "openai-completions" as const,
apiKey,
baseUrl: "https://models.example/v1",
models: [
{
id: "example-model",
name: "Example model",
reasoning: false,
input: ["text" as const],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128_000,
maxTokens: 16_384,
},
],
};
}
function migrationParams(state: ReturnType<typeof createState>, cfg: OpenClawConfig) {
return {
cfg,
env: state.env,
prompter: { shouldRepair: true } as DoctorPrompter,
runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() } as unknown as RuntimeEnv,
};
}
afterEach(() => {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
for (const dir of tempDirs.splice(0)) {
fs.rmSync(dir, { recursive: true, force: true });
}
});
describe("doctor model catalog credential migration", () => {
it("copies config, root, and plugin catalog keys before runtime retires plaintext", async () => {
const state = createState();
const { agentDir } = state;
const cfg: OpenClawConfig = {
models: { providers: { configured: provider("configured-secret") } },
};
const rootContents = `{
// Root catalogs use the same comment-tolerant syntax as ModelRegistry.
"providers": { "root": ${JSON.stringify(provider("root-secret"))}, },
}\n`;
fs.writeFileSync(path.join(agentDir, "models.json"), rootContents);
const pluginContents = `${JSON.stringify(
{
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
providers: { plugin: provider("plugin-secret") },
},
null,
2,
)}\n`;
replacePersistedPluginModelCatalogs({
agentDir,
pluginCatalogWrites: {
[encodePluginModelCatalogRelativePath("plugin-owner")]: pluginContents,
},
});
const first = await maybeMigrateModelCatalogCredentials(migrationParams(state, cfg));
expect(first.detected).toBe(3);
expect(first.migrated).toBe(3);
expect(first.warnings).toEqual([]);
expect(cfg.models?.providers?.configured?.apiKey).toBe("configured-secret");
expect(loadPersistedAuthProfileStore(agentDir)?.profiles).toMatchObject({
"configured:default": {
type: "api_key",
provider: "configured",
key: "configured-secret",
},
"root:default": { type: "api_key", provider: "root", key: "root-secret" },
"plugin:default": { type: "api_key", provider: "plugin", key: "plugin-secret" },
});
expect(fs.readFileSync(path.join(agentDir, "models.json"), "utf8")).toBe(rootContents);
const pluginCatalog = loadPersistedPluginModelCatalogsReadOnly(agentDir)[0];
expect(pluginCatalog?.contents).toBe(pluginContents);
const second = await maybeMigrateModelCatalogCredentials(migrationParams(state, cfg));
expect(second).toMatchObject({ detected: 0, migrated: 0, warnings: [] });
});
it("never overwrites an occupied default profile while preserving the catalog key", async () => {
const state = createState();
const { agentDir } = state;
saveAuthProfileStore(
{
version: 1,
profiles: {
"custom:default": {
type: "api_key",
provider: "custom",
key: "existing-secret",
},
},
order: { custom: ["custom:default"] },
},
agentDir,
);
fs.writeFileSync(
path.join(agentDir, "models.json"),
`${JSON.stringify({ providers: { custom: provider("catalog-secret") } }, null, 2)}\n`,
);
await maybeMigrateModelCatalogCredentials(migrationParams(state, {}));
const store = loadPersistedAuthProfileStore(agentDir);
expect(store?.profiles["custom:default"]).toMatchObject({ key: "existing-secret" });
expect(store?.profiles["custom:models-json"]).toMatchObject({ key: "catalog-secret" });
expect(store?.order?.custom).toEqual(["custom:default"]);
});
it("refuses to overwrite an unreadable canonical auth store", async () => {
const state = createState();
const { agentDir } = state;
const unreadable = { version: 1, profiles: "not-a-profile-map" };
writePersistedAuthProfileStoreRaw(unreadable, agentDir);
const rootContents = `${JSON.stringify({ providers: { custom: provider("catalog-secret") } })}\n`;
fs.writeFileSync(path.join(agentDir, "models.json"), rootContents);
const result = await maybeMigrateModelCatalogCredentials(migrationParams(state, {}));
expect(result.migrated).toBe(0);
expect(result.warnings).toHaveLength(1);
expect(readPersistedAuthProfileStoreRaw(agentDir)).toEqual(unreadable);
expect(fs.readFileSync(path.join(agentDir, "models.json"), "utf8")).toBe(rootContents);
});
it("recognizes profile references inherited from the shared main store", async () => {
const state = createState();
const childAgentDir = path.join(state.stateDir, "agents", "child", "agent");
fs.mkdirSync(childAgentDir, { recursive: true });
saveAuthProfileStore(
{
version: 1,
profiles: {
"custom:default": { type: "api_key", provider: "custom", key: "stored-secret" },
},
},
state.agentDir,
);
fs.writeFileSync(
path.join(childAgentDir, "models.json"),
`${JSON.stringify({ providers: { custom: provider("custom:default") } })}\n`,
);
const result = await maybeMigrateModelCatalogCredentials(migrationParams(state, {}));
expect(result).toMatchObject({ detected: 0, migrated: 0, warnings: [] });
expect(loadPersistedAuthProfileStore(childAgentDir)).toBeNull();
});
it("allocates a global config profile that child stores cannot shadow", async () => {
const state = createState();
const childAgentDir = path.join(state.stateDir, "agents", "child", "agent");
fs.mkdirSync(childAgentDir, { recursive: true });
saveAuthProfileStore(
{
version: 1,
profiles: {
"custom:default": { type: "api_key", provider: "custom", key: "configured-secret" },
},
},
state.agentDir,
);
saveAuthProfileStore(
{
version: 1,
profiles: {
"custom:default": { type: "api_key", provider: "custom", key: "child-secret" },
},
},
childAgentDir,
);
const result = await maybeMigrateModelCatalogCredentials(
migrationParams(state, {
models: { providers: { custom: provider("configured-secret") } },
}),
);
expect(result).toMatchObject({ detected: 1, migrated: 1, warnings: [] });
expect(
loadPersistedAuthProfileStore(state.agentDir)?.profiles["custom:models-json"],
).toMatchObject({ key: "configured-secret" });
expect(loadPersistedAuthProfileStore(childAgentDir)?.profiles["custom:default"]).toMatchObject({
key: "child-secret",
});
});
});
@@ -0,0 +1,321 @@
/** Doctor-owned migration of plaintext model-catalog credentials into agent SQLite. */
import fs from "node:fs";
import path from "node:path";
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { note } from "../../packages/terminal-core/src/note.js";
import { resolveDefaultAgentDir } from "../agents/agent-scope.js";
import { AUTH_STORE_VERSION } from "../agents/auth-profiles/constants.js";
import { mergeAuthProfileStores } from "../agents/auth-profiles/persisted.js";
import { loadPersistedAuthProfileStore } from "../agents/auth-profiles/persisted.js";
import { resolveSharedMainAuthAgentDir } from "../agents/auth-profiles/shared-main-dir.js";
import { updateAuthProfileStoreWithLock } from "../agents/auth-profiles/store.js";
import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles/types.js";
import { isNonSecretApiKeyMarker } from "../agents/model-auth-markers.js";
import { parseModelCatalogJson } from "../agents/model-catalog-json.js";
import {
isGeneratedPluginModelCatalog,
loadPersistedPluginModelCatalogsReadOnly,
} from "../agents/plugin-model-catalog.js";
import { resolveStateDir } from "../config/paths.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { RuntimeEnv } from "../runtime.js";
import { listAgentModelsJsonPaths } from "../secrets/storage-scan.js";
import { shortenHomePath } from "../utils.js";
import type { DoctorPrompter } from "./doctor-prompter.js";
type PlaintextCredential = { key: string; provider: string };
type AgentCatalogs = {
agentDir: string;
localStore: AuthProfileStore;
providers: Record<string, unknown>[];
};
function emptyStore(): AuthProfileStore {
return { version: AUTH_STORE_VERSION, profiles: {} };
}
function credentialMatches(
credential: AuthProfileCredential | undefined,
{ provider, key }: PlaintextCredential,
): boolean {
if (normalizeProviderId(credential?.provider ?? "") !== normalizeProviderId(provider)) {
return false;
}
return (
(credential?.type === "api_key" && credential.key === key) ||
(credential?.type === "token" && credential.token === key)
);
}
function collectCredentials(
providers: unknown,
store: AuthProfileStore,
blockedStores: readonly AuthProfileStore[] = [],
): PlaintextCredential[] {
if (!isRecord(providers)) {
return [];
}
return Object.entries(providers).flatMap(([provider, entry]) => {
if (!isRecord(entry) || typeof entry.apiKey !== "string") {
return [];
}
const key = entry.apiKey;
const credential = { provider, key };
if (
!key.trim() ||
isNonSecretApiKeyMarker(key) ||
store.profiles[key] !== undefined ||
findMatchingProfileId(store, credential, blockedStores) !== undefined
) {
return [];
}
return [credential];
});
}
function uniqueCredentials(credentials: readonly PlaintextCredential[]): PlaintextCredential[] {
return [
...new Map(
credentials.map((credential) => [
`${normalizeProviderId(credential.provider)}\0${credential.key}`,
credential,
]),
).values(),
];
}
function findMatchingProfileId(
store: AuthProfileStore,
credential: PlaintextCredential,
blockedStores: readonly AuthProfileStore[],
): string | undefined {
return Object.entries(store.profiles).find(
([profileId, stored]) =>
credentialMatches(stored, credential) &&
blockedStores.every(
(blocked) =>
blocked.profiles[profileId] === undefined ||
credentialMatches(blocked.profiles[profileId], credential),
),
)?.[0];
}
function allocateProfileId(
store: AuthProfileStore,
credential: PlaintextCredential,
blockedStores: readonly AuthProfileStore[],
): string {
const provider = normalizeProviderId(credential.provider);
for (let suffix = 1; ; suffix += 1) {
const profileId =
suffix === 1
? `${provider}:default`
: `${provider}:models-json${suffix === 2 ? "" : `-${suffix}`}`;
if (
(!store.profiles[profileId] || credentialMatches(store.profiles[profileId], credential)) &&
blockedStores.every(
(blocked) =>
!blocked.profiles[profileId] ||
credentialMatches(blocked.profiles[profileId], credential),
)
) {
return profileId;
}
}
}
async function persistCredentials(params: {
agentDir: string;
blockedStores?: readonly AuthProfileStore[];
credentials: readonly PlaintextCredential[];
inheritedStore?: AuthProfileStore;
stateDir: string;
}): Promise<number> {
const credentials = uniqueCredentials(params.credentials);
if (credentials.length === 0) {
return 0;
}
const blockedStores = params.blockedStores ?? [];
const profileIds = new Map<string, PlaintextCredential>();
let added = 0;
const updated = await updateAuthProfileStoreWithLock({
agentDir: params.agentDir,
stateDir: params.stateDir,
saveOptions: { filterExternalAuthProfiles: false, syncExternalCli: false },
updater: (localStore) => {
const effectiveStore = params.inheritedStore
? mergeAuthProfileStores(params.inheritedStore, localStore)
: localStore;
for (const credential of credentials) {
const profileId =
findMatchingProfileId(effectiveStore, credential, blockedStores) ??
allocateProfileId(effectiveStore, credential, blockedStores);
profileIds.set(profileId, credential);
if (credentialMatches(effectiveStore.profiles[profileId], credential)) {
continue;
}
localStore.profiles[profileId] = {
type: "api_key",
provider: normalizeProviderId(credential.provider),
key: credential.key,
};
effectiveStore.profiles[profileId] = localStore.profiles[profileId];
added += 1;
}
return added > 0;
},
});
if (!updated) {
throw new Error("auth profile store could not be updated");
}
const persisted = loadPersistedAuthProfileStore(params.agentDir);
const effectivePersisted = params.inheritedStore
? mergeAuthProfileStores(params.inheritedStore, persisted ?? emptyStore())
: persisted;
for (const [profileId, credential] of profileIds) {
if (!credentialMatches(effectivePersisted?.profiles[profileId], credential)) {
throw new Error(`credential verification failed for provider "${credential.provider}"`);
}
}
return added;
}
function collectAgentCatalogs(agentDir: string, warnings: string[]): AgentCatalogs {
const localStore = loadPersistedAuthProfileStore(agentDir) ?? emptyStore();
const providers: Record<string, unknown>[] = [];
const rootPath = path.join(agentDir, "models.json");
try {
const root = parseModelCatalogJson(fs.readFileSync(rootPath, "utf8"));
if (isRecord(root) && isRecord(root.providers)) {
providers.push(root.providers);
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
warnings.push(
`Could not read model catalog ${shortenHomePath(rootPath)}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
try {
for (const catalog of loadPersistedPluginModelCatalogsReadOnly(agentDir)) {
try {
const parsed = JSON.parse(catalog.contents) as unknown;
if (
isGeneratedPluginModelCatalog(parsed) &&
isRecord(parsed) &&
isRecord(parsed.providers)
) {
providers.push(parsed.providers);
}
} catch {
warnings.push(`Could not parse generated model catalog for plugin ${catalog.pluginId}.`);
}
}
} catch (error) {
warnings.push(
`Could not read generated model catalogs for ${shortenHomePath(agentDir)}: ${error instanceof Error ? error.message : String(error)}`,
);
}
return { agentDir, localStore, providers };
}
/** Copies and verifies catalog credentials before the runtime retires plaintext catalog auth. */
export async function maybeMigrateModelCatalogCredentials(params: {
cfg: OpenClawConfig;
env?: NodeJS.ProcessEnv;
prompter: DoctorPrompter;
runtime: RuntimeEnv;
}): Promise<{ detected: number; migrated: number; warnings: string[] }> {
const warnings: string[] = [];
const env = params.env ?? process.env;
const stateDir = resolveStateDir(env);
const mainAgentDir = resolveSharedMainAuthAgentDir(env);
const discoveredAgentDirs = listAgentModelsJsonPaths(params.cfg, stateDir, env).map(
(modelsPath) => path.dirname(modelsPath),
);
const agentDirs = [
...new Set([mainAgentDir, resolveDefaultAgentDir(params.cfg, env), ...discoveredAgentDirs]),
];
const mainStore = loadPersistedAuthProfileStore(mainAgentDir) ?? emptyStore();
const catalogs = agentDirs.map((agentDir) => collectAgentCatalogs(agentDir, warnings));
const effectiveStores = catalogs.map(({ agentDir, localStore }) =>
agentDir === mainAgentDir ? mainStore : mergeAuthProfileStores(mainStore, localStore),
);
const childStores = catalogs
.filter((catalog) => catalog.agentDir !== mainAgentDir)
.map((catalog) => catalog.localStore);
const configCredentials = collectCredentials(
params.cfg.models?.providers,
mainStore,
childStores,
);
const catalogCredentials = catalogs.map((catalog, index) =>
uniqueCredentials(
catalog.providers.flatMap((providers) =>
collectCredentials(providers, effectiveStores[index] ?? mainStore),
),
),
);
const detected =
configCredentials.length + catalogCredentials.reduce((sum, entries) => sum + entries.length, 0);
for (const warning of warnings) {
params.runtime.error(warning);
}
if (detected === 0) {
return { detected, migrated: 0, warnings };
}
note(
`Found ${detected} plaintext model credential${detected === 1 ? "" : "s"}. Run openclaw doctor --fix to copy and verify them in agent SQLite before plaintext catalog authentication is retired.`,
"Model catalog credentials",
);
const shouldRepair =
params.prompter.shouldRepair ||
(await params.prompter.confirmAutoFix({
message: "Copy model credentials into agent SQLite now?",
initialValue: true,
}));
if (!shouldRepair) {
return { detected, migrated: 0, warnings };
}
let migrated = 0;
try {
migrated += await persistCredentials({
agentDir: mainAgentDir,
blockedStores: childStores,
credentials: configCredentials,
stateDir,
});
} catch (error) {
const warning = `Could not migrate configured model credentials: ${error instanceof Error ? error.message : String(error)}`;
warnings.push(warning);
params.runtime.error(warning);
}
const migratedMainStore = loadPersistedAuthProfileStore(mainAgentDir) ?? mainStore;
for (const [index, catalog] of catalogs.entries()) {
try {
migrated += await persistCredentials({
agentDir: catalog.agentDir,
credentials: catalogCredentials[index] ?? [],
...(catalog.agentDir === mainAgentDir ? {} : { inheritedStore: migratedMainStore }),
stateDir,
});
} catch (error) {
const warning = `Could not migrate model credentials for ${shortenHomePath(catalog.agentDir)}: ${error instanceof Error ? error.message : String(error)}`;
warnings.push(warning);
params.runtime.error(warning);
}
}
if (migrated > 0) {
note(
`Copied and verified ${migrated} model credential${migrated === 1 ? "" : "s"} in agent SQLite. Existing catalog values remain active until the runtime migration lands.`,
"Doctor changes",
);
}
return { detected, migrated, warnings };
}
@@ -39,6 +39,11 @@ const mocks = vi.hoisted(() => ({
migrated: 0,
warnings: [],
}),
maybeMigrateModelCatalogCredentials: vi.fn(async () => ({
detected: 0,
migrated: 0,
warnings: [],
})),
maybeRepairGatewayDaemon: vi.fn().mockResolvedValue(undefined),
maybeRepairLegacyOAuthProfileIds: vi.fn(async (cfg: unknown) => cfg),
maybeRepairLegacyOAuthSidecarProfiles: vi.fn().mockResolvedValue(undefined),
@@ -248,6 +253,10 @@ vi.mock("../commands/doctor-plugin-model-catalog.js", () => ({
maybeMigrateLegacyPluginModelCatalogs: mocks.maybeMigrateLegacyPluginModelCatalogs,
}));
vi.mock("../commands/doctor-model-catalog-credentials.js", () => ({
maybeMigrateModelCatalogCredentials: mocks.maybeMigrateModelCatalogCredentials,
}));
vi.mock("../commands/doctor-gateway-daemon-flow.js", () => ({
maybeRepairGatewayDaemon: mocks.maybeRepairGatewayDaemon,
}));
@@ -1984,6 +1993,11 @@ describe("doctor health contributions", () => {
prompter: ctx.prompter,
runtime: ctx.runtime,
});
expect(mocks.maybeMigrateModelCatalogCredentials).toHaveBeenCalledWith({
cfg: ctx.cfg,
prompter: ctx.prompter,
runtime: ctx.runtime,
});
});
it("registers auth profile health as an opt-in structured check", async () => {
+8
View File
@@ -82,6 +82,14 @@ async function runAuthProfileHealth(ctx: DoctorHealthFlowContext): Promise<void>
prompter: ctx.prompter,
runtime: ctx.runtime,
});
const { maybeMigrateModelCatalogCredentials } =
await import("../commands/doctor-model-catalog-credentials.js");
await maybeMigrateModelCatalogCredentials({
cfg: ctx.cfg,
...(ctx.env ? { env: ctx.env } : {}),
prompter: ctx.prompter,
runtime: ctx.runtime,
});
ctx.cfg = await maybeRepairLegacyOAuthProfileIds(ctx.cfg, ctx.prompter);
await noteAuthProfileHealth({
cfg: ctx.cfg,