ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-27 21:07:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

refactor(models): consolidate config normalization (#130439)

Browse Source
This commit is contained in:
Peter Steinberger
2026-08-26 15:14:01 -07:00
committed by GitHub
parent 0e05d8205a
commit a0791fd289
7 changed files with 111 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/assertion-safety-baseline.txt
+1 -1
View File
@@ -2652,7 +2652,7 @@ src/config/config-env-vars.ts 5
src/config/config-journal-snapshot.ts 2
src/config/config-path-mutation.ts 1
src/config/context-visibility.ts 1
src/config/defaults.ts 7
src/config/defaults.ts 6
src/config/doc-baseline.ts 8
src/config/docs-config-examples.ts 1
src/config/future-version-guard.ts 2
src/agents/models-config.merge.ts
+10 -33
View File
@@ -4,6 +4,7 @@
* model catalogs without discarding existing credentials.
*/
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
import { asPositiveFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { isNonSecretApiKeyMarker } from "./model-auth-markers.js";
import { resolveCatalogOwnedModelCompat } from "./model-compat-catalog.js";
@@ -43,24 +44,6 @@ export type ExistingProviderConfig = ProviderConfig & {
api?: string;
};
function isPositiveFiniteTokenLimit(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value) && value > 0;
}
function resolvePreferredTokenLimit(params: {
explicitPresent: boolean;
explicitValue: unknown;
implicitValue: unknown;
}): number | undefined {
if (params.explicitPresent && isPositiveFiniteTokenLimit(params.explicitValue)) {
return params.explicitValue;
}
if (isPositiveFiniteTokenLimit(params.implicitValue)) {
return params.implicitValue;
}
return isPositiveFiniteTokenLimit(params.explicitValue) ? params.explicitValue : undefined;
}
function getProviderModelId(model: unknown): string {
if (!model || typeof model !== "object") {
return "";
@@ -117,21 +100,15 @@ export function mergeProviderModels(
return explicitModel;
}
const contextWindow = resolvePreferredTokenLimit({
explicitPresent: "contextWindow" in explicitModel,
explicitValue: explicitModel.contextWindow,
implicitValue: implicitModel.contextWindow,
});
const contextTokens = resolvePreferredTokenLimit({
explicitPresent: "contextTokens" in explicitModel,
explicitValue: explicitModel.contextTokens,
implicitValue: implicitModel.contextTokens,
});
const maxTokens = resolvePreferredTokenLimit({
explicitPresent: "maxTokens" in explicitModel,
explicitValue: explicitModel.maxTokens,
implicitValue: implicitModel.maxTokens,
});
const contextWindow =
asPositiveFiniteNumber(explicitModel.contextWindow) ??
asPositiveFiniteNumber(implicitModel.contextWindow);
const contextTokens =
asPositiveFiniteNumber(explicitModel.contextTokens) ??
asPositiveFiniteNumber(implicitModel.contextTokens);
const maxTokens =
asPositiveFiniteNumber(explicitModel.maxTokens) ??
asPositiveFiniteNumber(implicitModel.maxTokens);
const compat = resolveCatalogOwnedModelCompat({
catalogRoute: {
api: implicitModel.api ?? implicit.api,
src/agents/models-config.providers.normalize-keys.test.ts
+17 -2
View File
@@ -171,7 +171,7 @@ describe("normalizeProviders", () => {
}
});
it("deduplicates Google Gemini provider rows after model id normalization", async () => {
it("deduplicates model rows and keeps repeated publication stable with secret ownership", async () => {
const agentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-agent-"));
try {
const providers: NonNullable<NonNullable<OpenClawConfig["models"]>["providers"]> = {
@@ -195,9 +195,10 @@ describe("normalizeProviders", () => {
}),
],
},
custom: { baseUrl: "https://models.example/v1", models: [] },
};
const normalized = normalizeProviders({ providers, agentDir });
const normalized = normalizeProviders({ providers, agentDir, env: {} });
expect(normalized?.google?.models).toHaveLength(1);
// The first normalized row wins so explicit config details are not replaced by discovery.
@@ -208,6 +209,20 @@ describe("normalizeProviders", () => {
expect(model?.maxTokens).toBe(2048);
expect(model?.reasoning).toBe(false);
expect(model?.cost).toEqual({ input: 1, output: 2, cacheRead: 3, cacheWrite: 4 });
const published = normalizeProviderCatalogModelsForConfig(normalized);
expect(published).toBe(normalized);
const secretRefManagedProviders = new Set<string>();
const repeated = normalizeProviders({
providers: published,
agentDir,
env: {},
secretRefManagedProviders,
});
// A no-op object pass must still record marker ownership for secret preservation.
expect(repeated).toBe(published);
expect(normalizeProviderCatalogModelsForConfig(repeated)).toBe(published);
expect(secretRefManagedProviders.has("google")).toBe(true);
} finally {
await fs.rm(agentDir, { recursive: true, force: true });
}
src/agents/models-config.providers.normalize.ts
+13 -41
View File
@@ -76,9 +76,9 @@ function normalizeProviderModelsForConfig(
provider: ProviderConfig,
options: ProviderModelNormalizationOptions = {},
completeCatalogCosts = false,
): { provider: ProviderConfig; mutated: boolean } {
): ProviderConfig {
if (!Array.isArray(provider.models) || provider.models.length === 0) {
return { provider, mutated: false };
return provider;
}
let mutated = false;
@@ -122,9 +122,7 @@ function normalizeProviderModelsForConfig(
}
}
return mutated
? { provider: { ...provider, models: nextModels }, mutated }
: { provider, mutated };
return mutated ? { ...provider, models: nextModels } : provider;
}
export function normalizeProviderCatalogModelsForConfig(
@@ -141,10 +139,8 @@ export function normalizeProviderCatalogModelsForConfig(
// Complete the publication schema after duplicate rows merge, or synthetic
// zeroes can mask explicit cache prices supplied by a later row.
const normalized = normalizeProviderModelsForConfig(providerKey, provider, options, true);
if (normalized.mutated) {
mutated = true;
}
next[providerKey] = normalized.provider;
mutated ||= normalized !== provider;
next[providerKey] = normalized;
}
return mutated ? next : providers;
@@ -195,35 +191,26 @@ export function normalizeProviders(params: {
secretDefaults: params.secretDefaults,
});
if (normalizedHeaders.mutated) {
mutated = true;
normalizedProvider = { ...normalizedProvider, headers: normalizedHeaders.headers };
}
const providerWithConfiguredApiKey = normalizeConfiguredProviderApiKey({
normalizedProvider = normalizeConfiguredProviderApiKey({
providerKey: normalizedKey,
provider: normalizedProvider,
secretDefaults: params.secretDefaults,
profileApiKey: undefined,
secretRefManagedProviders: params.secretRefManagedProviders,
});
if (providerWithConfiguredApiKey !== normalizedProvider) {
mutated = true;
normalizedProvider = providerWithConfiguredApiKey;
}
// Reverse-lookup: if apiKey looks like a resolved secret value (not an env
// var name), check whether it matches the canonical env var for this provider.
// This prevents resolveConfigEnvVars()-resolved secrets from being persisted
// to models.json as plaintext. (Fixes #38757)
const providerWithResolvedEnvApiKey = normalizeResolvedEnvApiKey({
normalizedProvider = normalizeResolvedEnvApiKey({
providerKey: normalizedKey,
provider: normalizedProvider,
env,
secretRefManagedProviders: params.secretRefManagedProviders,
});
if (providerWithResolvedEnvApiKey !== normalizedProvider) {
mutated = true;
normalizedProvider = providerWithResolvedEnvApiKey;
}
const needsProfileApiKey =
Array.isArray(normalizedProvider.models) &&
@@ -236,7 +223,7 @@ export function normalizeProviders(params: {
const providerApiKeyResolver = needsProfileApiKey
? resolveProviderConfigApiKeyResolver(normalizedKey, undefined, params.manifestRegistry)
: undefined;
const providerWithApiKey = resolveMissingProviderApiKey({
normalizedProvider = resolveMissingProviderApiKey({
providerKey: normalizedKey,
provider: normalizedProvider,
env,
@@ -244,32 +231,17 @@ export function normalizeProviders(params: {
secretRefManagedProviders: params.secretRefManagedProviders,
providerApiKeyResolver,
});
if (providerWithApiKey !== normalizedProvider) {
mutated = true;
normalizedProvider = providerWithApiKey;
}
const providerSpecificNormalized = normalizeProviderSpecificConfig(
normalizedProvider = normalizeProviderSpecificConfig(
normalizedKey,
normalizedProvider,
params.manifestRegistry,
);
if (providerSpecificNormalized !== normalizedProvider) {
mutated = true;
normalizedProvider = providerSpecificNormalized;
}
const providerWithNormalizedModels = normalizeProviderModelsForConfig(
normalizedKey,
normalizedProvider,
{
manifestPlugins: params.manifestPlugins,
},
);
if (providerWithNormalizedModels.mutated) {
mutated = true;
normalizedProvider = providerWithNormalizedModels.provider;
}
normalizedProvider = normalizeProviderModelsForConfig(normalizedKey, normalizedProvider, {
manifestPlugins: params.manifestPlugins,
});
mutated ||= normalizedProvider !== provider;
const existing = next[normalizedKey];
if (existing) {
src/config/defaults.ts
+29 -88
View File
@@ -4,6 +4,7 @@ import {
collectManifestModelIdNormalizationPolicies,
normalizeConfiguredProviderCatalogModelId,
} from "@openclaw/model-catalog-core/provider-model-id-normalization";
import { asPositiveFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import { DEFAULT_CONTEXT_TOKENS } from "../agents/defaults.js";
@@ -13,7 +14,10 @@ import {
DEFAULT_SUBAGENT_MAX_CONCURRENT,
resolveAgentMaxConcurrent,
} from "./agent-limits.js";
import { normalizeAgentModelMapForConfig, normalizeAgentModelRefForConfig } from "./model-input.js";
import {
normalizeAgentModelMapForConfig,
normalizeAgentModelSelectionForConfig,
} from "./model-input.js";
import {
applyProviderConfigDefaultsForConfig,
normalizeProviderConfigForConfigDefaults,
@@ -66,10 +70,6 @@ const MISTRAL_SAFE_MAX_TOKENS_BY_MODEL = {
type ModelDefinitionLike = Partial<ModelDefinitionConfig> &
Pick<ModelDefinitionConfig, "id" | "name">;
function isPositiveNumber(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value) && value > 0;
}
function resolveModelCost(
raw?: Partial<ModelDefinitionConfig["cost"]>,
): ModelDefinitionConfig["cost"] {
@@ -243,9 +243,7 @@ export function applyModelDefaults(
continue;
}
const providerApi = normalizedProvider.api;
const providerMaxTokens = isPositiveNumber(normalizedProvider.maxTokens)
? normalizedProvider.maxTokens
: undefined;
const providerMaxTokens = asPositiveFiniteNumber(normalizedProvider.maxTokens);
const nextProvider = normalizedProvider;
if (nextProvider !== provider) {
mutated = true;
@@ -253,15 +251,11 @@ export function applyModelDefaults(
let providerMutated = false;
const nextModels = models.map((model) => {
const raw = model as ModelDefinitionLike;
let modelMutated = false;
const id = normalizeConfiguredProviderCatalogModelId(
providerId,
raw.id,
modelIdNormalizationPolicies,
);
if (id !== raw.id) {
modelMutated = true;
}
// Config entries are overrides, not full definitions: authored fields
// win, the owning catalog row fills omitted fields, and only then do
@@ -271,14 +265,8 @@ export function applyModelDefaults(
const catalogModel = resolveCatalogModel(providerId, id);
const reasoning =
typeof raw.reasoning === "boolean" ? raw.reasoning : (catalogModel?.reasoning ?? false);
if (raw.reasoning !== reasoning) {
modelMutated = true;
}
const input = raw.input ?? catalogModel?.input ?? [...DEFAULT_MODEL_INPUT];
if (raw.input === undefined) {
modelMutated = true;
}
const cost = resolveModelCost(
raw.cost || catalogModel?.cost ? { ...catalogModel?.cost, ...raw.cost } : undefined,
@@ -297,51 +285,29 @@ export function applyModelDefaults(
raw.cost.cacheRead !== cost.cacheRead ||
raw.cost.cacheWrite !== cost.cacheWrite ||
raw.cost.tieredPricing !== cost.tieredPricing;
if (costMutated) {
modelMutated = true;
}
const contextWindow = isPositiveNumber(raw.contextWindow)
? raw.contextWindow
: isPositiveNumber(catalogModel?.contextWindow)
? catalogModel.contextWindow
: undefined;
if (raw.contextWindow !== contextWindow) {
modelMutated = true;
}
const contextTokens = isPositiveNumber(raw.contextTokens)
? raw.contextTokens
: isPositiveNumber(catalogModel?.contextTokens)
? catalogModel.contextTokens
: undefined;
if (raw.contextTokens !== contextTokens) {
modelMutated = true;
}
const contextWindow =
asPositiveFiniteNumber(raw.contextWindow) ??
asPositiveFiniteNumber(catalogModel?.contextWindow);
const contextTokens =
asPositiveFiniteNumber(raw.contextTokens) ??
asPositiveFiniteNumber(catalogModel?.contextTokens);
const maxTokenContextWindow = contextWindow ?? DEFAULT_CONTEXT_TOKENS;
const defaultMaxTokens = Math.min(
providerMaxTokens ?? DEFAULT_MODEL_MAX_TOKENS,
maxTokenContextWindow,
);
const rawMaxTokens = isPositiveNumber(raw.maxTokens)
? raw.maxTokens
: isPositiveNumber(catalogModel?.maxTokens)
? catalogModel.maxTokens
: defaultMaxTokens;
const rawMaxTokens =
asPositiveFiniteNumber(raw.maxTokens) ??
asPositiveFiniteNumber(catalogModel?.maxTokens) ??
defaultMaxTokens;
const maxTokens = resolveNormalizedProviderModelMaxTokens({
providerId,
modelId: id,
contextWindow: maxTokenContextWindow,
rawMaxTokens,
});
if (raw.maxTokens !== maxTokens) {
modelMutated = true;
}
const api = raw.api ?? providerApi;
if (raw.api !== api) {
modelMutated = true;
}
const thinkingLevelMap =
raw.thinkingLevelMap === undefined && catalogModel?.thinkingLevelMap !== undefined
@@ -351,10 +317,17 @@ export function applyModelDefaults(
raw.compat === undefined && catalogModel?.compat !== undefined
? catalogModel.compat
: undefined;
if (thinkingLevelMap !== undefined || compat !== undefined) {
modelMutated = true;
}
const modelMutated =
id !== raw.id ||
raw.reasoning !== reasoning ||
raw.input === undefined ||
costMutated ||
raw.contextWindow !== contextWindow ||
raw.contextTokens !== contextTokens ||
raw.maxTokens !== maxTokens ||
raw.api !== api ||
thinkingLevelMap !== undefined ||
compat !== undefined;
if (!modelMutated) {
return model;
}
@@ -408,7 +381,7 @@ export function applyModelDefaults(
}
let nextAgent = agent;
if (Object.hasOwn(agent, "model")) {
const normalizedModel = normalizeAgentModelConfigForDefaults(agent.model);
const normalizedModel = normalizeAgentModelSelectionForConfig(agent.model);
if (normalizedModel !== agent.model) {
nextAgent = { ...nextAgent, model: normalizedModel as typeof agent.model };
listMutated = true;
@@ -438,7 +411,7 @@ export function applyModelDefaults(
}
let nextAgent = existingAgent;
const normalizedModel = normalizeAgentModelConfigForDefaults(existingAgent.model);
const normalizedModel = normalizeAgentModelSelectionForConfig(existingAgent.model);
if (normalizedModel !== existingAgent.model) {
nextAgent = { ...nextAgent, model: normalizedModel as typeof existingAgent.model };
mutated = true;
@@ -499,38 +472,6 @@ export function applyModelDefaults(
};
}
function normalizeAgentModelConfigForDefaults(value: unknown): unknown {
if (typeof value === "string") {
const normalized = normalizeAgentModelRefForConfig(value);
return normalized === value ? value : normalized;
}
if (!value || typeof value !== "object" || Array.isArray(value)) {
return value;
}
const raw = value as Record<string, unknown>;
let mutated = false;
const next: Record<string, unknown> = { ...raw };
if (typeof raw.primary === "string") {
const primary = normalizeAgentModelRefForConfig(raw.primary);
if (primary !== raw.primary) {
next.primary = primary;
mutated = true;
}
}
if (Array.isArray(raw.fallbacks)) {
const rawFallbacks = raw.fallbacks;
const fallbacks = rawFallbacks.map((fallback) =>
typeof fallback === "string" ? normalizeAgentModelRefForConfig(fallback) : fallback,
);
if (fallbacks.some((fallback, index) => fallback !== rawFallbacks[index])) {
next.fallbacks = fallbacks;
mutated = true;
}
}
return mutated ? next : value;
}
export function applyAgentDefaults(cfg: OpenClawConfig): OpenClawConfig {
const agents = cfg.agents;
const defaults = agents?.defaults;
src/config/model-input-normalization.ts
+11 -34
View File
@@ -4,42 +4,16 @@ import {
type ManifestModelIdNormalizationProvider,
} from "@openclaw/model-catalog-core/provider-model-id-normalization";
import { isRecord } from "../utils.js";
import { normalizeAgentModelMapForConfig, normalizeAgentModelRefForConfig } from "./model-input.js";
import {
normalizeAgentModelMapForConfig,
normalizeAgentModelRefForConfig,
normalizeAgentModelSelectionForConfig,
} from "./model-input.js";
import type { OpenClawConfig } from "./types.openclaw.js";
const MODEL_SELECTION_KEYS = ["model", "imageModel", "voiceModel", "pdfModel"] as const;
const MEDIA_MODEL_KEYS = ["image", "video", "music"] as const;
function normalizeModelSelection(value: unknown): unknown {
if (typeof value === "string") {
return normalizeAgentModelRefForConfig(value);
}
if (!isRecord(value)) {
return value;
}
let next = value;
const assign = (key: string, candidate: unknown) => {
if (candidate === next[key]) {
return;
}
next = { ...next, [key]: candidate };
};
if (typeof value.primary === "string") {
assign("primary", normalizeAgentModelRefForConfig(value.primary));
}
if (Array.isArray(value.fallbacks)) {
const originalFallbacks = value.fallbacks;
const fallbacks = originalFallbacks.map((fallback) =>
typeof fallback === "string" ? normalizeAgentModelRefForConfig(fallback) : fallback,
);
if (fallbacks.some((fallback, index) => fallback !== originalFallbacks[index])) {
assign("fallbacks", fallbacks);
}
}
return next;
}
function normalizeStringModelRef(value: unknown): unknown {
return typeof value === "string" ? normalizeAgentModelRefForConfig(value) : value;
}
@@ -71,7 +45,7 @@ function normalizeAgentModelScope(value: unknown): unknown {
for (const key of MODEL_SELECTION_KEYS) {
if (Object.hasOwn(value, key)) {
assign(key, normalizeModelSelection(value[key]));
assign(key, normalizeAgentModelSelectionForConfig(value[key]));
}
}
if (Object.hasOwn(value, "utilityModel")) {
@@ -85,7 +59,7 @@ function normalizeAgentModelScope(value: unknown): unknown {
if (!Object.hasOwn(originalMediaModels, key)) {
continue;
}
const normalized = normalizeModelSelection(originalMediaModels[key]);
const normalized = normalizeAgentModelSelectionForConfig(originalMediaModels[key]);
if (normalized !== mediaModels[key]) {
mediaModels[key] = normalized;
mediaModelsChanged = true;
@@ -96,7 +70,10 @@ function normalizeAgentModelScope(value: unknown): unknown {
}
}
assign("heartbeat", normalizeNestedModelField(value.heartbeat, "model", normalizeStringModelRef));
assign("subagents", normalizeNestedModelField(value.subagents, "model", normalizeModelSelection));
assign(
"subagents",
normalizeNestedModelField(value.subagents, "model", normalizeAgentModelSelectionForConfig),
);
if (isRecord(value.compaction)) {
let compaction = normalizeNestedModelField(value.compaction, "model", normalizeStringModelRef);
src/config/model-input.ts
+30
View File
@@ -76,6 +76,36 @@ export function normalizeAgentModelRefForConfig(model: string): string {
return modelKey(provider, normalizedModel);
}
/** Normalizes primary/fallback refs without replacing unchanged config values. */
export function normalizeAgentModelSelectionForConfig(value: unknown): unknown {
if (typeof value === "string") {
return normalizeAgentModelRefForConfig(value);
}
if (!isPlainRecord(value)) {
return value;
}
let next = value;
const assign = (key: string, candidate: unknown) => {
if (candidate !== next[key]) {
next = { ...next, [key]: candidate };
}
};
if (typeof value.primary === "string") {
assign("primary", normalizeAgentModelRefForConfig(value.primary));
}
if (Array.isArray(value.fallbacks)) {
const originalFallbacks = value.fallbacks;
const fallbacks = originalFallbacks.map((fallback) =>
typeof fallback === "string" ? normalizeAgentModelRefForConfig(fallback) : fallback,
);
if (fallbacks.some((fallback, index) => fallback !== originalFallbacks[index])) {
assign("fallbacks", fallbacks);
}
}
return next;
}
function mergeAgentModelEntryForConfig(existing: unknown, incoming: unknown): unknown {
if (!isPlainRecord(existing) || !isPlainRecord(incoming)) {
return incoming;