mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
improve(models): source pricing from hosted catalog (#114060)
* feat(model-catalog): serve hosted fallback pricing * refactor(config): retire client pricing bootstrap settings * refactor(gateway): delete client pricing refresh runtime * docs(models): explain hosted catalog pricing * fix(model-catalog): preserve pricing privacy and aliases * fix(model-catalog): fingerprint pricing eligibility * fix(model-catalog): harden pricing endpoint checks * fix(model-catalog): materialize source-safe pricing aliases * fix(model-catalog): keep unknown pricing fallbacks safe * fix(model-catalog): reject zero-only hosted prices * fix(model-catalog): fail closed without pricing policy metadata * refactor(utils): extract usage pricing normalization * fix(model-catalog): rebuild policy-owned pricing namespaces * test(model-catalog): type publisher cost fixtures * chore(config): regenerate schema baselines * fix(utils): keep raw pricing tiers private
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
0951d0dd30
commit
5347285d6b
@@ -730,8 +730,6 @@ src/gateway/gateway-models.profiles.live.test.ts
|
||||
src/gateway/managed-image-attachments.test.ts
|
||||
src/gateway/managed-image-attachments.ts
|
||||
src/gateway/mcp-http.test.ts
|
||||
src/gateway/model-pricing-cache.test.ts
|
||||
src/gateway/model-pricing-cache.ts
|
||||
src/gateway/node-registry.test.ts
|
||||
src/gateway/node-registry.ts
|
||||
src/gateway/openai-http.test.ts
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"core": 2306,
|
||||
"core": 2304,
|
||||
"channel": 3630,
|
||||
"plugin": 3556
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
b5a60d7a0cfb860835eb4f38acb6421db736c76121a891311f37dede4c26e5a4 config-baseline.json
|
||||
e51de41a04b71b26f3ed5edd8e663982f2dee84026b7ecfab6ef75ecb2e7eb4d config-baseline.core.json
|
||||
8a8f87d6d3b350de300c9ce32d628fcd2f6e90b3e1dfb8f2f6ff915f56e83642 config-baseline.json
|
||||
2079c0b778fafecfcb2bea1a2eb6d54fbbe00f44959a0a3a0393c5c60e1f9089 config-baseline.core.json
|
||||
af6ca0e70007113462270d46fa14ef0551e577e2fa157d2b9f6d0a93d36f96f1 config-baseline.channel.json
|
||||
d47f7eb4268480e99ab85b6c1b5ee07f6c570ce0555e3db682ec26037014b47e config-baseline.plugin.json
|
||||
|
||||
@@ -231,10 +231,11 @@ The hosted file is published from the public
|
||||
Its scheduled workflow refreshes from OpenClaw's shipped plugin manifests and
|
||||
pricing sources; every catalog content change is preserved as a public commit.
|
||||
|
||||
Run `openclaw models refresh` for an immediate check, or disable every hosted
|
||||
catalog request with `models.catalogRefresh.enabled: false`. A self-hosted mirror
|
||||
can be selected with an HTTPS `models.catalogRefresh.url` (or localhost HTTP
|
||||
for testing); see
|
||||
Run `openclaw models refresh` for an immediate metadata and pricing check, or
|
||||
disable every hosted catalog request with `models.catalogRefresh.enabled:
|
||||
false`. When disabled, pricing stays at bundled and explicitly configured
|
||||
values. A self-hosted mirror can be selected with an HTTPS
|
||||
`models.catalogRefresh.url` (or localhost HTTP for testing); see
|
||||
[configuration reference](/gateway/configuration-reference#models).
|
||||
|
||||
Custom providers configured under `models.providers` are written into `models.json` under the agent directory (default `~/.openclaw/agents/<agentId>/agent/models.json`). Provider-plugin catalogs are stored separately as generated plugin-owned catalog shards and load automatically. This file is merged with config by default; set `models.mode: "replace"` to use only your configured providers.
|
||||
|
||||
@@ -64,8 +64,6 @@ The `models` root also owns global model-catalog behavior.
|
||||
```json5
|
||||
{
|
||||
models: {
|
||||
// Optional. Default: true. Requires a Gateway restart when changed.
|
||||
pricing: { enabled: false },
|
||||
// Optional. Hosted catalog updates default on.
|
||||
catalogRefresh: {
|
||||
enabled: true,
|
||||
@@ -81,19 +79,21 @@ The `models` root also owns global model-catalog behavior.
|
||||
local model servers. OpenClaw probes the configured health endpoint, starts
|
||||
the absolute `command` when needed, waits for readiness, then sends the model
|
||||
request. See [Local model services](/gateway/local-model-services).
|
||||
- `models.pricing.enabled`: controls the background pricing bootstrap that
|
||||
starts after sidecars and channels reach the Gateway ready path. When `false`,
|
||||
the Gateway skips OpenRouter and LiteLLM pricing-catalog fetches; configured
|
||||
`models.providers.*.models[].cost` values still work for local cost estimates.
|
||||
- `models.catalogRefresh.enabled`: controls the hosted model catalog refresh
|
||||
(default: `true`). Set it to `false` to prevent all remote catalog requests;
|
||||
only catalog data shipped in the installed release is then used.
|
||||
model metadata and pricing then stay at the values shipped in the installed
|
||||
release or declared under `models.providers.*.models[].cost`.
|
||||
- `models.catalogRefresh.url`: optional HTTPS mirror override (plain HTTP is
|
||||
accepted only for explicit localhost testing). The Gateway
|
||||
checks in the background at startup and every six hours. A downloaded catalog
|
||||
applies on the next Gateway restart; a release whose bundled catalog is newer
|
||||
always wins.
|
||||
|
||||
Pricing updates ship in the same hosted catalog file as model metadata. The
|
||||
retired `models.pricing` toggle is removed automatically by `openclaw doctor
|
||||
--fix`; use `models.catalogRefresh.enabled: false` when OpenClaw must avoid all
|
||||
hosted catalog traffic.
|
||||
|
||||
## MCP
|
||||
|
||||
OpenClaw-managed MCP server definitions live under `mcp.servers` and are
|
||||
|
||||
@@ -147,7 +147,7 @@ See [Plugins](/tools/plugin) for the full plugin system guide, and [Capability m
|
||||
| `providerCatalogEntry` | No | `string` | Lightweight provider-catalog module path, relative to the plugin root, for manifest-scoped provider catalog metadata that can be loaded without activating the full plugin runtime. |
|
||||
| `modelSupport` | No | `object` | Manifest-owned shorthand model-family metadata used to auto-load the plugin before runtime. |
|
||||
| `modelCatalog` | No | `object` | Declarative model catalog metadata for providers owned by this plugin. This is the control-plane contract for future read-only listing, onboarding, model pickers, aliases, and suppression without loading plugin runtime. |
|
||||
| `modelPricing` | No | `object` | Provider-owned external pricing lookup policy. Use it to opt local/self-hosted providers out of remote pricing catalogs or map provider refs to OpenRouter/LiteLLM catalog ids without hardcoding provider ids in core. |
|
||||
| `modelPricing` | No | `object` | Provider-owned hosted-pricing publication policy. Use it to opt local/self-hosted providers out of published pricing or map provider refs to OpenRouter/LiteLLM catalog ids without hardcoding provider ids in core. |
|
||||
| `modelIdNormalization` | No | `object` | Provider-owned model-id alias/prefix cleanup that must run before provider runtime loads. |
|
||||
| `providerEndpoints` | No | `object[]` | Manifest-owned endpoint host/baseUrl metadata for provider routes that core must classify before provider runtime loads. |
|
||||
| `providerRequest` | No | `object` | Cheap provider-family and request-compatibility metadata used by generic request policy before provider runtime loads. |
|
||||
@@ -1133,7 +1133,7 @@ OpenClaw derives `trustedDirs` for manifest presets from the plugin root and, fo
|
||||
|
||||
## modelPricing reference
|
||||
|
||||
Use `modelPricing` when a provider needs control-plane pricing behavior before runtime loads. The Gateway pricing cache reads this metadata without importing provider runtime code.
|
||||
Use `modelPricing` when the hosted catalog publisher needs provider-specific pricing-key behavior. The publisher reads this metadata without importing provider runtime code.
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -1156,11 +1156,11 @@ Use `modelPricing` when a provider needs control-plane pricing behavior before r
|
||||
|
||||
Provider fields:
|
||||
|
||||
| Field | Type | What it means |
|
||||
| ------------ | ----------------- | -------------------------------------------------------------------------------------------------- |
|
||||
| `external` | `boolean` | Set `false` for local/self-hosted providers that should never fetch OpenRouter or LiteLLM pricing. |
|
||||
| `openRouter` | `false \| object` | OpenRouter pricing lookup mapping. `false` disables OpenRouter lookup for this provider. |
|
||||
| `liteLLM` | `false \| object` | LiteLLM pricing lookup mapping. `false` disables LiteLLM lookup for this provider. |
|
||||
| Field | Type | What it means |
|
||||
| ------------ | ----------------- | --------------------------------------------------------------------------------------------- |
|
||||
| `external` | `boolean` | Set `false` for local/self-hosted providers that should never use published external pricing. |
|
||||
| `openRouter` | `false \| object` | OpenRouter publication-key mapping. `false` disables OpenRouter matching for this provider. |
|
||||
| `liteLLM` | `false \| object` | LiteLLM publication-key mapping. `false` disables LiteLLM matching for this provider. |
|
||||
|
||||
Source fields:
|
||||
|
||||
|
||||
@@ -183,11 +183,10 @@ auth: non-API-key providers such as `aws-sdk` can show estimated cost when
|
||||
their configured model entry includes local pricing and the provider
|
||||
returns usage metadata.
|
||||
|
||||
After sidecars and channels reach the Gateway ready path, OpenClaw starts an
|
||||
optional background pricing bootstrap for configured model refs that do not
|
||||
already have local pricing. That bootstrap fetches remote OpenRouter and
|
||||
LiteLLM pricing catalogs. Set `models.pricing.enabled: false` to skip those
|
||||
catalog fetches on offline or restricted networks; explicit
|
||||
Pricing updates ship in the hosted model catalog alongside model metadata.
|
||||
OpenClaw does not fetch OpenRouter or LiteLLM directly. Set
|
||||
`models.catalogRefresh.enabled: false` to disable hosted catalog traffic on
|
||||
offline or restricted networks; bundled pricing and explicit
|
||||
`models.providers.*.models[].cost` entries still drive local cost estimates.
|
||||
|
||||
## Cache TTL and pruning impact
|
||||
|
||||
@@ -25,6 +25,9 @@ const validBundle = {
|
||||
],
|
||||
},
|
||||
},
|
||||
pricing: {
|
||||
"openai/gpt-external": { input: 2.5, output: 10, cacheRead: 1.25 },
|
||||
},
|
||||
} as const;
|
||||
|
||||
describe("remote model catalog bundle", () => {
|
||||
@@ -44,6 +47,11 @@ describe("remote model catalog bundle", () => {
|
||||
expect(anthropic.models[0]).not.toHaveProperty("baseUrl");
|
||||
expect(anthropic.models[0]).not.toHaveProperty("headers");
|
||||
expect(anthropic.models[0]?.compat).toEqual({ nested: {} });
|
||||
expect(parsed.pricing?.["openai/gpt-external"]).toEqual({
|
||||
input: 2.5,
|
||||
output: 10,
|
||||
cacheRead: 1.25,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unsupported versions, invalid timestamps, and malformed providers", () => {
|
||||
@@ -64,5 +72,17 @@ describe("remote model catalog bundle", () => {
|
||||
providers: { anthropic: { models: [{ id: " duplicate " }, { id: "duplicate" }] } },
|
||||
}),
|
||||
).toThrow("duplicate model id: duplicate");
|
||||
expect(() =>
|
||||
parseRemoteModelCatalogBundle({
|
||||
...validBundle,
|
||||
pricing: { "openai/bad": { input: -1, output: 2 } },
|
||||
}),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
parseRemoteModelCatalogBundle({
|
||||
...validBundle,
|
||||
pricing: { "openai/bad": { input: 1, output: 2, baseUrl: "https://bad.test" } },
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,29 +5,41 @@ import type { ModelCatalogProvider } from "./model-catalog-types.js";
|
||||
export const REMOTE_CATALOG_MAX_FUTURE_SKEW_MS = 24 * 60 * 60_000;
|
||||
|
||||
const stringMapSchema = z.record(z.string(), z.string());
|
||||
const pricingTierSchema = z
|
||||
.object({
|
||||
input: z.number().finite().nonnegative(),
|
||||
output: z.number().finite().nonnegative(),
|
||||
cacheRead: z.number().finite().nonnegative(),
|
||||
cacheWrite: z.number().finite().nonnegative(),
|
||||
range: z.union([
|
||||
z.tuple([z.number().finite().nonnegative()]),
|
||||
z.tuple([z.number().finite().nonnegative(), z.number().finite().nonnegative()]),
|
||||
]),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const costSchema = z
|
||||
.object({
|
||||
input: z.number().finite().nonnegative().optional(),
|
||||
output: z.number().finite().nonnegative().optional(),
|
||||
cacheRead: z.number().finite().nonnegative().optional(),
|
||||
cacheWrite: z.number().finite().nonnegative().optional(),
|
||||
tieredPricing: z
|
||||
.array(
|
||||
z.object({
|
||||
input: z.number().finite().nonnegative(),
|
||||
output: z.number().finite().nonnegative(),
|
||||
cacheRead: z.number().finite().nonnegative(),
|
||||
cacheWrite: z.number().finite().nonnegative(),
|
||||
range: z.union([
|
||||
z.tuple([z.number().finite().nonnegative()]),
|
||||
z.tuple([z.number().finite().nonnegative(), z.number().finite().nonnegative()]),
|
||||
]),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
tieredPricing: z.array(pricingTierSchema).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const hostedPricingSchema = z
|
||||
.object({
|
||||
input: z.number().finite().nonnegative(),
|
||||
output: z.number().finite().nonnegative(),
|
||||
cacheRead: z.number().finite().nonnegative().optional(),
|
||||
cacheWrite: z.number().finite().nonnegative().optional(),
|
||||
tieredPricing: z.array(pricingTierSchema).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type RemoteModelCatalogPricing = z.infer<typeof hostedPricingSchema>;
|
||||
|
||||
const modelSchema = z.object({
|
||||
id: z.string().trim().min(1),
|
||||
name: z.string().optional(),
|
||||
@@ -89,6 +101,7 @@ export const remoteModelCatalogBundleSchema = z
|
||||
minVersion: z.string().trim().min(1).optional(),
|
||||
sourceCommit: z.string().trim().min(1),
|
||||
providers: z.record(z.string().trim().min(1), remoteModelCatalogProviderSchema),
|
||||
pricing: z.record(z.string().trim().min(1), hostedPricingSchema).optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
|
||||
@@ -25,6 +25,21 @@ export type ModelCatalogBundleSummary = {
|
||||
providers: number;
|
||||
models: number;
|
||||
costModels: number;
|
||||
pricingEntries: number;
|
||||
};
|
||||
|
||||
export type PublishedModelPricing = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
tieredPricing?: Array<{
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
range: [number] | [number, number];
|
||||
}>;
|
||||
};
|
||||
|
||||
export type PublishedModelCatalogBundle = {
|
||||
@@ -39,6 +54,7 @@ export type PublishedModelCatalogBundle = {
|
||||
[key: string]: unknown;
|
||||
}
|
||||
>;
|
||||
pricing?: Record<string, PublishedModelPricing>;
|
||||
};
|
||||
|
||||
export function parsePublishModelCatalogArgs(args: string[]): PublishModelCatalogArgs;
|
||||
@@ -59,5 +75,6 @@ export function enrichModelCatalogPricing(options: {
|
||||
bundle: PublishedModelCatalogBundle;
|
||||
manifests: ModelCatalogManifestInput[];
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<number>;
|
||||
validateBundle?: (bundle: unknown) => PublishedModelCatalogBundle;
|
||||
}): Promise<{ modelsEnriched: number; pricingEntries: number }>;
|
||||
export function serializeModelCatalogBundle(bundle: PublishedModelCatalogBundle): string;
|
||||
|
||||
@@ -12,6 +12,8 @@ export const LITELLM_PRICING_URL =
|
||||
const SCRIPT_LABEL = "publish-model-catalog";
|
||||
const PRICING_FETCH_TIMEOUT_MS = 60_000;
|
||||
const MAX_PRICING_CATALOG_BYTES = 5 * 1024 * 1024;
|
||||
const BUNDLE_SIZE_WARNING_BYTES = 2 * 1024 * 1024;
|
||||
const CLIENT_BUNDLE_LIMIT_BYTES = 4 * 1024 * 1024;
|
||||
const defaultRootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
function requireOptionValue(args, index, flag) {
|
||||
@@ -129,6 +131,7 @@ export function summarizeModelCatalogBundle(bundle) {
|
||||
(total, provider) => total + provider.models.filter((model) => model.cost).length,
|
||||
0,
|
||||
),
|
||||
pricingEntries: Object.keys(bundle.pricing ?? {}).length,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -212,6 +215,43 @@ function parseLiteLLMPricing(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function compactPricing(pricing) {
|
||||
return {
|
||||
input: pricing.input,
|
||||
output: pricing.output,
|
||||
...(pricing.cacheRead > 0 ? { cacheRead: pricing.cacheRead } : {}),
|
||||
...(pricing.cacheWrite > 0 ? { cacheWrite: pricing.cacheWrite } : {}),
|
||||
...(pricing.tieredPricing ? { tieredPricing: pricing.tieredPricing } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function mergePricing(primary, secondary) {
|
||||
if (!primary || !hasKnownPricing(primary)) {
|
||||
return secondary;
|
||||
}
|
||||
if (!secondary || !hasKnownPricing(secondary)) {
|
||||
return primary;
|
||||
}
|
||||
if (!secondary?.tieredPricing) {
|
||||
return primary;
|
||||
}
|
||||
return { ...primary, tieredPricing: secondary.tieredPricing };
|
||||
}
|
||||
|
||||
function hasKnownPricing(pricing) {
|
||||
return (
|
||||
(pricing.input ?? 0) > 0 ||
|
||||
(pricing.output ?? 0) > 0 ||
|
||||
(pricing.cacheRead ?? 0) > 0 ||
|
||||
(pricing.cacheWrite ?? 0) > 0 ||
|
||||
Boolean(
|
||||
pricing.tieredPricing?.some(
|
||||
(tier) => tier.input > 0 || tier.output > 0 || tier.cacheRead > 0 || tier.cacheWrite > 0,
|
||||
),
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function applyModelIdTransforms(modelId, transforms) {
|
||||
const variants = new Set([modelId]);
|
||||
for (const transform of transforms ?? []) {
|
||||
@@ -264,6 +304,106 @@ function buildPricingCandidates({ providerId, modelId, source, policies, seen =
|
||||
return [...candidates];
|
||||
}
|
||||
|
||||
function reverseModelIdTransforms(modelId, transforms) {
|
||||
const variants = new Set([modelId]);
|
||||
for (const transform of transforms ?? []) {
|
||||
if (transform !== "version-dots") {
|
||||
continue;
|
||||
}
|
||||
for (const variant of Array.from(variants)) {
|
||||
variants.add(
|
||||
variant
|
||||
.replace(/^claude-(\d+)\.(\d+)-/u, "claude-$1-$2-")
|
||||
.replace(/^claude-([a-z]+)-(\d+)\.(\d+)$/u, "claude-$1-$2-$3"),
|
||||
);
|
||||
}
|
||||
}
|
||||
return [...variants];
|
||||
}
|
||||
|
||||
function setRuntimePricingSource(entries, key, source, pricing) {
|
||||
const entry = entries.get(key) ?? {};
|
||||
entry[source] = pricing;
|
||||
entries.set(key, entry);
|
||||
}
|
||||
|
||||
function collectPolicyRuntimePricingAliases({ providerId, policy, source, catalog, entries }) {
|
||||
const rawSourcePolicy = policy?.[source];
|
||||
const sourceEnabled = Boolean(rawSourcePolicy);
|
||||
const sourcePolicy = sourceEnabled ? rawSourcePolicy : {};
|
||||
const sourceProvider = sourcePolicy.provider ?? providerId;
|
||||
for (const [sourceKey, pricing] of catalog) {
|
||||
const slash = sourceKey.indexOf("/");
|
||||
if (slash <= 0 || slash === sourceKey.length - 1) {
|
||||
continue;
|
||||
}
|
||||
const keyProvider = sourceKey.slice(0, slash);
|
||||
const sourceModel = sourceKey.slice(slash + 1);
|
||||
if (keyProvider === sourceProvider) {
|
||||
for (const runtimeModel of reverseModelIdTransforms(
|
||||
sourceModel,
|
||||
sourcePolicy.modelIdTransforms,
|
||||
)) {
|
||||
setRuntimePricingSource(
|
||||
entries,
|
||||
`${providerId}/${runtimeModel}`,
|
||||
source,
|
||||
sourceEnabled ? pricing : undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (sourceEnabled && sourcePolicy.passthroughProviderModel) {
|
||||
setRuntimePricingSource(entries, `${providerId}/${sourceKey}`, source, pricing);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function materializePolicyRuntimePricing({
|
||||
hostedPricing,
|
||||
policies,
|
||||
openRouterCatalog,
|
||||
liteLlmCatalog,
|
||||
pricedProviderModelKeys,
|
||||
}) {
|
||||
for (const [providerId, policy] of policies) {
|
||||
for (const key of hostedPricing.keys()) {
|
||||
if (key.startsWith(`${providerId}/`)) {
|
||||
hostedPricing.delete(key);
|
||||
}
|
||||
}
|
||||
if (policy?.external === false) {
|
||||
continue;
|
||||
}
|
||||
const entries = new Map();
|
||||
collectPolicyRuntimePricingAliases({
|
||||
providerId,
|
||||
policy,
|
||||
source: "openRouter",
|
||||
catalog: openRouterCatalog,
|
||||
entries,
|
||||
});
|
||||
collectPolicyRuntimePricingAliases({
|
||||
providerId,
|
||||
policy,
|
||||
source: "liteLLM",
|
||||
catalog: liteLlmCatalog,
|
||||
entries,
|
||||
});
|
||||
for (const [key, entry] of entries) {
|
||||
if (pricedProviderModelKeys.has(key)) {
|
||||
hostedPricing.delete(key);
|
||||
continue;
|
||||
}
|
||||
const pricing = mergePricing(entry.openRouter, entry.liteLLM);
|
||||
if (pricing) {
|
||||
hostedPricing.set(key, pricing);
|
||||
} else {
|
||||
hostedPricing.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function readPricingPolicies(manifests) {
|
||||
const policies = new Map();
|
||||
for (const entry of manifests) {
|
||||
@@ -360,50 +500,94 @@ export async function enrichModelCatalogPricing(options) {
|
||||
}
|
||||
}
|
||||
const liteLlmCatalog = new Map();
|
||||
const liteLlmAliasGroups = [];
|
||||
for (const [id, row] of Object.entries(sources.liteLlm)) {
|
||||
const pricing = parseLiteLLMPricing(row);
|
||||
if (pricing) {
|
||||
liteLlmCatalog.set(id, pricing);
|
||||
const aliases = [id];
|
||||
if (typeof row?.litellm_provider === "string" && !id.includes("/")) {
|
||||
liteLlmCatalog.set(`${row.litellm_provider}/${id}`, pricing);
|
||||
aliases.push(`${row.litellm_provider}/${id}`);
|
||||
}
|
||||
for (const alias of aliases) {
|
||||
liteLlmCatalog.set(alias, pricing);
|
||||
}
|
||||
liteLlmAliasGroups.push(aliases);
|
||||
}
|
||||
}
|
||||
|
||||
let enriched = 0;
|
||||
const coveredPricingKeys = new Set();
|
||||
const pricedProviderModelKeys = new Set();
|
||||
for (const [providerId, provider] of Object.entries(options.bundle.providers)) {
|
||||
for (const model of provider.models) {
|
||||
const openRouterPricing = buildPricingCandidates({
|
||||
const openRouterCandidates = buildPricingCandidates({
|
||||
providerId,
|
||||
modelId: model.id,
|
||||
source: "openRouter",
|
||||
policies,
|
||||
})
|
||||
});
|
||||
const openRouterPricing = openRouterCandidates
|
||||
.map((candidate) => openRouterCatalog.get(candidate))
|
||||
.find(Boolean);
|
||||
const liteLlmPricing = buildPricingCandidates({
|
||||
const liteLlmCandidates = buildPricingCandidates({
|
||||
providerId,
|
||||
modelId: model.id,
|
||||
source: "liteLLM",
|
||||
policies,
|
||||
})
|
||||
});
|
||||
const liteLlmPricing = liteLlmCandidates
|
||||
.map((candidate) => liteLlmCatalog.get(candidate))
|
||||
.find(Boolean);
|
||||
const cost = openRouterPricing
|
||||
? {
|
||||
...openRouterPricing,
|
||||
...(liteLlmPricing?.tieredPricing
|
||||
? { tieredPricing: liteLlmPricing.tieredPricing }
|
||||
: {}),
|
||||
}
|
||||
: liteLlmPricing;
|
||||
if (cost) {
|
||||
const cost = mergePricing(openRouterPricing, liteLlmPricing);
|
||||
if (cost && hasKnownPricing(cost)) {
|
||||
model.cost = cost;
|
||||
enriched += 1;
|
||||
}
|
||||
if (model.cost && hasKnownPricing(model.cost)) {
|
||||
const providerModelKey = `${providerId}/${model.id}`;
|
||||
coveredPricingKeys.add(providerModelKey);
|
||||
pricedProviderModelKeys.add(providerModelKey);
|
||||
for (const candidate of [...openRouterCandidates, ...liteLlmCandidates]) {
|
||||
coveredPricingKeys.add(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return enriched;
|
||||
|
||||
const hostedPricing = new Map();
|
||||
for (const [key, pricing] of openRouterCatalog) {
|
||||
hostedPricing.set(key, pricing);
|
||||
}
|
||||
for (const [key, pricing] of liteLlmCatalog) {
|
||||
hostedPricing.set(key, mergePricing(hostedPricing.get(key), pricing));
|
||||
}
|
||||
for (const aliases of liteLlmAliasGroups) {
|
||||
if (aliases.some((alias) => coveredPricingKeys.has(alias))) {
|
||||
for (const alias of aliases) {
|
||||
coveredPricingKeys.add(alias);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const key of coveredPricingKeys) {
|
||||
hostedPricing.delete(key);
|
||||
}
|
||||
materializePolicyRuntimePricing({
|
||||
hostedPricing,
|
||||
policies,
|
||||
openRouterCatalog,
|
||||
liteLlmCatalog,
|
||||
pricedProviderModelKeys,
|
||||
});
|
||||
options.bundle.pricing = Object.fromEntries(
|
||||
[...hostedPricing.entries()]
|
||||
.toSorted(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, pricing]) => [key, compactPricing(pricing)]),
|
||||
);
|
||||
const validateBundle = options.validateBundle ?? (await loadClientBundleValidator());
|
||||
const validated = validateBundle(options.bundle);
|
||||
options.bundle.providers = validated.providers;
|
||||
options.bundle.pricing = validated.pricing;
|
||||
return { modelsEnriched: enriched, pricingEntries: hostedPricing.size };
|
||||
}
|
||||
|
||||
function sortCatalogValue(value) {
|
||||
@@ -454,24 +638,35 @@ async function runPublishModelCatalog(options = {}) {
|
||||
generatedAt,
|
||||
sourceCommit,
|
||||
});
|
||||
const enriched = args.pricing
|
||||
const pricingResult = args.pricing
|
||||
? await enrichModelCatalogPricing({ bundle, manifests, fetchImpl: options.fetchImpl })
|
||||
: 0;
|
||||
: { modelsEnriched: 0, pricingEntries: 0 };
|
||||
const summary = summarizeModelCatalogBundle(bundle);
|
||||
const stats = `schemaVersion=1 providers=${summary.providers} models=${summary.models} costModels=${summary.costModels} pricingEnriched=${enriched} generatedAt=${bundle.generatedAt} minVersion=${bundle.minVersion} sourceCommit=${bundle.sourceCommit}`;
|
||||
const serialized = serializeModelCatalogBundle(bundle);
|
||||
const bundleBytes = Buffer.byteLength(serialized);
|
||||
if (bundleBytes > BUNDLE_SIZE_WARNING_BYTES) {
|
||||
process.stderr.write(
|
||||
`[${SCRIPT_LABEL}] warning: bundle size ${bundleBytes} bytes exceeds ${BUNDLE_SIZE_WARNING_BYTES} bytes\n`,
|
||||
);
|
||||
}
|
||||
if (bundleBytes > CLIENT_BUNDLE_LIMIT_BYTES) {
|
||||
throw new Error(
|
||||
`catalog bundle ${bundleBytes} bytes exceeds client limit ${CLIENT_BUNDLE_LIMIT_BYTES} bytes`,
|
||||
);
|
||||
}
|
||||
const stats = `schemaVersion=1 providers=${summary.providers} models=${summary.models} costModels=${summary.costModels} pricingEnriched=${pricingResult.modelsEnriched} pricingEntries=${pricingResult.pricingEntries} bundleBytes=${bundleBytes} generatedAt=${bundle.generatedAt} minVersion=${bundle.minVersion} sourceCommit=${bundle.sourceCommit}`;
|
||||
if (args.dryRun) {
|
||||
process.stdout.write(`[${SCRIPT_LABEL}] dry-run ${stats}\n`);
|
||||
return { bundle, summary, pricingEnriched: enriched, wrote: false };
|
||||
return { bundle, summary, pricingEnriched: pricingResult.modelsEnriched, wrote: false };
|
||||
}
|
||||
|
||||
const serialized = serializeModelCatalogBundle(bundle);
|
||||
const outputFile = path.resolve(rootDir, args.out);
|
||||
if (args.out) {
|
||||
fs.mkdirSync(path.dirname(outputFile), { recursive: true });
|
||||
fs.writeFileSync(outputFile, serialized);
|
||||
}
|
||||
process.stdout.write(`[${SCRIPT_LABEL}] published ${stats} out=${args.out}\n`);
|
||||
return { bundle, summary, pricingEnriched: enriched, wrote: true };
|
||||
return { bundle, summary, pricingEnriched: pricingResult.modelsEnriched, wrote: true };
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
|
||||
@@ -69,9 +69,8 @@ vi.mock("./config-model-validation.js", () => ({
|
||||
|
||||
vi.mock("../gateway/config-reload-plan.js", () => ({
|
||||
buildGatewayReloadPlan: (changedPaths: string[]) => {
|
||||
const restartReasons = changedPaths.filter(
|
||||
(changedPath) =>
|
||||
changedPath.startsWith("models.pricing.") || changedPath.startsWith("plugins.load."),
|
||||
const restartReasons = changedPaths.filter((changedPath) =>
|
||||
changedPath.startsWith("plugins.load."),
|
||||
);
|
||||
const hotReasons = changedPaths.filter(
|
||||
(changedPath) =>
|
||||
@@ -3985,34 +3984,6 @@ describe("config cli", () => {
|
||||
expectLogExcludes("Restart the gateway to apply.");
|
||||
});
|
||||
|
||||
it("keeps the restart hint for broad models writes that change pricing bootstrap", async () => {
|
||||
const resolved: OpenClawConfig = {
|
||||
models: {
|
||||
pricing: {
|
||||
enabled: false,
|
||||
},
|
||||
providers: {
|
||||
openai: {
|
||||
agentRuntime: { id: "node" },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
setSnapshot(resolved, resolved);
|
||||
|
||||
await runConfigCommand([
|
||||
"config",
|
||||
"set",
|
||||
"models",
|
||||
'{"pricing":{"enabled":true},"providers":{"openai":{"agentRuntime":{"id":"node"}}}}',
|
||||
"--strict-json",
|
||||
"--replace",
|
||||
]);
|
||||
|
||||
expectLogIncludes("Updated models. Restart the gateway to apply.");
|
||||
expectLogExcludes("Change will apply without restarting the gateway.");
|
||||
});
|
||||
|
||||
it("keeps the restart hint for broad plugins writes that change load paths", async () => {
|
||||
const resolved: OpenClawConfig = {
|
||||
plugins: {
|
||||
@@ -4039,27 +4010,6 @@ describe("config cli", () => {
|
||||
expectLogExcludes("Change will apply without restarting the gateway.");
|
||||
});
|
||||
|
||||
it("keeps the restart hint for broad models unsets that remove pricing bootstrap", async () => {
|
||||
const resolved: OpenClawConfig = {
|
||||
models: {
|
||||
pricing: {
|
||||
enabled: false,
|
||||
},
|
||||
providers: {
|
||||
openai: {
|
||||
agentRuntime: { id: "node" },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
setSnapshot(resolved, resolved);
|
||||
|
||||
await runConfigCommand(["config", "unset", "models"]);
|
||||
|
||||
expectLogIncludes("Removed models. Restart the gateway to apply.");
|
||||
expectLogExcludes("Change will apply without restarting the gateway.");
|
||||
});
|
||||
|
||||
it("keeps the restart hint for broad plugins unsets that remove load paths", async () => {
|
||||
const resolved: OpenClawConfig = {
|
||||
plugins: {
|
||||
|
||||
@@ -10,6 +10,30 @@ import {
|
||||
LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS,
|
||||
} from "./legacy-config-migrations.runtime.models.js";
|
||||
|
||||
describe("retired model pricing config migration", () => {
|
||||
const migration = LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS.find(
|
||||
(entry) => entry.id === "models.pricing-retired",
|
||||
);
|
||||
|
||||
it("drops models.pricing while preserving hosted catalog config", () => {
|
||||
const raw = {
|
||||
models: {
|
||||
pricing: { enabled: false },
|
||||
catalogRefresh: { enabled: true },
|
||||
},
|
||||
};
|
||||
const changes: string[] = [];
|
||||
|
||||
expect(migration?.legacyRules?.[0]?.path).toEqual(["models", "pricing"]);
|
||||
migration?.apply(raw, changes);
|
||||
|
||||
expect(raw.models).toEqual({ catalogRefresh: { enabled: true } });
|
||||
expect(changes).toEqual([
|
||||
"Removed models.pricing (pricing now ships with the hosted model catalog).",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("model compat catalog ownership migration", () => {
|
||||
const migration = LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS.find(
|
||||
(entry) => entry.id === "models.providers.*.models.*.compat->provider-catalog",
|
||||
|
||||
@@ -41,6 +41,25 @@ const LEGACY_DEFAULT_MODEL_MIGRATION = defineLegacyConfigMigration({
|
||||
|
||||
export const LEGACY_CONFIG_MIGRATIONS_RUNTIME_MODELS = [
|
||||
LEGACY_DEFAULT_MODEL_MIGRATION,
|
||||
defineLegacyConfigMigration({
|
||||
id: "models.pricing-retired",
|
||||
describe: "Remove the retired client-side model pricing bootstrap toggle",
|
||||
legacyRules: [
|
||||
{
|
||||
path: ["models", "pricing"],
|
||||
message:
|
||||
'models.pricing is retired because pricing ships with the hosted catalog; run "openclaw doctor --fix" to remove it.',
|
||||
},
|
||||
],
|
||||
apply: (raw, changes) => {
|
||||
const models = getRecord(raw.models);
|
||||
if (!models || !Object.hasOwn(models, "pricing")) {
|
||||
return;
|
||||
}
|
||||
delete models.pricing;
|
||||
changes.push("Removed models.pricing (pricing now ships with the hosted model catalog).");
|
||||
},
|
||||
}),
|
||||
defineLegacyConfigMigration({
|
||||
id: "models.providers.*.models.*.compat->provider-catalog",
|
||||
describe: "Move known-model compatibility capability ownership into provider catalogs",
|
||||
|
||||
@@ -458,60 +458,6 @@ describe("gateway-status command", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces degraded model-pricing health as a warning", async () => {
|
||||
const { runtime, runtimeLogs, runtimeErrors } = createRuntimeCapture();
|
||||
const defaultProbeGateway = probeGateway.getMockImplementation();
|
||||
try {
|
||||
probeGateway.mockImplementation(async (opts: { url: string }) => {
|
||||
const result = defaultProbeGateway
|
||||
? await defaultProbeGateway(opts)
|
||||
: await mocks.probeGateway(opts);
|
||||
return {
|
||||
...result,
|
||||
health: {
|
||||
ok: true,
|
||||
modelPricing: {
|
||||
state: "degraded",
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
sources: [
|
||||
{
|
||||
source: "openrouter",
|
||||
state: "degraded",
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
await runGatewayStatus(runtime, { timeout: "1000", json: true });
|
||||
} finally {
|
||||
probeGateway.mockReset();
|
||||
if (defaultProbeGateway) {
|
||||
probeGateway.mockImplementation(defaultProbeGateway);
|
||||
}
|
||||
}
|
||||
|
||||
expect(runtimeErrors).toHaveLength(0);
|
||||
const parsed = JSON.parse(runtimeLogs.join("\n")) as {
|
||||
degraded?: boolean;
|
||||
warnings?: Array<{ code?: string; message?: string; targetIds?: string[] }>;
|
||||
};
|
||||
expect(parsed.degraded).toBe(false);
|
||||
const pricingWarnings =
|
||||
parsed.warnings?.filter((warning) => warning.code === "model_pricing_degraded") ?? [];
|
||||
expect(pricingWarnings).toHaveLength(2);
|
||||
expect(pricingWarnings.map((warning) => warning.message)).toEqual([
|
||||
"Model pricing warning: optional pricing refresh degraded: OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
"Model pricing warning: optional pricing refresh degraded: OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
]);
|
||||
expect(pricingWarnings.map((warning) => warning.targetIds)).toEqual([
|
||||
["sshTunnel"],
|
||||
["configRemote"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("includes diagnostic next steps when no gateway is reachable or discoverable", async () => {
|
||||
const { runtime, runtimeLogs, runtimeErrors } = createRuntimeCapture();
|
||||
const defaultProbeGateway = probeGateway.getMockImplementation();
|
||||
|
||||
@@ -53,23 +53,6 @@ function hasMultipleReachableGatewayIdentities(reachable: GatewayStatusProbedTar
|
||||
return new Set(identityKeys).size > 1;
|
||||
}
|
||||
|
||||
function readModelPricingDegradedDetail(health: unknown): string | null {
|
||||
if (!health || typeof health !== "object") {
|
||||
return null;
|
||||
}
|
||||
const modelPricing = (health as { modelPricing?: unknown }).modelPricing;
|
||||
if (!modelPricing || typeof modelPricing !== "object") {
|
||||
return null;
|
||||
}
|
||||
const record = modelPricing as { state?: unknown; detail?: unknown };
|
||||
if (record.state !== "degraded") {
|
||||
return null;
|
||||
}
|
||||
return typeof record.detail === "string" && record.detail.trim()
|
||||
? record.detail.trim()
|
||||
: "pricing bootstrap or refresh failed";
|
||||
}
|
||||
|
||||
/** Chooses the reachable target that best represents the user's requested gateway. */
|
||||
export function pickPrimaryProbedTarget(probed: GatewayStatusProbedTarget[]) {
|
||||
const reachable = probed.filter((entry) => isProbeReachable(entry.probe));
|
||||
@@ -159,17 +142,6 @@ export function buildGatewayStatusWarnings(params: {
|
||||
targetIds: [result.target.id],
|
||||
});
|
||||
}
|
||||
for (const result of reachable) {
|
||||
const detail = readModelPricingDegradedDetail(result.probe.health);
|
||||
if (!detail) {
|
||||
continue;
|
||||
}
|
||||
warnings.push({
|
||||
code: "model_pricing_degraded",
|
||||
message: `Model pricing warning: optional pricing refresh degraded: ${detail}`,
|
||||
targetIds: [result.target.id],
|
||||
});
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
formatContextEngineHealthLine,
|
||||
formatDeliveryQueueHealthLine,
|
||||
formatHealthChannelLines,
|
||||
formatModelPricingHealthLine,
|
||||
healthCommand,
|
||||
} from "./health.js";
|
||||
|
||||
@@ -426,31 +425,6 @@ describe("healthCommand", () => {
|
||||
expect(runtime.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("formats degraded model-pricing health as a warning", () => {
|
||||
const snapshot = createHealthSummary({
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
});
|
||||
snapshot.modelPricing = {
|
||||
state: "degraded",
|
||||
sources: [
|
||||
{
|
||||
source: "openrouter",
|
||||
state: "degraded",
|
||||
lastFailureAt: Date.now(),
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
},
|
||||
],
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
lastFailureAt: Date.now(),
|
||||
};
|
||||
|
||||
expect(formatModelPricingHealthLine(snapshot)).toBe(
|
||||
"Model pricing: warning (optional pricing refresh degraded) (OpenRouter pricing fetch failed: TypeError: fetch failed)",
|
||||
);
|
||||
});
|
||||
|
||||
it("formats per-account probe timings", () => {
|
||||
const summary = createHealthSummary({
|
||||
channels: {
|
||||
|
||||
@@ -36,8 +36,6 @@ import {
|
||||
} from "../gateway/channel-health-policy.js";
|
||||
import type { GatewayHotReloadStatus } from "../gateway/config-reload-status.types.js";
|
||||
import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
|
||||
import { getGatewayModelPricingHealth } from "../gateway/model-pricing-cache-state.js";
|
||||
import { isGatewayModelPricingEnabled } from "../gateway/model-pricing-config.js";
|
||||
import type { ChannelRuntimeSnapshot } from "../gateway/server-channel-runtime.types.js";
|
||||
import { info } from "../globals.js";
|
||||
import { countFailedDeliveryQueueEntries } from "../infra/delivery-queue-sqlite.js";
|
||||
@@ -214,19 +212,6 @@ function formatEventLoopHealthLine(summary: HealthSummary): string | null {
|
||||
}`;
|
||||
}
|
||||
|
||||
/** Formats optional model-pricing cache degradation for text health output. */
|
||||
export function formatModelPricingHealthLine(summary: HealthSummary): string | null {
|
||||
const modelPricing = summary.modelPricing;
|
||||
if (!modelPricing || modelPricing.state === "disabled") {
|
||||
return null;
|
||||
}
|
||||
if (modelPricing.state === "ok") {
|
||||
return null;
|
||||
}
|
||||
const detail = modelPricing.detail ? ` (${modelPricing.detail})` : "";
|
||||
return `Model pricing: warning (optional pricing refresh degraded)${detail}`;
|
||||
}
|
||||
|
||||
function buildContextEngineHealthSummary(): ContextEngineHealthSummary | undefined {
|
||||
const quarantined: ContextEngineHealthSummary["quarantined"] = [];
|
||||
for (const entry of listContextEngineQuarantines()) {
|
||||
@@ -788,7 +773,6 @@ export async function getHealthSnapshot(params?: {
|
||||
...(params?.configReloadHotReloadStatus
|
||||
? { configReload: { hotReloadStatus: params.configReloadHotReloadStatus } }
|
||||
: {}),
|
||||
modelPricing: getGatewayModelPricingHealth({ enabled: isGatewayModelPricingEnabled(cfg) }),
|
||||
channels,
|
||||
channelOrder,
|
||||
channelLabels,
|
||||
@@ -1015,10 +999,6 @@ export async function healthCommand(
|
||||
if (eventLoopLine) {
|
||||
runtime.log(styleHealthChannelLine(eventLoopLine, rich));
|
||||
}
|
||||
const modelPricingLine = formatModelPricingHealthLine(summary);
|
||||
if (modelPricingLine) {
|
||||
runtime.log(styleHealthChannelLine(modelPricingLine, rich));
|
||||
}
|
||||
const contextEngineLine = formatContextEngineHealthLine(summary);
|
||||
if (contextEngineLine) {
|
||||
runtime.log(styleHealthChannelLine(contextEngineLine, rich));
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
// Shared summary types returned by gateway health and rendered by the CLI.
|
||||
import type { GatewayModelPricingHealth } from "../gateway/model-pricing-cache.types.js";
|
||||
/** Health snapshot for one configured channel account. */
|
||||
export type ChannelAccountHealthSummary = {
|
||||
accountId: string;
|
||||
@@ -80,9 +79,6 @@ export type DeliveryQueueHealthSummary = {
|
||||
}>;
|
||||
};
|
||||
|
||||
/** Optional model pricing cache health reported by the gateway. */
|
||||
type ModelPricingHealthSummary = GatewayModelPricingHealth;
|
||||
|
||||
/** Config hot-reload watcher status, present only when a reloader is running. */
|
||||
type ConfigReloadHealthSummary = {
|
||||
hotReloadStatus: import("../gateway/config-reload-status.types.js").GatewayHotReloadStatus;
|
||||
@@ -97,7 +93,6 @@ export type HealthSummary = {
|
||||
plugins?: PluginHealthSummary;
|
||||
contextEngines?: ContextEngineHealthSummary;
|
||||
deliveryQueues?: DeliveryQueueHealthSummary;
|
||||
modelPricing?: ModelPricingHealthSummary;
|
||||
configReload?: ConfigReloadHealthSummary;
|
||||
channels: Record<string, ChannelHealthSummary>;
|
||||
channelOrder: string[];
|
||||
|
||||
@@ -512,14 +512,6 @@ export function buildGatewayStatusJsonPayload(params: {
|
||||
self: params.gatewaySelf ?? null,
|
||||
error: params.gatewayProbe?.error ?? null,
|
||||
authWarning: params.gatewayProbeAuthWarning ?? null,
|
||||
...(params.gatewayProbe?.health &&
|
||||
typeof params.gatewayProbe.health === "object" &&
|
||||
"modelPricing" in params.gatewayProbe.health
|
||||
? {
|
||||
// Preserve model pricing when the gateway already returned it; do not synthesize pricing locally.
|
||||
modelPricing: (params.gatewayProbe.health as { modelPricing?: unknown }).modelPricing,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -169,51 +169,4 @@ describe("status-json-payload", () => {
|
||||
}),
|
||||
).not.toHaveProperty("securityAudit");
|
||||
});
|
||||
|
||||
it("includes model-pricing health from the gateway probe", () => {
|
||||
const payload = buildStatusJsonPayload({
|
||||
summary: { ok: true },
|
||||
surface: {
|
||||
cfg: { gateway: {} },
|
||||
update: {
|
||||
root: "/tmp/openclaw",
|
||||
installKind: "package",
|
||||
packageManager: "npm",
|
||||
} as never,
|
||||
tailscaleMode: "off",
|
||||
gatewayMode: "local",
|
||||
remoteUrlMissing: false,
|
||||
gatewayConnection: { url: "ws://127.0.0.1:18789" },
|
||||
gatewayReachable: true,
|
||||
gatewayProbe: {
|
||||
connectLatencyMs: 42,
|
||||
error: null,
|
||||
health: {
|
||||
ok: true,
|
||||
modelPricing: {
|
||||
state: "degraded",
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
sources: [{ source: "openrouter", state: "degraded" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
gatewayProbeAuth: null,
|
||||
gatewaySelf: null,
|
||||
gatewayProbeAuthWarning: null,
|
||||
gatewayService: { label: "LaunchAgent", installed: false, loadedText: "not installed" },
|
||||
nodeService: { label: "node", installed: false, loadedText: "not installed" },
|
||||
},
|
||||
osSummary: { platform: "linux" },
|
||||
memory: null,
|
||||
memoryPlugin: null,
|
||||
agents: [],
|
||||
secretDiagnostics: [],
|
||||
});
|
||||
|
||||
const modelPricing = payload.gateway.modelPricing as
|
||||
| { state?: string; detail?: string }
|
||||
| undefined;
|
||||
expect(modelPricing?.state).toBe("degraded");
|
||||
expect(modelPricing?.detail).toBe("OpenRouter pricing fetch failed: TypeError: fetch failed");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -30,46 +30,6 @@ import {
|
||||
import type { MemoryPluginStatus, MemoryStatusSnapshot } from "./status.scan.shared.js";
|
||||
import type { StatusSummary } from "./status.types.js";
|
||||
|
||||
function readModelPricingHealth(params: {
|
||||
health?: HealthSummary;
|
||||
surface: StatusOverviewSurface;
|
||||
}): HealthSummary["modelPricing"] | undefined {
|
||||
if (params.health?.modelPricing) {
|
||||
return params.health.modelPricing;
|
||||
}
|
||||
// Fast status can receive model pricing through the gateway probe before deep health is requested.
|
||||
const probeHealth = params.surface.gatewayProbe?.health;
|
||||
if (!probeHealth || typeof probeHealth !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const modelPricing = (probeHealth as { modelPricing?: unknown }).modelPricing;
|
||||
if (!modelPricing || typeof modelPricing !== "object") {
|
||||
return undefined;
|
||||
}
|
||||
const state = (modelPricing as { state?: unknown }).state;
|
||||
if (state !== "ok" && state !== "degraded" && state !== "disabled") {
|
||||
return undefined;
|
||||
}
|
||||
return modelPricing as HealthSummary["modelPricing"];
|
||||
}
|
||||
|
||||
function buildModelPricingOverviewValue(params: {
|
||||
health?: HealthSummary["modelPricing"];
|
||||
ok: (value: string) => string;
|
||||
warn: (value: string) => string;
|
||||
muted: (value: string) => string;
|
||||
}): string | null {
|
||||
const health = params.health;
|
||||
if (!health) {
|
||||
return null;
|
||||
}
|
||||
if (health.state !== "degraded") {
|
||||
return null;
|
||||
}
|
||||
const detail = health.detail ? ` · ${health.detail}` : "";
|
||||
return params.warn(`warning · optional pricing refresh degraded${detail}`);
|
||||
}
|
||||
|
||||
/** Builds the default `openclaw status` overview rows from scan, health, memory, and session inputs. */
|
||||
export function buildStatusCommandOverviewRows(
|
||||
params: {
|
||||
@@ -159,16 +119,6 @@ export function buildStatusCommandOverviewRows(
|
||||
ok: params.ok,
|
||||
warn: params.warn,
|
||||
});
|
||||
const modelPricingValue = buildModelPricingOverviewValue({
|
||||
health: readModelPricingHealth({
|
||||
health: params.health,
|
||||
surface: params.surface,
|
||||
}),
|
||||
ok: params.ok,
|
||||
warn: params.warn,
|
||||
muted: params.muted,
|
||||
});
|
||||
|
||||
return buildStatusOverviewRowsFromSurface({
|
||||
surface: params.surface,
|
||||
decorateOk: params.ok,
|
||||
@@ -179,7 +129,6 @@ export function buildStatusCommandOverviewRows(
|
||||
updateValue: params.updateValue,
|
||||
agentsValue,
|
||||
suffixRows: [
|
||||
...(modelPricingValue ? [{ Item: "Model pricing", Value: modelPricingValue }] : []),
|
||||
...(params.updateRestartValue
|
||||
? [{ Item: "Update restart", Value: params.updateRestartValue }]
|
||||
: []),
|
||||
|
||||
@@ -109,39 +109,6 @@ describe("buildStatusCommandReportData", () => {
|
||||
expect(result.retainedLostTaskLine).toBe("muted(2 lost tasks retained until cleanupAfter)");
|
||||
});
|
||||
|
||||
it("adds model-pricing degradation from gateway probe health to overview rows", async () => {
|
||||
const baseParams = createStatusCommandReportDataParams();
|
||||
const result = await buildStatusCommandReportData(
|
||||
createStatusCommandReportDataParams({
|
||||
surface: {
|
||||
...baseParams.surface,
|
||||
gatewayProbe: {
|
||||
connectLatencyMs: 123,
|
||||
error: null,
|
||||
health: {
|
||||
ok: true,
|
||||
modelPricing: {
|
||||
state: "degraded",
|
||||
detail: "OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
sources: [{ source: "openrouter", state: "degraded" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
health: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
const modelPricingIndex = result.overviewRows.findIndex((row) => row.Item === "Model pricing");
|
||||
expect(modelPricingIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(result.overviewRows[modelPricingIndex]).toStrictEqual({
|
||||
Item: "Model pricing",
|
||||
Value:
|
||||
"warn(warning · optional pricing refresh degraded · OpenRouter pricing fetch failed: TypeError: fetch failed)",
|
||||
});
|
||||
expect(result.overviewRows[modelPricingIndex + 1]?.Item).toBe("Memory");
|
||||
});
|
||||
|
||||
it("adds pinned-session model selection lines", async () => {
|
||||
const baseParams = createStatusCommandReportDataParams();
|
||||
const result = await buildStatusCommandReportData(
|
||||
|
||||
@@ -286,15 +286,6 @@ export function buildStatusHealthRows(params: {
|
||||
Detail: formatEventLoopHealthDetail(params.health.eventLoop),
|
||||
});
|
||||
}
|
||||
if (params.health.modelPricing?.state === "degraded") {
|
||||
rows.push({
|
||||
Item: "Model pricing",
|
||||
Status: params.warn("WARN"),
|
||||
Detail: `optional pricing refresh degraded${
|
||||
params.health.modelPricing.detail ? `: ${params.health.modelPricing.detail}` : ""
|
||||
}`,
|
||||
});
|
||||
}
|
||||
for (const line of params.formatHealthChannelLines(params.health, { accountMode: "all" })) {
|
||||
const colon = line.indexOf(":");
|
||||
if (colon === -1) {
|
||||
|
||||
@@ -411,28 +411,6 @@ describe("plugins.slots.contextEngine", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("models.pricing", () => {
|
||||
it("accepts the model pricing bootstrap toggle", () => {
|
||||
for (const enabled of [true, false]) {
|
||||
const result = OpenClawSchema.safeParse({
|
||||
models: {
|
||||
pricing: { enabled },
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects non-boolean model pricing bootstrap values", () => {
|
||||
const result = OpenClawSchema.safeParse({
|
||||
models: {
|
||||
pricing: { enabled: "false" },
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("models.catalogRefresh", () => {
|
||||
it("accepts the refresh toggle and an http(s) override", () => {
|
||||
expect(
|
||||
|
||||
@@ -6,10 +6,6 @@ export const MODEL_FIELD_HELP: Record<string, string> = {
|
||||
'Controls provider catalog behavior: "merge" keeps built-ins and overlays your custom providers, while "replace" uses only your configured providers. In "merge", matching provider IDs preserve non-empty agent models.json baseUrl values, while apiKey values are preserved only when the provider is not SecretRef-managed in current config/auth-profile context; SecretRef-managed providers refresh apiKey from current source markers, and matching model contextWindow/maxTokens use the higher value between explicit and implicit entries.',
|
||||
"models.providers":
|
||||
"Provider map keyed by provider ID containing connection/auth settings and concrete model definitions. Built-in providers may be tuned with provider-level overlays; custom providers must include baseUrl and models. Use stable provider keys so references from agents and tooling remain portable across environments.",
|
||||
"models.pricing":
|
||||
"Controls the optional background model-pricing bootstrap that fetches remote per-token cost catalogs.",
|
||||
"models.pricing.enabled":
|
||||
"Enable the background model-pricing bootstrap. Set to false to skip OpenRouter and LiteLLM catalog fetches during Gateway startup; changing this value requires a Gateway restart.",
|
||||
"models.catalogRefresh":
|
||||
"Controls background updates to the bundled model catalog. Remote rows can update model metadata but cannot change provider endpoints or headers.",
|
||||
"models.catalogRefresh.enabled":
|
||||
|
||||
@@ -498,8 +498,6 @@ export const FIELD_LABELS: Record<string, string> = {
|
||||
"acp.runtime.installCommand": "ACP Runtime Install Command",
|
||||
models: "Models",
|
||||
"models.mode": "Model Catalog Mode",
|
||||
"models.pricing": "Model Pricing",
|
||||
"models.pricing.enabled": "Model Pricing Enabled",
|
||||
"models.catalogRefresh": "Model Catalog Refresh",
|
||||
"models.catalogRefresh.enabled": "Model Catalog Refresh Enabled",
|
||||
"models.catalogRefresh.url": "Model Catalog Refresh URL",
|
||||
|
||||
@@ -269,11 +269,6 @@ export type DiscoveryToggleConfig = {
|
||||
enabled?: boolean;
|
||||
};
|
||||
|
||||
export type ModelPricingConfig = {
|
||||
/** Enable external or generated pricing enrichment. */
|
||||
enabled?: boolean;
|
||||
};
|
||||
|
||||
export type ModelCatalogRefreshConfig = {
|
||||
/** Fetch model catalog updates from the hosted OpenClaw catalog. Default: true. */
|
||||
enabled?: boolean;
|
||||
@@ -286,8 +281,6 @@ export type ModelsConfig = {
|
||||
mode?: "merge" | "replace";
|
||||
/** Configured provider catalog keyed by provider id. */
|
||||
providers?: Record<string, ModelProviderConfig>;
|
||||
/** Pricing enrichment settings. */
|
||||
pricing?: ModelPricingConfig;
|
||||
/** Hosted model catalog refresh settings. */
|
||||
catalogRefresh?: ModelCatalogRefreshConfig;
|
||||
};
|
||||
|
||||
@@ -548,13 +548,6 @@ const ModelProvidersSchema = z
|
||||
}
|
||||
});
|
||||
|
||||
const ModelPricingConfigSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
const ModelCatalogRefreshConfigSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
@@ -586,7 +579,6 @@ export const ModelsConfigSchema = z
|
||||
.object({
|
||||
mode: z.union([z.literal("merge"), z.literal("replace")]).optional(),
|
||||
providers: ModelProvidersSchema.optional(),
|
||||
pricing: ModelPricingConfigSchema,
|
||||
catalogRefresh: ModelCatalogRefreshConfigSchema,
|
||||
})
|
||||
.strict()
|
||||
|
||||
@@ -111,10 +111,6 @@ const BASE_RELOAD_RULES: ReloadRule[] = [
|
||||
kind: "hot",
|
||||
actions: ["restart-heartbeat"],
|
||||
},
|
||||
{
|
||||
prefix: "models.pricing",
|
||||
kind: "restart",
|
||||
},
|
||||
{
|
||||
prefix: "models",
|
||||
kind: "hot",
|
||||
|
||||
@@ -282,11 +282,6 @@ describe("buildGatewayReloadPlan", () => {
|
||||
restart: true,
|
||||
reason: "gateway.auth.token",
|
||||
},
|
||||
{
|
||||
path: "models.pricing.enabled",
|
||||
restart: true,
|
||||
reason: "models.pricing.enabled",
|
||||
},
|
||||
{
|
||||
path: "agents.defaults.model",
|
||||
restart: false,
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
// Gateway model-pricing cache state.
|
||||
// Stores normalized pricing rows and source-health failures for runtime reads.
|
||||
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import { normalizeModelRef } from "../agents/model-selection.js";
|
||||
import type { GatewayModelPricingHealth } from "./model-pricing-cache.types.js";
|
||||
|
||||
export type CachedPricingTier = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
/** [startTokens, endTokens) — half-open interval on the input token axis. */
|
||||
range: [number, number];
|
||||
};
|
||||
|
||||
export type CachedModelPricing = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
/** Optional tiered pricing tiers sourced from LiteLLM or local config. */
|
||||
tieredPricing?: CachedPricingTier[];
|
||||
};
|
||||
|
||||
type GatewayModelPricingHealthSource = GatewayModelPricingHealth["sources"][number]["source"];
|
||||
|
||||
let cachedPricing = new Map<string, CachedModelPricing>();
|
||||
let cachedAt = 0;
|
||||
const sourceFailures = new Map<
|
||||
GatewayModelPricingHealthSource,
|
||||
{ lastFailureAt: number; detail: string }
|
||||
>();
|
||||
|
||||
function modelPricingCacheKey(provider: string, model: string): string {
|
||||
// Keys accept both provider/model and provider-prefixed model ids so external
|
||||
// catalogs can be queried without double-prefixing.
|
||||
const providerId = normalizeProviderId(provider);
|
||||
const modelId = model.trim();
|
||||
if (!providerId || !modelId) {
|
||||
return "";
|
||||
}
|
||||
return normalizeLowercaseStringOrEmpty(modelId).startsWith(
|
||||
`${normalizeLowercaseStringOrEmpty(providerId)}/`,
|
||||
)
|
||||
? modelId
|
||||
: `${providerId}/${modelId}`;
|
||||
}
|
||||
|
||||
export function replaceGatewayModelPricingCache(
|
||||
nextPricing: Map<string, CachedModelPricing>,
|
||||
nextCachedAt = Date.now(),
|
||||
): void {
|
||||
cachedPricing = nextPricing;
|
||||
cachedAt = nextCachedAt;
|
||||
}
|
||||
|
||||
export function recordGatewayModelPricingSourceFailure(
|
||||
source: GatewayModelPricingHealthSource,
|
||||
detail: string,
|
||||
failedAt = Date.now(),
|
||||
): void {
|
||||
sourceFailures.set(source, {
|
||||
lastFailureAt: failedAt,
|
||||
detail,
|
||||
});
|
||||
}
|
||||
|
||||
export function clearGatewayModelPricingSourceFailure(
|
||||
source: GatewayModelPricingHealthSource,
|
||||
): void {
|
||||
sourceFailures.delete(source);
|
||||
}
|
||||
|
||||
export function clearGatewayModelPricingFailures(): void {
|
||||
sourceFailures.clear();
|
||||
}
|
||||
|
||||
export function getGatewayModelPricingHealth(params?: {
|
||||
enabled?: boolean;
|
||||
}): GatewayModelPricingHealth {
|
||||
if (params?.enabled === false) {
|
||||
return {
|
||||
state: "disabled",
|
||||
sources: [],
|
||||
};
|
||||
}
|
||||
const sources: GatewayModelPricingHealth["sources"] = Array.from(sourceFailures.entries())
|
||||
.map(([source, failure]) => ({
|
||||
source,
|
||||
state: "degraded" as const,
|
||||
lastFailureAt: failure.lastFailureAt,
|
||||
detail: failure.detail,
|
||||
}))
|
||||
.toSorted((left, right) => left.source.localeCompare(right.source));
|
||||
const latest = sources.reduce<(typeof sources)[number] | undefined>((current, source) => {
|
||||
if (!current || (source.lastFailureAt ?? 0) > (current.lastFailureAt ?? 0)) {
|
||||
return source;
|
||||
}
|
||||
return current;
|
||||
}, undefined);
|
||||
return {
|
||||
state: sources.length > 0 ? "degraded" : "ok",
|
||||
sources,
|
||||
...(latest?.lastFailureAt ? { lastFailureAt: latest.lastFailureAt } : {}),
|
||||
...(latest?.detail ? { detail: latest.detail } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function getCachedGatewayModelPricing(params: {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
}): CachedModelPricing | undefined {
|
||||
const provider = params.provider?.trim();
|
||||
const model = params.model?.trim();
|
||||
if (!provider || !model) {
|
||||
return undefined;
|
||||
}
|
||||
const key = modelPricingCacheKey(provider, model);
|
||||
const direct = key ? cachedPricing.get(key) : undefined;
|
||||
if (direct) {
|
||||
return direct;
|
||||
}
|
||||
const normalized = normalizeModelRef(provider, model);
|
||||
const normalizedKey = modelPricingCacheKey(normalized.provider, normalized.model);
|
||||
if (normalizedKey === key) {
|
||||
return undefined;
|
||||
}
|
||||
return normalizedKey ? cachedPricing.get(normalizedKey) : undefined;
|
||||
}
|
||||
|
||||
export function getGatewayModelPricingCacheMeta(): {
|
||||
cachedAt: number;
|
||||
ttlMs: number;
|
||||
size: number;
|
||||
} {
|
||||
return {
|
||||
cachedAt,
|
||||
ttlMs: 0,
|
||||
size: cachedPricing.size,
|
||||
};
|
||||
}
|
||||
|
||||
function stablePricingValue(value: unknown): string {
|
||||
if (typeof value === "number") {
|
||||
return Number.isFinite(value) ? JSON.stringify(value) : JSON.stringify(String(value));
|
||||
}
|
||||
if (value === null || typeof value !== "object") {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return `[${value.map((entry) => stablePricingValue(entry)).join(",")}]`;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record)
|
||||
.filter((key) => record[key] !== undefined)
|
||||
.toSorted()
|
||||
.map((key) => `${JSON.stringify(key)}:${stablePricingValue(record[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
|
||||
export function getGatewayModelPricingCacheFingerprint(): string {
|
||||
const entries = Array.from(cachedPricing.entries()).toSorted(([a], [b]) => a.localeCompare(b));
|
||||
return stablePricingValue(entries);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,12 +0,0 @@
|
||||
/** Health of the gateway model-pricing sources exposed through health summaries. */
|
||||
export type GatewayModelPricingHealth = {
|
||||
state: "ok" | "degraded" | "disabled";
|
||||
sources: Array<{
|
||||
source: "openrouter" | "litellm" | "bootstrap" | "refresh";
|
||||
state: "ok" | "degraded";
|
||||
lastFailureAt?: number;
|
||||
detail?: string;
|
||||
}>;
|
||||
lastFailureAt?: number;
|
||||
detail?: string;
|
||||
};
|
||||
@@ -1,8 +0,0 @@
|
||||
// Gateway model-pricing config helper.
|
||||
// Resolves whether cost/pricing metadata should be available to Gateway surfaces.
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
|
||||
/** Returns whether gateway model pricing/cost metadata should be shown. */
|
||||
export function isGatewayModelPricingEnabled(config: OpenClawConfig): boolean {
|
||||
return config.models?.pricing?.enabled !== false;
|
||||
}
|
||||
@@ -617,7 +617,6 @@ export async function runGatewayClosePrelude(params: {
|
||||
skillsChangeUnsub?: () => void;
|
||||
disposeAuthRateLimiter?: () => void;
|
||||
disposeBrowserAuthRateLimiter: () => void;
|
||||
stopModelPricingRefresh?: () => void;
|
||||
stopChannelHealthMonitor?: () => Promise<void>;
|
||||
stopReadinessEventLoopHealth?: () => void;
|
||||
clearSecretsRuntimeSnapshot?: () => void;
|
||||
@@ -628,7 +627,6 @@ export async function runGatewayClosePrelude(params: {
|
||||
params.skillsChangeUnsub?.();
|
||||
params.disposeAuthRateLimiter?.();
|
||||
params.disposeBrowserAuthRateLimiter();
|
||||
params.stopModelPricingRefresh?.();
|
||||
await params.stopChannelHealthMonitor?.();
|
||||
params.stopReadinessEventLoopHealth?.();
|
||||
params.clearSecretsRuntimeSnapshot?.();
|
||||
|
||||
@@ -364,7 +364,6 @@ export async function prepareGatewayLifecycle(params: {
|
||||
nodeReapprovalCoordinator.dispose();
|
||||
},
|
||||
disposeBrowserAuthRateLimiter: () => browserAuthRateLimiter.dispose(),
|
||||
stopModelPricingRefresh: runtimeState.stopModelPricingRefresh,
|
||||
stopChannelHealthMonitor: async () => {
|
||||
const monitor = runtimeState?.channelHealthMonitor;
|
||||
monitor?.shutdown();
|
||||
|
||||
@@ -7,7 +7,6 @@ import type { ChannelHealthSummary, HealthSummary } from "../../commands/health.
|
||||
import { getStatusSummary } from "../../commands/status.js";
|
||||
import { listContextEngineQuarantines } from "../../context-engine/registry.js";
|
||||
import type { GatewayHotReloadStatus } from "../config-reload-status.types.js";
|
||||
import { getGatewayModelPricingHealth } from "../model-pricing-cache-state.js";
|
||||
import type { ChannelRuntimeSnapshot } from "../server-channel-runtime.types.js";
|
||||
import { HEALTH_REFRESH_INTERVAL_MS } from "../server-constants.js";
|
||||
import { formatError } from "../server-utils.js";
|
||||
@@ -127,9 +126,6 @@ function mergeCachedHealthRuntimeState(params: {
|
||||
...(params.configReloadHotReloadStatus
|
||||
? { configReload: { hotReloadStatus: params.configReloadHotReloadStatus } }
|
||||
: {}),
|
||||
modelPricing: getGatewayModelPricingHealth({
|
||||
enabled: params.cached.modelPricing?.state !== "disabled",
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -5028,25 +5028,19 @@ describe("gateway healthHandlers.status scope handling", () => {
|
||||
|
||||
describe("gateway healthHandlers.health cache freshness", () => {
|
||||
let healthHandlers: typeof import("./health.js").healthHandlers;
|
||||
let pricingState: typeof import("../model-pricing-cache-state.js");
|
||||
const contextEngineTestOwner = "plugin:health-test";
|
||||
|
||||
beforeAll(async () => {
|
||||
({ healthHandlers } = await import("./health.js"));
|
||||
pricingState = await import("../model-pricing-cache-state.js");
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
pricingState.replaceGatewayModelPricingCache(new Map(), 0);
|
||||
pricingState.clearGatewayModelPricingFailures();
|
||||
registerLegacyContextEngine();
|
||||
clearContextEnginesForOwner(contextEngineTestOwner);
|
||||
resetContextEngineRuntimeQuarantineForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
pricingState.replaceGatewayModelPricingCache(new Map(), 0);
|
||||
pricingState.clearGatewayModelPricingFailures();
|
||||
clearContextEnginesForOwner(contextEngineTestOwner);
|
||||
resetContextEngineRuntimeQuarantineForTests();
|
||||
});
|
||||
@@ -5198,69 +5192,6 @@ describe("gateway healthHandlers.health cache freshness", () => {
|
||||
expect(mockCallArg(respond, 0, 2)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("merges live model-pricing state into cached health responses", async () => {
|
||||
const cached = {
|
||||
ok: true,
|
||||
ts: Date.now(),
|
||||
durationMs: 1,
|
||||
channels: {},
|
||||
channelOrder: [],
|
||||
channelLabels: {},
|
||||
heartbeatSeconds: 0,
|
||||
defaultAgentId: "main",
|
||||
agents: [],
|
||||
sessions: { path: "/tmp/sessions.json", count: 0, recent: [] },
|
||||
modelPricing: { state: "ok", sources: [] },
|
||||
};
|
||||
pricingState.recordGatewayModelPricingSourceFailure(
|
||||
"openrouter",
|
||||
"OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
123,
|
||||
);
|
||||
const respond = vi.fn();
|
||||
const refreshHealthSnapshot = vi.fn().mockResolvedValue(cached);
|
||||
|
||||
await expectDefined(healthHandlers.health, "healthHandlers.health test invariant").call(
|
||||
healthHandlers,
|
||||
{
|
||||
req: {} as never,
|
||||
params: {} as never,
|
||||
respond: respond as never,
|
||||
context: {
|
||||
getHealthCache: () => cached,
|
||||
refreshHealthSnapshot,
|
||||
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
|
||||
logHealth: { error: vi.fn() },
|
||||
} as never,
|
||||
client: { connect: { role: "operator", scopes: ["operator.read"] } } as never,
|
||||
isWebchatConnect: () => false,
|
||||
},
|
||||
);
|
||||
|
||||
const payload = mockCallArg(respond, 0, 1) as
|
||||
| {
|
||||
modelPricing?: {
|
||||
state?: string;
|
||||
detail?: string;
|
||||
sources?: Array<{ source?: string; state?: string; lastFailureAt?: number }>;
|
||||
};
|
||||
}
|
||||
| undefined;
|
||||
expect(payload?.modelPricing?.state).toBe("degraded");
|
||||
expect(payload?.modelPricing?.detail).toBe(
|
||||
"OpenRouter pricing fetch failed: TypeError: fetch failed",
|
||||
);
|
||||
expect(payload?.modelPricing?.sources).toHaveLength(1);
|
||||
expect(payload?.modelPricing?.sources?.[0]?.source).toBe("openrouter");
|
||||
expect(payload?.modelPricing?.sources?.[0]?.state).toBe("degraded");
|
||||
expect(payload?.modelPricing?.sources?.[0]?.lastFailureAt).toBe(123);
|
||||
expect(mockCallArg(respond, 0, 3)).toEqual({ cached: true });
|
||||
expect(refreshHealthSnapshot).toHaveBeenCalledWith({
|
||||
probe: false,
|
||||
includeSensitive: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("merges live context-engine quarantine state into cached health responses", async () => {
|
||||
const engineId = `health-context-engine-${Date.now()}`;
|
||||
const consoleError = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
||||
@@ -35,7 +35,6 @@ export type GatewayServerMutableState = {
|
||||
skillsRefreshDelayMs: number;
|
||||
skillsChangeUnsub: () => void;
|
||||
channelHealthMonitor: ChannelHealthMonitor | null;
|
||||
stopModelPricingRefresh: () => void;
|
||||
mcpServer: { port: number; close: () => Promise<void> } | undefined;
|
||||
configReloader: GatewayConfigReloaderHandle;
|
||||
agentUnsub: (() => Promise<void> | void) | null;
|
||||
@@ -74,7 +73,6 @@ export function createGatewayServerMutableState(): GatewayServerMutableState {
|
||||
skillsRefreshDelayMs: 30_000,
|
||||
skillsChangeUnsub: () => {},
|
||||
channelHealthMonitor: null as ChannelHealthMonitor | null,
|
||||
stopModelPricingRefresh: () => {},
|
||||
mcpServer: undefined as { port: number; close: () => Promise<void> } | undefined,
|
||||
configReloader: {
|
||||
stop: async () => {},
|
||||
|
||||
@@ -23,7 +23,6 @@ const hoisted = vi.hoisted(() => {
|
||||
stop: vi.fn(),
|
||||
updateConfig: vi.fn(),
|
||||
};
|
||||
const stopModelPricingRefresh = vi.fn();
|
||||
const stopSessionUpstreamMonitor = vi.fn();
|
||||
const stopSessionDeliveryRuntime = vi.fn();
|
||||
return {
|
||||
@@ -34,7 +33,6 @@ const hoisted = vi.hoisted(() => {
|
||||
shutdown: vi.fn(),
|
||||
waitForIdle: vi.fn(async () => {}),
|
||||
})),
|
||||
stopModelPricingRefresh,
|
||||
stopSessionUpstreamMonitor,
|
||||
stopSessionDeliveryRuntime,
|
||||
startSessionDeliveryRuntime: vi.fn<StartSessionDeliveryRuntime>(
|
||||
@@ -42,9 +40,6 @@ const hoisted = vi.hoisted(() => {
|
||||
),
|
||||
schedulePendingSessionDeliveries: vi.fn(async () => undefined),
|
||||
startSessionUpstreamMonitor: vi.fn(() => ({ stop: stopSessionUpstreamMonitor })),
|
||||
startGatewayModelPricingRefresh: vi.fn(() => stopModelPricingRefresh),
|
||||
loadModelPricingCacheModule: vi.fn(),
|
||||
isVitestRuntimeEnv: vi.fn(() => false),
|
||||
recoverPendingDeliveries: vi.fn(async () => undefined),
|
||||
recoverPendingRestartContinuationDeliveries: vi.fn(async () => undefined),
|
||||
deliverQueuedSessionDelivery: vi.fn(async () => undefined),
|
||||
@@ -67,7 +62,6 @@ vi.mock("../sessions/session-upstream-monitor.js", () => ({
|
||||
vi.mock("../infra/env.js", () => ({
|
||||
isTruthyEnvValue: (value?: string) =>
|
||||
["1", "true", "yes", "on"].includes(value?.trim().toLowerCase() ?? ""),
|
||||
isVitestRuntimeEnv: hoisted.isVitestRuntimeEnv,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/outbound/deliver.js", () => ({
|
||||
@@ -97,14 +91,6 @@ vi.mock("./channel-health-monitor.js", () => ({
|
||||
startChannelHealthMonitor: hoisted.startChannelHealthMonitor,
|
||||
}));
|
||||
|
||||
vi.mock("./model-pricing-cache.js", () => ({
|
||||
...(() => {
|
||||
hoisted.loadModelPricingCacheModule();
|
||||
return {};
|
||||
})(),
|
||||
startGatewayModelPricingRefresh: hoisted.startGatewayModelPricingRefresh,
|
||||
}));
|
||||
|
||||
const {
|
||||
activateGatewayScheduledServices,
|
||||
runGatewayPostReadyMaintenance,
|
||||
@@ -127,15 +113,11 @@ describe("server-runtime-services", () => {
|
||||
hoisted.heartbeatRunner.updateConfig.mockClear();
|
||||
hoisted.startHeartbeatRunner.mockClear();
|
||||
hoisted.startChannelHealthMonitor.mockClear();
|
||||
hoisted.startGatewayModelPricingRefresh.mockClear();
|
||||
hoisted.stopModelPricingRefresh.mockClear();
|
||||
hoisted.startSessionUpstreamMonitor.mockClear();
|
||||
hoisted.stopSessionUpstreamMonitor.mockClear();
|
||||
hoisted.stopSessionDeliveryRuntime.mockClear();
|
||||
hoisted.startSessionDeliveryRuntime.mockClear();
|
||||
hoisted.schedulePendingSessionDeliveries.mockClear();
|
||||
hoisted.loadModelPricingCacheModule.mockClear();
|
||||
hoisted.isVitestRuntimeEnv.mockReset().mockReturnValue(false);
|
||||
hoisted.recoverPendingDeliveries.mockClear();
|
||||
hoisted.recoverPendingRestartContinuationDeliveries.mockClear();
|
||||
hoisted.deliverQueuedSessionDelivery.mockClear();
|
||||
@@ -149,25 +131,7 @@ describe("server-runtime-services", () => {
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
it("skips model pricing bootstrap import when pricing is disabled", async () => {
|
||||
activateGatewayScheduledServices({
|
||||
minimalTestGateway: false,
|
||||
cfgAtStart: { models: { pricing: { enabled: false } } } as never,
|
||||
deps: {} as never,
|
||||
sessionDeliveryRecoveryMaxEnqueuedAt: 123,
|
||||
cronState: createTestCronState(),
|
||||
cronReconciliation: createTestCronReconciliation(),
|
||||
logCron: { error: vi.fn() },
|
||||
log: createLog(),
|
||||
});
|
||||
|
||||
await vi.dynamicImportSettled();
|
||||
|
||||
expect(hoisted.loadModelPricingCacheModule).not.toHaveBeenCalled();
|
||||
expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps scheduled services and pricing refresh inert during initial runtime setup", async () => {
|
||||
it("keeps scheduled services inert during initial runtime setup", () => {
|
||||
const services = startGatewayRuntimeServices({
|
||||
minimalTestGateway: false,
|
||||
cfgAtStart: {} as never,
|
||||
@@ -180,8 +144,6 @@ describe("server-runtime-services", () => {
|
||||
});
|
||||
|
||||
expect(hoisted.startChannelHealthMonitor).toHaveBeenCalledTimes(1);
|
||||
expect(hoisted.loadModelPricingCacheModule).not.toHaveBeenCalled();
|
||||
expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled();
|
||||
expect(hoisted.startHeartbeatRunner).not.toHaveBeenCalled();
|
||||
expect(hoisted.startSessionUpstreamMonitor).not.toHaveBeenCalled();
|
||||
expect(hoisted.recoverPendingDeliveries).not.toHaveBeenCalled();
|
||||
@@ -204,30 +166,6 @@ describe("server-runtime-services", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("starts model pricing refresh after scheduled services activate", async () => {
|
||||
const pluginLookUpTable = {
|
||||
index: { plugins: [] },
|
||||
manifestRegistry: { plugins: [], diagnostics: [] },
|
||||
};
|
||||
const { cronStart, services } = activateScheduledServicesForTest({
|
||||
pluginLookUpTable: pluginLookUpTable as never,
|
||||
});
|
||||
|
||||
expect(hoisted.startHeartbeatRunner).toHaveBeenCalledTimes(1);
|
||||
expect(hoisted.startSessionUpstreamMonitor).toHaveBeenCalledTimes(1);
|
||||
expect(cronStart).toHaveBeenCalledTimes(1);
|
||||
await vi.dynamicImportSettled();
|
||||
expect(hoisted.startGatewayModelPricingRefresh).toHaveBeenCalledWith({
|
||||
config: {},
|
||||
pluginLookUpTable,
|
||||
});
|
||||
services.stopModelPricingRefresh();
|
||||
expect(hoisted.stopModelPricingRefresh).toHaveBeenCalledTimes(1);
|
||||
services.heartbeatRunner.stop();
|
||||
expect(hoisted.stopSessionUpstreamMonitor).toHaveBeenCalledTimes(1);
|
||||
expect(hoisted.heartbeatRunner.stop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("warns when cron is disabled but scheduled heartbeats remain enabled", () => {
|
||||
const warn = vi.fn();
|
||||
const log = {
|
||||
@@ -382,16 +320,6 @@ describe("server-runtime-services", () => {
|
||||
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("does not start model pricing refresh after scheduled services stop before import settles", async () => {
|
||||
const { services } = activateScheduledServicesForTest();
|
||||
|
||||
services.stopModelPricingRefresh();
|
||||
await vi.dynamicImportSettled();
|
||||
|
||||
expect(hoisted.startGatewayModelPricingRefresh).not.toHaveBeenCalled();
|
||||
expect(hoisted.stopModelPricingRefresh).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("activates heartbeat, cron, and delivery recovery after sidecars are ready", async () => {
|
||||
vi.useFakeTimers();
|
||||
const log = createLog();
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// Starts delayed maintenance, cron, heartbeat, recovery, and pricing refresh work.
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isVitestRuntimeEnv } from "../infra/env.js";
|
||||
import {
|
||||
resolveHeartbeatAgents,
|
||||
startHeartbeatRunner,
|
||||
@@ -13,11 +12,9 @@ import {
|
||||
schedulePendingSessionDeliveries,
|
||||
startSessionDeliveryRuntime,
|
||||
} from "../infra/session-delivery-queue-runtime.js";
|
||||
import type { PluginMetadataRegistryView } from "../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { startSessionUpstreamMonitor } from "../sessions/session-upstream-monitor.js";
|
||||
import { removeCronRunContinuationSessionIfIdle } from "../tasks/cron-run-continuation-cleanup.js";
|
||||
import { isGatewayModelPricingEnabled } from "./model-pricing-config.js";
|
||||
import type { GatewayCronReconciliation } from "./server-cron-reconciled.js";
|
||||
import type { GatewayCronState } from "./server-cron.js";
|
||||
import type { startGatewayMaintenanceTimers } from "./server-maintenance.js";
|
||||
@@ -305,41 +302,6 @@ function startPendingSessionDeliveryRuntime(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function startGatewayModelPricingRefreshOnDemand(params: {
|
||||
config: OpenClawConfig;
|
||||
pluginLookUpTable?: PluginMetadataRegistryView;
|
||||
log: GatewayRuntimeServiceLogger;
|
||||
}): () => void {
|
||||
if (!isGatewayModelPricingEnabled(params.config)) {
|
||||
return () => {};
|
||||
}
|
||||
let stopped = false;
|
||||
let stopRefresh: (() => void) | undefined;
|
||||
// Import pricing refresh lazily; many gateway starts never use model-pricing metadata.
|
||||
// The stopped flag closes the race where shutdown happens before the import resolves.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { startGatewayModelPricingRefresh } = await import("./model-pricing-cache.js");
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
stopRefresh = startGatewayModelPricingRefresh({
|
||||
config: params.config,
|
||||
...(params.pluginLookUpTable ? { pluginLookUpTable: params.pluginLookUpTable } : {}),
|
||||
});
|
||||
if (stopped) {
|
||||
stopRefresh();
|
||||
stopRefresh = undefined;
|
||||
}
|
||||
}).catch((err: unknown) =>
|
||||
params.log.error(`Model pricing refresh failed to start: ${String(err)}`),
|
||||
);
|
||||
return () => {
|
||||
stopped = true;
|
||||
stopRefresh?.();
|
||||
stopRefresh = undefined;
|
||||
};
|
||||
}
|
||||
|
||||
/** Activates background gateway services after core runtime startup is ready. */
|
||||
export function activateGatewayScheduledServices(params: {
|
||||
minimalTestGateway: boolean;
|
||||
@@ -351,14 +313,12 @@ export function activateGatewayScheduledServices(params: {
|
||||
startCron?: boolean;
|
||||
logCron: { error: (message: string) => void };
|
||||
log: GatewayRuntimeServiceLogger;
|
||||
pluginLookUpTable?: PluginMetadataRegistryView;
|
||||
}): { heartbeatRunner: HeartbeatRunner; stopModelPricingRefresh: () => void } {
|
||||
}): { heartbeatRunner: HeartbeatRunner } {
|
||||
if (params.minimalTestGateway) {
|
||||
// Minimal gateways keep handles callable but inert so tests can share shutdown paths with
|
||||
// production starts without launching background loops.
|
||||
return {
|
||||
heartbeatRunner: createNoopHeartbeatRunner(),
|
||||
stopModelPricingRefresh: () => {},
|
||||
};
|
||||
}
|
||||
if (
|
||||
@@ -404,15 +364,7 @@ export function activateGatewayScheduledServices(params: {
|
||||
cfg: params.cfgAtStart,
|
||||
log: params.log,
|
||||
});
|
||||
const stopModelPricingRefresh = !isVitestRuntimeEnv()
|
||||
? startGatewayModelPricingRefreshOnDemand({
|
||||
config: params.cfgAtStart,
|
||||
...(params.pluginLookUpTable ? { pluginLookUpTable: params.pluginLookUpTable } : {}),
|
||||
log: params.log,
|
||||
})
|
||||
: () => {};
|
||||
return {
|
||||
heartbeatRunner: heartbeatRunnerWithUpstreamMonitor,
|
||||
stopModelPricingRefresh,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -45,7 +45,6 @@ export function startGatewayRuntimeServices(params: {
|
||||
}): {
|
||||
heartbeatRunner: ReturnType<typeof createNoopHeartbeatRunner>;
|
||||
channelHealthMonitor: ChannelHealthMonitor | null;
|
||||
stopModelPricingRefresh: () => void;
|
||||
} {
|
||||
const channelHealthMonitor = startGatewayChannelHealthMonitor({
|
||||
cfg: params.cfgAtStart,
|
||||
@@ -55,6 +54,5 @@ export function startGatewayRuntimeServices(params: {
|
||||
return {
|
||||
heartbeatRunner: createNoopHeartbeatRunner(),
|
||||
channelHealthMonitor,
|
||||
stopModelPricingRefresh: () => {},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -411,10 +411,8 @@ export async function finishGatewayStartup(params: {
|
||||
startCron: false,
|
||||
logCron,
|
||||
log,
|
||||
pluginLookUpTable,
|
||||
});
|
||||
runtimeState.heartbeatRunner = activated.heartbeatRunner;
|
||||
runtimeState.stopModelPricingRefresh = activated.stopModelPricingRefresh;
|
||||
});
|
||||
};
|
||||
({
|
||||
|
||||
@@ -10,10 +10,6 @@ import {
|
||||
persistSessionTranscriptTurn,
|
||||
upsertSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
clearGatewayModelPricingFailures,
|
||||
replaceGatewayModelPricingCache,
|
||||
} from "../gateway/model-pricing-cache-state.js";
|
||||
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import * as usageFormat from "../utils/usage-format.js";
|
||||
@@ -85,11 +81,6 @@ async function refreshSessionCostUsageForTest(sessionFile: string): Promise<void
|
||||
});
|
||||
}
|
||||
|
||||
function clearGatewayModelPricingState(): void {
|
||||
replaceGatewayModelPricingCache(new Map(), 0);
|
||||
clearGatewayModelPricingFailures();
|
||||
}
|
||||
|
||||
describe("session cost usage", () => {
|
||||
const suiteRootTracker = createSuiteTempRootTracker({ prefix: "openclaw-session-cost-" });
|
||||
const withStateDir = async <T>(stateDir: string, fn: () => Promise<T>): Promise<T> =>
|
||||
@@ -552,16 +543,14 @@ describe("session cost usage", () => {
|
||||
const sessionsDir = path.join(root, "agents", "main", "sessions");
|
||||
await fs.mkdir(sessionsDir, { recursive: true });
|
||||
|
||||
// A real assistant turn that burned tokens. The transport recorded cost.total: 0,
|
||||
// derived from an all-zero catalog price — exactly what codex/gpt-5.x models produce,
|
||||
// since the Codex backend exposes no per-token price and the operator never set one.
|
||||
// A real assistant turn that burned tokens for a model absent from every pricing source.
|
||||
const entry = {
|
||||
type: "message",
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
provider: "custom",
|
||||
model: "unpriced-model",
|
||||
usage: {
|
||||
input: 881,
|
||||
output: 6,
|
||||
@@ -581,7 +570,6 @@ describe("session cost usage", () => {
|
||||
|
||||
// No operator-configured pricing for this model, so its all-zero cost is unknown,
|
||||
// not an intentional "free" price.
|
||||
clearGatewayModelPricingState();
|
||||
await withStateDir(root, async () => {
|
||||
const summary = await loadCostUsageSummary();
|
||||
expect(summary.totals.totalTokens).toBe(23287);
|
||||
@@ -605,8 +593,8 @@ describe("session cost usage", () => {
|
||||
timestamp: new Date().toISOString(),
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
provider: "custom",
|
||||
model: "unpriced-model",
|
||||
usage: {
|
||||
input: 881,
|
||||
output: 6,
|
||||
@@ -629,10 +617,10 @@ describe("session cost usage", () => {
|
||||
const config = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
custom: {
|
||||
models: [
|
||||
{
|
||||
id: "gpt-5.5",
|
||||
id: "unpriced-model",
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
],
|
||||
@@ -641,7 +629,6 @@ describe("session cost usage", () => {
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
clearGatewayModelPricingState();
|
||||
await withStateDir(root, async () => {
|
||||
const summary = await loadCostUsageSummary({ config });
|
||||
expect(summary.totals.totalTokens).toBe(23287);
|
||||
@@ -657,9 +644,9 @@ describe("session cost usage", () => {
|
||||
const sessionFile = path.join(sessionsDir, "sess-missing-by-model.jsonl");
|
||||
const timestamp = Date.now() - 1_000;
|
||||
const entries = [
|
||||
["openai", "gpt-5.6-sol"],
|
||||
["openai", "gpt-5.6-sol"],
|
||||
["openai-codex", "gpt-5.5"],
|
||||
["custom", "unpriced-a"],
|
||||
["custom", "unpriced-a"],
|
||||
["other", "unpriced-b"],
|
||||
].map(([provider, model], index) => ({
|
||||
type: "message",
|
||||
timestamp: new Date(timestamp + index).toISOString(),
|
||||
@@ -681,13 +668,12 @@ describe("session cost usage", () => {
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
clearGatewayModelPricingState();
|
||||
await withStateDir(root, async () => {
|
||||
const summary = await loadCostUsageSummary();
|
||||
expect(summary.totals.missingCostEntries).toBe(3);
|
||||
expect(summary.totals.missingCostByModel).toEqual({
|
||||
"openai/gpt-5.6-sol": 2,
|
||||
"openai-codex/gpt-5.5": 1,
|
||||
"custom/unpriced-a": 2,
|
||||
"other/unpriced-b": 1,
|
||||
});
|
||||
|
||||
const sessionSummary = await loadSessionCostSummary({ sessionFile });
|
||||
@@ -1631,8 +1617,8 @@ describe("session cost usage", () => {
|
||||
timestamp: `2026-02-05T12:0${index}:00.000Z`,
|
||||
message: {
|
||||
role: "assistant",
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
provider: "custom",
|
||||
model: "unpriced-batch",
|
||||
usage: { input: index + 1, output: 0, totalTokens: index + 1 },
|
||||
},
|
||||
}),
|
||||
@@ -1649,7 +1635,7 @@ describe("session cost usage", () => {
|
||||
refreshMode: "sync-when-empty",
|
||||
});
|
||||
expect(warmed.cacheStatus?.status).toBe("fresh");
|
||||
expect(warmed.totals.missingCostByModel).toEqual({ "openai/gpt-5.5": 2 });
|
||||
expect(warmed.totals.missingCostByModel).toEqual({ "custom/unpriced-batch": 2 });
|
||||
|
||||
await loadSessionCostSummariesFromCache({
|
||||
sessions,
|
||||
@@ -1664,8 +1650,8 @@ describe("session cost usage", () => {
|
||||
});
|
||||
expect(cached.cacheStatus.status).toBe("fresh");
|
||||
expect(cached.summaries.map((summary) => summary?.missingCostByModel)).toEqual([
|
||||
{ "openai/gpt-5.5": 1 },
|
||||
{ "openai/gpt-5.5": 1 },
|
||||
{ "custom/unpriced-batch": 1 },
|
||||
{ "custom/unpriced-batch": 1 },
|
||||
]);
|
||||
},
|
||||
{ interval: 10, timeout: 2_000 },
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
resolveModelCostConfig,
|
||||
resolveModelCostConfigFingerprint,
|
||||
} from "../utils/usage-format.js";
|
||||
import {
|
||||
resetRemoteModelCatalogOverlayForTest,
|
||||
setRemoteModelCatalogOverlaySourcesForTest,
|
||||
} from "./remote-overlay.test-support.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const readStoredCatalog = vi.fn();
|
||||
|
||||
beforeEach(() => {
|
||||
resetRemoteModelCatalogOverlayForTest();
|
||||
readStoredCatalog.mockReset().mockReturnValue({
|
||||
source_url: "https://catalog.openclaw.ai/models/v1/catalog.json",
|
||||
bundle_json: JSON.stringify({
|
||||
schemaVersion: 1,
|
||||
generatedAt: 200,
|
||||
minVersion: "2026.7.0",
|
||||
sourceCommit: "pricing-test",
|
||||
providers: {
|
||||
openai: {
|
||||
models: [
|
||||
{ id: "gpt-catalog", cost: { input: 1, output: 2 } },
|
||||
{
|
||||
id: "gpt-zero-tier",
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
tieredPricing: [{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, range: [0] }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
pricing: {
|
||||
"openai/gpt-external": { input: 2.5, output: 10, cacheRead: 1.25 },
|
||||
"openai/gpt-zero-hosted": {
|
||||
input: 0,
|
||||
output: 0,
|
||||
tieredPricing: [{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, range: [0] }],
|
||||
},
|
||||
"openai/gpt-zero-tier": { input: 4, output: 8 },
|
||||
"openrouter/openai/gpt-catalog": { input: 1, output: 2 },
|
||||
"z-ai/forbidden": { input: 9, output: 18 },
|
||||
},
|
||||
}),
|
||||
});
|
||||
setRemoteModelCatalogOverlaySourcesForTest({
|
||||
bundledGeneratedAt: () => 100,
|
||||
readStoredCatalog,
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
setRemoteModelCatalogOverlaySourcesForTest();
|
||||
resetRemoteModelCatalogOverlayForTest();
|
||||
});
|
||||
|
||||
function configFor(baseUrl: string): OpenClawConfig {
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl,
|
||||
models: [{ id: "gpt-external", name: "External GPT" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
}
|
||||
|
||||
describe("hosted model pricing", () => {
|
||||
it("resolves a non-catalog model from the stored hosted pricing map", () => {
|
||||
const agentDir = tempDirs.make("openclaw-hosted-pricing-");
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("https://api.openai.com/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-external",
|
||||
}),
|
||||
).toEqual({ input: 2.5, output: 10, cacheRead: 1.25, cacheWrite: 0 });
|
||||
});
|
||||
|
||||
it("prefers merged catalog pricing over configured pricing", () => {
|
||||
const agentDir = tempDirs.make("openclaw-catalog-pricing-");
|
||||
const config = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [
|
||||
{
|
||||
id: "gpt-catalog",
|
||||
name: "Catalog GPT",
|
||||
cost: { input: 99, output: 99, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
expect(
|
||||
resolveModelCostConfig({ config, agentDir, provider: "openai", model: "gpt-catalog" }),
|
||||
).toEqual({ input: 1, output: 2, cacheRead: 0, cacheWrite: 0 });
|
||||
});
|
||||
|
||||
it("does not apply hosted pricing to private endpoints or unknown models", () => {
|
||||
const agentDir = tempDirs.make("openclaw-private-pricing-");
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("http://127.0.0.1:8080/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-external",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(resolveModelCostConfigFingerprint(configFor("https://api.openai.com/v1"))).not.toBe(
|
||||
resolveModelCostConfigFingerprint(configFor("http://127.0.0.1:8080/v1")),
|
||||
);
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("https://fc-proxy.example.com/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-external",
|
||||
}),
|
||||
).toEqual({ input: 2.5, output: 10, cacheRead: 1.25, cacheWrite: 0 });
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("http://127.0.0.1:8080/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-catalog",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("https://api.openai.com/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "unknown-model",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("https://api.openai.com/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-zero-hosted",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
const disabled = configFor("https://api.openai.com/v1");
|
||||
disabled.models = {
|
||||
...disabled.models,
|
||||
catalogRefresh: { enabled: false },
|
||||
};
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: disabled,
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-external",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("resolves passthrough provider aliases through a priced catalog row", () => {
|
||||
const agentDir = tempDirs.make("openclaw-passthrough-pricing-");
|
||||
const config = {
|
||||
models: {
|
||||
providers: {
|
||||
openrouter: {
|
||||
baseUrl: "https://openrouter.ai/api/v1",
|
||||
models: [{ id: "openai/gpt-catalog", name: "Catalog GPT through OpenRouter" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config,
|
||||
agentDir,
|
||||
provider: "openrouter",
|
||||
model: "openai/gpt-catalog",
|
||||
}),
|
||||
).toEqual({ input: 1, output: 2, cacheRead: 0, cacheWrite: 0 });
|
||||
});
|
||||
|
||||
it("falls through zero-only catalog tiers without reviving disabled source aliases", () => {
|
||||
const agentDir = tempDirs.make("openclaw-zero-tier-pricing-");
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: configFor("https://api.openai.com/v1"),
|
||||
agentDir,
|
||||
provider: "openai",
|
||||
model: "gpt-zero-tier",
|
||||
}),
|
||||
).toEqual({ input: 4, output: 8, cacheRead: 0, cacheWrite: 0 });
|
||||
|
||||
const zaiConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://api.z.ai/api/paas/v4",
|
||||
models: [{ id: "forbidden", name: "Forbidden source alias" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
config: zaiConfig,
|
||||
agentDir,
|
||||
provider: "zai",
|
||||
model: "forbidden",
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("fingerprints provider overlays without explicit model rows", () => {
|
||||
const config = {
|
||||
models: { providers: { openai: { baseUrl: "https://api.openai.com/v1" } } },
|
||||
} as unknown as OpenClawConfig;
|
||||
expect(() => resolveModelCostConfigFingerprint(config)).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,267 @@
|
||||
import { isIP } from "node:net";
|
||||
import type { RemoteModelCatalogPricing } from "@openclaw/model-catalog-core";
|
||||
import type { ModelCatalogCost } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import { modelKey, normalizeModelRef } from "../agents/model-selection.js";
|
||||
import type { ModelDefinitionConfig } from "../config/types.models.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isInstalledPluginEnabled } from "../plugins/installed-plugin-index.js";
|
||||
import type { PluginManifestRegistry } from "../plugins/manifest-registry.js";
|
||||
import {
|
||||
resolvePluginMetadataSnapshot,
|
||||
type PluginMetadataSnapshot,
|
||||
} from "../plugins/plugin-metadata-snapshot.js";
|
||||
import { planEffectiveModelCatalogRows } from "./index.js";
|
||||
import { getRemoteModelCatalogPricing } from "./remote-overlay.js";
|
||||
|
||||
type PricingValue = RemoteModelCatalogPricing | ModelCatalogCost;
|
||||
type ManifestPlugins = readonly PluginManifestRegistry["plugins"][number][];
|
||||
type ExternalPricingPolicy = {
|
||||
external: boolean;
|
||||
};
|
||||
type PricingContext = {
|
||||
snapshot?: PluginMetadataSnapshot;
|
||||
catalog: ReadonlyMap<string, PricingValue>;
|
||||
hosted: Readonly<Record<string, RemoteModelCatalogPricing>>;
|
||||
normalizedHosted: ReadonlyMap<string, RemoteModelCatalogPricing>;
|
||||
policies: ReadonlyMap<string, ExternalPricingPolicy>;
|
||||
fingerprint: string;
|
||||
};
|
||||
|
||||
const EMPTY_CONFIG: OpenClawConfig = {};
|
||||
const pricingContextByConfig = new WeakMap<OpenClawConfig, PricingContext>();
|
||||
|
||||
function normalizePolicy(
|
||||
policy: { external?: boolean } | undefined,
|
||||
): ExternalPricingPolicy | undefined {
|
||||
if (!policy) {
|
||||
return undefined;
|
||||
}
|
||||
return { external: policy.external !== false };
|
||||
}
|
||||
|
||||
function activeManifestRegistry(
|
||||
snapshot: PluginMetadataSnapshot,
|
||||
config: OpenClawConfig,
|
||||
): PluginManifestRegistry {
|
||||
if (config.plugins?.enabled === false) {
|
||||
return { plugins: [], diagnostics: [] };
|
||||
}
|
||||
return {
|
||||
diagnostics: snapshot.manifestRegistry.diagnostics,
|
||||
plugins: snapshot.manifestRegistry.plugins.filter((plugin) =>
|
||||
isInstalledPluginEnabled(snapshot.index, plugin.id, config),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizedHostedKey(key: string, manifestPlugins?: ManifestPlugins): string | undefined {
|
||||
const slash = key.indexOf("/");
|
||||
if (slash <= 0 || slash === key.length - 1) {
|
||||
return undefined;
|
||||
}
|
||||
const normalized = normalizeModelRef(key.slice(0, slash), key.slice(slash + 1), {
|
||||
manifestPlugins,
|
||||
});
|
||||
return modelKey(normalized.provider, normalized.model);
|
||||
}
|
||||
|
||||
function buildPricingContext(config: OpenClawConfig): PricingContext {
|
||||
let snapshot: PluginMetadataSnapshot | undefined;
|
||||
try {
|
||||
snapshot = resolvePluginMetadataSnapshot({
|
||||
config,
|
||||
env: process.env,
|
||||
allowWorkspaceScopedCurrent: true,
|
||||
});
|
||||
} catch {
|
||||
snapshot = undefined;
|
||||
}
|
||||
const registry = snapshot
|
||||
? activeManifestRegistry(snapshot, config)
|
||||
: ({ plugins: [], diagnostics: [] } satisfies PluginManifestRegistry);
|
||||
const catalog = new Map<string, PricingValue>();
|
||||
for (const row of planEffectiveModelCatalogRows({ registry, config }).rows) {
|
||||
if (row.cost) {
|
||||
catalog.set(modelKey(row.provider, row.id), row.cost);
|
||||
}
|
||||
}
|
||||
const policies = new Map<string, ExternalPricingPolicy>();
|
||||
for (const plugin of registry.plugins) {
|
||||
for (const [provider, rawPolicy] of Object.entries(plugin.modelPricing?.providers ?? {})) {
|
||||
const policy = normalizePolicy(rawPolicy);
|
||||
if (policy) {
|
||||
policies.set(provider, policy);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Hosted aliases are policy-resolved against installed manifests. If that metadata is
|
||||
// unavailable, fail closed instead of treating every provider as policy-free.
|
||||
const hosted = snapshot ? (getRemoteModelCatalogPricing(config) ?? {}) : {};
|
||||
const normalizedHosted = new Map<string, RemoteModelCatalogPricing>();
|
||||
for (const [key, pricing] of Object.entries(hosted).toSorted(([a], [b]) => a.localeCompare(b))) {
|
||||
const normalized = normalizedHostedKey(key, snapshot?.plugins);
|
||||
if (normalized && !normalizedHosted.has(normalized)) {
|
||||
normalizedHosted.set(normalized, pricing);
|
||||
}
|
||||
}
|
||||
const fingerprint = JSON.stringify({
|
||||
catalog: [...catalog.entries()].toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
hosted: Object.entries(hosted).toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
normalizedHosted: [...normalizedHosted.entries()].toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
policies: [...policies.entries()].toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
});
|
||||
return { snapshot, catalog, hosted, normalizedHosted, policies, fingerprint };
|
||||
}
|
||||
|
||||
function getPricingContext(config: OpenClawConfig): PricingContext {
|
||||
const existing = pricingContextByConfig.get(config);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
const context = buildPricingContext(config);
|
||||
pricingContextByConfig.set(config, context);
|
||||
return context;
|
||||
}
|
||||
|
||||
function hasKnownPricing(pricing: PricingValue): boolean {
|
||||
return (
|
||||
Boolean(
|
||||
pricing.tieredPricing?.some(
|
||||
(tier) => tier.input > 0 || tier.output > 0 || tier.cacheRead > 0 || tier.cacheWrite > 0,
|
||||
),
|
||||
) ||
|
||||
(pricing.input ?? 0) > 0 ||
|
||||
(pricing.output ?? 0) > 0 ||
|
||||
(pricing.cacheRead ?? 0) > 0 ||
|
||||
(pricing.cacheWrite ?? 0) > 0
|
||||
);
|
||||
}
|
||||
|
||||
function isPrivateOrLoopbackHost(hostname: string): boolean {
|
||||
const host = hostname
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/^\[|\]$/gu, "");
|
||||
if (
|
||||
host === "localhost" ||
|
||||
host === "localhost.localdomain" ||
|
||||
host.endsWith(".localhost") ||
|
||||
host.endsWith(".local")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
const addressFamily = isIP(host);
|
||||
if (addressFamily === 6) {
|
||||
return (
|
||||
host === "::1" ||
|
||||
host === "0:0:0:0:0:0:0:1" ||
|
||||
host.startsWith("fe80:") ||
|
||||
host.startsWith("fc") ||
|
||||
host.startsWith("fd")
|
||||
);
|
||||
}
|
||||
if (addressFamily === 4) {
|
||||
return (
|
||||
host.startsWith("127.") ||
|
||||
host.startsWith("10.") ||
|
||||
host.startsWith("192.168.") ||
|
||||
host.startsWith("169.254.") ||
|
||||
/^172\.(1[6-9]|2\d|3[0-1])\./u.test(host)
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isPrivateOrLoopbackUrl(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return isPrivateOrLoopbackHost(new URL(value).hostname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findConfiguredModel(
|
||||
config: OpenClawConfig,
|
||||
provider: string,
|
||||
model: string,
|
||||
manifestPlugins?: ManifestPlugins,
|
||||
): ModelDefinitionConfig | undefined {
|
||||
return config.models?.providers?.[provider]?.models?.find((entry) => {
|
||||
const normalized = normalizeModelRef(provider, entry.id, { manifestPlugins });
|
||||
return modelKey(normalized.provider, normalized.model) === modelKey(provider, model);
|
||||
});
|
||||
}
|
||||
|
||||
function allowsHostedPricing(
|
||||
config: OpenClawConfig,
|
||||
provider: string,
|
||||
model: string,
|
||||
manifestPlugins?: ManifestPlugins,
|
||||
): boolean {
|
||||
const providerConfig = config.models?.providers?.[provider];
|
||||
const configuredModel = findConfiguredModel(config, provider, model, manifestPlugins);
|
||||
return !(
|
||||
isPrivateOrLoopbackUrl(configuredModel?.baseUrl) ||
|
||||
isPrivateOrLoopbackUrl(providerConfig?.baseUrl)
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveCatalogModelPricing(params: {
|
||||
config?: OpenClawConfig;
|
||||
provider: string;
|
||||
model: string;
|
||||
}): PricingValue | undefined {
|
||||
const config = params.config ?? EMPTY_CONFIG;
|
||||
const context = getPricingContext(config);
|
||||
const normalized = normalizeModelRef(params.provider, params.model, {
|
||||
manifestPlugins: context.snapshot?.plugins,
|
||||
});
|
||||
if (
|
||||
!allowsHostedPricing(config, normalized.provider, normalized.model, context.snapshot?.plugins)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const pricing = context.catalog.get(modelKey(normalized.provider, normalized.model));
|
||||
return pricing && hasKnownPricing(pricing) ? pricing : undefined;
|
||||
}
|
||||
|
||||
export function resolveHostedModelPricing(params: {
|
||||
config?: OpenClawConfig;
|
||||
provider: string;
|
||||
model: string;
|
||||
}): PricingValue | undefined {
|
||||
const config = params.config ?? EMPTY_CONFIG;
|
||||
const context = getPricingContext(config);
|
||||
const normalized = normalizeModelRef(params.provider, params.model, {
|
||||
manifestPlugins: context.snapshot?.plugins,
|
||||
});
|
||||
if (
|
||||
context.policies.get(normalized.provider)?.external === false ||
|
||||
!allowsHostedPricing(config, normalized.provider, normalized.model, context.snapshot?.plugins)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const key = modelKey(normalized.provider, normalized.model);
|
||||
const pricing =
|
||||
context.hosted[key] ??
|
||||
(context.policies.has(normalized.provider) ? undefined : context.normalizedHosted.get(key));
|
||||
return pricing && hasKnownPricing(pricing) ? pricing : undefined;
|
||||
}
|
||||
|
||||
export function modelCatalogPricingFingerprint(config?: OpenClawConfig): string {
|
||||
const resolvedConfig = config ?? EMPTY_CONFIG;
|
||||
const context = getPricingContext(resolvedConfig);
|
||||
const configuredEndpoints = Object.entries(resolvedConfig.models?.providers ?? {})
|
||||
.toSorted(([a], [b]) => a.localeCompare(b))
|
||||
.map(([provider, providerConfig]) => ({
|
||||
provider,
|
||||
baseUrl: providerConfig.baseUrl,
|
||||
models: (providerConfig.models ?? [])
|
||||
.map((model) => ({ id: model.id, baseUrl: model.baseUrl }))
|
||||
.toSorted((a, b) => a.id.localeCompare(b.id)),
|
||||
}));
|
||||
return JSON.stringify({ pricing: context.fingerprint, configuredEndpoints });
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getRemoteModelCatalogOverlay } from "./remote-overlay.js";
|
||||
import { getRemoteModelCatalogOverlay, getRemoteModelCatalogPricing } from "./remote-overlay.js";
|
||||
import {
|
||||
resetRemoteModelCatalogOverlayForTest,
|
||||
setRemoteModelCatalogOverlaySourcesForTest,
|
||||
@@ -16,6 +16,7 @@ const bundle = {
|
||||
minVersion: "2026.7.0",
|
||||
sourceCommit: "abc",
|
||||
providers: { anthropic: { models: [{ id: "new" }] } },
|
||||
pricing: { "openai/gpt-external": { input: 2.5, output: 10 } },
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -40,6 +41,10 @@ describe("remote model catalog overlay", () => {
|
||||
it("loads a newer compatible bundle once", () => {
|
||||
expect(getRemoteModelCatalogOverlay({})).toHaveProperty("anthropic");
|
||||
expect(getRemoteModelCatalogOverlay({})).toHaveProperty("anthropic");
|
||||
expect(getRemoteModelCatalogPricing({})?.["openai/gpt-external"]).toEqual({
|
||||
input: 2.5,
|
||||
output: 10,
|
||||
});
|
||||
expect(mocks.read).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
validateAndSanitizeRemoteModelCatalogBundle,
|
||||
type RemoteModelCatalogBundle,
|
||||
type RemoteModelCatalogPricing,
|
||||
} from "@openclaw/model-catalog-core";
|
||||
import type { ModelCatalogProvider } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
@@ -11,8 +12,12 @@ import { isRemoteModelCatalogRefreshEnabled, resolveRemoteCatalogUrl } from "./r
|
||||
import { readRemoteModelCatalog } from "./remote-store.js";
|
||||
|
||||
type RemoteModelCatalogOverlay = Readonly<Record<string, ModelCatalogProvider>>;
|
||||
type ActiveRemoteModelCatalog = {
|
||||
providers: RemoteModelCatalogOverlay;
|
||||
pricing?: Readonly<Record<string, RemoteModelCatalogPricing>>;
|
||||
};
|
||||
|
||||
let cachedOverlay: { sourceUrl: string; value: RemoteModelCatalogOverlay | null } | undefined;
|
||||
let cachedOverlay: { sourceUrl: string; value: ActiveRemoteModelCatalog | null } | undefined;
|
||||
let readBundledGeneratedAt = bundledCatalogGeneratedAt;
|
||||
let readStoredCatalog = readRemoteModelCatalog;
|
||||
|
||||
@@ -24,9 +29,7 @@ function isCompatible(bundle: RemoteModelCatalogBundle): boolean {
|
||||
return comparison !== null && comparison >= 0;
|
||||
}
|
||||
|
||||
export function getRemoteModelCatalogOverlay(
|
||||
config: OpenClawConfig,
|
||||
): RemoteModelCatalogOverlay | undefined {
|
||||
function getActiveRemoteModelCatalog(config: OpenClawConfig): ActiveRemoteModelCatalog | undefined {
|
||||
if (!isRemoteModelCatalogRefreshEnabled(config)) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -50,14 +53,30 @@ export function getRemoteModelCatalogOverlay(
|
||||
cachedOverlay = { sourceUrl, value: null };
|
||||
return undefined;
|
||||
}
|
||||
cachedOverlay = { sourceUrl, value: bundle.providers };
|
||||
return bundle.providers;
|
||||
const value = {
|
||||
providers: bundle.providers,
|
||||
...(bundle.pricing ? { pricing: bundle.pricing } : {}),
|
||||
};
|
||||
cachedOverlay = { sourceUrl, value };
|
||||
return value;
|
||||
} catch {
|
||||
cachedOverlay = undefined;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function getRemoteModelCatalogOverlay(
|
||||
config: OpenClawConfig,
|
||||
): RemoteModelCatalogOverlay | undefined {
|
||||
return getActiveRemoteModelCatalog(config)?.providers;
|
||||
}
|
||||
|
||||
export function getRemoteModelCatalogPricing(
|
||||
config: OpenClawConfig,
|
||||
): Readonly<Record<string, RemoteModelCatalogPricing>> | undefined {
|
||||
return getActiveRemoteModelCatalog(config)?.pricing;
|
||||
}
|
||||
|
||||
function resetRemoteModelCatalogOverlayForTest(): void {
|
||||
cachedOverlay = undefined;
|
||||
}
|
||||
|
||||
@@ -222,7 +222,7 @@ describe("secrets runtime provider and media surfaces", () => {
|
||||
},
|
||||
models: {
|
||||
...initial.config.models,
|
||||
pricing: { enabled: true },
|
||||
catalogRefresh: { enabled: false },
|
||||
},
|
||||
},
|
||||
runtimeSourceConfig,
|
||||
@@ -237,7 +237,7 @@ describe("secrets runtime provider and media surfaces", () => {
|
||||
"https://runtime-only.example",
|
||||
]);
|
||||
expect(active?.config.auth?.order?.openai).toEqual(["runtime-only-profile"]);
|
||||
expect(active?.config.models?.pricing?.enabled).toBe(true);
|
||||
expect(active?.config.models?.catalogRefresh?.enabled).toBe(false);
|
||||
expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new");
|
||||
expect(getRuntimeConfigSnapshot()).toEqual(active?.config);
|
||||
expect(getRuntimeConfigSourceSnapshot()).toEqual(runtimeSourceConfig);
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/** One normalized tier in a per-million-token pricing schedule. */
|
||||
export type PricingTier = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
/** Half-open input-token interval `[start, end)`. */
|
||||
range: [number, number];
|
||||
};
|
||||
|
||||
type RawPricingTier = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
range: [number, number] | [number];
|
||||
};
|
||||
|
||||
/** Per-million-token pricing used by usage summaries and cost estimates. */
|
||||
export type ModelCostConfig = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
tieredPricing?: PricingTier[];
|
||||
};
|
||||
|
||||
export type RawModelCostConfig = Omit<ModelCostConfig, "tieredPricing"> & {
|
||||
tieredPricing?: RawPricingTier[];
|
||||
};
|
||||
|
||||
function normalizeTieredPricing(raw: RawPricingTier[] | undefined): PricingTier[] | undefined {
|
||||
if (!raw || raw.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const result: PricingTier[] = [];
|
||||
for (const tier of raw) {
|
||||
const range = tier.range;
|
||||
const start = Array.isArray(range) && typeof range[0] === "number" ? range[0] : Number.NaN;
|
||||
if (!Number.isFinite(start)) {
|
||||
continue;
|
||||
}
|
||||
const rawEnd = range.length >= 2 ? range[1] : null;
|
||||
const end =
|
||||
typeof rawEnd === "number" && Number.isFinite(rawEnd) && rawEnd > start ? rawEnd : Infinity;
|
||||
if (
|
||||
!Number.isFinite(tier.input) ||
|
||||
!Number.isFinite(tier.output) ||
|
||||
!Number.isFinite(tier.cacheRead) ||
|
||||
!Number.isFinite(tier.cacheWrite)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
result.push({
|
||||
input: tier.input,
|
||||
output: tier.output,
|
||||
cacheRead: tier.cacheRead,
|
||||
cacheWrite: tier.cacheWrite,
|
||||
range: [start, end],
|
||||
});
|
||||
}
|
||||
return result.length > 0 ? result.toSorted((a, b) => a.range[0] - b.range[0]) : undefined;
|
||||
}
|
||||
|
||||
export function normalizeModelCostConfig(cost: RawModelCostConfig): ModelCostConfig {
|
||||
const normalizedTiers = normalizeTieredPricing(cost.tieredPricing);
|
||||
return {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
cacheRead: cost.cacheRead,
|
||||
cacheWrite: cost.cacheWrite,
|
||||
...(normalizedTiers ? { tieredPricing: normalizedTiers } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeResolvedPricing(cost: {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
tieredPricing?: RawPricingTier[];
|
||||
}): ModelCostConfig {
|
||||
const finiteOrZero = (value: number | undefined) =>
|
||||
typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
return normalizeModelCostConfig({
|
||||
input: finiteOrZero(cost.input),
|
||||
output: finiteOrZero(cost.output),
|
||||
cacheRead: finiteOrZero(cost.cacheRead),
|
||||
cacheWrite: finiteOrZero(cost.cacheWrite),
|
||||
...(cost.tieredPricing ? { tieredPricing: cost.tieredPricing } : {}),
|
||||
});
|
||||
}
|
||||
@@ -6,11 +6,6 @@ import path from "node:path";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
clearGatewayModelPricingFailures,
|
||||
replaceGatewayModelPricingCache,
|
||||
type CachedModelPricing,
|
||||
} from "../gateway/model-pricing-cache-state.js";
|
||||
import * as manifestModelIdNormalization from "../plugins/manifest-model-id-normalization.js";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import {
|
||||
@@ -25,23 +20,6 @@ import {
|
||||
type ModelCostConfig = NonNullable<ReturnType<typeof resolveModelCostConfig>>;
|
||||
type PricingTier = NonNullable<ModelCostConfig["tieredPricing"]>[number];
|
||||
|
||||
function setGatewayModelPricing(
|
||||
entries: Array<{
|
||||
provider: string;
|
||||
model: string;
|
||||
pricing: CachedModelPricing;
|
||||
}>,
|
||||
): void {
|
||||
replaceGatewayModelPricingCache(
|
||||
new Map(entries.map((entry) => [`${entry.provider}/${entry.model}`, entry.pricing])),
|
||||
);
|
||||
}
|
||||
|
||||
function clearGatewayModelPricingState(): void {
|
||||
replaceGatewayModelPricingCache(new Map(), 0);
|
||||
clearGatewayModelPricingFailures();
|
||||
}
|
||||
|
||||
function requireCostConfig(
|
||||
cost: ReturnType<typeof resolveModelCostConfig>,
|
||||
label: string,
|
||||
@@ -75,14 +53,12 @@ describe("usage-format", () => {
|
||||
delete process.env.OPENCLAW_AGENT_DIR;
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
resetUsageFormatCachesForTest();
|
||||
clearGatewayModelPricingState();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
envSnapshot?.restore();
|
||||
envSnapshot = undefined;
|
||||
resetUsageFormatCachesForTest();
|
||||
clearGatewayModelPricingState();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -216,14 +192,6 @@ describe("usage-format", () => {
|
||||
"utf8",
|
||||
);
|
||||
|
||||
setGatewayModelPricing([
|
||||
{
|
||||
provider: "demo-preferred",
|
||||
model: "demo-model",
|
||||
pricing: { input: 30, output: 31, cacheRead: 32, cacheWrite: 33 },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
provider: "demo-preferred",
|
||||
@@ -353,14 +321,6 @@ describe("usage-format", () => {
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
|
||||
setGatewayModelPricing([
|
||||
{
|
||||
provider: "demo-config-provider",
|
||||
model: "demo-model",
|
||||
pricing: { input: 3, output: 4, cacheRead: 0.3, cacheWrite: 0.4 },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
provider: "demo-config-provider",
|
||||
@@ -375,28 +335,6 @@ describe("usage-format", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("falls back to cached gateway pricing when no configured cost exists", () => {
|
||||
setGatewayModelPricing([
|
||||
{
|
||||
provider: "demo-cached-provider",
|
||||
model: "demo-model",
|
||||
pricing: { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(
|
||||
resolveModelCostConfig({
|
||||
provider: "demo-cached-provider",
|
||||
model: "demo-model",
|
||||
}),
|
||||
).toEqual({
|
||||
input: 2.5,
|
||||
output: 15,
|
||||
cacheRead: 0.25,
|
||||
cacheWrite: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("can skip plugin-backed model normalization for display-only cost lookup", () => {
|
||||
const config = {
|
||||
models: {
|
||||
@@ -1088,43 +1026,4 @@ describe("usage-format", () => {
|
||||
expect(expectDefined(tiers[0], "tiers[0] test invariant").range).toEqual([0, 32000]);
|
||||
expect(expectDefined(tiers[1], "tiers[1] test invariant").input).toBe(0.7);
|
||||
});
|
||||
|
||||
it("resolves tiered pricing from cached gateway (LiteLLM)", () => {
|
||||
setGatewayModelPricing([
|
||||
{
|
||||
provider: "volcengine",
|
||||
model: "doubao-seed",
|
||||
pricing: {
|
||||
input: 0.46,
|
||||
output: 2.3,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
tieredPricing: [
|
||||
{
|
||||
input: 0.46,
|
||||
output: 2.3,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
range: [0, 32000] as [number, number],
|
||||
},
|
||||
{
|
||||
input: 0.7,
|
||||
output: 3.5,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
range: [32000, 128000] as [number, number],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]);
|
||||
|
||||
const cost = resolveModelCostConfig({
|
||||
provider: "volcengine",
|
||||
model: "doubao-seed",
|
||||
});
|
||||
const tiers = requireTieredPricing(requireCostConfig(cost, "cached gateway"), "cached gateway");
|
||||
|
||||
expect(tiers).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
+30
-98
@@ -15,48 +15,21 @@ import type { NormalizedUsage } from "../agents/usage.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import type { ModelProviderConfig } from "../config/types.models.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { getGatewayModelPricingCacheFingerprint } from "../gateway/model-pricing-cache-state.js";
|
||||
import { getCachedGatewayModelPricing } from "../gateway/model-pricing-cache.js";
|
||||
import { tryReadJsonSync } from "../infra/json-files.js";
|
||||
import {
|
||||
modelCatalogPricingFingerprint,
|
||||
resolveCatalogModelPricing,
|
||||
resolveHostedModelPricing,
|
||||
} from "../model-catalog/pricing.js";
|
||||
import {
|
||||
normalizeModelCostConfig,
|
||||
normalizeResolvedPricing,
|
||||
type ModelCostConfig,
|
||||
type PricingTier,
|
||||
type RawModelCostConfig,
|
||||
} from "./usage-format-pricing.js";
|
||||
export { formatTokenCount } from "./token-format.js";
|
||||
|
||||
/**
|
||||
* A single tier in a tiered-pricing schedule. Prices are expressed as
|
||||
* USD per-million tokens, just like the flat `ModelCostConfig` fields.
|
||||
*
|
||||
* `range` is a half-open interval `[start, end)` expressed in *input*
|
||||
* token counts. The tiers MUST be sorted in ascending `range[0]` order
|
||||
* with no gaps.
|
||||
*/
|
||||
type PricingTier = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
/** [startTokens, endTokens) — half-open interval on the input token axis. */
|
||||
range: [number, number];
|
||||
};
|
||||
|
||||
type RawPricingTier = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
range: [number, number] | [number];
|
||||
};
|
||||
|
||||
/** Per-million-token model pricing used by usage summaries and cost estimates. */
|
||||
export type ModelCostConfig = {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead: number;
|
||||
cacheWrite: number;
|
||||
/** Optional tiered pricing tiers. When present, `estimateUsageCost`
|
||||
* uses them instead of the flat rates above. The flat rates still
|
||||
* serve as the "default / first-tier" fallback for callers that are
|
||||
* unaware of tiered pricing. */
|
||||
tieredPricing?: PricingTier[];
|
||||
};
|
||||
export type { ModelCostConfig } from "./usage-format-pricing.js";
|
||||
|
||||
type UsageTotals = {
|
||||
input?: number;
|
||||
@@ -91,10 +64,6 @@ type ProviderCostIndex = {
|
||||
structureFingerprint: string;
|
||||
};
|
||||
|
||||
type RawModelCostConfig = Omit<ModelCostConfig, "tieredPricing"> & {
|
||||
tieredPricing?: RawPricingTier[];
|
||||
};
|
||||
|
||||
const EMPTY_PROVIDER_COST_INDEX = new Map<string, ModelCostConfig>();
|
||||
const MODELS_JSON_COST_CACHE_LIMIT = 128;
|
||||
const MODEL_KEY_CACHE_LIMIT = 4096;
|
||||
@@ -179,58 +148,6 @@ function shouldUseNormalizedCostLookup(params: { provider?: string; model?: stri
|
||||
return provider === "anthropic" || provider === "openrouter" || provider === "vercel-ai-gateway";
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a raw tieredPricing array from models.json / config.
|
||||
* Supports open-ended ranges such as `[128000]` or `[128000, -1]`,
|
||||
* which are converted to `[128000, Infinity]`.
|
||||
*/
|
||||
function normalizeTieredPricing(raw: RawPricingTier[] | undefined): PricingTier[] | undefined {
|
||||
if (!raw || raw.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const result: PricingTier[] = [];
|
||||
for (const tier of raw) {
|
||||
const range = tier.range;
|
||||
if (!Array.isArray(range) || range.length < 1) {
|
||||
continue;
|
||||
}
|
||||
const start = typeof range[0] === "number" ? range[0] : Number.NaN;
|
||||
if (!Number.isFinite(start)) {
|
||||
continue;
|
||||
}
|
||||
const rawEnd = range.length >= 2 ? range[1] : null;
|
||||
const end =
|
||||
typeof rawEnd === "number" && Number.isFinite(rawEnd) && rawEnd > start ? rawEnd : Infinity;
|
||||
if (
|
||||
!Number.isFinite(tier.input) ||
|
||||
!Number.isFinite(tier.output) ||
|
||||
!Number.isFinite(tier.cacheRead) ||
|
||||
!Number.isFinite(tier.cacheWrite)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
result.push({
|
||||
input: tier.input,
|
||||
output: tier.output,
|
||||
cacheRead: tier.cacheRead,
|
||||
cacheWrite: tier.cacheWrite,
|
||||
range: [start, end],
|
||||
});
|
||||
}
|
||||
return result.length > 0 ? result.toSorted((a, b) => a.range[0] - b.range[0]) : undefined;
|
||||
}
|
||||
|
||||
function normalizeModelCostConfig(cost: RawModelCostConfig): ModelCostConfig {
|
||||
const normalizedTiers = normalizeTieredPricing(cost.tieredPricing);
|
||||
return {
|
||||
input: cost.input,
|
||||
output: cost.output,
|
||||
cacheRead: cost.cacheRead,
|
||||
cacheWrite: cost.cacheWrite,
|
||||
...(normalizedTiers ? { tieredPricing: normalizedTiers } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function isRawModelCostConfig(value: unknown): value is RawModelCostConfig {
|
||||
return value !== null && typeof value === "object";
|
||||
}
|
||||
@@ -626,7 +543,7 @@ export function resolveModelCostConfigFingerprint(
|
||||
modelsJsonNormalized: serializeCostIndex(
|
||||
loadModelsJsonCostIndex({ agentDir: resolvedAgentDir }),
|
||||
),
|
||||
gatewayPricing: getGatewayModelPricingCacheFingerprint(),
|
||||
catalogPricing: modelCatalogPricingFingerprint(config),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -646,6 +563,16 @@ export function resolveModelCostConfig(params: {
|
||||
return undefined;
|
||||
}
|
||||
const agentDir = resolveCostAgentDir(params.config, params.agentDir);
|
||||
if (params.allowPluginNormalization !== false) {
|
||||
const catalogPricing = resolveCatalogModelPricing({
|
||||
config: params.config,
|
||||
provider: params.provider ?? "",
|
||||
model: params.model ?? "",
|
||||
});
|
||||
if (catalogPricing) {
|
||||
return normalizeResolvedPricing(catalogPricing);
|
||||
}
|
||||
}
|
||||
|
||||
// Favor direct configured keys first so local pricing/status lookups stay
|
||||
// synchronous and do not drag plugin/provider discovery into the hot path.
|
||||
@@ -684,7 +611,12 @@ export function resolveModelCostConfig(params: {
|
||||
}
|
||||
}
|
||||
|
||||
return getCachedGatewayModelPricing(params);
|
||||
const hostedPricing = resolveHostedModelPricing({
|
||||
config: params.config,
|
||||
provider: params.provider ?? "",
|
||||
model: params.model ?? "",
|
||||
});
|
||||
return hostedPricing ? normalizeResolvedPricing(hostedPricing) : undefined;
|
||||
}
|
||||
|
||||
const toNumber = (value: number | undefined): number =>
|
||||
|
||||
@@ -23,7 +23,12 @@ afterEach(() => {
|
||||
}
|
||||
});
|
||||
|
||||
function fixtureProvider(prefix: string, count: number) {
|
||||
function fixtureProvider(
|
||||
prefix: string,
|
||||
count: number,
|
||||
): {
|
||||
models: Array<{ id: string; cost?: { input: number; output: number } }>;
|
||||
} {
|
||||
return { models: Array.from({ length: count }, (_, index) => ({ id: `${prefix}-${index}` })) };
|
||||
}
|
||||
|
||||
@@ -56,6 +61,7 @@ describe("publish model catalog", () => {
|
||||
providers: 2,
|
||||
models: 200,
|
||||
costModels: 0,
|
||||
pricingEntries: 0,
|
||||
});
|
||||
expect(MODEL_CATALOG_MIN_MODELS).toBe(200);
|
||||
});
|
||||
@@ -123,11 +129,12 @@ describe("publish model catalog", () => {
|
||||
expect(fs.existsSync(out)).toBe(false);
|
||||
});
|
||||
|
||||
it("enriches only existing models with OpenRouter flat and LiteLLM tier pricing", async () => {
|
||||
it("enriches catalog models and emits unmatched hosted pricing keys", async () => {
|
||||
const anthropic = fixtureProvider("claude", 100);
|
||||
anthropic.models[0] = { id: "claude-3-5-sonnet" };
|
||||
const openai = fixtureProvider("gpt", 100);
|
||||
openai.models[0] = { id: "gpt-special" };
|
||||
openai.models[1] = { id: "zero-upstream", cost: { input: 5, output: 6 } };
|
||||
const manifests = [
|
||||
{
|
||||
pluginId: "anthropic",
|
||||
@@ -144,6 +151,31 @@ describe("publish model catalog", () => {
|
||||
manifestPath: "openai.json",
|
||||
manifest: { modelCatalog: { providers: { openai } } },
|
||||
},
|
||||
{
|
||||
pluginId: "openrouter",
|
||||
manifestPath: "openrouter.json",
|
||||
manifest: {
|
||||
modelPricing: {
|
||||
providers: {
|
||||
openrouter: {
|
||||
openRouter: { passthroughProviderModel: true },
|
||||
liteLLM: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
pluginId: "mapped",
|
||||
manifestPath: "mapped.json",
|
||||
manifest: {
|
||||
modelPricing: {
|
||||
providers: {
|
||||
mapped: { openRouter: { provider: "approved-source" }, liteLLM: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
const bundle = await assembleModelCatalogBundle({
|
||||
manifests,
|
||||
@@ -165,6 +197,8 @@ describe("publish model catalog", () => {
|
||||
},
|
||||
{ id: "openai/gpt-2", pricing: { prompt: "-1", completion: "0.000004" } },
|
||||
{ id: "unknown/new-model", pricing: { prompt: "1", completion: "1" } },
|
||||
{ id: "custom/secondary-wins", pricing: { prompt: "0", completion: "0" } },
|
||||
{ id: "mapped/wrong-source", pricing: { prompt: "0.000013", completion: "0.000014" } },
|
||||
],
|
||||
});
|
||||
}
|
||||
@@ -184,18 +218,63 @@ describe("publish model catalog", () => {
|
||||
output_cost_per_token: 0.000004,
|
||||
},
|
||||
"unknown/new-model": { input_cost_per_token: 1, output_cost_per_token: 1 },
|
||||
"external-model": {
|
||||
litellm_provider: "custom",
|
||||
input_cost_per_token: 0.000007,
|
||||
output_cost_per_token: 0.000008,
|
||||
},
|
||||
"forbidden-model": {
|
||||
litellm_provider: "openrouter",
|
||||
input_cost_per_token: 0.000009,
|
||||
output_cost_per_token: 0.00001,
|
||||
},
|
||||
"secondary-wins": {
|
||||
litellm_provider: "custom",
|
||||
input_cost_per_token: 0.000011,
|
||||
output_cost_per_token: 0.000012,
|
||||
},
|
||||
"zero-upstream": {
|
||||
litellm_provider: "openai",
|
||||
input_cost_per_token: 0,
|
||||
output_cost_per_token: 0,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
await expect(enrichModelCatalogPricing({ bundle, manifests, fetchImpl })).resolves.toBe(2);
|
||||
await expect(enrichModelCatalogPricing({ bundle, manifests, fetchImpl })).resolves.toEqual({
|
||||
modelsEnriched: 2,
|
||||
pricingEntries: 11,
|
||||
});
|
||||
expect(bundle.providers.anthropic?.models[0]?.cost).toMatchObject({ input: 1, output: 2 });
|
||||
expect(bundle.providers.openai?.models[0]?.cost).toMatchObject({
|
||||
input: 3,
|
||||
output: 4,
|
||||
tieredPricing: [{ input: 5, output: 6, range: [1000] }],
|
||||
});
|
||||
expect(bundle.providers.openai?.models[1]?.cost).toEqual({ input: 5, output: 6 });
|
||||
expect(bundle.providers.openai?.models[2]?.cost).toBeUndefined();
|
||||
expect(summarizeModelCatalogBundle(bundle)).toMatchObject({ models: 200, costModels: 2 });
|
||||
expect(bundle.pricing).toEqual({
|
||||
"anthropic/claude-3.5-sonnet": { input: 1, output: 2 },
|
||||
"custom/external-model": { input: 7, output: 8 },
|
||||
"custom/secondary-wins": { input: 11, output: 12 },
|
||||
"external-model": { input: 7, output: 8 },
|
||||
"forbidden-model": { input: 9, output: 10 },
|
||||
"openrouter/anthropic/claude-3.5-sonnet": { input: 1, output: 2 },
|
||||
"openrouter/mapped/wrong-source": { input: 13, output: 14 },
|
||||
"openrouter/openai/gpt-special": { input: 3, output: 4 },
|
||||
"openrouter/unknown/new-model": { input: 1_000_000, output: 1_000_000 },
|
||||
"secondary-wins": { input: 11, output: 12 },
|
||||
"unknown/new-model": { input: 1_000_000, output: 1_000_000 },
|
||||
});
|
||||
expect(bundle.pricing).not.toHaveProperty("openrouter/forbidden-model");
|
||||
expect(bundle.pricing).not.toHaveProperty("mapped/wrong-source");
|
||||
expect(bundle.pricing).not.toHaveProperty("gpt-special");
|
||||
expect(bundle.pricing).not.toHaveProperty("openai/gpt-special");
|
||||
expect(summarizeModelCatalogBundle(bundle)).toMatchObject({
|
||||
models: 200,
|
||||
costModels: 3,
|
||||
pricingEntries: 11,
|
||||
});
|
||||
expect(Object.hasOwn(bundle.providers, "unknown")).toBe(false);
|
||||
});
|
||||
|
||||
@@ -236,7 +315,7 @@ describe("publish model catalog", () => {
|
||||
return new Response("not-json", { status: 200 });
|
||||
},
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
).resolves.toEqual({ modelsEnriched: 0, pricingEntries: 0 });
|
||||
} finally {
|
||||
stderr.mockRestore();
|
||||
}
|
||||
@@ -255,10 +334,12 @@ describe("publish model catalog", () => {
|
||||
const left = {
|
||||
...base,
|
||||
providers: { zeta: { models: [{ id: "b" }, { id: "a" }] }, alpha: { models: [{ id: "c" }] } },
|
||||
pricing: { "z/model": { input: 2, output: 3 }, "a/model": { input: 1, output: 2 } },
|
||||
};
|
||||
const right = {
|
||||
...base,
|
||||
providers: { alpha: { models: [{ id: "c" }] }, zeta: { models: [{ id: "a" }, { id: "b" }] } },
|
||||
pricing: { "a/model": { output: 2, input: 1 }, "z/model": { output: 3, input: 2 } },
|
||||
};
|
||||
expect(serializeModelCatalogBundle(left)).toBe(serializeModelCatalogBundle(right));
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user