Files
openclaw/src/plugins/web-fetch-providers.runtime.ts
T
Vincent Koc 2f6459093a improve(web-fetch): speed up provider fallback loading (#98559)
* perf(web-fetch): add web fetch benchmark

* perf(web-fetch): cache provider fallback resolution

* perf(firecrawl): avoid duplicate scrape payload wrapping

* perf(web-readability): reuse void tag lookup

* fix(web-fetch): keep provider cache live

* fix(web-fetch): invalidate provider cache on registry swap

* perf(web-fetch): reduce html extraction overhead

* fix(web-fetch): bound provider discovery cache

* perf(web-fetch): avoid full plugin load on fallback
2026-07-04 12:47:10 -07:00

92 lines
3.3 KiB
TypeScript

/** Runtime resolver for plugin-contributed web fetch providers. */
import { loadOpenClawPlugins } from "./loader.js";
import type { PluginLoadOptions } from "./loader.js";
import type { PluginManifestRecord } from "./manifest-registry.js";
import type { PluginWebFetchProviderEntry } from "./types.js";
import {
resolveBundledWebFetchResolutionConfig,
sortWebFetchProviders,
} from "./web-fetch-providers.shared.js";
import {
resolveBundledRuntimeWebFetchProvidersFromPublicArtifacts,
resolveBundledWebFetchProvidersFromPublicArtifacts,
} from "./web-provider-public-artifacts.js";
import {
mapRegistryProviders,
resolveManifestDeclaredWebProviderCandidatePluginIds,
} from "./web-provider-resolution-shared.js";
import {
resolvePluginWebProviders,
resolveRuntimeWebProviders,
} from "./web-provider-runtime-shared.js";
function resolveWebFetchCandidatePluginIds(params: {
config?: PluginLoadOptions["config"];
workspaceDir?: string;
env?: PluginLoadOptions["env"];
onlyPluginIds?: readonly string[];
origin?: PluginManifestRecord["origin"];
sandboxed?: boolean;
}): string[] | undefined {
return resolveManifestDeclaredWebProviderCandidatePluginIds({
contract: "webFetchProviders",
configKey: "webFetch",
config: params.config,
workspaceDir: params.workspaceDir,
env: params.env,
onlyPluginIds: params.onlyPluginIds,
origin: params.origin,
sandboxed: params.sandboxed,
});
}
function mapRegistryWebFetchProviders(params: {
registry: ReturnType<typeof loadOpenClawPlugins>;
onlyPluginIds?: readonly string[];
}): PluginWebFetchProviderEntry[] {
return mapRegistryProviders({
entries: params.registry.webFetchProviders,
onlyPluginIds: params.onlyPluginIds,
sortProviders: sortWebFetchProviders,
});
}
/** Resolves web fetch providers, activating plugin runtimes when requested. */
export function resolvePluginWebFetchProviders(params: {
config?: PluginLoadOptions["config"];
workspaceDir?: string;
env?: PluginLoadOptions["env"];
onlyPluginIds?: readonly string[];
activate?: boolean;
cache?: boolean;
mode?: "runtime" | "setup";
origin?: PluginManifestRecord["origin"];
sandboxed?: boolean;
}): PluginWebFetchProviderEntry[] {
return resolvePluginWebProviders(params, {
resolveBundledResolutionConfig: resolveBundledWebFetchResolutionConfig,
resolveCandidatePluginIds: resolveWebFetchCandidatePluginIds,
mapRegistryProviders: mapRegistryWebFetchProviders,
resolveBundledPublicArtifactProviders: resolveBundledWebFetchProvidersFromPublicArtifacts,
resolveBundledRuntimeArtifactProviders:
resolveBundledRuntimeWebFetchProvidersFromPublicArtifacts,
});
}
/** Resolves already-eligible runtime web fetch providers without setup-mode activation. */
export function resolveRuntimeWebFetchProviders(params: {
config?: PluginLoadOptions["config"];
workspaceDir?: string;
env?: PluginLoadOptions["env"];
onlyPluginIds?: readonly string[];
origin?: PluginManifestRecord["origin"];
}): PluginWebFetchProviderEntry[] {
return resolveRuntimeWebProviders(params, {
resolveBundledResolutionConfig: resolveBundledWebFetchResolutionConfig,
resolveCandidatePluginIds: resolveWebFetchCandidatePluginIds,
mapRegistryProviders: mapRegistryWebFetchProviders,
resolveBundledRuntimeArtifactProviders:
resolveBundledRuntimeWebFetchProvidersFromPublicArtifacts,
});
}