diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index 3f9440d1a994..adf0e55e7ef1 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -340,7 +340,7 @@ session establishment, not on every turn; use `/new`, `/reset`, or a gateway restart after changing native plugin config. - `plugins.entries.firecrawl.config.webFetch`: Firecrawl web-fetch provider settings. - - `apiKey`: Firecrawl API key (accepts SecretRef). Falls back to `plugins.entries.firecrawl.config.webSearch.apiKey`, legacy `tools.web.fetch.firecrawl.apiKey`, or `FIRECRAWL_API_KEY` env var. + - `apiKey`: Optional Firecrawl API key for higher limits (accepts SecretRef). Falls back to `plugins.entries.firecrawl.config.webSearch.apiKey`, legacy `tools.web.fetch.firecrawl.apiKey`, or `FIRECRAWL_API_KEY` env var. - `baseUrl`: Firecrawl API base URL (default: `https://api.firecrawl.dev`; self-hosted overrides must target private/internal endpoints). - `onlyMainContent`: extract only the main content from pages (default: `true`). - `maxAgeMs`: maximum cache age in milliseconds (default: `172800000` / 2 days). diff --git a/docs/help/faq.md b/docs/help/faq.md index 2a8ad0e0664d..f4817a823c04 100644 --- a/docs/help/faq.md +++ b/docs/help/faq.md @@ -857,7 +857,7 @@ lives on the [First-run FAQ](/help/faq-first-run). - If you use allowlists, add `web_search`/`web_fetch`/`x_search` or `group:web`. - `web_fetch` is enabled by default (unless explicitly disabled). - - If `tools.web.fetch.provider` is omitted, OpenClaw auto-detects the first ready fetch fallback provider from available credentials. Today the bundled provider is Firecrawl. + - If `tools.web.fetch.provider` is omitted, OpenClaw auto-detects the first ready fetch fallback provider from configured credentials. Today the bundled provider is Firecrawl. - Daemons read env vars from `~/.openclaw/.env` (or the service environment). Docs: [Web tools](/tools/web). diff --git a/docs/reference/api-usage-costs.md b/docs/reference/api-usage-costs.md index 48654d22fec4..0ad2538e5e06 100644 --- a/docs/reference/api-usage-costs.md +++ b/docs/reference/api-usage-costs.md @@ -154,7 +154,8 @@ See [Web tools](/tools/web). ### 5) Web fetch tool (Firecrawl) -`web_fetch` can call **Firecrawl** when an API key is present: +`web_fetch` can call **Firecrawl** with keyless starter access. Add an API key +for higher limits: - `FIRECRAWL_API_KEY` or `plugins.entries.firecrawl.config.webFetch.apiKey` diff --git a/docs/tools/firecrawl.md b/docs/tools/firecrawl.md index bfba75d45994..44896881c315 100644 --- a/docs/tools/firecrawl.md +++ b/docs/tools/firecrawl.md @@ -2,7 +2,8 @@ summary: "Firecrawl search, scrape, and web_fetch fallback" read_when: - You want Firecrawl-backed web extraction - - You need a Firecrawl API key + - You want keyless Firecrawl web_fetch + - You need a Firecrawl API key for search or higher limits - You want Firecrawl as a web_search provider - You want anti-bot extraction for web_fetch title: "Firecrawl" @@ -17,10 +18,12 @@ OpenClaw can use **Firecrawl** in three ways: It is a hosted extraction/search service that supports bot circumvention and caching, which helps with JS-heavy sites or pages that block plain HTTP fetches. -## Get an API key +## Keyless web_fetch and API keys -1. Create a Firecrawl account and generate an API key. -2. Store it in config or set `FIRECRAWL_API_KEY` in the gateway environment. +The explicitly selected hosted Firecrawl `web_fetch` fallback supports starter +access without an API key. Add `FIRECRAWL_API_KEY` in the gateway environment +or configure it when you need higher limits. Firecrawl `web_search` and +`firecrawl_scrape` require an API key. ## Configure Firecrawl search @@ -57,17 +60,23 @@ Notes: - `baseUrl` defaults to hosted Firecrawl at `https://api.firecrawl.dev`. Self-hosted overrides are allowed only for private/internal endpoints; HTTP is accepted only for those private targets. - `FIRECRAWL_BASE_URL` is the shared env fallback for Firecrawl search and scrape base URLs. -## Configure Firecrawl scrape + web_fetch fallback +## Configure Firecrawl web_fetch fallback ```json5 { + tools: { + web: { + fetch: { + provider: "firecrawl", // explicit selection enables keyless fallback + }, + }, + }, plugins: { entries: { firecrawl: { enabled: true, config: { webFetch: { - apiKey: "FIRECRAWL_API_KEY_HERE", baseUrl: "https://api.firecrawl.dev", onlyMainContent: true, maxAgeMs: 172800000, @@ -82,13 +91,15 @@ Notes: Notes: -- Firecrawl fallback attempts run only when an API key is available (`plugins.entries.firecrawl.config.webFetch.apiKey` or `FIRECRAWL_API_KEY`). +- The explicitly selected Firecrawl `web_fetch` fallback works without an API key. When configured, OpenClaw sends `plugins.entries.firecrawl.config.webFetch.apiKey` or `FIRECRAWL_API_KEY` for higher limits. +- Choosing Firecrawl during onboarding or `openclaw configure --section web` enables the plugin and selects Firecrawl for `web_fetch` unless another fetch provider is already configured. +- `firecrawl_scrape` requires an API key. - `maxAgeMs` controls how old cached results can be (ms). Default is 2 days. - Legacy `tools.web.fetch.firecrawl.*` config is auto-migrated by `openclaw doctor --fix`. - Firecrawl scrape/base URL overrides follow the same hosted/private rule as search: public hosted traffic uses `https://api.firecrawl.dev`; self-hosted overrides must resolve to private/internal endpoints. - `firecrawl_scrape` rejects obvious private, loopback, metadata, and non-HTTP(S) target URLs before forwarding them to Firecrawl, matching the `web_fetch` target-safety contract for explicit Firecrawl scrape calls. -`firecrawl_scrape` reuses the same `plugins.entries.firecrawl.config.webFetch.*` settings and env vars. +`firecrawl_scrape` reuses the same `plugins.entries.firecrawl.config.webFetch.*` settings and env vars, including its required API key. ### Self-hosted Firecrawl @@ -141,7 +152,7 @@ than basic-only scraping. `web_fetch` extraction order: 1. Readability (local) -2. Firecrawl (if selected or auto-detected as the active web-fetch fallback) +2. Firecrawl (when selected, or auto-detected from configured credentials) 3. Basic HTML cleanup (last fallback) The selection knob is `tools.web.fetch.provider`. If you omit it, OpenClaw diff --git a/docs/tools/web-fetch.md b/docs/tools/web-fetch.md index 39fcc032e2ab..13a7d12f206e 100644 --- a/docs/tools/web-fetch.md +++ b/docs/tools/web-fetch.md @@ -48,7 +48,7 @@ Truncate output to this many characters. Runs Readability (main-content extraction) on the HTML response. - If Readability fails and Firecrawl is configured, retries through the + If Readability fails and Firecrawl is selected, retries through the Firecrawl API with bot-circumvention mode. @@ -120,7 +120,7 @@ If Readability extraction fails, `web_fetch` can fall back to enabled: true, config: { webFetch: { - apiKey: "fc-...", // optional if FIRECRAWL_API_KEY is set + // apiKey: "fc-...", // optional; omit for keyless starter access baseUrl: "https://api.firecrawl.dev", onlyMainContent: true, maxAgeMs: 86400000, // cache duration (1 day) @@ -133,11 +133,11 @@ If Readability extraction fails, `web_fetch` can fall back to } ``` -`plugins.entries.firecrawl.config.webFetch.apiKey` supports SecretRef objects. +`plugins.entries.firecrawl.config.webFetch.apiKey` is optional and supports SecretRef objects. Legacy `tools.web.fetch.firecrawl.*` config is auto-migrated by `openclaw doctor --fix`. - If Firecrawl is enabled and its SecretRef is unresolved with no + If you configure a Firecrawl API-key SecretRef and it is unresolved with no `FIRECRAWL_API_KEY` env fallback, gateway startup fails fast. @@ -151,7 +151,7 @@ Current runtime behavior: - `tools.web.fetch.provider` selects the fetch fallback provider explicitly. - If `provider` is omitted, OpenClaw auto-detects the first ready web-fetch - provider from available credentials. Non-sandboxed `web_fetch` can use + provider from configured credentials. Non-sandboxed `web_fetch` can use installed plugins that declare `contracts.webFetchProviders` and register a matching provider at runtime. Today the bundled provider is Firecrawl. - Sandboxed `web_fetch` calls stay limited to bundled providers. diff --git a/docs/tools/web.md b/docs/tools/web.md index a3b99873cac5..c28bb38da709 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -307,7 +307,7 @@ plugin or run `openclaw doctor --fix` to clean up the stale config. - choose it with `tools.web.fetch.provider` - or omit that field and let OpenClaw auto-detect the first ready web-fetch - provider from available credentials + provider from configured credentials - non-sandboxed `web_fetch` can use installed plugin providers that declare `contracts.webFetchProviders`; sandboxed fetches stay bundled-only - today the bundled web-fetch provider is Firecrawl, configured under diff --git a/extensions/browser/package.json b/extensions/browser/package.json index 5dbf4baebf76..3d98091418f5 100644 --- a/extensions/browser/package.json +++ b/extensions/browser/package.json @@ -14,7 +14,7 @@ }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*", - "undici": "8.3.0" + "undici": "8.5.0" }, "openclaw": { "extensions": [ diff --git a/extensions/discord/package.json b/extensions/discord/package.json index 43f996bfb33e..bd570cabab6a 100644 --- a/extensions/discord/package.json +++ b/extensions/discord/package.json @@ -12,7 +12,7 @@ "discord-api-types": "0.38.48", "libopus-wasm": "0.2.0", "typebox": "1.1.39", - "undici": "8.3.0", + "undici": "8.5.0", "ws": "8.21.0" }, "devDependencies": { diff --git a/extensions/firecrawl/openclaw.plugin.json b/extensions/firecrawl/openclaw.plugin.json index ee1276da4e9b..1d7ebb077209 100644 --- a/extensions/firecrawl/openclaw.plugin.json +++ b/extensions/firecrawl/openclaw.plugin.json @@ -24,7 +24,7 @@ }, "webFetch.apiKey": { "label": "Firecrawl Fetch API Key", - "help": "Firecrawl API key for web fetch fallback (fallback: FIRECRAWL_API_KEY env var).", + "help": "Optional for hosted keyless scraping; add a key for higher limits (fallback: FIRECRAWL_API_KEY env var).", "sensitive": true, "placeholder": "fc-..." }, diff --git a/extensions/firecrawl/src/firecrawl-client.ts b/extensions/firecrawl/src/firecrawl-client.ts index dbbc12a4e4d9..162bf6042d27 100644 --- a/extensions/firecrawl/src/firecrawl-client.ts +++ b/extensions/firecrawl/src/firecrawl-client.ts @@ -87,6 +87,7 @@ export type FirecrawlScrapeParams = { cfg?: OpenClawConfig; url: string; extractMode: "markdown" | "text"; + access?: "credential" | "keyless"; maxChars?: number; onlyMainContent?: boolean; maxAgeMs?: number; @@ -184,7 +185,7 @@ async function postFirecrawlJson( url: string; mode?: FirecrawlEndpointMode; timeoutSeconds: number; - apiKey: string; + apiKey?: string; body: Record; errorLabel: string; }, @@ -201,8 +202,10 @@ async function postFirecrawlJson( init: { method: "POST", headers: { - Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", + // Hosted Firecrawl accepts starter scrape requests without a token. + // Send one only when configured so higher-limit accounts still apply. + ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}), }, body: JSON.stringify(params.body), }, @@ -522,7 +525,9 @@ export async function runFirecrawlScrape( assertFirecrawlScrapeTargetAllowed(params.url); const apiKey = resolveFirecrawlApiKey(params.cfg); - if (!apiKey) { + // Hosted v2/scrape accepts starter requests without a bearer token. + // Only the selected web_fetch provider opts into that access mode. + if (!apiKey && params.access !== "keyless") { throw new Error( "firecrawl_scrape needs a Firecrawl API key. Set FIRECRAWL_API_KEY in the Gateway environment, or configure plugins.entries.firecrawl.config.webFetch.apiKey.", ); diff --git a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts index 7ac3cfb52182..fd9694d509c5 100644 --- a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts +++ b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts @@ -14,7 +14,9 @@ function ensureRecord(target: Record, key: string): Record { throw new Error("expected Firecrawl plugin entry"); } expect(pluginEntry.enabled).toBe(true); + expect(applied.tools?.web?.fetch?.provider).toBe("firecrawl"); + + const preservedFetchProvider = provider.applySelectionConfig({ + tools: { + web: { + fetch: { + provider: "other", + }, + }, + }, + } as OpenClawConfig); + expect(preservedFetchProvider.tools?.web?.fetch?.provider).toBe("other"); }); it("parses scrape payloads into wrapped external-content results", () => { @@ -241,6 +253,72 @@ describe("firecrawl tools", () => { expect(authHeader).toBe("Bearer firecrawl-test-key"); }); + it("omits Firecrawl authorization for keyless scrape requests", async () => { + let capturedInit: RequestInit | undefined; + global.fetch = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { + capturedInit = init; + return new Response( + JSON.stringify({ + success: true, + data: { + markdown: "# Keyless", + metadata: { + sourceURL: "https://example.com/keyless-firecrawl", + statusCode: 200, + }, + }, + }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ); + }) as typeof fetch; + + await runActualFirecrawlScrape({ + cfg: { + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + baseUrl: "https://api.firecrawl.dev", + }, + }, + }, + }, + }, + } as OpenClawConfig, + url: "https://example.com/keyless-firecrawl", + extractMode: "markdown", + access: "keyless", + }); + + expect(new Headers(capturedInit?.headers).has("Authorization")).toBe(false); + }); + + it("requires credentials for direct scrape requests", async () => { + await expect( + runActualFirecrawlScrape({ + cfg: { + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + baseUrl: "https://api.firecrawl.dev", + }, + }, + }, + }, + }, + } as OpenClawConfig, + url: "https://example.com/direct-scrape", + extractMode: "markdown", + }), + ).rejects.toThrow("firecrawl_scrape needs a Firecrawl API key"); + }); + it("blocks private and non-http scrape targets before Firecrawl requests", () => { expect( firecrawlClientTesting.assertFirecrawlScrapeTargetAllowed("https://example.com/page"), @@ -402,6 +480,7 @@ describe("firecrawl tools", () => { expect(provider.id).toBe("firecrawl"); expect(provider.credentialPath).toBe("plugins.entries.firecrawl.config.webFetch.apiKey"); + expect(provider.requiresCredential).toBe(false); const pluginEntry = applied.plugins?.entries?.firecrawl; if (!pluginEntry) { throw new Error("expected Firecrawl fetch plugin entry"); @@ -430,6 +509,7 @@ describe("firecrawl tools", () => { cfg: { test: true }, url: "https://docs.openclaw.ai", extractMode: "markdown", + access: "keyless", maxChars: 1500, proxy: "stealth", storeInCache: false, @@ -454,6 +534,7 @@ describe("firecrawl tools", () => { cfg: { test: true }, url: "https://docs.openclaw.ai", extractMode: "markdown", + access: "keyless", maxChars: 1500, }); await expect( diff --git a/extensions/firecrawl/web-search-shared.ts b/extensions/firecrawl/web-search-shared.ts index f4c23b738c25..1fa4ebd879e6 100644 --- a/extensions/firecrawl/web-search-shared.ts +++ b/extensions/firecrawl/web-search-shared.ts @@ -1,6 +1,7 @@ // Firecrawl plugin module implements web search shared behavior. import { createWebSearchProviderContractFields, + enablePluginInConfig, type WebSearchProviderPlugin, } from "openclaw/plugin-sdk/provider-web-search-contract"; @@ -22,6 +23,12 @@ export function getConfiguredFirecrawlFetchCredentialFallback(config?: { } export function buildFirecrawlWebSearchProviderBase(): Omit { + const contractFields = createWebSearchProviderContractFields({ + credentialPath: FIRECRAWL_CREDENTIAL_PATH, + searchCredential: { type: "scoped", scopeId: "firecrawl" }, + configuredCredential: { pluginId: "firecrawl" }, + }); + return { id: "firecrawl", label: "Firecrawl Search", @@ -34,12 +41,26 @@ export function buildFirecrawlWebSearchProviderBase(): Omit { + const enabled = enablePluginInConfig(config, "firecrawl"); + if (!enabled.enabled || enabled.config.tools?.web?.fetch?.provider) { + return enabled.config; + } + return { + ...enabled.config, + tools: { + ...enabled.config.tools, + web: { + ...enabled.config.tools?.web, + fetch: { + ...enabled.config.tools?.web?.fetch, + provider: "firecrawl", + }, + }, + }, + }; + }, getConfiguredCredentialFallback: getConfiguredFirecrawlFetchCredentialFallback, }; } diff --git a/extensions/qa-matrix/package.json b/extensions/qa-matrix/package.json index d808ae3f7efb..04b3c2f667fa 100644 --- a/extensions/qa-matrix/package.json +++ b/extensions/qa-matrix/package.json @@ -5,7 +5,7 @@ "description": "OpenClaw Matrix QA runner plugin", "type": "module", "dependencies": { - "undici": "8.3.0" + "undici": "8.5.0" }, "devDependencies": { "@openclaw/matrix": "workspace:*", diff --git a/extensions/telegram/package.json b/extensions/telegram/package.json index cfbd665af5ed..a003007e5971 100644 --- a/extensions/telegram/package.json +++ b/extensions/telegram/package.json @@ -9,7 +9,7 @@ "@grammyjs/transformer-throttler": "1.2.1", "grammy": "1.43.0", "typebox": "1.1.39", - "undici": "8.3.0" + "undici": "8.5.0" }, "devDependencies": { "@openclaw/plugin-sdk": "workspace:*" diff --git a/package.json b/package.json index a8ab41d12325..d90b90562fff 100644 --- a/package.json +++ b/package.json @@ -1949,7 +1949,7 @@ "tslog": "4.10.2", "typebox": "1.1.39", "typescript": "6.0.3", - "undici": "8.3.0", + "undici": "8.5.0", "web-push": "3.6.7", "web-tree-sitter": "0.26.9", "ws": "8.21.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 86dc61657fb1..c754a5edec9c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,7 +87,7 @@ importers: version: 0.3.0 '@openclaw/proxyline': specifier: 0.3.3 - version: 0.3.3(undici@8.3.0) + version: 0.3.3(undici@8.5.0) chalk: specifier: 5.6.2 version: 5.6.2 @@ -188,8 +188,8 @@ importers: specifier: 6.0.3 version: 6.0.3 undici: - specifier: 8.3.0 - version: 8.3.0 + specifier: 8.5.0 + version: 8.5.0 web-push: specifier: 3.6.7 version: 3.6.7 @@ -442,8 +442,8 @@ importers: specifier: workspace:* version: link:../../packages/plugin-sdk undici: - specifier: 8.3.0 - version: 8.3.0 + specifier: 8.5.0 + version: 8.5.0 extensions/byteplus: devDependencies: @@ -689,8 +689,8 @@ importers: specifier: 1.1.39 version: 1.1.39 undici: - specifier: 8.3.0 - version: 8.3.0 + specifier: 8.5.0 + version: 8.5.0 ws: specifier: 8.21.0 version: 8.21.0 @@ -1364,8 +1364,8 @@ importers: extensions/qa-matrix: dependencies: undici: - specifier: 8.3.0 - version: 8.3.0 + specifier: 8.5.0 + version: 8.5.0 devDependencies: '@openclaw/matrix': specifier: workspace:* @@ -1533,8 +1533,8 @@ importers: specifier: 1.1.39 version: 1.1.39 undici: - specifier: 8.3.0 - version: 8.3.0 + specifier: 8.5.0 + version: 8.5.0 devDependencies: '@openclaw/plugin-sdk': specifier: workspace:* @@ -7375,12 +7375,12 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} - undici@7.27.2: - resolution: {integrity: sha512-uZsKNuzQxDMUY6M3pIMvy5tvlGmtq8XJ2oLAkfRKGNu+1VQAIvLy2xIVG5ATZl5wDXl/tddByAWCizRbOme+TA==} + undici@7.28.0: + resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} engines: {node: '>=20.18.1'} - undici@8.3.0: - resolution: {integrity: sha512-TkUDgb6tl7KOGZ+7e8E3d2FYgUQgF6z5YypqjWmixVQSQERFcVrVg0ySADm2LVLRh5ljAaHTCR5Fmz3Q34rB7Q==} + undici@8.5.0: + resolution: {integrity: sha512-xamtWoB1EshgjpmlXd7GGm2VfdDtw1+rD8uhry8pSNW3If6S8E0m2T2+orSKeZXEn/aPJMviCpDBA65WJt8zhg==} engines: {node: '>=22.19.0'} unhomoglyph@1.0.6: @@ -9223,9 +9223,9 @@ snapshots: jszip: 3.10.1 tar: 7.5.16 - '@openclaw/proxyline@0.3.3(undici@8.3.0)': + '@openclaw/proxyline@0.3.3(undici@8.5.0)': dependencies: - undici: 8.3.0 + undici: 8.5.0 '@opentelemetry/api-logs@0.219.0': dependencies: @@ -11895,7 +11895,7 @@ snapshots: saxes: 6.0.0 symbol-tree: 3.2.4 tough-cookie: 4.1.3 - undici: 7.27.2 + undici: 7.28.0 w3c-xmlserializer: 5.0.0 webidl-conversions: 8.0.1 whatwg-mimetype: 5.0.0 @@ -12763,7 +12763,7 @@ snapshots: '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) '@mozilla/readability': 0.6.0 '@openclaw/fs-safe': 0.3.0 - '@openclaw/proxyline': 0.3.3(undici@8.3.0) + '@openclaw/proxyline': 0.3.3(undici@8.5.0) '@silvia-odwyer/photon-node': 0.3.4 chalk: 5.6.2 chokidar: 5.0.0 @@ -12801,7 +12801,7 @@ snapshots: tslog: 4.10.2 typebox: 1.1.39 typescript: 6.0.3 - undici: 8.3.0 + undici: 8.5.0 web-push: 3.6.7 web-tree-sitter: 0.26.9 ws: 8.21.0 @@ -13873,9 +13873,9 @@ snapshots: undici-types@7.24.6: {} - undici@7.27.2: {} + undici@7.28.0: {} - undici@8.3.0: {} + undici@8.5.0: {} unhomoglyph@1.0.6: {} diff --git a/src/commands/configure.wizard.test.ts b/src/commands/configure.wizard.test.ts index 8c6df841dcdb..0536b1a58b47 100644 --- a/src/commands/configure.wizard.test.ts +++ b/src/commands/configure.wizard.test.ts @@ -420,15 +420,25 @@ describe("runConfigureWizard", () => { it("persists provider-owned web search config changes returned by setupSearch", async () => { setupBaseWizardState(); - mocks.setupSearch.mockImplementation(async (cfg: OpenClawConfig) => - createEnabledWebSearchConfig("firecrawl", { + mocks.setupSearch.mockImplementation(async (cfg: OpenClawConfig) => { + const configured = createEnabledWebSearchConfig("firecrawl", { enabled: true, config: { webSearch: { apiKey: "fc-entered-key" } }, - })(cfg), - ); + })(cfg); + return { + ...configured, + tools: { + ...configured.tools, + web: { + ...configured.tools?.web, + fetch: { provider: "firecrawl" }, + }, + }, + }; + }); queueWizardPrompts({ select: [], - confirm: [true, false], + confirm: [true, true], }); await runWebConfigureWizard(); @@ -442,6 +452,12 @@ describe("runConfigureWizard", () => { const search = getWebSearch(written); expect(search.provider).toBe("firecrawl"); expect(search.enabled).toBe(true); + const tools = requireRecord(written.tools, "tools config"); + const web = requireRecord(tools.web, "web config"); + expect(requireRecord(web.fetch, "web fetch config")).toEqual({ + enabled: true, + provider: "firecrawl", + }); const firecrawl = getPluginEntry(written, "firecrawl"); expect(firecrawl.enabled).toBe(true); const firecrawlConfig = requireRecord(firecrawl.config, "firecrawl config"); @@ -449,6 +465,45 @@ describe("runConfigureWizard", () => { "fc-entered-key", ); expect(mocks.setupSearch).toHaveBeenCalledOnce(); + expect(mocks.setupSearch).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + { preserveDisabledSearchState: false }, + ); + }); + + it("keeps web_search disabled when provider setup has no credential", async () => { + setupBaseWizardState(); + mocks.setupSearch.mockImplementation(async (cfg: OpenClawConfig) => ({ + ...cfg, + tools: { + ...cfg.tools, + web: { + ...cfg.tools?.web, + fetch: { provider: "firecrawl" }, + search: { enabled: false, provider: "firecrawl" }, + }, + }, + })); + queueWizardPrompts({ + select: [], + confirm: [true, true], + }); + + await runWebConfigureWizard(); + + const written = requireWriteConfig(); + expect(getWebSearch(written)).toMatchObject({ + enabled: false, + provider: "firecrawl", + }); + const tools = requireRecord(written.tools, "tools config"); + const web = requireRecord(tools.web, "web config"); + expect(requireRecord(web.fetch, "web fetch config")).toEqual({ + enabled: true, + provider: "firecrawl", + }); }); it("notes unavailable web search providers under plugin policy", async () => { diff --git a/src/commands/configure.wizard.ts b/src/commands/configure.wizard.ts index 2fdfe224b56d..bc5642160f05 100644 --- a/src/commands/configure.wizard.ts +++ b/src/commands/configure.wizard.ts @@ -337,10 +337,14 @@ async function promptWebToolsConfig( }; } } else { - workingConfig = await setupSearch(workingConfig, runtime, prompter); + workingConfig = await setupSearch(workingConfig, runtime, prompter, { + preserveDisabledSearchState: false, + }); + const selectedSearch = workingConfig.tools?.web?.search; nextSearch = { - ...workingConfig.tools?.web?.search, - enabled: workingConfig.tools?.web?.search?.provider ? true : existingSearch?.enabled, + ...selectedSearch, + enabled: + selectedSearch?.enabled ?? (selectedSearch?.provider ? true : existingSearch?.enabled), openaiCodex: { ...existingSearch?.openaiCodex, ...(nextSearch.openaiCodex as Record | undefined), @@ -359,7 +363,7 @@ async function promptWebToolsConfig( ); const nextFetch = { - ...existingFetch, + ...workingConfig.tools?.web?.fetch, enabled: enableFetch, }; diff --git a/src/commands/onboard-search.test.ts b/src/commands/onboard-search.test.ts index 1f2186bea78a..3b1d48883004 100644 --- a/src/commands/onboard-search.test.ts +++ b/src/commands/onboard-search.test.ts @@ -96,7 +96,19 @@ function createSearchProviderEntry(id: string): PluginWebSearchProviderEntry { >; entries[metadata.pluginId] = { ...entries[metadata.pluginId], enabled: true }; next.plugins = { ...next.plugins, entries }; - return next; + if (id !== "firecrawl" || next.tools?.web?.fetch?.provider) { + return next; + } + return { + ...next, + tools: { + ...next.tools, + web: { + ...next.tools?.web, + fetch: { provider: "firecrawl" }, + }, + }, + }; }, }; if (id === "kimi") { @@ -391,7 +403,7 @@ describe("setupSearch", () => { }); const result = await setupSearch(cfg, runtime, prompter); expect(result.tools?.web?.search?.provider).toBe("brave"); - expect(result.tools?.web?.search?.enabled).toBeUndefined(); + expect(result.tools?.web?.search?.enabled).toBe(false); const missingNote = notes.find((n) => n.message.includes("No Brave Search API key stored")); expect(missingNote?.message).toContain("No Brave Search API key stored"); } finally { @@ -403,6 +415,30 @@ describe("setupSearch", () => { } }); + it("keeps keyless Firecrawl fetch configured when search setup has no key", async () => { + const original = process.env.FIRECRAWL_API_KEY; + delete process.env.FIRECRAWL_API_KEY; + try { + const { prompter } = createPrompter({ + selectValue: "firecrawl", + textValue: "", + }); + + const result = await setupSearch({}, runtime, prompter); + + expect(result.tools?.web?.search?.provider).toBe("firecrawl"); + expect(result.tools?.web?.search?.enabled).toBe(false); + expect(result.tools?.web?.fetch?.provider).toBe("firecrawl"); + expect(result.plugins?.entries?.firecrawl?.enabled).toBe(true); + } finally { + if (original === undefined) { + delete process.env.FIRECRAWL_API_KEY; + } else { + process.env.FIRECRAWL_API_KEY = original; + } + } + }); + it("keeps existing key when user leaves input blank", async () => { const result = await runBlankPerplexityKeyEntry( "existing-key", // pragma: allowlist secret @@ -481,7 +517,7 @@ describe("setupSearch", () => { }); expect(prompter.text).toHaveBeenCalled(); expect(result.tools?.web?.search?.provider).toBe("grok"); - expect(result.tools?.web?.search?.enabled).toBeUndefined(); + expect(result.tools?.web?.search?.enabled).toBe(false); } finally { if (original === undefined) { delete process.env.XAI_API_KEY; diff --git a/src/flows/search-setup.test.ts b/src/flows/search-setup.test.ts index 503e338d2bde..3f58da30bdcb 100644 --- a/src/flows/search-setup.test.ts +++ b/src/flows/search-setup.test.ts @@ -396,6 +396,46 @@ describe("runSearchSetupFlow", () => { expect(xaiConfig?.xSearch?.model).toBe("grok-4-1-fast"); }); + it("allows an explicit setup flow to reenable credential-ready web_search", async () => { + const select = vi.fn().mockResolvedValueOnce("grok").mockResolvedValueOnce("no"); + const prompter = createWizardPrompter({ + select: select as never, + }); + + const next = await runSearchSetupFlow( + { + plugins: { + allow: ["xai"], + entries: { + xai: { + config: { + webSearch: { + apiKey: "xai-test-key", + }, + }, + }, + }, + }, + tools: { + web: { + search: { + enabled: false, + provider: "grok", + }, + }, + }, + }, + createNonExitingRuntime(), + prompter, + { preserveDisabledSearchState: false }, + ); + + expect(next.tools?.web?.search).toMatchObject({ + enabled: true, + provider: "grok", + }); + }); + it("installs an external catalog search provider before enabling it", async () => { const select = vi.fn().mockResolvedValueOnce("brave"); const text = vi.fn().mockResolvedValue("brave-test-key"); diff --git a/src/flows/search-setup.ts b/src/flows/search-setup.ts index e44ff69e765a..75107e88b4a1 100644 --- a/src/flows/search-setup.ts +++ b/src/flows/search-setup.ts @@ -353,6 +353,7 @@ function preserveDisabledState(original: OpenClawConfig, result: OpenClawConfig) type SetupSearchOptions = { quickstartDefaults?: boolean; + preserveDisabledSearchState?: boolean; secretInputMode?: SecretInputMode; }; @@ -388,7 +389,9 @@ async function finalizeSearchProviderSetup(params: { } next = installed.cfg; } - next = preserveDisabledState(params.originalConfig, next); + if (params.opts?.preserveDisabledSearchState !== false) { + next = preserveDisabledState(params.originalConfig, next); + } if (!params.entry.runSetup) { return next; } @@ -399,7 +402,9 @@ async function finalizeSearchProviderSetup(params: { quickstartDefaults: params.opts?.quickstartDefaults, secretInputMode: params.opts?.secretInputMode, }); - return preserveDisabledState(params.originalConfig, next); + return params.opts?.preserveDisabledSearchState === false + ? next + : preserveDisabledState(params.originalConfig, next); } export async function runSearchSetupFlow( @@ -686,16 +691,20 @@ export async function runSearchSetupFlow( const search: SearchConfig = { ...config.tools?.web?.search, + enabled: false, provider: choice, }; - return { - ...config, - tools: { - ...config.tools, - web: { - ...config.tools?.web, - search, + return applySearchProviderSelectionConfig( + { + ...config, + tools: { + ...config.tools, + web: { + ...config.tools?.web, + search, + }, }, }, - }; + entry, + ); } diff --git a/src/secrets/runtime-web-tools.shared.ts b/src/secrets/runtime-web-tools.shared.ts index 05d551f1a6ab..37d1fe324060 100644 --- a/src/secrets/runtime-web-tools.shared.ts +++ b/src/secrets/runtime-web-tools.shared.ts @@ -404,7 +404,8 @@ export async function resolveRuntimeWebProviderSelection< let keylessFallbackProvider: TProvider | undefined; for (const provider of candidates) { - if (provider.requiresCredential === false) { + const isKeyless = provider.requiresCredential === false; + if (isKeyless) { if (!params.configuredProvider && !params.allowKeylessAutoSelect) { continue; } @@ -412,13 +413,6 @@ export async function resolveRuntimeWebProviderSelection< keylessFallbackProvider ||= provider; continue; } - selectedProvider = provider.id; - selectedResolution = { - source: "missing" as TSource, - secretRefConfigured: false, - fallbackUsedAfterRefFailure: false, - }; - break; } const path = params.inactivePathsForProvider(provider)[0] ?? ""; @@ -507,6 +501,18 @@ export async function resolveRuntimeWebProviderSelection< }); } + if ( + isKeyless && + selectedCandidateResolution.secretRefConfigured && + !selectedCandidateResolution.value + ) { + continue; + } + + if (isKeyless && !params.configuredProvider && !selectedCandidateResolution.value) { + continue; + } + if (params.configuredProvider) { selectedProvider = provider.id; selectedResolution = selectedCandidateResolution; @@ -525,6 +531,24 @@ export async function resolveRuntimeWebProviderSelection< break; } + if (isKeyless) { + selectedProvider = provider.id; + selectedResolution = selectedCandidateResolution; + if (selectedCandidateResolution.value) { + setResolvedCredentialPath({ + resolvedConfig: params.resolvedConfig, + path: selectedCandidatePath, + value: selectedCandidateResolution.value, + }); + params.setResolvedCredential({ + resolvedConfig: params.resolvedConfig, + provider, + value: selectedCandidateResolution.value, + }); + } + break; + } + if (selectedCandidateResolution.value) { selectedProvider = provider.id; selectedResolution = selectedCandidateResolution; diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts index 795b47ef25df..7e851fc485b1 100644 --- a/src/secrets/runtime-web-tools.test.ts +++ b/src/secrets/runtime-web-tools.test.ts @@ -256,6 +256,7 @@ function buildTestWebFetchProviders(): PluginWebFetchProviderEntry[] { id: "firecrawl", label: "firecrawl", hint: "firecrawl test provider", + requiresCredential: false, envVars: ["FIRECRAWL_API_KEY"], placeholder: "fc-...", signupUrl: "https://example.com/firecrawl", @@ -582,6 +583,41 @@ describe("runtime web tools resolution", () => { }); }); + it("selects the configured keyless Firecrawl fetch provider without an API key", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + }), + }); + + expect(metadata.fetch.providerSource).toBe("configured"); + expect(metadata.fetch.selectedProvider).toBe("firecrawl"); + expect(metadata.fetch.selectedProviderKeySource).toBe("missing"); + }); + + it("does not auto-select keyless Firecrawl fetch without a credential", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: true, + }, + }, + }, + }), + }); + + expect(metadata.fetch.providerSource).toBe("none"); + expect(metadata.fetch.selectedProvider).toBeUndefined(); + }); + it("does not auto-select a keyless provider when no credentials are configured", async () => { const { metadata } = await runRuntimeWebTools({ config: asConfig({ diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index b65c750ddf6c..c02ac2628b77 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -512,12 +512,11 @@ function readConfiguredFetchProviderCredentialFallback(params: { } function inactivePathsForFetchProvider(provider: PluginWebFetchProviderEntry): string[] { - if (provider.requiresCredential === false) { - return []; - } return provider.inactiveSecretPaths?.length ? provider.inactiveSecretPaths - : [provider.credentialPath]; + : provider.credentialPath + ? [provider.credentialPath] + : []; } /** diff --git a/src/web-fetch/runtime.test.ts b/src/web-fetch/runtime.test.ts index 2723db4b24f7..5f4e9259f8b0 100644 --- a/src/web-fetch/runtime.test.ts +++ b/src/web-fetch/runtime.test.ts @@ -179,6 +179,36 @@ describe("web fetch runtime", () => { }); }); + it("uses an explicitly configured keyless provider without an API key", () => { + const provider = createFirecrawlProvider({ + requiresCredential: false, + }); + resolvePluginWebFetchProvidersMock.mockReturnValue([provider]); + + const resolved = resolveWebFetchDefinition({ + config: { + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + } as OpenClawConfig, + }); + + expect(requireResolvedWebFetch(resolved).provider.id).toBe("firecrawl"); + }); + + it("does not auto-detect a keyless provider without a credential", () => { + const provider = createFirecrawlProvider({ + requiresCredential: false, + }); + resolvePluginWebFetchProvidersMock.mockReturnValue([provider]); + + expect(resolveWebFetchDefinition({ config: {} })).toBeNull(); + }); + it("auto-detects providers from configured fallback credentials", () => { const provider = createFirecrawlProvider({ getConfiguredCredentialFallback: (config) => { diff --git a/src/web-fetch/runtime.ts b/src/web-fetch/runtime.ts index 76fdfc8a0485..6dd0fc84e601 100644 --- a/src/web-fetch/runtime.ts +++ b/src/web-fetch/runtime.ts @@ -1,5 +1,12 @@ /** Runtime provider selection and tool construction for the `web_fetch` tool. */ import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; +import { + hasWebProviderEntryCredential, + providerRequiresCredential, + readWebProviderEnvValue, + resolveWebProviderConfig, + resolveWebProviderDefinition, +} from "../../packages/web-content-core/src/provider-runtime-shared.js"; import type { OpenClawConfig } from "../config/types.js"; import { logVerbose } from "../globals.js"; import type { @@ -13,13 +20,6 @@ import { import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js"; import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime-web-tools-state.js"; import type { RuntimeWebFetchMetadata } from "../secrets/runtime-web-tools.types.js"; -import { - hasWebProviderEntryCredential, - providerRequiresCredential, - readWebProviderEnvValue, - resolveWebProviderConfig, - resolveWebProviderDefinition, -} from "../../packages/web-content-core/src/provider-runtime-shared.js"; // Runtime provider selection for the web_fetch tool. It resolves config, // credentials, runtime metadata, and sandbox-safe bundled provider scopes. @@ -38,10 +38,7 @@ type ResolveWebFetchDefinitionParams = { }; /** Resolves whether web_fetch is enabled for the current config/sandbox. */ -function resolveWebFetchEnabled(params: { - fetch?: WebFetchConfig; - sandboxed?: boolean; -}): boolean { +function resolveWebFetchEnabled(params: { fetch?: WebFetchConfig; sandboxed?: boolean }): boolean { if (typeof params.fetch?.enabled === "boolean") { return params.fetch.enabled; } @@ -78,6 +75,28 @@ function hasEntryCredential( }); } +function hasAutoDetectCredential( + provider: Pick< + PluginWebFetchProviderEntry, + | "envVars" + | "getConfiguredCredentialFallback" + | "getConfiguredCredentialValue" + | "getCredentialValue" + | "requiresCredential" + >, + config: OpenClawConfig | undefined, + fetch: WebFetchConfig | undefined, +): boolean { + return hasEntryCredential( + { + ...provider, + requiresCredential: true, + }, + config, + fetch, + ); +} + /** Reports whether a web_fetch provider has usable credentials. */ export function isWebFetchProviderConfigured(params: { provider: Pick< @@ -128,6 +147,9 @@ function resolveWebFetchProviderId(params: { for (const provider of providers) { if (!providerRequiresCredential(provider)) { + if (!hasAutoDetectCredential(provider, params.config, params.fetch)) { + continue; + } logVerbose( `web_fetch: ${raw ? `invalid configured provider "${raw}", ` : ""}auto-detected keyless provider "${provider.id}"`, );