mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
test(web): trim duplicate provider test surfaces (#121727)
This commit is contained in:
committed by
GitHub
parent
c39abcecda
commit
18e5b0e3e3
@@ -214,10 +214,8 @@ export async function runDuckDuckGoSearch(params: {
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
decodeDuckDuckGoUrl,
|
||||
decodeHtmlEntities,
|
||||
isBotChallenge,
|
||||
parseDuckDuckGoHtml,
|
||||
readDuckDuckGoHtmlResponse,
|
||||
};
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -254,18 +254,6 @@ describe("duckduckgo web search provider", () => {
|
||||
).toBe("off");
|
||||
});
|
||||
|
||||
it("decodes direct and redirect urls plus common html entities", () => {
|
||||
expect(
|
||||
ddgClientTesting.decodeDuckDuckGoUrl(
|
||||
"https://duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fsearch%3Fq%3Dclaw",
|
||||
),
|
||||
).toBe("https://example.com/search?q=claw");
|
||||
expect(ddgClientTesting.decodeDuckDuckGoUrl("https://example.com")).toBe("https://example.com");
|
||||
expect(ddgClientTesting.decodeHtmlEntities("Fish & Chips … 'ok'")).toBe(
|
||||
"Fish & Chips ... 'ok'",
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves out-of-range numeric html entities intact instead of throwing", () => {
|
||||
expect(() => ddgClientTesting.decodeHtmlEntities("Result � end")).not.toThrow();
|
||||
expect(ddgClientTesting.decodeHtmlEntities("Result � end")).toBe(
|
||||
@@ -347,14 +335,6 @@ describe("duckduckgo web search provider", () => {
|
||||
`;
|
||||
|
||||
expect(ddgClientTesting.isBotChallenge(challengeHtml)).toBe(true);
|
||||
expect(ddgClientTesting.parseDuckDuckGoHtml(challengeHtml)).toStrictEqual([]);
|
||||
expect(ddgClientTesting.isBotChallenge(normalHtml)).toBe(false);
|
||||
expect(ddgClientTesting.parseDuckDuckGoHtml(normalHtml)).toEqual([
|
||||
{
|
||||
title: "Coding Challenge",
|
||||
url: "https://example.com/challenge",
|
||||
snippet: "A fun coding challenge for interview prep.",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -621,12 +621,9 @@ export async function executeExaWebSearchProviderTool(
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
normalizeExaResults,
|
||||
normalizeExaFreshness,
|
||||
parseExaContents,
|
||||
buildExaCacheKey,
|
||||
resolveExaApiKey,
|
||||
resolveExaConfig,
|
||||
resolveExaDescription,
|
||||
resolveExaSearchCount,
|
||||
resolveExaSearchEndpoint,
|
||||
@@ -634,4 +631,3 @@ export const testing = {
|
||||
readExaErrorDetail,
|
||||
readExaSearchResults,
|
||||
} as const;
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
// Exa API module exposes the plugin public contract.
|
||||
export { testing, testing as __testing } from "./src/exa-web-search-provider.runtime.js";
|
||||
export { testing } from "./src/exa-web-search-provider.runtime.js";
|
||||
|
||||
@@ -729,7 +729,5 @@ export const testing = {
|
||||
postFirecrawlJson,
|
||||
readFirecrawlJsonResponse,
|
||||
resolveEndpoint,
|
||||
validateFirecrawlBaseUrl,
|
||||
resolveSearchItems,
|
||||
};
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -124,85 +124,6 @@ describe("firecrawl tools", () => {
|
||||
expect(preservedFetchProvider.tools?.web?.fetch?.provider).toBe("other");
|
||||
});
|
||||
|
||||
it("parses scrape payloads into wrapped external-content results", () => {
|
||||
const result = firecrawlClientTesting.parseFirecrawlScrapePayload({
|
||||
payload: {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "# Hello\n\nWorld",
|
||||
metadata: {
|
||||
title: "Example page",
|
||||
sourceURL: "https://example.com/final",
|
||||
statusCode: 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
url: "https://example.com/start",
|
||||
extractMode: "text",
|
||||
maxChars: 1000,
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://example.com/final");
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.extractor).toBe("firecrawl");
|
||||
expect(String(result.text)).toContain("Hello");
|
||||
expect(String(result.text)).toContain("World");
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("extracts search items from flexible Firecrawl payload shapes", () => {
|
||||
const items = firecrawlClientTesting.resolveSearchItems({
|
||||
success: true,
|
||||
data: [
|
||||
{
|
||||
title: "Docs",
|
||||
url: "https://docs.example.com/path",
|
||||
description: "Reference docs",
|
||||
markdown: "Body",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(items).toEqual([
|
||||
{
|
||||
title: "Docs",
|
||||
url: "https://docs.example.com/path",
|
||||
description: "Reference docs",
|
||||
content: "Body",
|
||||
published: undefined,
|
||||
siteName: "docs.example.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("extracts search items from Firecrawl v2 data.web payloads", () => {
|
||||
const items = firecrawlClientTesting.resolveSearchItems({
|
||||
success: true,
|
||||
data: {
|
||||
web: [
|
||||
{
|
||||
title: "API Platform - OpenAI",
|
||||
url: "https://openai.com/api/",
|
||||
description: "Build on the OpenAI API platform.",
|
||||
markdown: "# API Platform",
|
||||
position: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(items).toEqual([
|
||||
{
|
||||
title: "API Platform - OpenAI",
|
||||
url: "https://openai.com/api/",
|
||||
description: "Build on the OpenAI API platform.",
|
||||
content: "# API Platform",
|
||||
published: undefined,
|
||||
siteName: "openai.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds canonical provider URLs after percent-encoding hostile Unicode", () => {
|
||||
const expandedUrl = `https://example.com/${"🦀".repeat(1_000)}`;
|
||||
expect(expandedUrl.length).toBeLessThan(2_048);
|
||||
@@ -1623,78 +1544,5 @@ describe("firecrawl tools", () => {
|
||||
expect(resolveFirecrawlScrapeTimeoutSeconds(cfg, 0)).toBe(42);
|
||||
expect(resolveFirecrawlSearchTimeoutSeconds(0.5)).toBe(1);
|
||||
});
|
||||
|
||||
it("normalizes mixed search payload shapes into search items", () => {
|
||||
expect(
|
||||
firecrawlClientTesting.resolveSearchItems({
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
sourceURL: "https://www.example.com/post",
|
||||
snippet: "Snippet text",
|
||||
markdown: "# Title\nBody",
|
||||
metadata: {
|
||||
title: "Example title",
|
||||
publishedDate: "2026-03-22",
|
||||
},
|
||||
},
|
||||
{
|
||||
url: "",
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
title: "Example title",
|
||||
url: "https://www.example.com/post",
|
||||
description: "Snippet text",
|
||||
content: "# Title\nBody",
|
||||
published: "2026-03-22",
|
||||
siteName: "example.com",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("parses scrape payloads, extracts text, and marks truncation", () => {
|
||||
const result = firecrawlClientTesting.parseFirecrawlScrapePayload({
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Hello\n\nThis is a long body for scraping.",
|
||||
metadata: {
|
||||
title: "Example page",
|
||||
sourceURL: "https://docs.example.com/page",
|
||||
statusCode: 200,
|
||||
},
|
||||
},
|
||||
warning: "cached result",
|
||||
},
|
||||
url: "https://docs.example.com/page",
|
||||
extractMode: "text",
|
||||
maxChars: 12,
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://docs.example.com/page");
|
||||
expect(result.status).toBe(200);
|
||||
expect(result.extractMode).toBe("text");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength).toBeGreaterThan(12);
|
||||
expect(String(result.text)).toContain("Hello");
|
||||
expect(String(result.title)).toContain("Example page");
|
||||
expect(String(result.warning)).toContain("cached result");
|
||||
});
|
||||
|
||||
it("throws when scrape payload has no usable content", () => {
|
||||
expect(() =>
|
||||
firecrawlClientTesting.parseFirecrawlScrapePayload({
|
||||
payload: {
|
||||
data: {},
|
||||
},
|
||||
url: "https://docs.example.com/page",
|
||||
extractMode: "markdown",
|
||||
maxChars: 100,
|
||||
}),
|
||||
).toThrow("Firecrawl scrape returned no content.");
|
||||
});
|
||||
});
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
// Google API module exposes the plugin public contract.
|
||||
export { buildGoogleGeminiCliBackend } from "./cli-backend.js";
|
||||
export { buildGoogleImageGenerationProvider } from "./image-generation-provider.js";
|
||||
export { buildGoogleMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
export { buildGoogleSpeechProvider } from "./speech-provider.js";
|
||||
export { googleMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
export { buildGoogleVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
@@ -281,4 +281,3 @@ export const testing = {
|
||||
resolveMiniMaxRegion,
|
||||
readMiniMaxSearchJsonResponse: readProviderJsonResponse<MiniMaxSearchResponse>,
|
||||
} as const;
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -515,7 +515,5 @@ export const testing = {
|
||||
resolveKimiModel,
|
||||
resolveKimiBaseUrl,
|
||||
extractKimiCitations,
|
||||
hasKimiSearchResults,
|
||||
extractKimiToolResultContent,
|
||||
} as const;
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
// Moonshot API module exposes the plugin public contract.
|
||||
export { testing, testing as __testing } from "./src/kimi-web-search-provider.runtime.js";
|
||||
export { testing } from "./src/kimi-web-search-provider.runtime.js";
|
||||
export { moonshotMediaUnderstandingProvider } from "./media-understanding-provider.js";
|
||||
|
||||
@@ -279,7 +279,6 @@ export async function executeParallelWebSearchProviderTool(
|
||||
|
||||
export const testing = {
|
||||
buildParallelCacheKey,
|
||||
invalidSearchQueriesPayload,
|
||||
missingParallelKeyPayload,
|
||||
normalizeParallelClientModel,
|
||||
normalizeParallelObjective,
|
||||
@@ -287,12 +286,8 @@ export const testing = {
|
||||
normalizeParallelSearchQueries,
|
||||
normalizeParallelSessionId,
|
||||
resolveParallelApiKey,
|
||||
resolveParallelConfig,
|
||||
resolveParallelSearchCount,
|
||||
resolveParallelSearchEndpoint,
|
||||
PARALLEL_ERROR_BODY_LIMIT_BYTES,
|
||||
PARALLEL_SEARCH_RESPONSE_LIMIT_BYTES,
|
||||
USER_AGENT,
|
||||
} as const;
|
||||
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -1 +1 @@
|
||||
export { testing, testing as __testing } from "./src/parallel-web-search-provider.runtime.js";
|
||||
export { testing } from "./src/parallel-web-search-provider.runtime.js";
|
||||
|
||||
@@ -558,11 +558,7 @@ export const testing = {
|
||||
resolvePerplexityBaseUrl,
|
||||
resolvePerplexityModel,
|
||||
resolvePerplexityTransport,
|
||||
isDirectPerplexityBaseUrl,
|
||||
resolvePerplexityRequestModel,
|
||||
resolvePerplexityApiKey,
|
||||
readPerplexityJsonResponse,
|
||||
normalizeToIsoDate,
|
||||
isoToPerplexityDate,
|
||||
} as const;
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
// Perplexity API module exposes the plugin public contract.
|
||||
export { testing, testing as __testing } from "./src/perplexity-web-search-provider.runtime.js";
|
||||
@@ -332,4 +332,3 @@ export const testing = {
|
||||
validateSearxngBaseUrl,
|
||||
SEARXNG_SEARCH_CACHE,
|
||||
};
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -413,4 +413,3 @@ export const testing = {
|
||||
readTavilyJsonResponse,
|
||||
resolveEndpoint,
|
||||
};
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -424,12 +424,8 @@ export const testing = {
|
||||
buildXaiWebSearchPayload,
|
||||
extractXaiWebSearchContent,
|
||||
resolveXaiToolSearchConfig,
|
||||
resolveXaiWebSearchAuth,
|
||||
resolveXaiInlineCitations,
|
||||
resolveXaiWebSearchCredential,
|
||||
resolveXaiWebSearchEndpoint,
|
||||
resolveXaiWebSearchModel,
|
||||
resolveXaiWebSearchTimeoutSeconds,
|
||||
requestXaiWebSearch,
|
||||
};
|
||||
export { testing as __testing };
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
// Xai API module exposes the plugin public contract.
|
||||
export { testing, testing as __testing } from "./src/web-search-provider.runtime.js";
|
||||
@@ -7,8 +7,8 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { buildXaiCatalogModels, resolveXaiCatalogEntry } from "./model-definitions.js";
|
||||
import { isModernXaiModel, resolveXaiForwardCompatModel } from "./provider-models.js";
|
||||
import { resolveFallbackXaiAuth } from "./src/tool-auth-shared.js";
|
||||
import { testing } from "./src/web-search-provider.runtime.js";
|
||||
import { requestXaiWebSearch } from "./src/web-search-shared.js";
|
||||
import { testing } from "./test-api.js";
|
||||
import { createXaiWebSearchProvider as createXaiWebSearchContractProvider } from "./web-search-contract-api.js";
|
||||
import { createXaiWebSearchProvider } from "./web-search.js";
|
||||
|
||||
|
||||
@@ -194,7 +194,6 @@ export const testing = {
|
||||
DEFAULT_GATEWAY_SCHEMA_ERROR,
|
||||
DEFAULT_RAW_SCHEMA_ERROR,
|
||||
SUCCESS_MARKER,
|
||||
extractSuccessReplyTexts,
|
||||
resolveGatewayPort,
|
||||
validateSuccessResult,
|
||||
validateRejectResult,
|
||||
|
||||
@@ -174,8 +174,6 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
FETCH_HTML_MAX_BYTES,
|
||||
fetchHtml,
|
||||
readBoundedResponseText,
|
||||
truncate,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user