From ff0113cdb2d8151ed3152c70cc1cfb779b2b8b96 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 20 Aug 2026 11:33:03 -0700 Subject: [PATCH] fix(perplexity): reject empty grounded search answers (#126780) --- .../perplexity-web-search-provider.runtime.ts | 10 +- .../perplexity-web-search-provider.test.ts | 131 ++++++++++++++++-- 2 files changed, 130 insertions(+), 11 deletions(-) diff --git a/extensions/perplexity/src/perplexity-web-search-provider.runtime.ts b/extensions/perplexity/src/perplexity-web-search-provider.runtime.ts index 7fc28ce7a5fe..d846dbf5252f 100644 --- a/extensions/perplexity/src/perplexity-web-search-provider.runtime.ts +++ b/extensions/perplexity/src/perplexity-web-search-provider.runtime.ts @@ -298,8 +298,14 @@ async function runPerplexitySearch(params: { return await throwWebSearchApiError(res, "Perplexity"); } const data = await readProviderJsonResponse(res, "Perplexity"); + const content = data.choices?.[0]?.message?.content; + if (typeof content !== "string" || !content.trim()) { + throw new Error( + "Perplexity search returned no final answer. Retry the query or choose another search provider.", + ); + } return { - content: data.choices?.[0]?.message?.content ?? "No response", + content, citations: extractPerplexityCitations(data), }; }, @@ -461,7 +467,7 @@ export async function executePerplexitySearch( runtime.baseUrl, runtime.model, query, - resolveSearchCount(count, DEFAULT_SEARCH_COUNT), + structured ? resolveSearchCount(count, DEFAULT_SEARCH_COUNT) : undefined, country, language, freshness, diff --git a/extensions/perplexity/src/perplexity-web-search-provider.test.ts b/extensions/perplexity/src/perplexity-web-search-provider.test.ts index 2a87db69ed09..c85883334f18 100644 --- a/extensions/perplexity/src/perplexity-web-search-provider.test.ts +++ b/extensions/perplexity/src/perplexity-web-search-provider.test.ts @@ -21,6 +21,33 @@ const openRouterPerplexityApiKey = ["sk", "or", "v1", "test"].join("-"); const directPerplexityApiKey = ["pplx", "test"].join("-"); const enterprisePerplexityApiKey = ["enterprise", "perplexity", "test"].join("-"); +function mockPerplexityResponseOnce(body: unknown): void { + withTrustedWebSearchEndpointMock.mockImplementationOnce( + async (_params: { init: RequestInit }, run: (response: Response) => Promise) => + await run( + new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ), + ); +} + +function createConfiguredPerplexityTool(structured: boolean) { + const webSearch = { + apiKey: directPerplexityApiKey, + ...(structured ? {} : { baseUrl: "https://api.perplexity.ai" }), + }; + const tool = createPerplexityWebSearchProvider().createTool({ + config: { plugins: { entries: { perplexity: { config: { webSearch } } } } }, + searchConfig: {}, + }); + if (!tool) { + throw new Error("Expected tool definition"); + } + return tool; +} + describe("perplexity web search provider", () => { it("points missing-key users to fetch/browser alternatives", async () => { await withEnvAsync( @@ -67,6 +94,100 @@ describe("perplexity web search provider", () => { expect(withTrustedWebSearchEndpointMock).not.toHaveBeenCalled(); }); + it.each([ + { name: "missing choices", response: {} }, + { name: "empty choices", response: { choices: [] } }, + { name: "missing message", response: { choices: [{}] } }, + { name: "missing content", response: { choices: [{ message: {} }] } }, + { name: "null content", response: { choices: [{ message: { content: null } }] } }, + { name: "empty content", response: { choices: [{ message: { content: "" } }] } }, + { name: "whitespace content", response: { choices: [{ message: { content: " \n " } }] } }, + { + name: "citations without an answer", + response: { + choices: [{ message: { content: null } }], + citations: ["https://example.test/source"], + }, + }, + { + name: "tool calls without an answer", + response: { + choices: [{ finish_reason: "tool_calls", message: { content: null, tool_calls: [] } }], + }, + }, + { + name: "audio without an answer", + response: { choices: [{ message: { content: null, audio: { id: "audio-response" } } }] }, + }, + ])("rejects and does not cache chat-completions $name", async ({ name, response }) => { + withTrustedWebSearchEndpointMock.mockReset(); + mockPerplexityResponseOnce(response); + mockPerplexityResponseOnce({ + choices: [{ message: { content: " Recovered grounded answer " } }], + citations: ["https://example.test/recovered"], + }); + + const tool = createConfiguredPerplexityTool(false); + const args = { query: `perplexity empty answer ${name}` }; + await expect(tool.execute(args)).rejects.toThrow( + "Perplexity search returned no final answer. Retry the query or choose another search provider.", + ); + + const recovered = await tool.execute(args); + expect(recovered.content).toContain(" Recovered grounded answer "); + expect(recovered.citations).toEqual(["https://example.test/recovered"]); + expect(withTrustedWebSearchEndpointMock).toHaveBeenCalledTimes(2); + }); + + it.each([ + { name: "chat completions", structured: false, expectedRequests: 1 }, + { name: "native Search API", structured: true, expectedRequests: 2 }, + ])( + "uses count as a cache dimension only when $name sends it upstream", + async ({ name, structured, expectedRequests }) => { + withTrustedWebSearchEndpointMock.mockReset(); + const response = structured + ? { results: [] } + : { + choices: [ + { + message: { + content: "Grounded answer", + annotations: [ + { + type: "url_citation", + url_citation: { url: "https://example.test/citation" }, + }, + ], + }, + }, + ], + }; + mockPerplexityResponseOnce(response); + if (structured) { + mockPerplexityResponseOnce(response); + } + + const tool = createConfiguredPerplexityTool(structured); + const query = `perplexity cache count ${name}`; + const first = await tool.execute({ query, count: 1 }); + const second = await tool.execute({ query, count: 7 }); + const third = await tool.execute({ query, count: 1 }); + + expect(first.cached).toBeUndefined(); + expect(second.cached).toBe(structured ? undefined : true); + expect(third.cached).toBe(true); + expect(withTrustedWebSearchEndpointMock).toHaveBeenCalledTimes(expectedRequests); + if (structured) { + expect(first.results).toEqual([]); + expect(first.count).toBe(0); + } else { + expect(first.content).toContain("Grounded answer"); + expect(first.citations).toEqual(["https://example.test/citation"]); + } + }, + ); + it.each([ { name: "native Search API", webSearch: { apiKey: "pplx-test" } }, { @@ -216,15 +337,7 @@ describe("perplexity web search provider", () => { }); it("sends official date filter fields in the Search API request body", async () => { - withTrustedWebSearchEndpointMock.mockImplementationOnce( - async (_params: { init: RequestInit }, run: (response: Response) => Promise) => - await run( - new Response(JSON.stringify({ results: [] }), { - status: 200, - headers: { "Content-Type": "application/json" }, - }), - ), - ); + mockPerplexityResponseOnce({ results: [] }); await withEnvAsync( { [perplexityApiKeyEnv]: directPerplexityApiKey, [openRouterApiKeyEnv]: undefined },