mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 12:26:38 -06:00
fix(exa): bound untrusted search JSON response reads (#96038)
Exa search success responses were read via an unbounded `await response.json()`, so a misbehaving or hostile endpoint could stream an arbitrarily large body into memory before parsing. Read the success body through the shared bounded reader (16 MiB cap, the same limit other bundled providers use) and cancel the stream on overflow. This mirrors the error-body bound already in place and the #95103/#95108 response -limit campaign on the success-JSON side. AI-assisted.
This commit is contained in:
@@ -20,6 +20,7 @@ import {
|
||||
wrapWebContent,
|
||||
writeCachedSearchPayload,
|
||||
} from "openclaw/plugin-sdk/provider-web-search";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import {
|
||||
normalizeOptionalLowercaseString,
|
||||
normalizeOptionalString,
|
||||
@@ -30,6 +31,10 @@ const EXA_SEARCH_TYPES = ["auto", "neural", "fast", "deep", "deep-reasoning", "i
|
||||
const EXA_FRESHNESS_VALUES = ["day", "week", "month", "year"] as const;
|
||||
const EXA_MAX_SEARCH_COUNT = 100;
|
||||
const EXA_ERROR_BODY_LIMIT_BYTES = 8 * 1024;
|
||||
// Exa search responses are untrusted external bodies. Cap the success JSON the
|
||||
// same way other bundled providers do (16 MiB) so a misbehaving or hostile
|
||||
// endpoint cannot stream an unbounded body into memory before we parse it.
|
||||
const EXA_SEARCH_JSON_MAX_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
type ExaConfig = {
|
||||
apiKey?: string;
|
||||
@@ -70,9 +75,17 @@ type ExaSearchResponse = {
|
||||
results?: unknown;
|
||||
};
|
||||
|
||||
async function readExaSearchResults(response: Response): Promise<ExaSearchResult[]> {
|
||||
async function readExaSearchResults(
|
||||
response: Response,
|
||||
opts?: { maxBytes?: number },
|
||||
): Promise<ExaSearchResult[]> {
|
||||
const maxBytes = opts?.maxBytes ?? EXA_SEARCH_JSON_MAX_BYTES;
|
||||
const bytes = await readResponseWithLimit(response, maxBytes, {
|
||||
onOverflow: ({ maxBytes: maxBytesLocal }) =>
|
||||
new Error(`Exa API response exceeds ${maxBytesLocal} bytes`),
|
||||
});
|
||||
try {
|
||||
return normalizeExaResults(await response.json());
|
||||
return normalizeExaResults(JSON.parse(new TextDecoder().decode(bytes)));
|
||||
} catch (cause) {
|
||||
throw new Error("Exa API returned malformed JSON", { cause });
|
||||
}
|
||||
|
||||
@@ -26,6 +26,33 @@ function cancelTrackedResponse(
|
||||
};
|
||||
}
|
||||
|
||||
function streamingJsonResponse(params: { chunkCount: number; chunkSize: number }): {
|
||||
response: Response;
|
||||
getReadCount: () => number;
|
||||
} {
|
||||
// Streaming fixture proves an oversized success body stops being read before
|
||||
// the whole payload is buffered into memory.
|
||||
let reads = 0;
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (reads >= params.chunkCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
reads += 1;
|
||||
controller.enqueue(encoder.encode("a".repeat(params.chunkSize)));
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new Response(stream, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
getReadCount: () => reads,
|
||||
};
|
||||
}
|
||||
|
||||
describe("exa web search provider", () => {
|
||||
it("exposes the expected metadata and selection wiring", () => {
|
||||
const provider = createExaWebSearchProvider();
|
||||
@@ -265,6 +292,27 @@ describe("exa web search provider", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("parses well-formed Exa search JSON under the byte cap", async () => {
|
||||
const response = new Response(
|
||||
JSON.stringify({ results: [{ url: "https://example.com", title: "Example" }] }),
|
||||
{ status: 200, headers: { "content-type": "application/json" } },
|
||||
);
|
||||
|
||||
await expect(testing.readExaSearchResults(response)).resolves.toEqual([
|
||||
{ url: "https://example.com", title: "Example" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("caps oversized Exa search JSON instead of buffering the whole body", async () => {
|
||||
const streamed = streamingJsonResponse({ chunkCount: 64, chunkSize: 1024 });
|
||||
|
||||
await expect(
|
||||
testing.readExaSearchResults(streamed.response, { maxBytes: 4096 }),
|
||||
).rejects.toThrow(/Exa API response exceeds 4096 bytes/);
|
||||
|
||||
expect(streamed.getReadCount()).toBeLessThan(64);
|
||||
});
|
||||
|
||||
it("bounds Exa API error bodies without using response.text()", async () => {
|
||||
const tracked = cancelTrackedResponse(`${"exa upstream unavailable ".repeat(1024)}tail`, {
|
||||
status: 503,
|
||||
|
||||
Reference in New Issue
Block a user