mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 13:26:04 -06:00
fix(parallel): bound successful web-search JSON response reads (#96035)
* fix(parallel): bound successful web-search JSON response reads
The Parallel web_search provider parsed its /v1/search success body with an
unbounded await res.json(). The body comes from an external web-search
upstream, so a hostile or malfunctioning endpoint streaming an unbounded JSON
payload could force the runtime to buffer the whole response before parsing,
creating memory pressure or a hang on the provider path.
Read the success body through the shared readProviderJsonResponse helper with a
16 MiB cap (matching the provider JSON cap from #95218); on overflow the stream
is cancelled and a bounded error is thrown. The error-body path was already
bounded (readResponseTextLimited, 8 KiB). Symmetric follow-up to the
#95103/#95108 response-limit campaign.
* docs(parallel): drop upstream PR ref from response-cap comment
Replace the PR-specific '#95218' annotation with a neutral description of
the shared provider JSON cap so the comment stays accurate independent of
upstream PR numbering.
(cherry picked from commit 6163b1977b)
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
import { createRequire } from "node:module";
|
||||
import { readPluginPackageVersion } from "openclaw/plugin-sdk/extension-shared";
|
||||
import { readResponseTextLimited } from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
readProviderJsonResponse,
|
||||
readResponseTextLimited,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
DEFAULT_SEARCH_COUNT,
|
||||
mergeScopedSearchConfig,
|
||||
@@ -36,6 +39,12 @@ import {
|
||||
const PARALLEL_BASE_URL = "https://api.parallel.ai";
|
||||
const PARALLEL_SEARCH_PATHNAME = "/v1/search";
|
||||
const PARALLEL_ERROR_BODY_LIMIT_BYTES = 8 * 1024;
|
||||
// Parallel's /v1/search returns a bounded result set, but the body is external
|
||||
// (web-search upstream) and untrusted. Cap the successful JSON read so a
|
||||
// hostile or malfunctioning endpoint streaming an unbounded body cannot force
|
||||
// the runtime to buffer the whole payload before parsing. 16 MiB matches the
|
||||
// shared provider JSON cap (readProviderJsonResponse default).
|
||||
const PARALLEL_SEARCH_RESPONSE_LIMIT_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const PLUGIN_VERSION = readPluginPackageVersion({ require });
|
||||
@@ -151,11 +160,9 @@ async function runParallelSearch(params: {
|
||||
);
|
||||
throw new Error(`Parallel API error (${res.status}): ${detail || res.statusText}`);
|
||||
}
|
||||
try {
|
||||
return (await res.json()) as ParallelSearchResponse;
|
||||
} catch (cause) {
|
||||
throw new Error("Parallel API returned malformed JSON", { cause });
|
||||
}
|
||||
return await readProviderJsonResponse<ParallelSearchResponse>(res, "Parallel API", {
|
||||
maxBytes: PARALLEL_SEARCH_RESPONSE_LIMIT_BYTES,
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -282,6 +289,7 @@ export const testing = {
|
||||
resolveParallelSearchCount,
|
||||
resolveParallelSearchEndpoint,
|
||||
PARALLEL_ERROR_BODY_LIMIT_BYTES,
|
||||
PARALLEL_SEARCH_RESPONSE_LIMIT_BYTES,
|
||||
USER_AGENT,
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -59,6 +59,40 @@ function cancelTrackedResponse(
|
||||
};
|
||||
}
|
||||
|
||||
function streamedJsonResponse(params: { chunkCount: number; chunkSize: number }): {
|
||||
response: Response;
|
||||
getReadCount: () => number;
|
||||
wasCanceled: () => boolean;
|
||||
} {
|
||||
// Multi-chunk fixture: proves the bounded read stops pulling chunks before
|
||||
// the whole (here syntactically broken / unbounded) body is buffered, and
|
||||
// that the stream is cancelled on overflow.
|
||||
let reads = 0;
|
||||
let canceled = false;
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (reads >= params.chunkCount) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
reads += 1;
|
||||
controller.enqueue(encoder.encode("a".repeat(params.chunkSize)));
|
||||
},
|
||||
cancel() {
|
||||
canceled = true;
|
||||
},
|
||||
});
|
||||
return {
|
||||
response: new Response(stream, {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
getReadCount: () => reads,
|
||||
wasCanceled: () => canceled,
|
||||
};
|
||||
}
|
||||
|
||||
import { testing } from "../test-api.js";
|
||||
import { createParallelWebSearchProvider as createContractParallelWebSearchProvider } from "../web-search-contract-api.js";
|
||||
import { createParallelWebSearchProvider } from "./parallel-web-search-provider.js";
|
||||
@@ -583,6 +617,65 @@ describe("parallel web search provider", () => {
|
||||
expect(textSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds successful Parallel JSON bodies instead of buffering the whole response", async () => {
|
||||
// 200-chunk x 1 MiB body (~200 MiB) caps at 16 MiB: the bounded reader must
|
||||
// stop pulling chunks and cancel the stream well before draining it, then
|
||||
// surface a bounded error rather than buffering the whole payload.
|
||||
const streamed = streamedJsonResponse({ chunkCount: 200, chunkSize: 1024 * 1024 });
|
||||
endpointMockState.responses.push(streamed.response);
|
||||
const provider = createParallelWebSearchProvider();
|
||||
const tool = provider.createTool({
|
||||
config: {},
|
||||
searchConfig: { parallel: { apiKey: "par-secret" } },
|
||||
});
|
||||
if (!tool) {
|
||||
throw new Error("Expected tool definition");
|
||||
}
|
||||
|
||||
const error = await tool
|
||||
.execute({
|
||||
objective: `parallel-success-body-${Date.now()}-${Math.random()}`,
|
||||
search_queries: ["openclaw"],
|
||||
})
|
||||
.catch((cause: unknown) => cause);
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).toMatch(
|
||||
new RegExp(
|
||||
`Parallel API: JSON response exceeds ${testing.PARALLEL_SEARCH_RESPONSE_LIMIT_BYTES} bytes`,
|
||||
),
|
||||
);
|
||||
// Stopped well before draining all 200 chunks, and cancelled the stream.
|
||||
expect(streamed.getReadCount()).toBeLessThan(200);
|
||||
expect(streamed.wasCanceled()).toBe(true);
|
||||
});
|
||||
|
||||
it("parses a well-formed Parallel JSON body under the byte cap", async () => {
|
||||
endpointMockState.responses.push(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
search_id: "ok",
|
||||
session_id: "ok-session",
|
||||
results: [{ url: "https://example.com/a", title: "A", excerpts: ["alpha"] }],
|
||||
}),
|
||||
{ status: 200, headers: { "Content-Type": "application/json" } },
|
||||
),
|
||||
);
|
||||
const provider = createParallelWebSearchProvider();
|
||||
const tool = provider.createTool({
|
||||
config: {},
|
||||
searchConfig: { parallel: { apiKey: "par-secret" } },
|
||||
});
|
||||
if (!tool) {
|
||||
throw new Error("Expected tool definition");
|
||||
}
|
||||
const result = (await tool.execute({
|
||||
objective: `parallel-success-ok-${Date.now()}-${Math.random()}`,
|
||||
search_queries: ["openclaw"],
|
||||
})) as { provider?: string; searchId?: string; count?: number };
|
||||
expect(result).toMatchObject({ provider: "parallel", searchId: "ok", count: 1 });
|
||||
});
|
||||
|
||||
it("does not surface a Parallel-generated sessionId on a cache hit", async () => {
|
||||
// Unique objective so this test does not collide with the SDK's
|
||||
// module-level web-search cache across other cases.
|
||||
|
||||
Reference in New Issue
Block a user