From 2655bf4dac8f64f9cf9a864ac1b669cbfdf004d0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Jul 2026 23:29:25 +0100 Subject: [PATCH] fix(clawhub): bound archive download bodies (#101176) * fix(clawhub): bound archive download bodies Co-authored-by: NIO * docs(changelog): note ClawHub archive bounds * chore(changelog): defer ClawHub release note * fix(clawhub): avoid response-limit shadowing --------- Co-authored-by: NIO --- src/infra/clawhub.test.ts | 121 ++++++++++++++++++++++++++++++++++++++ src/infra/clawhub.ts | 38 ++++++++++-- 2 files changed, 153 insertions(+), 6 deletions(-) diff --git a/src/infra/clawhub.test.ts b/src/infra/clawhub.test.ts index 2fcaa0811134..5b8b8884ab51 100644 --- a/src/infra/clawhub.test.ts +++ b/src/infra/clawhub.test.ts @@ -8,6 +8,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { + downloadClawHubGitHubSkillArchive, downloadClawHubPackageArchive, downloadClawHubSkillArchive, downloadClawHubSkillArchiveUrl, @@ -69,6 +70,92 @@ function createStalledBodyResponse(params: { }; } +function createOversizedArchiveResponse( + params: { + headers?: HeadersInit; + } = {}, +): { + response: Response; + cancel: ReturnType; +} { + const cancel = vi.fn(); + const body = new ReadableStream({ + cancel() { + cancel(); + }, + }); + const headers = new Headers(params.headers); + headers.set("content-type", headers.get("content-type") ?? "application/zip"); + headers.set("content-length", String(256 * 1024 * 1024 + 512 * 1024)); + return { + response: new Response(body, { + status: 200, + headers, + }), + cancel, + }; +} + +const oversizedArchiveCases: Array<{ + name: string; + headers?: HeadersInit; + download: (response: Response) => Promise; + expectedResource: string; +}> = [ + { + name: "package archive", + download: (response) => + downloadClawHubPackageArchive({ + name: "@hyf/zai-external-alpha", + version: "0.0.1", + fetchImpl: async () => response, + }), + expectedResource: "package archive download for @hyf/zai-external-alpha", + }, + { + name: "ClawPack artifact", + headers: { "content-type": "application/octet-stream" }, + download: (response) => + downloadClawHubPackageArchive({ + name: "demo", + version: "1.2.3", + artifact: "clawpack", + fetchImpl: async () => response, + }), + expectedResource: "ClawPack download for demo@1.2.3", + }, + { + name: "skill archive", + download: (response) => + downloadClawHubSkillArchive({ + slug: "agentreceipt", + version: "1.0.0", + fetchImpl: async () => response, + }), + expectedResource: "skill archive download for agentreceipt", + }, + { + name: "resolver URL archive", + download: (response) => + downloadClawHubSkillArchiveUrl({ + baseUrl: "https://clawhub.ai", + url: "https://downloads.example.com/skill.zip", + fetchImpl: async () => response, + }), + expectedResource: "skill archive download at /skill.zip", + }, + { + name: "GitHub source archive", + download: (response) => + downloadClawHubGitHubSkillArchive({ + repo: "owner/repo", + commit: "abc123", + fetchImpl: async () => response, + }), + expectedResource: "GitHub source archive for owner/repo@abc123", + }, +]; + describe("clawhub helpers", () => { const originalEnv = captureEnv(["HOME", "XDG_CONFIG_HOME"]); @@ -846,6 +933,40 @@ describe("clawhub helpers", () => { ).rejects.toThrow(/declared sha256/); }); + it.each(oversizedArchiveCases)( + "rejects and cancels oversized $name downloads", + async ({ headers, download, expectedResource }) => { + const oversized = createOversizedArchiveResponse({ headers }); + + await expect(download(oversized.response)).rejects.toThrow( + `ClawHub ${expectedResource} exceeded 268435456 bytes (268959744 bytes declared)`, + ); + expect(oversized.cancel).toHaveBeenCalledTimes(1); + }, + ); + + it("uses decoded stream bytes instead of encoded content length", async () => { + const bytes = new Uint8Array([1, 2, 3]); + const archive = await downloadClawHubPackageArchive({ + name: "encoded-package", + version: "1.0.0", + fetchImpl: async () => + new Response(bytes, { + status: 200, + headers: { + "content-encoding": "gzip", + "content-length": String(256 * 1024 * 1024 + 1), + "content-type": "application/zip", + }, + }), + }); + try { + await expect(fs.readFile(archive.archivePath)).resolves.toEqual(Buffer.from(bytes)); + } finally { + await archive.cleanup(); + } + }); + it("annotates 429 errors with the reset hint and a sign-in hint when unauthenticated", async () => { process.env.OPENCLAW_CLAWHUB_CONFIG_PATH = path.join(os.tmpdir(), "openclaw-no-clawhub-config"); await expect( diff --git a/src/infra/clawhub.ts b/src/infra/clawhub.ts index 37576a51eba0..d9de71fa78c2 100644 --- a/src/infra/clawhub.ts +++ b/src/infra/clawhub.ts @@ -11,7 +11,10 @@ import { import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; import { sha256Base64, sha256Hex as digestSha256Hex } from "./crypto-digest.js"; import { readResponseTextSnippet, readResponseWithLimit } from "./http-body.js"; -import { parseStrictPositiveInteger } from "./parse-finite-number.js"; +import { + parseStrictNonNegativeInteger, + parseStrictPositiveInteger, +} from "./parse-finite-number.js"; import { isAtLeast, parseSemver } from "./runtime-guard.js"; import { compareComparableSemver, parseComparableSemver } from "./semver-compare.js"; import { createTempDownloadTarget } from "./temp-download.js"; @@ -21,6 +24,8 @@ const DEFAULT_CLAWHUB_URL = "https://clawhub.ai"; const DEFAULT_GITHUB_CODELOAD_URL = "https://codeload.github.com"; const DEFAULT_FETCH_TIMEOUT_MS = 30_000; const SKILL_CARD_MAX_BYTES = 256 * 1024; +// Align with marketplace archive downloads (src/plugins/marketplace.ts). +const CLAWHUB_ARCHIVE_MAX_BYTES = 256 * 1024 * 1024; // ClawHub is an external marketplace: bound untrusted JSON and error bodies so // a hostile or malfunctioning host cannot exhaust memory with an endless stream. const CLAWHUB_JSON_MAX_BYTES = 16 * 1024 * 1024; @@ -798,17 +803,38 @@ async function readClawHubResponseBytes(params: { resourceLabel: string; }): Promise { const timeoutMs = resolveClawHubRequestTimeoutMs(params.timeoutMs); - return await readResponseWithLimit(params.response, params.maxBytes ?? Number.MAX_SAFE_INTEGER, { + const maxBytes = params.maxBytes ?? CLAWHUB_ARCHIVE_MAX_BYTES; + const contentEncoding = normalizeOptionalString(params.response.headers.get("content-encoding")); + const declaredSize = + !contentEncoding || contentEncoding.toLowerCase() === "identity" + ? parseStrictNonNegativeInteger(params.response.headers.get("content-length")) + : undefined; + if (declaredSize !== undefined && declaredSize > maxBytes) { + // Fetch may decode encoded bodies while retaining their wire length, so + // only identity lengths can safely short-circuit the decoded stream cap. + await params.response.body?.cancel().catch(() => undefined); + throw createClawHubBodyLimitError(params.resourceLabel, declaredSize, maxBytes, "declared"); + } + return await readResponseWithLimit(params.response, maxBytes, { chunkTimeoutMs: timeoutMs, - onOverflow: ({ size, maxBytes }) => - new Error( - `ClawHub ${params.resourceLabel} exceeded ${maxBytes} bytes (${size} bytes received)`, - ), + onOverflow: ({ size, maxBytes: limitBytes }) => + createClawHubBodyLimitError(params.resourceLabel, size, limitBytes), onIdleTimeout: ({ chunkTimeoutMs }) => new Error(`ClawHub ${params.resourceLabel} body stalled after ${chunkTimeoutMs}ms`), }); } +function createClawHubBodyLimitError( + resourceLabel: string, + size: number, + maxBytes: number, + measurement: "declared" | "received" = "received", +): Error { + return new Error( + `ClawHub ${resourceLabel} exceeded ${maxBytes} bytes (${size} bytes ${measurement})`, + ); +} + function isJsonObject(value: unknown): value is Record { return Boolean(value && typeof value === "object" && !Array.isArray(value)); }