mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-28 05:16:23 -06:00
feat(gateway): serve HTTP byte ranges on media routes (#115667)
* feat(gateway): support HTTP media ranges * fix(gateway): bound full media streams * fix(gateway): normalize byte range captures * fix(gateway): keep byte range helpers internal
This commit is contained in:
committed by
GitHub
parent
43127e12f9
commit
bdebdd0391
@@ -51,6 +51,36 @@ describe("Control UI assistant media e2e", () => {
|
||||
);
|
||||
expect(await ticketed.text()).toBe("ticketed control ui media\n");
|
||||
|
||||
const ranged = await fetch(
|
||||
`${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
{ headers: { Range: "bytes=9-15" } },
|
||||
);
|
||||
expect(ranged.status).toBe(206);
|
||||
expect(ranged.headers.get("accept-ranges")).toBe("bytes");
|
||||
expect(ranged.headers.get("content-range")).toBe("bytes 9-15/26");
|
||||
expect(ranged.headers.get("content-length")).toBe("7");
|
||||
expect(ranged.headers.get("etag")).toMatch(/^"[A-Za-z0-9_-]+"$/);
|
||||
expect(await ranged.text()).toBe("control");
|
||||
|
||||
const head = await fetch(
|
||||
`${route}?source=${sourceParam}&mediaTicket=${encodeURIComponent(payload.mediaTicket ?? "")}`,
|
||||
{ method: "HEAD" },
|
||||
);
|
||||
expect(head.status).toBe(200);
|
||||
expect(head.headers.get("accept-ranges")).toBe("bytes");
|
||||
expect(head.headers.get("content-length")).toBe("26");
|
||||
expect(head.headers.get("etag")).toBe(ranged.headers.get("etag"));
|
||||
expect(await head.text()).toBe("");
|
||||
|
||||
const emptyFilePath = path.join(mediaDir, "empty.bin");
|
||||
await fs.writeFile(emptyFilePath, Buffer.alloc(0));
|
||||
const empty = await fetch(`${route}?source=${encodeURIComponent(emptyFilePath)}`, {
|
||||
headers: { Authorization: `Bearer ${CONTROL_UI_E2E_TOKEN}` },
|
||||
});
|
||||
expect(empty.status).toBe(200);
|
||||
expect(empty.headers.get("content-length")).toBe("0");
|
||||
expect((await empty.arrayBuffer()).byteLength).toBe(0);
|
||||
|
||||
const otherFilePath = path.join(mediaDir, "other-preview.txt");
|
||||
await fs.writeFile(otherFilePath, "other media\n", "utf8");
|
||||
const wrongSource = await fetch(
|
||||
|
||||
@@ -75,6 +75,7 @@ import {
|
||||
sendControlUiHtmlBody,
|
||||
serveControlUiAsset,
|
||||
} from "./control-ui-static.js";
|
||||
import { resolveByteResponse, writeByteHeaders } from "./http-byte-range.js";
|
||||
import { buildMissingScopeForbiddenBody, sendGatewayAuthFailure } from "./http-common.js";
|
||||
import {
|
||||
getBearerToken,
|
||||
@@ -643,8 +644,23 @@ export async function handleControlUiAssistantMediaRequest(
|
||||
buildAssistantMediaContentDisposition(filename, contentType),
|
||||
);
|
||||
res.setHeader("Cache-Control", "no-cache");
|
||||
res.setHeader("Content-Length", String(opened.stat.size));
|
||||
const stream = opened.handle.createReadStream({ start: 0, autoClose: false });
|
||||
const byteResponse = resolveByteResponse({
|
||||
file: opened.stat,
|
||||
method: req.method,
|
||||
rangeHeader: req.headers.range,
|
||||
ifRangeHeader: req.headers["if-range"],
|
||||
});
|
||||
writeByteHeaders(res, byteResponse);
|
||||
if (req.method === "HEAD" || byteResponse.kind === "unsatisfiable" || opened.stat.size === 0) {
|
||||
await closeOpenedHandle();
|
||||
res.end();
|
||||
return true;
|
||||
}
|
||||
const stream = opened.handle.createReadStream({
|
||||
start: byteResponse.kind === "partial" ? byteResponse.range.start : 0,
|
||||
end: byteResponse.kind === "partial" ? byteResponse.range.end : opened.stat.size - 1,
|
||||
autoClose: false,
|
||||
});
|
||||
const finishClose = () => {
|
||||
void closeOpenedHandle();
|
||||
};
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { ServerResponse } from "node:http";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveByteResponse, writeByteHeaders } from "./http-byte-range.js";
|
||||
|
||||
const FILE = { size: 10, mtimeMs: 1_752_000_000_123.5 };
|
||||
|
||||
describe("resolveByteResponse", () => {
|
||||
it("resolves an open-ended range", () => {
|
||||
expect(
|
||||
resolveByteResponse({ file: FILE, method: "GET", rangeHeader: "bytes=4-" }),
|
||||
).toMatchObject({
|
||||
kind: "partial",
|
||||
statusCode: 206,
|
||||
contentLength: 6,
|
||||
range: { start: 4, end: 9 },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves a suffix range", () => {
|
||||
expect(
|
||||
resolveByteResponse({ file: FILE, method: "GET", rangeHeader: "bytes=-3" }),
|
||||
).toMatchObject({
|
||||
kind: "partial",
|
||||
statusCode: 206,
|
||||
contentLength: 3,
|
||||
range: { start: 7, end: 9 },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves an exact range", () => {
|
||||
expect(
|
||||
resolveByteResponse({ file: FILE, method: "GET", rangeHeader: "bytes=2-5" }),
|
||||
).toMatchObject({
|
||||
kind: "partial",
|
||||
statusCode: 206,
|
||||
contentLength: 4,
|
||||
range: { start: 2, end: 5 },
|
||||
});
|
||||
});
|
||||
|
||||
it("returns 416 with the complete file size for an out-of-bounds range", () => {
|
||||
const plan = resolveByteResponse({ file: FILE, method: "GET", rangeHeader: "bytes=10-20" });
|
||||
expect(plan).toMatchObject({
|
||||
kind: "unsatisfiable",
|
||||
statusCode: 416,
|
||||
contentLength: 0,
|
||||
size: 10,
|
||||
});
|
||||
|
||||
const setHeader = vi.fn();
|
||||
const res = { statusCode: 0, setHeader } as unknown as ServerResponse;
|
||||
writeByteHeaders(res, plan);
|
||||
expect(res.statusCode).toBe(416);
|
||||
expect(setHeader).toHaveBeenCalledWith("Content-Range", "bytes */10");
|
||||
});
|
||||
|
||||
it.each(["items=0-1", "bytes=broken", "bytes=0-1,4-5"])(
|
||||
"falls back to a full response for malformed or multipart range %s",
|
||||
(rangeHeader) => {
|
||||
expect(resolveByteResponse({ file: FILE, method: "GET", rangeHeader })).toMatchObject({
|
||||
kind: "full",
|
||||
statusCode: 200,
|
||||
contentLength: 10,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("honors a matching If-Range ETag", () => {
|
||||
const etag = resolveByteResponse({ file: FILE }).etag;
|
||||
expect(
|
||||
resolveByteResponse({
|
||||
file: FILE,
|
||||
method: "GET",
|
||||
rangeHeader: "bytes=1-2",
|
||||
ifRangeHeader: etag,
|
||||
}),
|
||||
).toMatchObject({ kind: "partial", statusCode: 206, range: { start: 1, end: 2 } });
|
||||
});
|
||||
|
||||
it("falls back to a full response for a mismatched If-Range ETag", () => {
|
||||
expect(
|
||||
resolveByteResponse({
|
||||
file: FILE,
|
||||
method: "GET",
|
||||
rangeHeader: "bytes=1-2",
|
||||
ifRangeHeader: '"different"',
|
||||
}),
|
||||
).toMatchObject({ kind: "full", statusCode: 200, contentLength: 10 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("byte ETag generation", () => {
|
||||
it("is stable for the same file identity and changes with size or mtime", () => {
|
||||
const etag = resolveByteResponse({ file: FILE }).etag;
|
||||
expect(resolveByteResponse({ file: { ...FILE } }).etag).toBe(etag);
|
||||
expect(resolveByteResponse({ file: { ...FILE, size: FILE.size + 1 } }).etag).not.toBe(etag);
|
||||
expect(resolveByteResponse({ file: { ...FILE, mtimeMs: FILE.mtimeMs + 1 } }).etag).not.toBe(
|
||||
etag,
|
||||
);
|
||||
expect(etag).toMatch(/^"[A-Za-z0-9_-]+"$/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { ServerResponse } from "node:http";
|
||||
|
||||
type FileIdentity = {
|
||||
size: number;
|
||||
mtimeMs: number;
|
||||
};
|
||||
|
||||
type ByteSlice = {
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
|
||||
type ByteResponsePlan =
|
||||
| {
|
||||
kind: "full";
|
||||
statusCode: 200;
|
||||
contentLength: number;
|
||||
etag: string;
|
||||
}
|
||||
| {
|
||||
kind: "partial";
|
||||
statusCode: 206;
|
||||
contentLength: number;
|
||||
etag: string;
|
||||
range: ByteSlice;
|
||||
size: number;
|
||||
}
|
||||
| {
|
||||
kind: "unsatisfiable";
|
||||
statusCode: 416;
|
||||
contentLength: 0;
|
||||
etag: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
function createByteEtag(file: FileIdentity): string {
|
||||
// Gateway media files are write-once, so size + mtimeMs uniquely version their bytes here.
|
||||
// Serving mutable files with a strong validator would instead require a content hash.
|
||||
const digest = createHash("sha256").update(`${file.size}:${file.mtimeMs}`).digest("base64url");
|
||||
return `"${digest}"`;
|
||||
}
|
||||
|
||||
function parseByteRange(value: string, size: number): ByteSlice | "invalid" | "unsatisfiable" {
|
||||
const normalized = value.trim();
|
||||
if (normalized.includes(",")) {
|
||||
// Multipart ranges are deliberately unsupported; serve the full representation instead.
|
||||
return "invalid";
|
||||
}
|
||||
const match = /^bytes=(\d*)-(\d*)$/i.exec(normalized);
|
||||
if (!match || (!match[1] && !match[2])) {
|
||||
return "invalid";
|
||||
}
|
||||
const [, rangeStart = "", rangeEnd = ""] = match;
|
||||
|
||||
const fileSize = BigInt(size);
|
||||
if (!rangeStart) {
|
||||
const suffixLength = BigInt(rangeEnd);
|
||||
if (suffixLength === 0n || fileSize === 0n) {
|
||||
return "unsatisfiable";
|
||||
}
|
||||
const start = suffixLength >= fileSize ? 0n : fileSize - suffixLength;
|
||||
return { start: Number(start), end: size - 1 };
|
||||
}
|
||||
|
||||
const start = BigInt(rangeStart);
|
||||
if (start >= fileSize) {
|
||||
return "unsatisfiable";
|
||||
}
|
||||
const requestedEnd = rangeEnd ? BigInt(rangeEnd) : fileSize - 1n;
|
||||
if (requestedEnd < start) {
|
||||
return "unsatisfiable";
|
||||
}
|
||||
const end = requestedEnd >= fileSize ? fileSize - 1n : requestedEnd;
|
||||
return { start: Number(start), end: Number(end) };
|
||||
}
|
||||
|
||||
export function resolveByteResponse(params: {
|
||||
file: FileIdentity;
|
||||
method?: string;
|
||||
rangeHeader?: string | string[];
|
||||
ifRangeHeader?: string | string[];
|
||||
}): ByteResponsePlan {
|
||||
const etag = createByteEtag(params.file);
|
||||
const full = {
|
||||
kind: "full",
|
||||
statusCode: 200,
|
||||
contentLength: params.file.size,
|
||||
etag,
|
||||
} as const;
|
||||
if (params.method !== "GET" || typeof params.rangeHeader !== "string") {
|
||||
return full;
|
||||
}
|
||||
if (params.ifRangeHeader !== undefined && params.ifRangeHeader !== etag) {
|
||||
return full;
|
||||
}
|
||||
|
||||
const range = parseByteRange(params.rangeHeader, params.file.size);
|
||||
if (range === "invalid") {
|
||||
return full;
|
||||
}
|
||||
if (range === "unsatisfiable") {
|
||||
return {
|
||||
kind: "unsatisfiable",
|
||||
statusCode: 416,
|
||||
contentLength: 0,
|
||||
etag,
|
||||
size: params.file.size,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "partial",
|
||||
statusCode: 206,
|
||||
contentLength: range.end - range.start + 1,
|
||||
etag,
|
||||
range,
|
||||
size: params.file.size,
|
||||
};
|
||||
}
|
||||
|
||||
export function writeByteHeaders(res: ServerResponse, plan: ByteResponsePlan): void {
|
||||
res.statusCode = plan.statusCode;
|
||||
res.setHeader("Accept-Ranges", "bytes");
|
||||
res.setHeader("ETag", plan.etag);
|
||||
res.setHeader("Content-Length", String(plan.contentLength));
|
||||
if (plan.kind === "partial") {
|
||||
res.setHeader("Content-Range", `bytes ${plan.range.start}-${plan.range.end}/${plan.size}`);
|
||||
} else if (plan.kind === "unsatisfiable") {
|
||||
res.setHeader("Content-Range", `bytes */${plan.size}`);
|
||||
}
|
||||
}
|
||||
@@ -324,6 +324,56 @@ describe("handleManagedOutgoingImageHttpRequest", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("serves a byte range from the validated managed image", async () => {
|
||||
const { attachmentId, sessionKey } = await createFixture(stateDir);
|
||||
|
||||
const { result } = await requestManagedImage({
|
||||
stateDir,
|
||||
pathName: `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`,
|
||||
authResponse: { authMethod: "token" },
|
||||
headers: { range: "bytes=9-13" },
|
||||
});
|
||||
|
||||
expect(result.statusCode).toBe(206);
|
||||
expect(result.headers["accept-ranges"]).toBe("bytes");
|
||||
expect(result.headers["content-range"]).toBe("bytes 9-13/14");
|
||||
expect(result.headers["content-length"]).toBe("5");
|
||||
expect(result.headers.etag).toMatch(/^"[A-Za-z0-9_-]+"$/);
|
||||
expect(result.body.toString("utf8")).toBe("image");
|
||||
});
|
||||
|
||||
it("advertises byte ranges without a body for HEAD", async () => {
|
||||
const { attachmentId, sessionKey } = await createFixture(stateDir);
|
||||
|
||||
const { result } = await requestManagedImage({
|
||||
stateDir,
|
||||
pathName: `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`,
|
||||
method: "HEAD",
|
||||
authResponse: { authMethod: "token" },
|
||||
});
|
||||
|
||||
expect(result.statusCode).toBe(200);
|
||||
expect(result.headers["accept-ranges"]).toBe("bytes");
|
||||
expect(result.headers["content-length"]).toBe("14");
|
||||
expect(result.headers.etag).toMatch(/^"[A-Za-z0-9_-]+"$/);
|
||||
expect(result.body).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("serves an empty managed image without a body", async () => {
|
||||
const { attachmentId, sessionKey, originalPath } = await createFixture(stateDir);
|
||||
await fs.writeFile(originalPath, Buffer.alloc(0));
|
||||
|
||||
const { result } = await requestManagedImage({
|
||||
stateDir,
|
||||
pathName: `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`,
|
||||
authResponse: { authMethod: "token" },
|
||||
});
|
||||
|
||||
expect(result.statusCode).toBe(200);
|
||||
expect(result.headers["content-length"]).toBe("0");
|
||||
expect(result.body).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("serves an exact transcript image through a short-lived artifact ticket", async () => {
|
||||
const { attachmentId, sessionKey } = await createFixture(stateDir);
|
||||
const canonicalPath = `/api/chat/media/outgoing/${encodeURIComponent(sessionKey)}/${attachmentId}/full`;
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
import { resolveDefaultAgentId } from "../agents/agent-scope-config.js";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { readLocalFileSafely } from "../infra/fs-safe.js";
|
||||
import { openLocalFileSafely, readLocalFileSafely } from "../infra/fs-safe.js";
|
||||
import { assertLocalMediaAllowed, resolveLocalMediaRoots } from "../media/local-media-access.js";
|
||||
import { resolveLocalMediaPath } from "../media/local-media-path.js";
|
||||
import {
|
||||
@@ -24,6 +24,7 @@ import { getMediaDir, MEDIA_MAX_BYTES, saveMediaBuffer, saveMediaSource } from "
|
||||
import { safeEqualSecret } from "../security/secret-equal.js";
|
||||
import type { AuthRateLimiter } from "./auth-rate-limit.js";
|
||||
import type { ResolvedGatewayAuth } from "./auth.js";
|
||||
import { resolveByteResponse, writeByteHeaders } from "./http-byte-range.js";
|
||||
import { sendJson, sendMethodNotAllowed, sendMissingScopeForbidden } from "./http-common.js";
|
||||
import {
|
||||
authorizeGatewayHttpRequestOrReply,
|
||||
@@ -1234,8 +1235,8 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
return false;
|
||||
}
|
||||
|
||||
if (req.method !== "GET") {
|
||||
sendMethodNotAllowed(res, "GET");
|
||||
if (req.method !== "GET" && req.method !== "HEAD") {
|
||||
sendMethodNotAllowed(res, "GET, HEAD");
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1303,21 +1304,25 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
return true;
|
||||
}
|
||||
|
||||
let body: Buffer;
|
||||
let opened: Awaited<ReturnType<typeof openLocalFileSafely>>;
|
||||
try {
|
||||
body = (
|
||||
await readLocalFileSafely({
|
||||
filePath: resolveManagedImageOriginalPath(record),
|
||||
})
|
||||
).buffer;
|
||||
opened = await openLocalFileSafely({
|
||||
filePath: resolveManagedImageOriginalPath(record),
|
||||
});
|
||||
} catch {
|
||||
sendStatus(res, 404, "not found");
|
||||
return true;
|
||||
}
|
||||
|
||||
res.statusCode = 200;
|
||||
let handleClosed = false;
|
||||
const closeOpenedHandle = async () => {
|
||||
if (handleClosed) {
|
||||
return;
|
||||
}
|
||||
handleClosed = true;
|
||||
await opened.handle.close().catch(() => {});
|
||||
};
|
||||
res.setHeader("content-type", record.original.contentType || "application/octet-stream");
|
||||
res.setHeader("content-length", String(body.byteLength));
|
||||
res.setHeader("x-content-type-options", "nosniff");
|
||||
res.setHeader("referrer-policy", "no-referrer");
|
||||
res.setHeader(
|
||||
@@ -1330,7 +1335,40 @@ export async function handleManagedOutgoingImageHttpRequest(
|
||||
"content-disposition",
|
||||
`inline; filename="${safeAttachmentFilename(record.original.filename)}"`,
|
||||
);
|
||||
res.end(body);
|
||||
const byteResponse = resolveByteResponse({
|
||||
file: opened.stat,
|
||||
method: req.method,
|
||||
rangeHeader: req.headers.range,
|
||||
ifRangeHeader: req.headers["if-range"],
|
||||
});
|
||||
writeByteHeaders(res, byteResponse);
|
||||
if (req.method === "HEAD" || byteResponse.kind === "unsatisfiable" || opened.stat.size === 0) {
|
||||
await closeOpenedHandle();
|
||||
res.end();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Stream from the verified descriptor so a path swap cannot bypass fs-safe after validation.
|
||||
const stream = opened.handle.createReadStream({
|
||||
start: byteResponse.kind === "partial" ? byteResponse.range.start : 0,
|
||||
end: byteResponse.kind === "partial" ? byteResponse.range.end : opened.stat.size - 1,
|
||||
autoClose: false,
|
||||
});
|
||||
const finishClose = () => {
|
||||
void closeOpenedHandle();
|
||||
};
|
||||
stream.once("end", finishClose);
|
||||
stream.once("close", finishClose);
|
||||
stream.once("error", () => {
|
||||
void closeOpenedHandle();
|
||||
if (!res.headersSent) {
|
||||
sendStatus(res, 404, "not found");
|
||||
} else {
|
||||
res.destroy();
|
||||
}
|
||||
});
|
||||
res.once("close", finishClose);
|
||||
stream.pipe(res);
|
||||
return true;
|
||||
}
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
Reference in New Issue
Block a user