ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(music): bound generated track downloads

Browse Source
This commit is contained in:
Vincent Koc
2026-05-29 12:03:51 +02:00
parent 3eca409456
commit 036298fbae
5 changed files with 232 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/fal/music-generation-provider.test.ts
+39
View File
@@ -45,6 +45,18 @@ function postRequest(): Record<string, unknown> {
return request as Record<string, unknown>;
}
function streamedAudioResponse(bytes: string): Response {
return new Response(
new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(bytes));
controller.close();
},
}),
{ headers: { "content-type": "audio/mpeg" } },
);
}
describe("fal music generation provider", () => {
afterEach(() => {
assertOkOrThrowHttpErrorMock.mockClear();
@@ -111,6 +123,33 @@ describe("fal music generation provider", () => {
expect(result.metadata?.audioUrl).toBe("https://v3b.fal.media/files/b/kangaroo/out.mp3");
});
it("rejects generated music downloads that exceed the configured media cap", async () => {
postJsonRequestMock.mockResolvedValue({
response: {
json: async () => ({
audio: {
url: "https://v3b.fal.media/files/b/out.mp3",
content_type: "audio/mpeg",
},
}),
},
release: vi.fn(async () => {}),
});
vi.stubGlobal(
"fetch",
vi.fn(async () => streamedAudioResponse("too-large")),
);
await expect(
buildFalMusicGenerationProvider().generateMusic({
provider: "fal",
model: "fal-ai/minimax-music/v2.6",
prompt: "short track",
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
}),
).rejects.toThrow("fal generated music download exceeds 1 bytes");
});
it("rejects MiniMax lyrics requests that also ask for instrumental output", async () => {
await expect(
buildFalMusicGenerationProvider().generateMusic({
extensions/fal/music-generation-provider.ts
+10
View File
@@ -13,6 +13,7 @@ const DEFAULT_FAL_MUSIC_MODEL = "fal-ai/minimax-music/v2.6";
const FAL_ACE_STEP_MODEL = "fal-ai/ace-step/prompt-to-audio";
const FAL_STABLE_AUDIO_MODEL = "fal-ai/stable-audio-25/text-to-audio";
const DEFAULT_TIMEOUT_MS = 180_000;
const DEFAULT_GENERATED_MUSIC_MAX_BYTES = 16 * 1024 * 1024;
const FAL_MUSIC_MODELS = [
DEFAULT_FAL_MUSIC_MODEL,
@@ -24,6 +25,14 @@ function resolveFalMusicModel(model: string | undefined): string {
return normalizeOptionalString(model) ?? DEFAULT_FAL_MUSIC_MODEL;
}
function resolveGeneratedMusicMaxBytes(req: MusicGenerationRequest): number {
const configured = req.cfg.agents?.defaults?.mediaMaxMb;
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
return Math.floor(configured * 1024 * 1024);
}
return DEFAULT_GENERATED_MUSIC_MAX_BYTES;
}
function buildFalMinimaxBody(req: MusicGenerationRequest): Record<string, unknown> {
const lyrics = normalizeOptionalString(req.lyrics);
if (lyrics && req.instrumental === true) {
@@ -162,6 +171,7 @@ export function buildFalMusicGenerationProvider(): MusicGenerationProvider {
fetchFn: fetch,
provider: "fal",
requestFailedMessage: "fal generated music download failed",
maxBytes: resolveGeneratedMusicMaxBytes(req),
});
const lyrics =
typeof payload === "object" && payload && !Array.isArray(payload)
extensions/minimax/music-generation-provider.test.ts
+90
View File
@@ -49,6 +49,18 @@ function mockCallArg(mock: { mock: { calls: unknown[][] } }, index = 0): Record<
return call[0] as Record<string, unknown>;
}
function streamedAudioResponse(bytes: string): Response {
return new Response(
new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(bytes));
controller.close();
},
}),
{ headers: { "content-type": "audio/mpeg" } },
);
}
describe("minimax music generation provider", () => {
it("declares explicit mode capabilities", () => {
expectExplicitMusicGenerationCapabilities(buildMinimaxMusicGenerationProvider());
@@ -161,6 +173,58 @@ describe("minimax music generation provider", () => {
expect(result.tracks[0]?.buffer).toEqual(terminalAudio);
});
it("rejects streamed generated music that exceeds the configured media cap", async () => {
postJsonRequestMock.mockResolvedValue({
response: new Response(
`data: ${JSON.stringify({
data: { status: 2, audio: Buffer.from("too-large").toString("hex") },
base_resp: { status_code: 0 },
})}`,
{
headers: { "content-type": "text/event-stream" },
},
),
release: vi.fn(async () => {}),
});
const provider = buildMinimaxMusicGenerationProvider();
await expect(
provider.generateMusic({
provider: "minimax",
model: "music-2.6",
prompt: "short track",
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
}),
).rejects.toThrow("MiniMax generated music download exceeds 1 bytes");
});
it("rejects inline generated music that exceeds the configured media cap before decoding", async () => {
postJsonRequestMock.mockResolvedValue({
response: new Response(
JSON.stringify({
data: {
audio: Buffer.from("too-large").toString("hex"),
},
base_resp: { status_code: 0 },
}),
{
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
});
const provider = buildMinimaxMusicGenerationProvider();
await expect(
provider.generateMusic({
provider: "minimax",
model: "music-2.6",
prompt: "short track",
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
}),
).rejects.toThrow("MiniMax generated music download exceeds 1 bytes");
});
it("downloads tracks when url output is returned in data.audio", async () => {
mockMusicGenerationResponse({
task_id: "task-url",
@@ -192,6 +256,32 @@ describe("minimax music generation provider", () => {
expect(result.metadata?.audioUrl).toBe("https://example.com/url-audio.mp3");
});
it("rejects generated music downloads that exceed the configured media cap", async () => {
postJsonRequestMock.mockResolvedValue({
response: new Response(
JSON.stringify({
data: {
audio: "https://example.com/too-large.mp3",
},
base_resp: { status_code: 0 },
}),
{ headers: { "content-type": "application/json" } },
),
release: vi.fn(async () => {}),
});
fetchWithTimeoutMock.mockResolvedValueOnce(streamedAudioResponse("too-large"));
const provider = buildMinimaxMusicGenerationProvider();
await expect(
provider.generateMusic({
provider: "minimax",
model: "music-2.6",
prompt: "short track",
cfg: { agents: { defaults: { mediaMaxMb: 0.000001 } } },
}),
).rejects.toThrow("MiniMax generated music download exceeds 1 bytes");
});
it("honors explicit long caller timeouts for request and download fallbacks", async () => {
mockMusicGenerationResponse({
data: {
extensions/minimax/music-generation-provider.ts
+85 -13
View File
@@ -2,6 +2,7 @@ import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
import type {
GeneratedMusicAsset,
MusicGenerationProvider,
MusicGenerationRequest,
} from "openclaw/plugin-sdk/music-generation";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
@@ -14,12 +15,16 @@ import {
resolveProviderHttpRequestConfig,
type ProviderOperationDeadline,
} from "openclaw/plugin-sdk/provider-http";
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
const DEFAULT_MINIMAX_MUSIC_BASE_URL = "https://api.minimax.io";
const DEFAULT_MINIMAX_MUSIC_MODEL = "music-2.6";
const DEFAULT_TIMEOUT_MS = 120_000;
const DEFAULT_OPERATION_TIMEOUT_MS = 300_000;
const DEFAULT_GENERATED_MUSIC_MAX_BYTES = 16 * 1024 * 1024;
const STREAM_ENVELOPE_MAX_BYTES_MULTIPLIER = 5;
const STREAM_ENVELOPE_OVERHEAD_BYTES = 64 * 1024;
type MinimaxBaseResp = {
status_code?: number;
@@ -71,11 +76,17 @@ function assertMinimaxBaseResp(baseResp: MinimaxBaseResp | undefined, context: s
);
}
function decodePossibleBinary(data: string): Buffer {
function decodePossibleBinaryWithLimit(data: string, maxBytes: number): Buffer {
const trimmed = data.trim();
if (/^[0-9a-f]+$/iu.test(trimmed) && trimmed.length % 2 === 0) {
if (trimmed.length / 2 > maxBytes) {
throw createGeneratedMusicTooLargeError(maxBytes);
}
return Buffer.from(trimmed, "hex");
}
if (Buffer.byteLength(trimmed, "base64") > maxBytes) {
throw createGeneratedMusicTooLargeError(maxBytes);
}
return Buffer.from(trimmed, "base64");
}
@@ -95,10 +106,19 @@ function isLikelyRemoteUrl(value: string | undefined): boolean {
return Boolean(trimmed && /^https?:\/\//iu.test(trimmed));
}
function resolveGeneratedMusicMaxBytes(req: MusicGenerationRequest): number {
const configured = req.cfg.agents?.defaults?.mediaMaxMb;
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) {
return Math.floor(configured * 1024 * 1024);
}
return DEFAULT_GENERATED_MUSIC_MAX_BYTES;
}
async function downloadTrackFromUrl(params: {
url: string;
timeoutMs?: number;
fetchFn: typeof fetch;
maxBytes: number;
}): Promise<GeneratedMusicAsset> {
const response = await fetchProviderDownloadResponse({
url: params.url,
@@ -111,7 +131,10 @@ async function downloadTrackFromUrl(params: {
const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "audio/mpeg";
const ext = extensionForMime(mimeType)?.replace(/^\./u, "") || "mp3";
return {
buffer: Buffer.from(await response.arrayBuffer()),
buffer: await readResponseWithLimit(response, params.maxBytes, {
onOverflow: ({ maxBytes }) =>
new Error(`MiniMax generated music download exceeds ${maxBytes} bytes`),
}),
mimeType,
fileName: `track-1.${ext}`,
};
@@ -130,9 +153,21 @@ function resolveBodyReadTimeoutMs(deadline: ProviderOperationDeadline): number {
});
}
function createGeneratedMusicTooLargeError(maxBytes: number): Error {
return new Error(`MiniMax generated music download exceeds ${maxBytes} bytes`);
}
function resolveStreamEnvelopeMaxBytes(maxBytes: number): number {
return Math.max(
STREAM_ENVELOPE_OVERHEAD_BYTES,
maxBytes * STREAM_ENVELOPE_MAX_BYTES_MULTIPLIER + STREAM_ENVELOPE_OVERHEAD_BYTES,
);
}
async function readResponseBufferWithDeadline(
response: Response,
deadline: ProviderOperationDeadline,
maxBytes: number,
): Promise<Buffer> {
const body = response.body;
if (!body) {
@@ -157,8 +192,18 @@ async function readResponseBufferWithDeadline(
if (!result.value || result.value.length === 0) {
continue;
}
const nextTotalBytes = totalBytes + result.value.byteLength;
if (nextTotalBytes > maxBytes) {
const error = createGeneratedMusicTooLargeError(maxBytes);
try {
await reader.cancel(error);
} catch {
// Preserve the size-limit failure that caused cancellation.
}
throw error;
}
chunks.push(result.value);
totalBytes += result.value.byteLength;
totalBytes = nextTotalBytes;
} catch (error) {
try {
await reader.cancel(error);
@@ -188,18 +233,26 @@ async function readResponseBufferWithDeadline(
async function readStreamingTrack(
response: Response,
deadline: ProviderOperationDeadline,
maxBytes: number,
): Promise<GeneratedMusicAsset> {
const contentType = normalizeOptionalString(response.headers.get("content-type")) ?? "";
if (contentType.toLowerCase().startsWith("audio/")) {
const ext = extensionForMime(contentType)?.replace(/^\./u, "") || "mp3";
return {
buffer: await readResponseBufferWithDeadline(response, deadline),
buffer: await readResponseBufferWithDeadline(response, deadline, maxBytes),
mimeType: contentType,
fileName: `track-1.${ext}`,
};
}
const chunks: Buffer[] = [];
const text = new TextDecoder().decode(await readResponseBufferWithDeadline(response, deadline));
let decodedBytes = 0;
const text = new TextDecoder().decode(
await readResponseBufferWithDeadline(
response,
deadline,
resolveStreamEnvelopeMaxBytes(maxBytes),
),
);
for (const rawLine of text.split(/\r?\n/u)) {
const line = rawLine.trim();
if (!line.startsWith("data:")) {
@@ -216,7 +269,13 @@ async function readStreamingTrack(
if (String(frame.data?.status ?? "") === "2" && chunks.length > 0) {
continue;
}
chunks.push(decodePossibleBinary(audio));
const chunk = decodePossibleBinaryWithLimit(audio, maxBytes - decodedBytes);
const nextDecodedBytes = decodedBytes + chunk.byteLength;
if (nextDecodedBytes > maxBytes) {
throw createGeneratedMusicTooLargeError(maxBytes);
}
chunks.push(chunk);
decodedBytes = nextDecodedBytes;
}
}
const buffer = Buffer.concat(chunks);
@@ -341,10 +400,19 @@ function buildMinimaxMusicProvider(providerId: string): MusicGenerationProvider
await assertOkOrThrowHttpError(res, "MiniMax music generation failed");
const contentType = normalizeOptionalString(res.headers.get("content-type")) ?? "";
const lowerContentType = contentType.toLowerCase();
const maxGeneratedMusicBytes = resolveGeneratedMusicMaxBytes(req);
const payload =
lowerContentType.includes("text/event-stream") || lowerContentType.startsWith("audio/")
? null
: ((await res.clone().json()) as MinimaxMusicCreateResponse);
: (JSON.parse(
new TextDecoder().decode(
await readResponseBufferWithDeadline(
res.clone(),
deadline,
resolveStreamEnvelopeMaxBytes(maxGeneratedMusicBytes),
),
),
) as MinimaxMusicCreateResponse);
if (payload) {
assertMinimaxBaseResp(payload.base_resp, "MiniMax music generation failed");
}
@@ -366,14 +434,18 @@ function buildMinimaxMusicProvider(providerId: string): MusicGenerationProvider
defaultTimeoutMs: req.timeoutMs ?? DEFAULT_TIMEOUT_MS,
}),
fetchFn,
maxBytes: resolveGeneratedMusicMaxBytes(req),
})
: inlineAudio
? {
buffer: decodePossibleBinary(inlineAudio),
mimeType: "audio/mpeg",
fileName: "track-1.mp3",
}
: await readStreamingTrack(res, deadline);
? (() => {
const buffer = decodePossibleBinaryWithLimit(inlineAudio, maxGeneratedMusicBytes);
return {
buffer,
mimeType: "audio/mpeg",
fileName: "track-1.mp3",
};
})()
: await readStreamingTrack(res, deadline, maxGeneratedMusicBytes);
if (!track) {
throw new Error("MiniMax music generation response missing audio output");
}
src/music-generation/provider-assets.ts
+8 -1
View File
@@ -1,5 +1,7 @@
import { fetchProviderDownloadResponse } from "../media-understanding/shared.js";
import { maxBytesForKind } from "../media/constants.js";
import { extensionForMime } from "../media/mime.js";
import { readResponseWithLimit } from "../media/read-response-with-limit.js";
import { isRecord } from "../shared/record-coerce.js";
import { normalizeOptionalString } from "../shared/string-coerce.js";
import type { GeneratedMusicAsset } from "./types.js";
@@ -82,6 +84,7 @@ export async function downloadGeneratedMusicAsset(params: {
provider: string;
requestFailedMessage: string;
index?: number;
maxBytes?: number;
}): Promise<GeneratedMusicAsset> {
const response = await fetchProviderDownloadResponse({
url: params.candidate.url,
@@ -96,8 +99,12 @@ export async function downloadGeneratedMusicAsset(params: {
normalizeSpecificAudioMimeType(params.candidate.mimeType) ??
"audio/mpeg";
const ext = extensionForMime(mimeType)?.replace(/^\./u, "") || "mp3";
const maxBytes = params.maxBytes ?? maxBytesForKind("audio");
return {
buffer: Buffer.from(await response.arrayBuffer()),
buffer: await readResponseWithLimit(response, maxBytes, {
onOverflow: ({ maxBytes }) =>
new Error(`${params.provider} generated music download exceeds ${maxBytes} bytes`),
}),
mimeType,
fileName: params.candidate.fileName ?? `track-${(params.index ?? 0) + 1}.${ext}`,
metadata: {