From 41ea5316aa989f975c7a171a7fac4a066c03c974 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Apr 2026 18:49:31 +0100 Subject: [PATCH] test: add shared media live harness --- docs/help/testing.md | 25 +- docs/tools/music-generation.md | 6 + docs/tools/video-generation.md | 38 +-- package.json | 4 + scripts/pnpm-runner.d.ts | 14 ++ scripts/test-live-media.ts | 347 ++++++++++++++++++++++++++++ src/scripts/test-live-media.test.ts | 73 ++++++ 7 files changed, 487 insertions(+), 20 deletions(-) create mode 100644 scripts/pnpm-runner.d.ts create mode 100644 scripts/test-live-media.ts create mode 100644 src/scripts/test-live-media.test.ts diff --git a/docs/help/testing.md b/docs/help/testing.md index 36cac632cae5..e5bfc8cbd640 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -454,6 +454,7 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Test: `src/image-generation/runtime.live.test.ts` - Command: `pnpm test:live src/image-generation/runtime.live.test.ts` +- Harness: `pnpm test:live:media image` - Scope: - Enumerates every registered image-generation provider plugin - Loads missing provider env vars from your login shell (`~/.profile`) before probing @@ -478,6 +479,7 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Test: `extensions/music-generation-providers.live.test.ts` - Enable: `OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/music-generation-providers.live.test.ts` +- Harness: `pnpm test:live:media music` - Scope: - Exercises the shared bundled music-generation provider path - Currently covers Google and MiniMax @@ -501,6 +503,7 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Test: `extensions/video-generation-providers.live.test.ts` - Enable: `OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/video-generation-providers.live.test.ts` +- Harness: `pnpm test:live:media video` - Scope: - Exercises the shared bundled video-generation provider path - Loads provider env vars from your login shell (`~/.profile`) before probing @@ -508,20 +511,36 @@ If you want to rely on env keys (e.g. exported in your `~/.profile`), run local - Skips providers with no usable auth/profile/model - Runs both declared runtime modes when available: - `generate` with prompt-only input - - `imageToVideo` when the provider declares `capabilities.imageToVideo.enabled` + - `imageToVideo` when the provider declares `capabilities.imageToVideo.enabled` and the selected provider/model accepts buffer-backed local image input in the shared sweep - `videoToVideo` when the provider declares `capabilities.videoToVideo.enabled` and the selected provider/model accepts buffer-backed local video input in the shared sweep + - Current declared-but-skipped `imageToVideo` providers in the shared sweep: + - `vydra` because bundled `veo3` is text-only and bundled `kling` requires a remote image URL - Current `videoToVideo` live coverage: - - `google` - - `openai` - `runway` only when the selected model is `runway/gen4_aleph` - Current declared-but-skipped `videoToVideo` providers in the shared sweep: - `alibaba`, `qwen`, `xai` because those paths currently require remote `http(s)` / MP4 reference URLs + - `google` because the current shared Gemini/Veo lane uses local buffer-backed input and that path is not accepted in the shared sweep + - `openai` because the current shared lane lacks org-specific video inpaint/remix access guarantees - Optional narrowing: - `OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS="google,openai,runway"` - `OPENCLAW_LIVE_VIDEO_GENERATION_MODELS="google/veo-3.1-fast-generate-preview,openai/sora-2,runway/gen4_aleph"` - Optional auth behavior: - `OPENCLAW_LIVE_REQUIRE_PROFILE_KEYS=1` to force profile-store auth and ignore env-only overrides +## Media live harness + +- Command: `pnpm test:live:media` +- Purpose: + - Runs the shared image, music, and video live suites through one repo-native entrypoint + - Auto-loads missing provider env vars from `~/.profile` + - Auto-narrows each suite to providers that currently have usable auth by default + - Reuses `scripts/test-live.mjs`, so heartbeat and quiet-mode behavior stay consistent +- Examples: + - `pnpm test:live:media` + - `pnpm test:live:media image video --providers openai,google,minimax` + - `pnpm test:live:media video --video-providers openai,runway --all-providers` + - `pnpm test:live:media music --quiet` + ## Docker runners (optional "works in Linux" checks) These Docker runners split into two buckets: diff --git a/docs/tools/music-generation.md b/docs/tools/music-generation.md index de390ecb5207..e8eddb4a3e28 100644 --- a/docs/tools/music-generation.md +++ b/docs/tools/music-generation.md @@ -248,6 +248,12 @@ Opt-in live coverage for the shared bundled providers: OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/music-generation-providers.live.test.ts ``` +Repo wrapper: + +```bash +pnpm test:live:media music +``` + This live file loads missing provider env vars from `~/.profile`, prefers live/env API keys ahead of stored auth profiles by default, and runs both `generate` and declared `edit` coverage when the provider enables edit mode. diff --git a/docs/tools/video-generation.md b/docs/tools/video-generation.md index 268ec4b2133d..8bd0938edecc 100644 --- a/docs/tools/video-generation.md +++ b/docs/tools/video-generation.md @@ -103,20 +103,20 @@ runtime modes at runtime. This is the explicit mode contract used by `video_generate`, contract tests, and the shared live sweep. -| Provider | `generate` | `imageToVideo` | `videoToVideo` | Shared live lanes today | -| -------- | ---------- | -------------- | -------------- | ---------------------------------------------------------------------------------------------------------- | -| Alibaba | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs | -| BytePlus | Yes | Yes | No | `generate`, `imageToVideo` | -| ComfyUI | Yes | Yes | No | Not in the shared sweep; workflow-specific coverage lives with Comfy tests | -| fal | Yes | Yes | No | `generate`, `imageToVideo` | -| Google | Yes | Yes | Yes | `generate`, `imageToVideo`, `videoToVideo` | -| MiniMax | Yes | Yes | No | `generate`, `imageToVideo` | -| OpenAI | Yes | Yes | Yes | `generate`, `imageToVideo`, `videoToVideo` | -| Qwen | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs | -| Runway | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` | -| Together | Yes | Yes | No | `generate`, `imageToVideo` | -| Vydra | Yes | Yes | No | `generate`, `imageToVideo` | -| xAI | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider currently needs a remote MP4 URL | +| Provider | `generate` | `imageToVideo` | `videoToVideo` | Shared live lanes today | +| -------- | ---------- | -------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| Alibaba | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs | +| BytePlus | Yes | Yes | No | `generate`, `imageToVideo` | +| ComfyUI | Yes | Yes | No | Not in the shared sweep; workflow-specific coverage lives with Comfy tests | +| fal | Yes | Yes | No | `generate`, `imageToVideo` | +| Google | Yes | Yes | Yes | `generate`, `imageToVideo`; shared `videoToVideo` skipped because the current buffer-backed Gemini/Veo sweep does not accept that input | +| MiniMax | Yes | Yes | No | `generate`, `imageToVideo` | +| OpenAI | Yes | Yes | Yes | `generate`, `imageToVideo`; shared `videoToVideo` skipped because this org/input path currently needs provider-side inpaint/remix access | +| Qwen | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider needs remote `http(s)` video URLs | +| Runway | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` runs only when the selected model is `runway/gen4_aleph` | +| Together | Yes | Yes | No | `generate`, `imageToVideo` | +| Vydra | Yes | Yes | No | `generate`; shared `imageToVideo` skipped because bundled `veo3` is text-only and bundled `kling` requires a remote image URL | +| xAI | Yes | Yes | Yes | `generate`, `imageToVideo`; `videoToVideo` skipped because this provider currently needs a remote MP4 URL | ## Tool parameters @@ -140,7 +140,7 @@ and the shared live sweep. | Parameter | Type | Description | | ----------------- | ------- | ------------------------------------------------------------------------ | | `aspectRatio` | string | `1:1`, `2:3`, `3:2`, `3:4`, `4:3`, `4:5`, `5:4`, `9:16`, `16:9`, `21:9` | -| `resolution` | string | `480P`, `720P`, or `1080P` | +| `resolution` | string | `480P`, `720P`, `768P`, or `1080P` | | `durationSeconds` | number | Target duration in seconds (rounded to nearest provider-supported value) | | `size` | string | Size hint when the provider supports it | | `audio` | boolean | Enable generated audio when supported | @@ -254,6 +254,12 @@ Opt-in live coverage for the shared bundled providers: OPENCLAW_LIVE_TEST=1 pnpm test:live -- extensions/video-generation-providers.live.test.ts ``` +Repo wrapper: + +```bash +pnpm test:live:media video +``` + This live file loads missing provider env vars from `~/.profile`, prefers live/env API keys ahead of stored auth profiles by default, and runs the declared modes it can exercise safely with local media: @@ -265,8 +271,6 @@ declared modes it can exercise safely with local media: Today the shared `videoToVideo` live lane covers: -- `google` -- `openai` - `runway` only when you select `runway/gen4_aleph` ## Configuration diff --git a/package.json b/package.json index 95eb6b96f20c..4ac73cede00f 100644 --- a/package.json +++ b/package.json @@ -1183,6 +1183,10 @@ "test:live": "node scripts/test-live.mjs", "test:live:cache": "bun scripts/check-live-cache.ts", "test:live:gateway-profiles": "node scripts/test-live.mjs -- src/gateway/gateway-models.profiles.live.test.ts", + "test:live:media": "node --import tsx scripts/test-live-media.ts", + "test:live:media:image": "node --import tsx scripts/test-live-media.ts image", + "test:live:media:music": "node --import tsx scripts/test-live-media.ts music", + "test:live:media:video": "node --import tsx scripts/test-live-media.ts video", "test:live:models-profiles": "node scripts/test-live.mjs -- src/agents/models.profiles.live.test.ts", "test:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/test-projects.mjs", "test:parallels:linux": "bash scripts/e2e/parallels-linux-smoke.sh", diff --git a/scripts/pnpm-runner.d.ts b/scripts/pnpm-runner.d.ts new file mode 100644 index 000000000000..f05bc1d7c842 --- /dev/null +++ b/scripts/pnpm-runner.d.ts @@ -0,0 +1,14 @@ +import type { ChildProcess, SpawnOptions } from "node:child_process"; + +export type PnpmRunnerParams = { + pnpmArgs?: string[]; + nodeArgs?: string[]; + npmExecPath?: string; + nodeExecPath?: string; + platform?: NodeJS.Platform; + comSpec?: string; + stdio?: SpawnOptions["stdio"]; + env?: NodeJS.ProcessEnv; +}; + +export function spawnPnpmRunner(params?: PnpmRunnerParams): ChildProcess; diff --git a/scripts/test-live-media.ts b/scripts/test-live-media.ts new file mode 100644 index 000000000000..e2079f7384bb --- /dev/null +++ b/scripts/test-live-media.ts @@ -0,0 +1,347 @@ +#!/usr/bin/env -S node --import tsx + +import { pathToFileURL } from "node:url"; +import { collectProviderApiKeys } from "../src/agents/live-auth-keys.js"; +import { loadShellEnvFallback } from "../src/infra/shell-env.js"; +import { getProviderEnvVars } from "../src/secrets/provider-env-vars.js"; +import { spawnPnpmRunner } from "./pnpm-runner.mjs"; + +export type MediaSuiteId = "image" | "music" | "video"; + +export type MediaSuiteConfig = { + id: MediaSuiteId; + testFile: string; + providerEnvVar: string; + providers: string[]; +}; + +export const MEDIA_SUITES: Record = { + image: { + id: "image", + testFile: "src/image-generation/runtime.live.test.ts", + providerEnvVar: "OPENCLAW_LIVE_IMAGE_GENERATION_PROVIDERS", + providers: ["fal", "google", "minimax", "openai", "vydra"], + }, + music: { + id: "music", + testFile: "extensions/music-generation-providers.live.test.ts", + providerEnvVar: "OPENCLAW_LIVE_MUSIC_GENERATION_PROVIDERS", + providers: ["google", "minimax"], + }, + video: { + id: "video", + testFile: "extensions/video-generation-providers.live.test.ts", + providerEnvVar: "OPENCLAW_LIVE_VIDEO_GENERATION_PROVIDERS", + providers: [ + "alibaba", + "byteplus", + "fal", + "google", + "minimax", + "openai", + "qwen", + "runway", + "together", + "vydra", + "xai", + ], + }, +}; + +const DEFAULT_SUITES: MediaSuiteId[] = ["image", "music", "video"]; + +export type CliOptions = { + suites: MediaSuiteId[]; + globalProviders: Set | null; + suiteProviders: Partial>>; + requireAuth: boolean; + quietArgs: string[]; + passthroughArgs: string[]; + help: boolean; +}; + +export type SuiteRunPlan = { + suite: MediaSuiteConfig; + providers: string[]; + skippedReason?: string; +}; + +function parseCsv(raw: string | undefined): Set | null { + const trimmed = raw?.trim(); + if (!trimmed) { + return null; + } + const values = trimmed + .split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean); + return values.length ? new Set(values) : null; +} + +function parseSuiteToken(raw: string): MediaSuiteId | null { + const normalized = raw.trim().toLowerCase(); + if (normalized === "image" || normalized === "music" || normalized === "video") { + return normalized; + } + return null; +} + +export function parseArgs(argv: string[]): CliOptions { + const suites = new Set(); + const suiteProviders: Partial>> = {}; + const passthroughArgs: string[] = []; + const quietArgs: string[] = []; + let globalProviders: Set | null = null; + let requireAuth = true; + let help = false; + + const readValue = (index: number): string => { + const value = argv[index + 1]?.trim(); + if (!value) { + throw new Error(`Missing value for ${argv[index]}`); + } + return value; + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index] ?? ""; + if (!arg || arg === "--") { + continue; + } + if (arg === "--help" || arg === "-h") { + help = true; + continue; + } + if ( + arg === "--quiet" || + arg === "--quiet-live" || + arg === "--no-quiet" || + arg === "--no-quiet-live" + ) { + quietArgs.push(arg); + continue; + } + if (arg === "--providers") { + globalProviders = parseCsv(readValue(index)); + index += 1; + continue; + } + if (arg === "--image-providers" || arg === "--music-providers" || arg === "--video-providers") { + const suite = parseSuiteToken(arg.slice(2, arg.indexOf("-providers"))); + if (!suite) { + throw new Error(`Unknown suite flag: ${arg}`); + } + suiteProviders[suite] = parseCsv(readValue(index)) ?? new Set(); + index += 1; + continue; + } + if (arg === "--with-auth" || arg === "--require-auth") { + requireAuth = true; + continue; + } + if (arg === "--all-providers" || arg === "--no-auth-filter") { + requireAuth = false; + continue; + } + if (arg.startsWith("--")) { + passthroughArgs.push(arg); + const next = argv[index + 1]; + if (next && !next.startsWith("--")) { + passthroughArgs.push(next); + index += 1; + } + continue; + } + const suite = parseSuiteToken(arg); + if (suite) { + suites.add(suite); + continue; + } + if (arg === "all") { + suites.add("image"); + suites.add("music"); + suites.add("video"); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + return { + suites: (suites.size ? [...suites] : DEFAULT_SUITES).toSorted(), + globalProviders, + suiteProviders, + requireAuth, + quietArgs, + passthroughArgs, + help, + }; +} + +function selectProviders(params: { + suite: MediaSuiteConfig; + globalProviders: Set | null; + suiteProviders: Set | undefined; + requireAuth: boolean; +}): string[] { + const explicit = params.suiteProviders ?? params.globalProviders; + let providers = params.suite.providers.filter((provider) => + explicit ? explicit.has(provider) : true, + ); + if (!params.requireAuth) { + return providers; + } + providers = providers.filter((provider) => collectProviderApiKeys(provider).length > 0); + return providers; +} + +export function buildRunPlan(options: CliOptions): SuiteRunPlan[] { + const expectedKeys = [ + ...new Set( + options.suites.flatMap((suiteId) => + MEDIA_SUITES[suiteId].providers.flatMap((provider) => getProviderEnvVars(provider)), + ), + ), + ]; + if (expectedKeys.length) { + loadShellEnvFallback({ + enabled: true, + env: process.env, + expectedKeys, + logger: { warn: (message: string) => console.warn(message) }, + }); + } + + return options.suites.map((suiteId) => { + const suite = MEDIA_SUITES[suiteId]; + const providers = selectProviders({ + suite, + globalProviders: options.globalProviders, + suiteProviders: options.suiteProviders[suiteId], + requireAuth: options.requireAuth, + }); + return { + suite, + providers, + ...(providers.length === 0 + ? { + skippedReason: options.requireAuth + ? "no providers with usable auth" + : "no providers selected", + } + : {}), + }; + }); +} + +function printHelp(): void { + console.log(`Media live harness + +Usage: + pnpm test:live:media + pnpm test:live:media image + pnpm test:live:media image video --providers openai,google,minimax + pnpm test:live:media video --video-providers openai,runway --all-providers + +Defaults: + - runs image + music + video + - auto-loads missing provider env vars from ~/.profile + - narrows each suite to providers that currently have usable auth + - forwards extra args to scripts/test-live.mjs + +Flags: + --providers global provider filter + --image-providers image-suite provider filter + --music-providers music-suite provider filter + --video-providers video-suite provider filter + --all-providers do not auto-filter by available auth + --quiet | --no-quiet passed through to test:live +`); +} + +async function runSuite(params: { + plan: SuiteRunPlan; + quietArgs: string[]; + passthroughArgs: string[]; +}): Promise { + const { plan } = params; + if (!plan.providers.length) { + console.log( + `[live:media] skip ${plan.suite.id}: ${plan.skippedReason ?? "no providers selected"}`, + ); + return 0; + } + + const env = { + ...process.env, + [plan.suite.providerEnvVar]: plan.providers.join(","), + }; + const args = [ + "test:live", + ...params.quietArgs, + "--", + plan.suite.testFile, + ...params.passthroughArgs, + ]; + console.log( + `[live:media] run ${plan.suite.id}: ${plan.suite.testFile} providers=${plan.providers.join(",")}`, + ); + + const child = spawnPnpmRunner({ + pnpmArgs: args, + stdio: "inherit", + env, + }); + + return await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("exit", (code, signal) => { + if (signal) { + reject(new Error(`${plan.suite.id} exited via signal ${signal}`)); + return; + } + resolve(code ?? 1); + }); + }); +} + +export async function runCli(argv: string[]): Promise { + const options = parseArgs(argv); + if (options.help) { + printHelp(); + return 0; + } + const plan = buildRunPlan(options); + const runnable = plan.filter((entry) => entry.providers.length > 0); + const skipped = plan.filter((entry) => entry.providers.length === 0); + + for (const entry of skipped) { + console.log( + `[live:media] skip ${entry.suite.id}: ${entry.skippedReason ?? "no providers selected"}`, + ); + } + if (runnable.length === 0) { + console.log("[live:media] nothing to run"); + return 0; + } + + for (const entry of runnable) { + const exitCode = await runSuite({ + plan: entry, + quietArgs: options.quietArgs, + passthroughArgs: options.passthroughArgs, + }); + if (exitCode !== 0) { + return exitCode; + } + } + return 0; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + runCli(process.argv.slice(2)) + .then((code) => process.exit(code)) + .catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/src/scripts/test-live-media.test.ts b/src/scripts/test-live-media.test.ts new file mode 100644 index 000000000000..b53b4dcf78c3 --- /dev/null +++ b/src/scripts/test-live-media.test.ts @@ -0,0 +1,73 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const loadShellEnvFallbackMock = vi.fn(); +const collectProviderApiKeysMock = vi.fn((provider: string) => + process.env[`TEST_AUTH_${provider.toUpperCase()}`] ? ["test-key"] : [], +); + +vi.mock("../../src/infra/shell-env.js", () => ({ + loadShellEnvFallback: loadShellEnvFallbackMock, +})); + +vi.mock("../../src/agents/live-auth-keys.js", () => ({ + collectProviderApiKeys: collectProviderApiKeysMock, +})); + +describe("test-live-media", () => { + afterEach(() => { + collectProviderApiKeysMock.mockClear(); + loadShellEnvFallbackMock.mockReset(); + vi.unstubAllEnvs(); + }); + + it("defaults to all suites with auth filtering", async () => { + vi.stubEnv("TEST_AUTH_OPENAI", "1"); + vi.stubEnv("TEST_AUTH_GOOGLE", "1"); + vi.stubEnv("TEST_AUTH_MINIMAX", "1"); + vi.stubEnv("TEST_AUTH_FAL", "1"); + vi.stubEnv("TEST_AUTH_VYDRA", "1"); + + const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); + const plan = buildRunPlan(parseArgs([])); + + expect(plan.map((entry) => entry.suite.id)).toEqual(["image", "music", "video"]); + expect(plan.find((entry) => entry.suite.id === "image")?.providers).toEqual([ + "fal", + "google", + "minimax", + "openai", + "vydra", + ]); + expect(plan.find((entry) => entry.suite.id === "music")?.providers).toEqual([ + "google", + "minimax", + ]); + expect(plan.find((entry) => entry.suite.id === "video")?.providers).toEqual([ + "fal", + "google", + "minimax", + "openai", + "vydra", + ]); + }); + + it("supports suite-specific provider filters without auth narrowing", async () => { + const { buildRunPlan, parseArgs } = await import("../../scripts/test-live-media.ts"); + const plan = buildRunPlan( + parseArgs(["video", "--video-providers", "openai,runway", "--all-providers"]), + ); + + expect(plan).toHaveLength(1); + expect(plan[0]?.suite.id).toBe("video"); + expect(plan[0]?.providers).toEqual(["openai", "runway"]); + }); + + it("forwards quiet flags separately from passthrough args", async () => { + const { parseArgs } = await import("../../scripts/test-live-media.ts"); + const options = parseArgs(["image", "--quiet", "--reporter", "dot"]); + + expect(options.suites).toEqual(["image"]); + expect(options.quietArgs).toEqual(["--quiet"]); + expect(options.passthroughArgs).toEqual(["--reporter", "dot"]); + }); +});