diff --git a/qa/scenarios/media/audio-proxy-and-limit-handling.yaml b/qa/scenarios/media/audio-proxy-and-limit-handling.yaml index 8c29de12c0b5..279d34874fc9 100644 --- a/qa/scenarios/media/audio-proxy-and-limit-handling.yaml +++ b/qa/scenarios/media/audio-proxy-and-limit-handling.yaml @@ -23,11 +23,11 @@ scenario: - src/media-understanding/attachments.cache.ts - src/infra/net/proxy-fetch.ts - extensions/openai/media-understanding-provider.ts - - test/e2e/qa-lab/media/audio-proxy-and-limit-handling.product.test.ts + - test/e2e/qa-lab/media/audio-proxy-and-limit-handling.e2e.test.ts regressionRefs: - https://github.com/openclaw/openclaw/pull/73817 - https://github.com/openclaw/openclaw/pull/97280 execution: kind: vitest - path: test/e2e/qa-lab/media/audio-proxy-and-limit-handling.product.test.ts + path: test/e2e/qa-lab/media/audio-proxy-and-limit-handling.e2e.test.ts summary: Run the apply-to-provider audio path against loopback proxy and provider servers, then prove maxBytes rejection performs no network IO. diff --git a/qa/scenarios/media/media-audio-selection.yaml b/qa/scenarios/media/media-audio-selection.yaml new file mode 100644 index 000000000000..94e1392e0416 --- /dev/null +++ b/qa/scenarios/media/media-audio-selection.yaml @@ -0,0 +1,27 @@ +title: Media audio selection and STT fallback + +scenario: + id: media-audio-selection + surface: media-understanding + coverage: + primary: + - media.audio-attachment-selection + - media.batch-stt-provider-and-cli-fallback + objective: Verify audio attachment policy and ordered provider-to-CLI transcription fallback through the media-understanding apply boundary. + successCriteria: + - Mixed attachments select only eligible audio under first/all modes, last preference, and configured caps. + - Multiple selected audio transcripts preserve deterministic selected-attachment order in inbound context. + - A failed provider attempt falls through to an actual temporary executable without mocking process execution. + docsRefs: + - docs/nodes/media-understanding.md + - docs/help/testing.md + codeRefs: + - src/media-understanding/apply.ts + - src/media-understanding/attachments.select.ts + - src/media-understanding/runner.ts + - src/media-understanding/runner.entries.ts + - test/e2e/qa-lab/media/media-audio-selection.e2e.test.ts + execution: + kind: vitest + path: test/e2e/qa-lab/media/media-audio-selection.e2e.test.ts + summary: Run product-boundary audio selection and real provider-to-CLI fallback proof. diff --git a/qa/scenarios/media/media-reference-intake.yaml b/qa/scenarios/media/media-reference-intake.yaml index b371d54c1471..6d6f002b1ddb 100644 --- a/qa/scenarios/media/media-reference-intake.yaml +++ b/qa/scenarios/media/media-reference-intake.yaml @@ -22,8 +22,8 @@ scenario: - src/media/fetch.ts - src/media/store.ts - packages/media-core/src/mime.ts - - test/e2e/qa-lab/media/media-reference-intake.product.test.ts + - test/e2e/qa-lab/media/media-reference-intake.e2e.test.ts execution: kind: vitest - path: test/e2e/qa-lab/media/media-reference-intake.product.test.ts + path: test/e2e/qa-lab/media/media-reference-intake.e2e.test.ts summary: Load plain-path, file URL, guarded loopback HTTP, and media-store references and verify byte-sniffed MIME and kind. diff --git a/qa/scenarios/media/pdf-document-extraction-dispatch.yaml b/qa/scenarios/media/pdf-document-extraction-dispatch.yaml index 74542e2790a2..dff289c8fb01 100644 --- a/qa/scenarios/media/pdf-document-extraction-dispatch.yaml +++ b/qa/scenarios/media/pdf-document-extraction-dispatch.yaml @@ -21,8 +21,8 @@ scenario: - src/media/document-extractors.runtime.ts - src/plugins/document-extractors.runtime.ts - extensions/document-extract/document-extractor.ts - - test/e2e/qa-lab/media/pdf-document-extraction-dispatch.product.test.ts + - test/e2e/qa-lab/media/pdf-document-extraction-dispatch.e2e.test.ts execution: kind: vitest - path: test/e2e/qa-lab/media/pdf-document-extraction-dispatch.product.test.ts + path: test/e2e/qa-lab/media/pdf-document-extraction-dispatch.e2e.test.ts summary: Run a parseable inline PDF through bundled extractor discovery, real clawpdf extraction, and the disabled-plugin failure path. diff --git a/qa/scenarios/media/vision-routing-core.yaml b/qa/scenarios/media/vision-routing-core.yaml index da6350e5ff4e..97f06762da02 100644 --- a/qa/scenarios/media/vision-routing-core.yaml +++ b/qa/scenarios/media/vision-routing-core.yaml @@ -20,8 +20,8 @@ scenario: - src/media-understanding/runner.ts - src/auto-reply/media-note.ts - src/auto-reply/reply/current-turn-images.ts - - test/e2e/qa-lab/media/vision-routing-core.product.test.ts + - test/e2e/qa-lab/media/vision-routing-core.e2e.test.ts execution: kind: vitest - path: test/e2e/qa-lab/media/vision-routing-core.product.test.ts + path: test/e2e/qa-lab/media/vision-routing-core.e2e.test.ts summary: Exercise native vision bypass and ordered provider fallback through apply, summary projection, and current-turn image resolution. diff --git a/test/e2e/qa-lab/media/audio-proxy-and-limit-handling.product.test.ts b/test/e2e/qa-lab/media/audio-proxy-and-limit-handling.e2e.test.ts similarity index 100% rename from test/e2e/qa-lab/media/audio-proxy-and-limit-handling.product.test.ts rename to test/e2e/qa-lab/media/audio-proxy-and-limit-handling.e2e.test.ts diff --git a/test/e2e/qa-lab/media/media-audio-selection.e2e.test.ts b/test/e2e/qa-lab/media/media-audio-selection.e2e.test.ts new file mode 100644 index 000000000000..2052f86ad643 --- /dev/null +++ b/test/e2e/qa-lab/media/media-audio-selection.e2e.test.ts @@ -0,0 +1,247 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { MsgContext } from "../../../../src/auto-reply/templating.js"; +import type { OpenClawConfig } from "../../../../src/config/types.js"; +import { applyMediaUnderstanding } from "../../../../src/media-understanding/apply.js"; +import { MIN_AUDIO_FILE_BYTES } from "../../../../src/media-understanding/defaults.constants.js"; +import type { MediaUnderstandingProvider } from "../../../../src/media-understanding/types.js"; +import { useAutoCleanupTempDirTracker } from "../../../helpers/temp-dir.js"; + +vi.mock("../../../../src/plugins/capability-provider-runtime.js", () => ({ + resolvePluginCapabilityProviders: () => [], +})); + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +async function writeAudioFixture(dir: string, name: string, fill: number): Promise { + const filePath = path.join(dir, name); + await fs.writeFile(filePath, Buffer.alloc(MIN_AUDIO_FILE_BYTES + 1, fill)); + return filePath; +} + +function createSelectionConfig( + attachments: NonNullable< + NonNullable["media"]>["audio"] + >["attachments"], +): OpenClawConfig { + return { + models: { + providers: { + "qa-stt": { + apiKey: "qa-test-key", // pragma: allowlist secret + baseUrl: "https://qa.invalid", + models: [], + }, + }, + }, + tools: { + media: { + models: [{ provider: "qa-stt", model: "fixture-stt", capabilities: ["audio"] }], + audio: { + enabled: true, + attachments, + }, + }, + }, + }; +} + +function createAudioProvider(seenFileNames: string[]): MediaUnderstandingProvider { + return { + id: "qa-stt", + capabilities: ["audio"], + resolveAuth: () => ({ kind: "none" as const, source: "qa-product-fixture" }), + transcribeAudio: async (request: { fileName: string }) => { + seenFileNames.push(request.fileName); + return { text: `transcript:${request.fileName}`, model: "fixture-stt" }; + }, + }; +} + +async function writePortableTranscriber(dir: string): Promise<{ + command: string; + markerPath: string; +}> { + const markerPath = path.join(dir, "cli-ran.txt"); + const scriptPath = path.join(dir, "qa-transcriber.cjs"); + await fs.writeFile( + scriptPath, + [ + "#!/usr/bin/env node", + 'const fs = require("node:fs");', + 'const path = require("node:path");', + "const [markerPath, mediaPath] = process.argv.slice(2);", + "if (!markerPath || !mediaPath) process.exit(2);", + "fs.writeFileSync(markerPath, mediaPath);", + "process.stdout.write(`cli transcript:${path.basename(mediaPath)}\\n`);", + "", + ].join("\n"), + { mode: 0o755 }, + ); + if (process.platform !== "win32") { + return { command: scriptPath, markerPath }; + } + + const command = path.join(dir, "qa-transcriber.cmd"); + await fs.writeFile(command, `@echo off\r\n"${process.execPath}" "${scriptPath}" %*\r\n`, "utf8"); + return { command, markerPath }; +} + +describe("QA media audio selection product proof", () => { + it("applies mixed-attachment preference, mode, caps, and transcript ordering", async () => { + const dir = tempDirs.make("openclaw-qa-media-audio-selection-"); + const firstAudio = await writeAudioFixture(dir, "first.ogg", 0x11); + const secondAudio = await writeAudioFixture(dir, "second.ogg", 0x22); + const thirdAudio = await writeAudioFixture(dir, "third.ogg", 0x33); + const preflightAudio = await writeAudioFixture(dir, "preflight.ogg", 0x44); + const imagePath = path.join(dir, "ignored.png"); + await fs.writeFile(imagePath, Buffer.from("not-an-audio-attachment")); + const media = [ + { path: firstAudio, contentType: "audio/ogg" }, + { path: imagePath, contentType: "image/png" }, + { path: secondAudio, contentType: "audio/ogg" }, + { path: thirdAudio, contentType: "audio/ogg" }, + { path: preflightAudio, contentType: "audio/ogg", transcribed: true }, + ]; + + const firstModeCalls: string[] = []; + const firstModeCtx: MsgContext = { Body: "", media }; + const firstModeResult = await applyMediaUnderstanding({ + ctx: firstModeCtx, + cfg: createSelectionConfig({ mode: "first", prefer: "last", maxAttachments: 3 }), + providers: { "qa-stt": createAudioProvider(firstModeCalls) }, + processingMode: "audio-only", + workspaceDir: dir, + }); + + expect(firstModeCalls).toEqual(["third.ogg"]); + expect(firstModeResult.outputs.map((output) => output.attachmentIndex)).toEqual([3]); + expect(firstModeCtx.Transcript).toBe("transcript:third.ogg"); + + const allModeCalls: string[] = []; + const allModeCtx: MsgContext = { Body: "", media }; + const allModeResult = await applyMediaUnderstanding({ + ctx: allModeCtx, + cfg: createSelectionConfig({ mode: "all", prefer: "last", maxAttachments: 2 }), + providers: { "qa-stt": createAudioProvider(allModeCalls) }, + processingMode: "audio-only", + workspaceDir: dir, + }); + + expect(allModeCalls).toEqual(["third.ogg", "second.ogg"]); + expect(allModeResult.outputs.map((output) => output.attachmentIndex)).toEqual([3, 2]); + expect( + allModeResult.decisions.find((decision) => decision.capability === "audio"), + ).toMatchObject({ + outcome: "success", + attachments: [{ attachmentIndex: 3 }, { attachmentIndex: 2 }], + }); + expect(allModeCtx.Transcript).toBe( + "Audio 1:\ntranscript:third.ogg\n\nAudio 2:\ntranscript:second.ogg", + ); + expect(allModeCtx.Body).toBe( + [ + "[Audio 1/2]\nTranscript:\ntranscript:third.ogg", + "[Audio 2/2]\nTranscript:\ntranscript:second.ogg", + ].join("\n\n"), + ); + }); + + it("falls through from a failed provider to an actual temporary CLI executable", async () => { + const dir = tempDirs.make("openclaw-qa-media-audio-fallback-"); + const audioPath = await writeAudioFixture(dir, "fallback.wav", 0x55); + const executable = await writePortableTranscriber(dir); + let providerAttempts = 0; + const ctx: MsgContext = { + Body: "", + media: [{ path: audioPath, contentType: "audio/wav" }], + }; + const cfg: OpenClawConfig = { + models: { + providers: { + "qa-stt": { + apiKey: "qa-test-key", // pragma: allowlist secret + baseUrl: "https://qa.invalid", + models: [], + }, + }, + }, + tools: { + media: { + models: [ + { provider: "qa-stt", model: "fixture-stt", capabilities: ["audio"] }, + { + type: "cli", + command: executable.command, + args: [executable.markerPath, "{{MediaPath}}"], + capabilities: ["audio"], + }, + ], + audio: { enabled: true }, + }, + }, + }; + + const result = await applyMediaUnderstanding({ + ctx, + cfg, + providers: { + "qa-stt": { + id: "qa-stt", + capabilities: ["audio"], + resolveAuth: () => ({ kind: "none", source: "qa-product-fixture" }), + transcribeAudio: async () => { + providerAttempts += 1; + throw new Error("intentional provider failure"); + }, + }, + }, + processingMode: "audio-only", + workspaceDir: dir, + }); + + expect(providerAttempts).toBe(1); + expect(await fs.readFile(executable.markerPath, "utf8")).toBe(await fs.realpath(audioPath)); + expect(ctx.Transcript).toBe("cli transcript:fallback.wav"); + const cliProvider = path.parse(executable.command).name; + expect(result.outputs).toEqual([ + expect.objectContaining({ + attachmentIndex: 0, + kind: "audio.transcription", + provider: cliProvider, + model: executable.command, + text: "cli transcript:fallback.wav", + }), + ]); + expect(result.decisions.find((decision) => decision.capability === "audio")).toMatchObject({ + outcome: "success", + attachments: [ + { + attachmentIndex: 0, + attempts: [ + { + type: "provider", + provider: "qa-stt", + model: "fixture-stt", + outcome: "failed", + reason: expect.stringContaining("intentional provider failure"), + }, + { + type: "cli", + provider: cliProvider, + model: executable.command, + outcome: "success", + }, + ], + chosen: { + type: "cli", + provider: cliProvider, + model: executable.command, + outcome: "success", + }, + }, + ], + }); + }); +}); diff --git a/test/e2e/qa-lab/media/media-reference-intake.product.test.ts b/test/e2e/qa-lab/media/media-reference-intake.e2e.test.ts similarity index 100% rename from test/e2e/qa-lab/media/media-reference-intake.product.test.ts rename to test/e2e/qa-lab/media/media-reference-intake.e2e.test.ts diff --git a/test/e2e/qa-lab/media/pdf-document-extraction-dispatch.product.test.ts b/test/e2e/qa-lab/media/pdf-document-extraction-dispatch.e2e.test.ts similarity index 100% rename from test/e2e/qa-lab/media/pdf-document-extraction-dispatch.product.test.ts rename to test/e2e/qa-lab/media/pdf-document-extraction-dispatch.e2e.test.ts diff --git a/test/e2e/qa-lab/media/vision-routing-core.product.test.ts b/test/e2e/qa-lab/media/vision-routing-core.e2e.test.ts similarity index 100% rename from test/e2e/qa-lab/media/vision-routing-core.product.test.ts rename to test/e2e/qa-lab/media/vision-routing-core.e2e.test.ts