From ffcca757eb3a02286fa9482e3a3c94c6280ca8ef Mon Sep 17 00:00:00 2001
From: Peter Steinberger <steipete@gmail.com>
Date: Mon, 3 Aug 2026 07:13:19 -0700
Subject: [PATCH] fix(media): fall back when providers return empty output
 (#118660)

* fix(media): fall back when providers return empty output

* test(media): type the synthetic attachment cache fixture

* test(media): avoid object spread in fallback test cases
---
 src/media-understanding/runner.ts            |   2 +-
 src/media-understanding/runner.video.test.ts | 125 +++++++++++++++++++
 2 files changed, 126 insertions(+), 1 deletion(-)

diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts
index df28188cf9b0..6e8febec4d45 100644
--- a/src/media-understanding/runner.ts
+++ b/src/media-understanding/runner.ts
@@ -782,7 +782,7 @@ async function runAttachmentEntries(params: {
               config: params.config,
               secretOwnerId: candidate.secretOwnerId,
             });
-      if (result) {
+      if (result?.text) {
         const decision = buildModelDecision({ entry, entryType, outcome: "success" });
         if (result.provider) {
           decision.provider = result.provider;
diff --git a/src/media-understanding/runner.video.test.ts b/src/media-understanding/runner.video.test.ts
index 9040472588a1..665c2a435e26 100644
--- a/src/media-understanding/runner.video.test.ts
+++ b/src/media-understanding/runner.video.test.ts
@@ -1,6 +1,10 @@
 // Video runner tests cover provider request wiring, auth/config precedence, and
 // provider output handling for video attachments.
 import { describe, expect, it, vi } from "vitest";
+import {
+  formatAudioTranscripts,
+  formatMediaUnderstandingBody,
+} from "../../packages/media-understanding-common/src/format.js";
 import type { OpenClawConfig } from "../config/types.js";
 import { withTempDir } from "../test-helpers/temp-dir.js";
 import { withEnvAsync } from "../test-utils/env.js";
@@ -465,3 +469,124 @@ describe("runCapability video provider wiring", () => {
     expect(firstCall?.modelApi).toBeUndefined();
   });
 });
+
+describe("runCapability provider output decisions", () => {
+  const outputs = [
+    { label: "empty", text: "" },
+    { label: "whitespace", text: " \t\n" },
+    { label: "usable", text: "  usable primary output  " },
+  ] as const;
+  const cases = (["audio", "video", "image"] as const).flatMap((capability) =>
+    outputs.flatMap((output) =>
+      (output.text.trim() ? [true] : [true, false]).map((configureFallback) => ({
+        capability,
+        configureFallback,
+        label: output.label,
+        text: output.text,
+      })),
+    ),
+  );
+
+  it.each(cases)(
+    "handles $label $capability provider output with fallback=$configureFallback",
+    async ({ capability, configureFallback, text }) => {
+      const extension = capability === "image" ? "png" : capability === "video" ? "mp4" : "wav";
+      const mime = `${capability}/${extension}`;
+      const buffer =
+        capability === "image"
+          ? Buffer.from(
+              "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8Xw8AAuMBg4n8tLwAAAAASUVORK5CYII=",
+              "base64",
+            )
+          : Buffer.alloc(2048, 1);
+      const primary = vi.fn(async () => ({ text, model: "primary-model" }));
+      const fallback = vi.fn(async () => ({
+        text: "usable fallback output",
+        model: "fallback-model",
+      }));
+      const createProvider = (
+        id: string,
+        run: () => Promise<{ text: string; model: string }>,
+      ): MediaUnderstandingProvider => ({
+        id,
+        capabilities: [capability],
+        ...(capability === "audio"
+          ? { transcribeAudio: run }
+          : capability === "video"
+            ? { describeVideo: run }
+            : { describeImage: run }),
+      });
+      const providerIds = ["qa-primary", ...(configureFallback ? ["qa-fallback"] : [])];
+      const cfg = {
+        models: {
+          providers: Object.fromEntries(
+            providerIds.map((provider) => [provider, { apiKey: "test-key", models: [] }]),
+          ),
+        },
+        tools: {
+          media: {
+            models: providerIds.map((provider) => ({
+              provider,
+              model: provider === "qa-primary" ? "primary-model" : "fallback-model",
+              capabilities: [capability],
+            })),
+            [capability]: { enabled: true },
+          },
+        },
+      } as unknown as OpenClawConfig;
+
+      const result = await runCapability({
+        capability,
+        cfg,
+        ctx: { Body: "" },
+        attachments: {
+          getBuffer: async () => ({
+            buffer,
+            mime,
+            fileName: `fixture.${extension}`,
+            size: buffer.length,
+          }),
+        } as unknown as Parameters<typeof runCapability>[0]["attachments"],
+        media: [{ index: 0, kind: capability, mime }],
+        agentDir: "/tmp/openclaw-media-provider-output-test",
+        providerRegistry: new Map<string, MediaUnderstandingProvider>([
+          ["qa-primary", createProvider("qa-primary", primary)],
+          ["qa-fallback", createProvider("qa-fallback", fallback)],
+        ]),
+      });
+
+      const usablePrimary = text.trim();
+      const expectedText = usablePrimary || (configureFallback ? "usable fallback output" : "");
+      const expectedFallbackCalls = !usablePrimary && configureFallback ? 1 : 0;
+      expect(primary).toHaveBeenCalledOnce();
+      expect(fallback).toHaveBeenCalledTimes(expectedFallbackCalls);
+      expect(result.outputs.map((output) => output.text)).toEqual(
+        expectedText ? [expectedText] : [],
+      );
+      expect(result.decision.outcome).toBe(expectedText ? "success" : "skipped");
+
+      const attempts = result.decision.attachments[0]?.attempts.map(
+        ({ provider, outcome, reason }) => ({
+          provider,
+          outcome,
+          ...(reason ? { reason } : {}),
+        }),
+      );
+      expect(attempts).toEqual([
+        usablePrimary
+          ? { provider: "qa-primary", outcome: "success" }
+          : { provider: "qa-primary", outcome: "skipped", reason: "empty output" },
+        ...(expectedFallbackCalls ? [{ provider: "qa-fallback", outcome: "success" }] : []),
+      ]);
+
+      if (capability === "audio") {
+        expect(formatMediaUnderstandingBody({ outputs: result.outputs })).toBe(
+          expectedText ? `[Audio]\nTranscript:\n${expectedText}` : "",
+        );
+        if (expectedText) {
+          expect(formatAudioTranscripts(result.outputs)).toBe(expectedText);
+        }
+      }
+    },
+  );
+});