diff --git a/extensions/feishu/src/delivery-trace.test.ts b/extensions/feishu/src/delivery-trace.test.ts index a74724ed3262..97fc273a4eb1 100644 --- a/extensions/feishu/src/delivery-trace.test.ts +++ b/extensions/feishu/src/delivery-trace.test.ts @@ -33,6 +33,7 @@ type FeishuTraceState = { reactionCount: number; cardCount: number; setupCount: number; + loadedMedia: { buffer: Buffer; fileName: string; contentType: string } | null; omitNextMessageReceipt: boolean; wireFaults: Array<{ fault: "rate-limit"; retryAfterMs: number }>; }; @@ -47,6 +48,7 @@ const traceState = vi.hoisted( reactionCount: 0, cardCount: 0, setupCount: 0, + loadedMedia: null, omitNextMessageReceipt: false, wireFaults: [], }), @@ -89,6 +91,14 @@ vi.mock("./runtime.js", async () => { const textChunking = await import("openclaw/plugin-sdk/text-chunking"); const markdownTables = await import("openclaw/plugin-sdk/markdown-table-runtime"); const runtime = { + media: { + loadWebMedia: async () => { + if (!traceState.loadedMedia) { + throw new Error("trace media not initialized"); + } + return traceState.loadedMedia; + }, + }, channel: { text: { resolveTextChunkLimit: replyChunking.resolveTextChunkLimit, @@ -127,6 +137,7 @@ vi.mock("./streaming-card.js", async (importOriginal) => { }); let createFeishuReplyDispatcher: CreateFeishuReplyDispatcher; +let feishuOutbound: typeof import("./outbound.js").feishuOutbound; let streamingStartBackoffUntilByAccount: StreamingStartBackoffMap; beforeAll(async () => { @@ -134,6 +145,7 @@ beforeAll(async () => { // Reload only after this file's hoisted mocks are registered. vi.resetModules(); ({ createFeishuReplyDispatcher } = await import("./reply-dispatcher.js")); + ({ feishuOutbound } = await import("./outbound.js")); ({ streamingStartBackoffUntilByAccount } = await import("./reply-dispatcher-state.js")); }); @@ -150,6 +162,7 @@ afterEach(() => { traceState.larkClient = null; traceState.cardKitFetch = null; traceState.omitNextMessageReceipt = false; + traceState.loadedMedia = null; traceState.wireFaults = []; streamingStartBackoffUntilByAccount.clear(); }); @@ -181,6 +194,16 @@ function createRecordingLarkClient() { }; return { im: { + file: { + create: (args: { data: { file_name: string; file_type: string } }) => { + traceState.recordWireCall({ + method: "im.file.create", + payload: { file_name: args.data.file_name, file_type: args.data.file_type }, + result: { file_key: "file-trace" }, + }); + return Promise.resolve({ file_key: "file-trace" }); + }, + }, message: { create: (args: { params: { receive_id_type: string }; @@ -438,6 +461,133 @@ describe("feishu delivery trace goldens", () => { expect(streamingStartBackoffUntilByAccount.has("main")).toBe(false); }); + it.each(["outbound", "dispatcher"] as const)( + "preserves caption visibility when %s voice-looking media resolves to a PDF", + async (surface) => { + const acceptedMessageIds: string[] = []; + const events = await runDeliveryTraceScenario({ + scenario: { + name: `feishu-${surface}-authoritative-voice-media`, + steps: [{ kind: "final", text: "Critical caption must remain visible" }], + }, + setup: (recorder) => { + setupFeishuTrace(recorder, "final-only"); + traceState.loadedMedia = { + buffer: Buffer.from("%PDF-1.7 trace document"), + fileName: "report.pdf", + contentType: "application/pdf", + }; + traceState.account = { + ...makeTraceAccount("final-only"), + config: FeishuConfigSchema.parse({ renderMode: "raw", streaming: { mode: "off" } }), + }; + return async (step) => { + if (step.kind !== "final") { + throw new Error("unexpected authoritative-media trace step"); + } + if (surface === "outbound") { + await feishuOutbound.sendMedia?.({ + cfg: {}, + to: "oc-trace-chat", + text: step.text ?? "", + mediaUrl: "https://example.com/download.ogg", + onDeliveryResult: async (result) => { + acceptedMessageIds.push(result.messageId); + }, + }); + return; + } + const dispatcher = createFeishuReplyDispatcher({ + cfg: {} as never, + agentId: "agent", + runtime: {} as never, + chatId: "oc-trace-chat", + sendTarget: "oc-trace-chat", + }); + await dispatcher.delivery.deliver( + { text: step.text, mediaUrl: "https://example.com/download.ogg" }, + { kind: "final" }, + ); + }; + }, + }); + const sends = events.filter((event) => event.kind === "im.message.create"); + + expect(events.find((event) => event.kind === "im.file.create")).toMatchObject({ + data: { payload: { file_name: "report.pdf", file_type: "pdf" } }, + }); + expect(sends).toHaveLength(2); + expect(sends).toMatchObject([ + { data: { payload: { msg_type: "file" } } }, + { + data: { + payload: { + msg_type: "post", + content: { + zh_cn: { content: [[{ tag: "md", text: "Critical caption must remain visible" }]] }, + }, + }, + }, + }, + ]); + if (surface === "outbound") { + expect(acceptedMessageIds).toEqual(["om-1", "om-2"]); + } + }, + ); + + it.each([ + { label: "inferred native voice", audioAsVoice: false }, + { label: "explicit voice intent", audioAsVoice: true }, + ])("does not repeat visible TTS text when $label resolves to a PDF", async (voice) => { + const events = await runDeliveryTraceScenario({ + scenario: { + name: `feishu-visible-tts-${voice.audioAsVoice ? "explicit" : "inferred"}`, + steps: [{ kind: "final", text: "Already-visible TTS answer" }], + }, + setup: (recorder) => { + setupFeishuTrace(recorder, "final-only"); + traceState.loadedMedia = { + buffer: Buffer.from("%PDF-1.7 trace document"), + fileName: "report.pdf", + contentType: "application/pdf", + }; + traceState.account = { + ...makeTraceAccount("final-only"), + config: FeishuConfigSchema.parse({ renderMode: "raw", streaming: { mode: "off" } }), + }; + const dispatcher = createFeishuReplyDispatcher({ + cfg: {} as never, + agentId: "agent", + runtime: {} as never, + chatId: "oc-trace-chat", + sendTarget: "oc-trace-chat", + }); + return async (step) => { + if (step.kind !== "final") { + throw new Error("unexpected visible-TTS trace step"); + } + await dispatcher.delivery.deliver( + { + text: step.text, + mediaUrl: "https://example.com/download.ogg", + ...(voice.audioAsVoice ? { audioAsVoice: true } : {}), + ttsSupplement: { + spokenText: step.text ?? "", + visibleTextAlreadyDelivered: true, + }, + }, + { kind: "final" }, + ); + }; + }, + }); + + const sends = events.filter((event) => event.kind === "im.message.create"); + expect(sends).toHaveLength(1); + expect(sends[0]).toMatchObject({ data: { payload: { msg_type: "file" } } }); + }); + for (const scenarioName of FEISHU_TRACE_SCENARIOS) { it(`records ${scenarioName}`, async () => { const events = await runDeliveryTraceScenario({ diff --git a/extensions/feishu/src/media-upload-contract.test.ts b/extensions/feishu/src/media-upload-contract.test.ts index b519359e4466..c8dd3df0e675 100644 --- a/extensions/feishu/src/media-upload-contract.test.ts +++ b/extensions/feishu/src/media-upload-contract.test.ts @@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => ({ imageCreate: vi.fn(), messageCreate: vi.fn(), runFfmpeg: vi.fn(), + runFfprobe: vi.fn(), })); vi.mock("./client.js", () => ({ createFeishuClient: mocks.createClient })); @@ -27,6 +28,7 @@ vi.mock("./runtime.js", () => ({ vi.mock("openclaw/plugin-sdk/media-runtime", async (importOriginal) => ({ ...(await importOriginal()), runFfmpeg: mocks.runFfmpeg, + runFfprobe: mocks.runFfprobe, })); let sendMediaFeishu: typeof import("./media.js").sendMediaFeishu; @@ -83,6 +85,81 @@ describe("Feishu upload contracts", () => { mocks.fileCreate.mockResolvedValue({ code: 0, data: { file_key: "file_1" } }); mocks.imageCreate.mockResolvedValue({ code: 0, data: { image_key: "image_1" } }); mocks.messageCreate.mockResolvedValue({ code: 0, data: { message_id: "message_1" } }); + mocks.runFfprobe.mockResolvedValue("1.25\n"); + }); + + it.each([ + { + label: "a voice-looking URL resolves to a PDF", + mediaUrl: "https://example.com/download.ogg", + fileName: "report.pdf", + contentType: "application/pdf", + buffer: Buffer.from("%PDF-1.7 document"), + messageType: "file", + degraded: true, + }, + { + label: "a voice-looking URL resolves to an image", + mediaUrl: "https://example.com/download.opus", + fileName: "photo.png", + contentType: "image/png", + buffer: pngImage, + messageType: "image", + degraded: true, + }, + { + label: "a voice-looking URL resolves to a video", + mediaUrl: "https://example.com/download.ogg", + fileName: "clip.mp4", + contentType: "video/mp4", + buffer: Buffer.from("video bytes"), + messageType: "media", + degraded: true, + }, + { + label: "actual native voice remains audio", + mediaUrl: "https://example.com/download.ogg", + fileName: "voice.ogg", + contentType: "audio/ogg", + buffer: Buffer.from("voice bytes"), + messageType: "audio", + degraded: false, + }, + { + label: "an ordinary PDF was never treated as voice", + mediaUrl: "https://example.com/report.pdf", + fileName: "report.pdf", + contentType: "application/pdf", + buffer: Buffer.from("%PDF-1.7 document"), + messageType: "file", + degraded: false, + }, + { + label: "explicit voice intent still reports a PDF degradation", + mediaUrl: "https://example.com/report.pdf", + fileName: "report.pdf", + contentType: "application/pdf", + buffer: Buffer.from("%PDF-1.7 document"), + audioAsVoice: true, + messageType: "file", + degraded: true, + }, + ])("reconciles voice visibility after loading when $label", async (media) => { + mocks.loadWebMedia.mockResolvedValueOnce({ + buffer: media.buffer, + fileName: media.fileName, + contentType: media.contentType, + }); + + const result = await sendMediaFeishu({ + cfg: emptyConfig, + to: "user:ou_target", + mediaUrl: media.mediaUrl, + ...(media.audioAsVoice ? { audioAsVoice: true } : {}), + }); + + expect(mockCallData(mocks.messageCreate).msg_type).toBe(media.messageType); + expect(result.voiceIntentDegradedToFile).toBe(media.degraded ? true : undefined); }); it.each([ diff --git a/extensions/feishu/src/media.ts b/extensions/feishu/src/media.ts index 3e43d8e4af4e..7ded2e0416a4 100644 --- a/extensions/feishu/src/media.ts +++ b/extensions/feishu/src/media.ts @@ -1057,7 +1057,9 @@ export async function sendMediaFeishu(params: { : await runBeforeFeishuMessageDispatch(() => resolveFeishuOutboundMediaKind({ buffer, fileName: name, contentType }), ); - const voiceIntentDegradedToFile = audioAsVoice === true && routing.msgType !== "audio"; + const voiceIntentDegradedToFile = + routing.msgType !== "audio" && + shouldSuppressFeishuTextForVoiceMedia({ mediaUrl, audioAsVoice }); await runBeforeFeishuMessageDispatch(() => assertFeishuUploadWithinEnvelope({ buffer, mediaMaxBytes, msgType: routing.msgType }), diff --git a/extensions/feishu/src/reply-dispatcher.ts b/extensions/feishu/src/reply-dispatcher.ts index b1bc01e76f24..b4c740c58429 100644 --- a/extensions/feishu/src/reply-dispatcher.ts +++ b/extensions/feishu/src/reply-dispatcher.ts @@ -1496,7 +1496,7 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP if (hasMedia) { await collectMediaDelivery( payload, - hasVoiceMedia && hasText ? { fallbackText: text } : undefined, + !ttsTextAlreadyVisible && hasVoiceMedia && hasText ? { fallbackText: text } : undefined, ); } const deliveredContent = hasVoiceMedia ? (deliveredResults.at(-1)?.content ?? text) : text;