ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-25 20:05:46 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(feishu): preserve captions after inferred voice degrades (#128771)

Browse Source
This commit is contained in:
Peter Steinberger
2026-08-24 10:38:26 -07:00
committed by GitHub
parent 7d08e40e2f
commit 86d2c35e2e
4 changed files with 231 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/feishu/src/delivery-trace.test.ts
+150
View File
@@ -33,6 +33,7 @@ type FeishuTraceState = {
reactionCount: number;
cardCount: number;
setupCount: number;
loadedMedia: { buffer: Buffer; fileName: string; contentType: string } | null;
omitNextMessageReceipt: boolean;
wireFaults: Array<{ fault: "rate-limit"; retryAfterMs: number }>;
};
@@ -47,6 +48,7 @@ const traceState = vi.hoisted(
reactionCount: 0,
cardCount: 0,
setupCount: 0,
loadedMedia: null,
omitNextMessageReceipt: false,
wireFaults: [],
}),
@@ -89,6 +91,14 @@ vi.mock("./runtime.js", async () => {
const textChunking = await import("openclaw/plugin-sdk/text-chunking");
const markdownTables = await import("openclaw/plugin-sdk/markdown-table-runtime");
const runtime = {
media: {
loadWebMedia: async () => {
if (!traceState.loadedMedia) {
throw new Error("trace media not initialized");
}
return traceState.loadedMedia;
},
},
channel: {
text: {
resolveTextChunkLimit: replyChunking.resolveTextChunkLimit,
@@ -127,6 +137,7 @@ vi.mock("./streaming-card.js", async (importOriginal) => {
});
let createFeishuReplyDispatcher: CreateFeishuReplyDispatcher;
let feishuOutbound: typeof import("./outbound.js").feishuOutbound;
let streamingStartBackoffUntilByAccount: StreamingStartBackoffMap;
beforeAll(async () => {
@@ -134,6 +145,7 @@ beforeAll(async () => {
// Reload only after this file's hoisted mocks are registered.
vi.resetModules();
({ createFeishuReplyDispatcher } = await import("./reply-dispatcher.js"));
({ feishuOutbound } = await import("./outbound.js"));
({ streamingStartBackoffUntilByAccount } = await import("./reply-dispatcher-state.js"));
});
@@ -150,6 +162,7 @@ afterEach(() => {
traceState.larkClient = null;
traceState.cardKitFetch = null;
traceState.omitNextMessageReceipt = false;
traceState.loadedMedia = null;
traceState.wireFaults = [];
streamingStartBackoffUntilByAccount.clear();
});
@@ -181,6 +194,16 @@ function createRecordingLarkClient() {
};
return {
im: {
file: {
create: (args: { data: { file_name: string; file_type: string } }) => {
traceState.recordWireCall({
method: "im.file.create",
payload: { file_name: args.data.file_name, file_type: args.data.file_type },
result: { file_key: "file-trace" },
});
return Promise.resolve({ file_key: "file-trace" });
},
},
message: {
create: (args: {
params: { receive_id_type: string };
@@ -438,6 +461,133 @@ describe("feishu delivery trace goldens", () => {
expect(streamingStartBackoffUntilByAccount.has("main")).toBe(false);
});
it.each(["outbound", "dispatcher"] as const)(
"preserves caption visibility when %s voice-looking media resolves to a PDF",
async (surface) => {
const acceptedMessageIds: string[] = [];
const events = await runDeliveryTraceScenario({
scenario: {
name: `feishu-${surface}-authoritative-voice-media`,
steps: [{ kind: "final", text: "Critical caption must remain visible" }],
},
setup: (recorder) => {
setupFeishuTrace(recorder, "final-only");
traceState.loadedMedia = {
buffer: Buffer.from("%PDF-1.7 trace document"),
fileName: "report.pdf",
contentType: "application/pdf",
};
traceState.account = {
...makeTraceAccount("final-only"),
config: FeishuConfigSchema.parse({ renderMode: "raw", streaming: { mode: "off" } }),
};
return async (step) => {
if (step.kind !== "final") {
throw new Error("unexpected authoritative-media trace step");
}
if (surface === "outbound") {
await feishuOutbound.sendMedia?.({
cfg: {},
to: "oc-trace-chat",
text: step.text ?? "",
mediaUrl: "https://example.com/download.ogg",
onDeliveryResult: async (result) => {
acceptedMessageIds.push(result.messageId);
},
});
return;
}
const dispatcher = createFeishuReplyDispatcher({
cfg: {} as never,
agentId: "agent",
runtime: {} as never,
chatId: "oc-trace-chat",
sendTarget: "oc-trace-chat",
});
await dispatcher.delivery.deliver(
{ text: step.text, mediaUrl: "https://example.com/download.ogg" },
{ kind: "final" },
);
};
},
});
const sends = events.filter((event) => event.kind === "im.message.create");
expect(events.find((event) => event.kind === "im.file.create")).toMatchObject({
data: { payload: { file_name: "report.pdf", file_type: "pdf" } },
});
expect(sends).toHaveLength(2);
expect(sends).toMatchObject([
{ data: { payload: { msg_type: "file" } } },
{
data: {
payload: {
msg_type: "post",
content: {
zh_cn: { content: [[{ tag: "md", text: "Critical caption must remain visible" }]] },
},
},
},
},
]);
if (surface === "outbound") {
expect(acceptedMessageIds).toEqual(["om-1", "om-2"]);
}
},
);
it.each([
{ label: "inferred native voice", audioAsVoice: false },
{ label: "explicit voice intent", audioAsVoice: true },
])("does not repeat visible TTS text when $label resolves to a PDF", async (voice) => {
const events = await runDeliveryTraceScenario({
scenario: {
name: `feishu-visible-tts-${voice.audioAsVoice ? "explicit" : "inferred"}`,
steps: [{ kind: "final", text: "Already-visible TTS answer" }],
},
setup: (recorder) => {
setupFeishuTrace(recorder, "final-only");
traceState.loadedMedia = {
buffer: Buffer.from("%PDF-1.7 trace document"),
fileName: "report.pdf",
contentType: "application/pdf",
};
traceState.account = {
...makeTraceAccount("final-only"),
config: FeishuConfigSchema.parse({ renderMode: "raw", streaming: { mode: "off" } }),
};
const dispatcher = createFeishuReplyDispatcher({
cfg: {} as never,
agentId: "agent",
runtime: {} as never,
chatId: "oc-trace-chat",
sendTarget: "oc-trace-chat",
});
return async (step) => {
if (step.kind !== "final") {
throw new Error("unexpected visible-TTS trace step");
}
await dispatcher.delivery.deliver(
{
text: step.text,
mediaUrl: "https://example.com/download.ogg",
...(voice.audioAsVoice ? { audioAsVoice: true } : {}),
ttsSupplement: {
spokenText: step.text ?? "",
visibleTextAlreadyDelivered: true,
},
},
{ kind: "final" },
);
};
},
});
const sends = events.filter((event) => event.kind === "im.message.create");
expect(sends).toHaveLength(1);
expect(sends[0]).toMatchObject({ data: { payload: { msg_type: "file" } } });
});
for (const scenarioName of FEISHU_TRACE_SCENARIOS) {
it(`records ${scenarioName}`, async () => {
const events = await runDeliveryTraceScenario({
extensions/feishu/src/media-upload-contract.test.ts
+77
View File
@@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => ({
imageCreate: vi.fn(),
messageCreate: vi.fn(),
runFfmpeg: vi.fn(),
runFfprobe: vi.fn(),
}));
vi.mock("./client.js", () => ({ createFeishuClient: mocks.createClient }));
@@ -27,6 +28,7 @@ vi.mock("./runtime.js", () => ({
vi.mock("openclaw/plugin-sdk/media-runtime", async (importOriginal) => ({
...(await importOriginal<typeof import("openclaw/plugin-sdk/media-runtime")>()),
runFfmpeg: mocks.runFfmpeg,
runFfprobe: mocks.runFfprobe,
}));
let sendMediaFeishu: typeof import("./media.js").sendMediaFeishu;
@@ -83,6 +85,81 @@ describe("Feishu upload contracts", () => {
mocks.fileCreate.mockResolvedValue({ code: 0, data: { file_key: "file_1" } });
mocks.imageCreate.mockResolvedValue({ code: 0, data: { image_key: "image_1" } });
mocks.messageCreate.mockResolvedValue({ code: 0, data: { message_id: "message_1" } });
mocks.runFfprobe.mockResolvedValue("1.25\n");
});
it.each([
{
label: "a voice-looking URL resolves to a PDF",
mediaUrl: "https://example.com/download.ogg",
fileName: "report.pdf",
contentType: "application/pdf",
buffer: Buffer.from("%PDF-1.7 document"),
messageType: "file",
degraded: true,
},
{
label: "a voice-looking URL resolves to an image",
mediaUrl: "https://example.com/download.opus",
fileName: "photo.png",
contentType: "image/png",
buffer: pngImage,
messageType: "image",
degraded: true,
},
{
label: "a voice-looking URL resolves to a video",
mediaUrl: "https://example.com/download.ogg",
fileName: "clip.mp4",
contentType: "video/mp4",
buffer: Buffer.from("video bytes"),
messageType: "media",
degraded: true,
},
{
label: "actual native voice remains audio",
mediaUrl: "https://example.com/download.ogg",
fileName: "voice.ogg",
contentType: "audio/ogg",
buffer: Buffer.from("voice bytes"),
messageType: "audio",
degraded: false,
},
{
label: "an ordinary PDF was never treated as voice",
mediaUrl: "https://example.com/report.pdf",
fileName: "report.pdf",
contentType: "application/pdf",
buffer: Buffer.from("%PDF-1.7 document"),
messageType: "file",
degraded: false,
},
{
label: "explicit voice intent still reports a PDF degradation",
mediaUrl: "https://example.com/report.pdf",
fileName: "report.pdf",
contentType: "application/pdf",
buffer: Buffer.from("%PDF-1.7 document"),
audioAsVoice: true,
messageType: "file",
degraded: true,
},
])("reconciles voice visibility after loading when $label", async (media) => {
mocks.loadWebMedia.mockResolvedValueOnce({
buffer: media.buffer,
fileName: media.fileName,
contentType: media.contentType,
});
const result = await sendMediaFeishu({
cfg: emptyConfig,
to: "user:ou_target",
mediaUrl: media.mediaUrl,
...(media.audioAsVoice ? { audioAsVoice: true } : {}),
});
expect(mockCallData(mocks.messageCreate).msg_type).toBe(media.messageType);
expect(result.voiceIntentDegradedToFile).toBe(media.degraded ? true : undefined);
});
it.each([
extensions/feishu/src/media.ts
+3 -1
View File
@@ -1057,7 +1057,9 @@ export async function sendMediaFeishu(params: {
: await runBeforeFeishuMessageDispatch(() =>
resolveFeishuOutboundMediaKind({ buffer, fileName: name, contentType }),
);
const voiceIntentDegradedToFile = audioAsVoice === true && routing.msgType !== "audio";
const voiceIntentDegradedToFile =
routing.msgType !== "audio" &&
shouldSuppressFeishuTextForVoiceMedia({ mediaUrl, audioAsVoice });
await runBeforeFeishuMessageDispatch(() =>
assertFeishuUploadWithinEnvelope({ buffer, mediaMaxBytes, msgType: routing.msgType }),
extensions/feishu/src/reply-dispatcher.ts
+1 -1
View File
@@ -1496,7 +1496,7 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP
if (hasMedia) {
await collectMediaDelivery(
payload,
hasVoiceMedia && hasText ? { fallbackText: text } : undefined,
!ttsTextAlreadyVisible && hasVoiceMedia && hasText ? { fallbackText: text } : undefined,
);
}
const deliveredContent = hasVoiceMedia ? (deliveredResults.at(-1)?.content ?? text) : text;