From fa3eb673cd0f4285a4d182a4997c9bde0633d443 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 10 Jul 2026 06:33:26 +0100 Subject: [PATCH] feat(slack): accept spoken mentions in audio clips (#103416) * feat(slack): support spoken audio mentions * chore: keep release notes in PR body --- docs/channels/slack.md | 20 +- extensions/slack/src/monitor/media.ts | 7 + .../preflight-audio.runtime.ts | 3 + .../message-handler/preflight-audio.test.ts | 179 +++++++++++ .../message-handler/preflight-audio.ts | 148 +++++++++ .../message-handler/prepare-content.ts | 2 + .../monitor/message-handler/prepare.test.ts | 268 +++++++++++++++- .../src/monitor/message-handler/prepare.ts | 296 +++++++++++++----- 8 files changed, 834 insertions(+), 89 deletions(-) create mode 100644 extensions/slack/src/monitor/message-handler/preflight-audio.runtime.ts create mode 100644 extensions/slack/src/monitor/message-handler/preflight-audio.test.ts create mode 100644 extensions/slack/src/monitor/message-handler/preflight-audio.ts diff --git a/docs/channels/slack.md b/docs/channels/slack.md index 7304f82fbb42..2f3d5674a604 100644 --- a/docs/channels/slack.md +++ b/docs/channels/slack.md @@ -1441,7 +1441,7 @@ To speak to OpenClaw in Slack today, send a Slack audio clip to the OpenClaw app Audio clips and Slackbot dictation have different privacy semantics: clips follow Slack file-retention policy and OpenClaw downloads them for transcription, while Slack says dictation audio is not stored. -In a channel with `requireMention: true`, include a typed mention of the bot with a captionless audio clip, or send the clip in a DM. Slack clip transcription currently happens after the channel mention gate. +In a channel with `requireMention: true`, a captionless audio clip can satisfy the gate by speaking a configured mention pattern (`agents.list[].groupChat.mentionPatterns`, falling back to `messages.groupChat.mentionPatterns`). OpenClaw authorizes the sender before downloading or transcribing the clip, then admits it only when the transcript matches. A failed or nonmatching speculative transcript is discarded with the downloaded clip; it is not retained in channel history. Native Slack `@bot` identity cannot be inferred from speech, so configure a spoken-name pattern or include a typed mention. If transcript echoing is enabled, the echo is sent only after admission. ## Media, chunking, and delivery @@ -1902,15 +1902,15 @@ When a single Slack message contains multiple file attachments: ### Known limits -| Scenario | Current behavior | Workaround | -| ------------------------------------------- | ---------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| Expired Slack file URL | File skipped; no error shown | Re-upload the file in Slack | -| Audio transcription unavailable | Clip remains attached but no transcript is produced | Configure `tools.media.audio` or install a supported local transcription CLI | -| Captionless clip in a mention-gated channel | Dropped before clip transcription | Add a typed bot mention or send the clip in a DM | -| Vision model not configured | Image attachments are stored as media references, but not analyzed as images | Configure `agents.defaults.imageModel` or use a vision-capable reply model | -| Very large images (> 20 MB by default) | Skipped per size cap | Increase `channels.slack.mediaMaxMb` if Slack allows | -| Forwarded/shared attachments | Text and Slack-hosted image/file media are best-effort | Re-share directly in the OpenClaw thread | -| PDF attachments | Stored as file/media context, not automatically routed through image vision | Use `download-file` for file metadata or the `pdf` tool for PDF analysis | +| Scenario | Current behavior | Workaround | +| --------------------------------------------- | ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Expired Slack file URL | File skipped; no error shown | Re-upload the file in Slack | +| Audio transcription unavailable | Clip remains attached but no transcript is produced | Configure `tools.media.audio` or install a supported local transcription CLI | +| Captionless clip does not pass a mention gate | Dropped after private speculative transcription; transcript and download discarded | Configure a spoken-name mention pattern, add a typed bot mention, or use a DM | +| Vision model not configured | Image attachments are stored as media references, but not analyzed as images | Configure `agents.defaults.imageModel` or use a vision-capable reply model | +| Very large images (> 20 MB by default) | Skipped per size cap | Increase `channels.slack.mediaMaxMb` if Slack allows | +| Forwarded/shared attachments | Text and Slack-hosted image/file media are best-effort | Re-share directly in the OpenClaw thread | +| PDF attachments | Stored as file/media context, not automatically routed through image vision | Use `download-file` for file metadata or the `pdf` tool for PDF analysis | ### Related documentation diff --git a/extensions/slack/src/monitor/media.ts b/extensions/slack/src/monitor/media.ts index c8a8e77036c9..8c67034c1c21 100644 --- a/extensions/slack/src/monitor/media.ts +++ b/extensions/slack/src/monitor/media.ts @@ -367,6 +367,7 @@ export async function resolveSlackMedia(params: { readIdleTimeoutMs?: number; totalTimeoutMs?: number; abortSignal?: AbortSignal; + preloadedMedia?: ReadonlyMap; }): Promise { const files = params.files ?? []; const limitedFiles = @@ -376,6 +377,12 @@ export async function resolveSlackMedia(params: { limitedFiles, MAX_SLACK_MEDIA_CONCURRENCY, async (file) => { + // Audio preflight keys the original event file object so admission can + // reuse that exact download without turning this into a persistent cache. + const preloaded = params.preloadedMedia?.get(file); + if (preloaded) { + return preloaded; + } const eventUrl = file.url_private_download ?? file.url_private; const url = eventUrl ?? (await fetchFreshSlackFileUrl({ file, client: params.client })); if (!url) { diff --git a/extensions/slack/src/monitor/message-handler/preflight-audio.runtime.ts b/extensions/slack/src/monitor/message-handler/preflight-audio.runtime.ts new file mode 100644 index 000000000000..3d4e6e78fa9d --- /dev/null +++ b/extensions/slack/src/monitor/message-handler/preflight-audio.runtime.ts @@ -0,0 +1,3 @@ +// Slack plugin module implements audio preflight runtime behavior. +export { sendDurableMessageBatch } from "openclaw/plugin-sdk/channel-outbound"; +export { transcribeFirstAudio } from "openclaw/plugin-sdk/media-runtime"; diff --git a/extensions/slack/src/monitor/message-handler/preflight-audio.test.ts b/extensions/slack/src/monitor/message-handler/preflight-audio.test.ts new file mode 100644 index 000000000000..32f7f8b9cff3 --- /dev/null +++ b/extensions/slack/src/monitor/message-handler/preflight-audio.test.ts @@ -0,0 +1,179 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { SlackMessageEvent } from "../../types.js"; +import type { SlackMediaResult } from "../media-types.js"; +import { + discardSlackPreflightMedia, + formatSlackAudioTranscriptForAgent, + hasCaptionlessSlackAudio, + resolveSlackPreflightAudioTranscript, + sendSlackPreflightAudioTranscriptEcho, +} from "./preflight-audio.js"; + +const { sendDurableMessageBatchMock, transcribeFirstAudioMock } = vi.hoisted(() => ({ + sendDurableMessageBatchMock: vi.fn(), + transcribeFirstAudioMock: vi.fn(), +})); + +vi.mock("./preflight-audio.runtime.js", () => ({ + sendDurableMessageBatch: sendDurableMessageBatchMock, + transcribeFirstAudio: transcribeFirstAudioMock, +})); + +function createSlackMessage(overrides: Partial): SlackMessageEvent { + return { + type: "message", + channel: "C1", + channel_type: "channel", + user: "U1", + text: "", + ts: "1.000", + ...overrides, + } as SlackMessageEvent; +} + +function createAudioConfig(overrides: Record = {}): OpenClawConfig { + return { + tools: { + media: { + audio: { + enabled: true, + echoTranscript: true, + ...overrides, + }, + }, + }, + } as OpenClawConfig; +} + +describe("Slack captionless audio preflight", () => { + beforeEach(() => { + sendDurableMessageBatchMock.mockReset(); + sendDurableMessageBatchMock.mockResolvedValue({ status: "sent", messageIds: ["1"] }); + transcribeFirstAudioMock.mockReset(); + }); + + it("recognizes captionless Slack audio independently of Slack's video MIME", () => { + const voiceClip = createSlackMessage({ + files: [ + { + id: "F1", + name: "voice.mp4", + mimetype: "video/mp4", + subtype: "slack_audio", + }, + ], + }); + + expect(hasCaptionlessSlackAudio(voiceClip)).toBe(true); + expect(hasCaptionlessSlackAudio({ ...voiceClip, text: "typed caption" })).toBe(false); + expect( + hasCaptionlessSlackAudio( + createSlackMessage({ + files: [{ id: "F2", name: "screen.mp4", mimetype: "video/mp4" }], + }), + ), + ).toBe(false); + }); + + it("frames machine transcripts as untrusted input without replacing the file placeholder", () => { + expect( + formatSlackAudioTranscriptForAgent({ + transcript: 'Bill said "review it"', + rawBody: "[Slack file: voice.mp4 (fileId: F1)]", + }), + ).toBe( + '[Audio transcript (machine-generated, untrusted)]: "Bill said \\"review it\\""\n' + + "[Slack file: voice.mp4 (fileId: F1)]", + ); + }); + + it("transcribes the first audio attachment once and suppresses speculative echo", async () => { + transcribeFirstAudioMock.mockResolvedValue("Bill please review this"); + const cfg = createAudioConfig(); + const media: SlackMediaResult[] = [ + { path: "/tmp/image.png", contentType: "image/png", placeholder: "[image]" }, + { path: "/tmp/voice.mp4", contentType: "audio/mp4", placeholder: "[voice]" }, + ]; + + await expect( + resolveSlackPreflightAudioTranscript({ + media, + cfg, + accountId: "work", + originatingTo: "channel:C1", + sessionKey: "agent:main:slack:channel:c1", + messageThreadId: "1.000", + }), + ).resolves.toEqual({ transcript: "Bill please review this", mediaIndex: 1 }); + + expect(transcribeFirstAudioMock).toHaveBeenCalledTimes(1); + expect(transcribeFirstAudioMock).toHaveBeenCalledWith({ + ctx: expect.objectContaining({ + MediaPaths: ["/tmp/image.png", "/tmp/voice.mp4"], + MediaTypes: ["image/png", "audio/mp4"], + OriginatingChannel: "slack", + OriginatingTo: "channel:C1", + AccountId: "work", + MessageThreadId: "1.000", + SessionKey: "agent:main:slack:channel:c1", + }), + cfg: expect.objectContaining({ + tools: expect.objectContaining({ + media: expect.objectContaining({ + audio: expect.objectContaining({ echoTranscript: false }), + }), + }), + }), + }); + expect(cfg.tools?.media?.audio?.echoTranscript).toBe(true); + }); + + it("echoes only an admitted transcript and preserves literal replacement tokens", async () => { + await sendSlackPreflightAudioTranscriptEcho({ + transcript: "cost is $& and $1", + cfg: createAudioConfig({ echoFormat: "heard: {transcript}" }), + accountId: "work", + originatingTo: "channel:C1", + messageThreadId: "1.000", + }); + + expect(sendDurableMessageBatchMock).toHaveBeenCalledWith({ + cfg: expect.any(Object), + channel: "slack", + to: "channel:C1", + accountId: "work", + threadId: "1.000", + payloads: [{ text: "heard: cost is $& and $1" }], + bestEffort: true, + durability: "best_effort", + }); + + sendDurableMessageBatchMock.mockClear(); + await sendSlackPreflightAudioTranscriptEcho({ + transcript: "not echoed", + cfg: createAudioConfig({ echoTranscript: false }), + accountId: "work", + originatingTo: "channel:C1", + }); + expect(sendDurableMessageBatchMock).not.toHaveBeenCalled(); + }); + + it("removes preflight downloads when the transcript does not admit the message", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-slack-audio-preflight-")); + const audioPath = path.join(root, "voice.mp4"); + await fs.writeFile(audioPath, "voice"); + + try { + await discardSlackPreflightMedia([ + { path: audioPath, contentType: "audio/mp4", placeholder: "[voice]" }, + ]); + await expect(fs.stat(audioPath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/extensions/slack/src/monitor/message-handler/preflight-audio.ts b/extensions/slack/src/monitor/message-handler/preflight-audio.ts new file mode 100644 index 000000000000..2751f27eb4a6 --- /dev/null +++ b/extensions/slack/src/monitor/message-handler/preflight-audio.ts @@ -0,0 +1,148 @@ +// Slack plugin module implements captionless audio mention preflight behavior. +import fs from "node:fs/promises"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; +import { mimeTypeFromFilePath } from "openclaw/plugin-sdk/media-mime"; +import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; +import type { SlackFile, SlackMessageEvent } from "../../types.js"; +import { MAX_SLACK_MEDIA_FILES, type SlackMediaResult } from "../media-types.js"; + +const SLACK_DEFAULT_ECHO_TRANSCRIPT_FORMAT = '📝 "{transcript}"'; + +const loadSlackPreflightAudioRuntime = createLazyRuntimeModule( + () => import("./preflight-audio.runtime.js"), +); + +function isSlackAudioFile(file: SlackFile): boolean { + if (file.subtype === "slack_audio") { + return true; + } + const mime = file.mimetype?.split(";")[0]?.trim().toLowerCase(); + if (mime?.startsWith("audio/")) { + return true; + } + return Boolean(mimeTypeFromFilePath(file.name)?.startsWith("audio/")); +} + +export function findCaptionlessSlackAudioFile(message: SlackMessageEvent): SlackFile | undefined { + if (message.text?.trim()) { + return undefined; + } + return message.files?.slice(0, MAX_SLACK_MEDIA_FILES).find(isSlackAudioFile); +} + +export function hasCaptionlessSlackAudio(message: SlackMessageEvent): boolean { + return Boolean(findCaptionlessSlackAudioFile(message)); +} + +export function formatSlackAudioTranscriptForAgent(params: { + transcript: string; + rawBody: string; +}): string { + const framed = `[Audio transcript (machine-generated, untrusted)]: ${JSON.stringify(params.transcript)}`; + return [framed, params.rawBody].filter(Boolean).join("\n"); +} + +function suppressSlackPreflightAudioEcho(cfg: OpenClawConfig): OpenClawConfig { + const audio = cfg.tools?.media?.audio; + if (!audio?.echoTranscript) { + return cfg; + } + return { + ...cfg, + tools: { + ...cfg.tools, + media: { + ...cfg.tools?.media, + audio: { + ...audio, + echoTranscript: false, + }, + }, + }, + }; +} + +export async function resolveSlackPreflightAudioTranscript(params: { + media: readonly SlackMediaResult[]; + cfg: OpenClawConfig; + accountId: string; + originatingTo: string; + sessionKey: string; + messageThreadId?: string; +}): Promise<{ transcript: string; mediaIndex: number } | null> { + const mediaIndex = params.media.findIndex((entry) => + entry.contentType?.toLowerCase().startsWith("audio/"), + ); + if (mediaIndex < 0) { + return null; + } + try { + const { transcribeFirstAudio } = await loadSlackPreflightAudioRuntime(); + const transcript = await transcribeFirstAudio({ + ctx: { + MediaPaths: params.media.map((entry) => entry.path), + MediaTypes: params.media.map((entry) => entry.contentType ?? ""), + Provider: "slack", + Surface: "slack", + OriginatingChannel: "slack", + OriginatingTo: params.originatingTo, + AccountId: params.accountId, + MessageThreadId: params.messageThreadId, + ChatType: "channel", + SessionKey: params.sessionKey, + }, + cfg: suppressSlackPreflightAudioEcho(params.cfg), + }); + return transcript ? { transcript, mediaIndex } : null; + } catch (err) { + logVerbose(`slack: audio preflight transcription failed: ${String(err)}`); + return null; + } +} + +function formatSlackAudioTranscriptEcho(transcript: string, format: string): string { + // Function replacement preserves literal `$` sequences in provider output. + return format.replace("{transcript}", () => transcript); +} + +export async function sendSlackPreflightAudioTranscriptEcho(params: { + transcript: string; + cfg: OpenClawConfig; + accountId: string; + originatingTo: string; + messageThreadId?: string; +}): Promise { + const audio = params.cfg.tools?.media?.audio; + if (!audio?.echoTranscript) { + return; + } + const text = formatSlackAudioTranscriptEcho( + params.transcript, + audio.echoFormat ?? SLACK_DEFAULT_ECHO_TRANSCRIPT_FORMAT, + ); + try { + const { sendDurableMessageBatch } = await loadSlackPreflightAudioRuntime(); + const send = await sendDurableMessageBatch({ + cfg: params.cfg, + channel: "slack", + to: params.originatingTo, + accountId: params.accountId, + threadId: params.messageThreadId, + payloads: [{ text }], + bestEffort: true, + durability: "best_effort", + }); + if (send.status === "failed") { + throw send.error; + } + } catch (err) { + logVerbose(`slack: audio transcript echo failed: ${String(err)}`); + } +} + +export async function discardSlackPreflightMedia( + media: readonly SlackMediaResult[] | null | undefined, +): Promise { + await Promise.allSettled((media ?? []).map((entry) => fs.rm(entry.path, { force: true }))); +} diff --git a/extensions/slack/src/monitor/message-handler/prepare-content.ts b/extensions/slack/src/monitor/message-handler/prepare-content.ts index 7131a2cf8fc9..659864a6c4eb 100644 --- a/extensions/slack/src/monitor/message-handler/prepare-content.ts +++ b/extensions/slack/src/monitor/message-handler/prepare-content.ts @@ -89,6 +89,7 @@ export async function resolveSlackMessageContent(params: { mediaReadIdleTimeoutMs?: number; mediaTotalTimeoutMs?: number; abortSignal?: AbortSignal; + preloadedMedia?: ReadonlyMap; }): Promise { const ownFiles = filterInheritedParentFiles({ files: params.message.files, @@ -107,6 +108,7 @@ export async function resolveSlackMessageContent(params: { readIdleTimeoutMs: params.mediaReadIdleTimeoutMs, totalTimeoutMs: params.mediaTotalTimeoutMs, abortSignal: params.abortSignal, + preloadedMedia: params.preloadedMedia, }), ) : Promise.resolve(null); diff --git a/extensions/slack/src/monitor/message-handler/prepare.test.ts b/extensions/slack/src/monitor/message-handler/prepare.test.ts index e948f926cda0..02c3d66bfe4e 100644 --- a/extensions/slack/src/monitor/message-handler/prepare.test.ts +++ b/extensions/slack/src/monitor/message-handler/prepare.test.ts @@ -1,4 +1,5 @@ // Slack tests cover prepare plugin behavior. +import fs from "node:fs/promises"; import type { App } from "@slack/bolt"; import { expectChannelInboundContextContract as expectInboundContextContract } from "openclaw/plugin-sdk/channel-contract-testing"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; @@ -32,10 +33,23 @@ import { } from "./prepare.test-helpers.js"; import { clearSlackSubteamMentionCacheForTest } from "./subteam-mentions.js"; -const { enqueueSystemEventMock, logVerboseMock, shouldLogVerboseMock } = vi.hoisted(() => ({ +const { + enqueueSystemEventMock, + logVerboseMock, + sendDurableMessageBatchMock, + shouldLogVerboseMock, + transcribeFirstAudioMock, +} = vi.hoisted(() => ({ enqueueSystemEventMock: vi.fn(), logVerboseMock: vi.fn(), + sendDurableMessageBatchMock: vi.fn(), shouldLogVerboseMock: vi.fn(() => false), + transcribeFirstAudioMock: vi.fn(), +})); + +vi.mock("./preflight-audio.runtime.js", () => ({ + sendDurableMessageBatch: sendDurableMessageBatchMock, + transcribeFirstAudio: transcribeFirstAudioMock, })); vi.mock("openclaw/plugin-sdk/runtime-env", async (importOriginal) => { @@ -69,8 +83,11 @@ describe("slack prepareSlackMessage inbound contract", () => { clearSlackSubteamMentionCacheForTest(); enqueueSystemEventMock.mockClear(); logVerboseMock.mockClear(); + sendDurableMessageBatchMock.mockReset(); + sendDurableMessageBatchMock.mockResolvedValue({ status: "sent", messageIds: ["1"] }); shouldLogVerboseMock.mockReset(); shouldLogVerboseMock.mockReturnValue(false); + transcribeFirstAudioMock.mockReset(); }); afterAll(() => { @@ -3501,6 +3518,255 @@ Second paragraph should still reach the agent after Slack's preview cutoff.`; expect(prepared).toBeNull(); }); + function createCaptionlessSlackAudioMessage( + overrides: Partial = {}, + ): SlackMessageEvent { + return createSlackMessage({ + channel: "C0AHZFCAS1K", + channel_type: "channel", + user: "U_BEK", + text: "", + ts: "1777244692.409919", + files: [ + { + id: "FPDF", + name: "report.pdf", + mimetype: "application/pdf", + url_private_download: "https://files.slack.com/files-pri/T1-FPDF/report.pdf", + }, + { + id: "FVOICE", + name: "voice.mp4", + mimetype: "video/mp4", + subtype: "slack_audio", + url_private_download: "https://files.slack.com/files-pri/T1-FVOICE/voice.mp4", + }, + ], + ...overrides, + }); + } + + function resolveFetchInputUrl(input: string | URL | Request): string { + return input instanceof Request ? input.url : String(input); + } + + function createAudioMentionSlackCtx(params: { + storePath?: string; + appClient?: App["client"]; + channelUsers?: string[]; + audioEnabled?: boolean; + }) { + const cfg = { + ...(params.storePath ? { session: { store: params.storePath } } : {}), + messages: { groupChat: { mentionPatterns: ["\\bbill\\b"] } }, + tools: { media: { audio: { enabled: params.audioEnabled ?? true } } }, + channels: { + slack: { + enabled: true, + replyToMode: "all", + groupPolicy: "open", + }, + }, + } as OpenClawConfig; + const slackCtx = createInboundSlackCtx({ + cfg, + ...(params.appClient ? { appClient: params.appClient } : {}), + channelsConfig: { + C0AHZFCAS1K: { + requireMention: true, + ...(params.channelUsers ? { users: params.channelUsers } : {}), + }, + }, + defaultRequireMention: true, + replyToMode: "all", + }); + slackCtx.resolveChannelName = async () => ({ name: "proj-openclaw", type: "channel" }); + slackCtx.resolveUserName = async () => ({ name: "Bek" }); + return slackCtx; + } + + it("admits a spoken-name audio root once and keeps its follow-up on the seeded thread session", async () => { + const originalFetch = globalThis.fetch; + const mockFetch = vi.fn( + async (_input: string | URL | Request) => + new Response(Buffer.from("voice clip"), { + status: 200, + headers: { "content-type": "video/mp4" }, + }), + ); + globalThis.fetch = mockFetch as typeof fetch; + const { storePath } = storeFixture.makeTmpStorePath(); + const rootTs = "1777244692.409919"; + const expectedSessionKey = `agent:main:slack:channel:c0ahzfcas1k:thread:${rootTs}`; + const replies = vi.fn().mockResolvedValue({ + messages: [{ text: "voice clip", user: "U_BEK", ts: rootTs }], + response_metadata: { next_cursor: "" }, + }); + const slackCtx = createAudioMentionSlackCtx({ + storePath, + appClient: { conversations: { replies } } as unknown as App["client"], + }); + let downloadedPath: string | undefined; + let downloadedPaths: string[] = []; + transcribeFirstAudioMock.mockImplementation( + async ({ ctx }: { ctx: { MediaPaths: string[] } }) => { + downloadedPath = ctx.MediaPaths[0]; + return "Bill /new please review this"; + }, + ); + + try { + const root = await prepareSlackMessage({ + ctx: slackCtx, + account: createSlackAccount({ replyToMode: "all" }), + message: createCaptionlessSlackAudioMessage(), + opts: { source: "message" }, + }); + recordSlackThreadParticipation("default", "C0AHZFCAS1K", rootTs); + const followUp = await prepareSlackMessage({ + ctx: slackCtx, + account: createSlackAccount({ replyToMode: "all" }), + message: createSlackMessage({ + channel: "C0AHZFCAS1K", + channel_type: "channel", + user: "U_BEK", + text: "and summarize the risks", + ts: "1777244714.000100", + thread_ts: rootTs, + }), + opts: { source: "message" }, + }); + + assertPrepared(root, "captionless audio root"); + assertPrepared(followUp, "audio-root follow-up"); + downloadedPaths = root.ctxPayload.MediaPaths ?? []; + expect(root.ctxPayload.SessionKey).toBe(expectedSessionKey); + expect(followUp.ctxPayload.SessionKey).toBe(expectedSessionKey); + expect(root.ctxPayload.MessageThreadId).toBe(rootTs); + expect(root.ctxPayload.WasMentioned).toBe(true); + expect(root.ctxPayload.MentionSource).toBe("mention_pattern"); + expect(root.ctxPayload.CommandBody).toBe(""); + expect(root.ctxPayload.Transcript).toBe("Bill /new please review this"); + expect(root.ctxPayload.MediaTranscribedIndexes).toEqual([1]); + expect(root.ctxPayload.RawBody).toContain("[Slack file: voice.mp4 (fileId: FVOICE)]"); + expect(root.ctxPayload.BodyForAgent).toContain( + '[Audio transcript (machine-generated, untrusted)]: "Bill /new please review this"', + ); + expect(transcribeFirstAudioMock).toHaveBeenCalledTimes(1); + expect(transcribeFirstAudioMock).toHaveBeenCalledWith({ + ctx: expect.objectContaining({ SessionKey: expectedSessionKey }), + cfg: expect.any(Object), + }); + const fetchedUrls = mockFetch.mock.calls.map(([input]) => resolveFetchInputUrl(input)); + expect(fetchedUrls).toHaveLength(2); + expect(fetchedUrls.filter((url) => url.includes("FVOICE"))).toHaveLength(1); + expect(fetchedUrls.filter((url) => url.includes("FPDF"))).toHaveLength(1); + } finally { + globalThis.fetch = originalFetch; + const pathsToRemove = new Set([ + ...downloadedPaths, + ...(downloadedPath ? [downloadedPath] : []), + ]); + for (const mediaPath of pathsToRemove) { + await fs.rm(mediaPath, { force: true }); + } + } + }); + + it("does not download or transcribe denied senders' captionless audio", async () => { + const originalFetch = globalThis.fetch; + const mockFetch = vi.fn(async () => { + throw new Error("denied audio must not be downloaded"); + }); + globalThis.fetch = mockFetch as typeof fetch; + const slackCtx = createAudioMentionSlackCtx({ channelUsers: ["U_OWNER"] }); + + try { + const prepared = await prepareMessageWith( + slackCtx, + createSlackAccount({ replyToMode: "all" }), + createCaptionlessSlackAudioMessage(), + ); + + expect(prepared).toBeNull(); + expect(mockFetch).not.toHaveBeenCalled(); + expect(transcribeFirstAudioMock).not.toHaveBeenCalled(); + expect(slackCtx.channelHistories.size).toBe(0); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("does not download captionless audio when audio understanding is disabled", async () => { + const originalFetch = globalThis.fetch; + const mockFetch = vi.fn(async () => { + throw new Error("disabled audio must not be downloaded"); + }); + globalThis.fetch = mockFetch as typeof fetch; + const slackCtx = createAudioMentionSlackCtx({ audioEnabled: false }); + + try { + const prepared = await prepareMessageWith( + slackCtx, + createSlackAccount({ replyToMode: "all" }), + createCaptionlessSlackAudioMessage(), + ); + + expect(prepared).toBeNull(); + expect(mockFetch).not.toHaveBeenCalled(); + expect(transcribeFirstAudioMock).not.toHaveBeenCalled(); + } finally { + globalThis.fetch = originalFetch; + } + }); + + it("drops nonmatching audio transcripts, keeps only the file marker, and removes the download", async () => { + const originalFetch = globalThis.fetch; + const mockFetch = vi.fn( + async (_input: string | URL | Request) => + new Response(Buffer.from("voice clip"), { + status: 200, + headers: { "content-type": "video/mp4" }, + }), + ); + globalThis.fetch = mockFetch as typeof fetch; + const slackCtx = createAudioMentionSlackCtx({}); + slackCtx.historyLimit = 5; + let downloadedPath: string | undefined; + transcribeFirstAudioMock.mockImplementation( + async ({ ctx }: { ctx: { MediaPaths: string[] } }) => { + downloadedPath = ctx.MediaPaths[0]; + return "please review this"; + }, + ); + + try { + const prepared = await prepareMessageWith( + slackCtx, + createSlackAccount({ replyToMode: "all" }), + createCaptionlessSlackAudioMessage({ ts: "1777244692.409920" }), + ); + + expect(prepared).toBeNull(); + expect(transcribeFirstAudioMock).toHaveBeenCalledTimes(1); + expect(mockFetch).toHaveBeenCalledTimes(1); + expect( + resolveFetchInputUrl(mockFetch.mock.calls[0]?.[0] as string | URL | Request), + ).toContain("FVOICE"); + expect(downloadedPath).toEqual(expect.any(String)); + await expect(fs.stat(downloadedPath as string)).rejects.toMatchObject({ code: "ENOENT" }); + const entries = Array.from(slackCtx.channelHistories.values()).flat(); + expect(entries).toHaveLength(1); + expect(entries[0]?.body).toBe("[Slack file: report.pdf (fileId: FPDF)]"); + expect(entries[0]?.media).toBeUndefined(); + } finally { + globalThis.fetch = originalFetch; + if (downloadedPath) { + await fs.rm(downloadedPath, { force: true }); + } + } + }); + it("keeps a regex-mentioned Slack thread root and URL-only follow-up on one parent session", async () => { const { storePath } = storeFixture.makeTmpStorePath(); const rootTs = "1777244692.409919"; diff --git a/extensions/slack/src/monitor/message-handler/prepare.ts b/extensions/slack/src/monitor/message-handler/prepare.ts index 2434d6c9af9b..ab4cec6ee22a 100644 --- a/extensions/slack/src/monitor/message-handler/prepare.ts +++ b/extensions/slack/src/monitor/message-handler/prepare.ts @@ -13,6 +13,7 @@ import { logInboundDrop, matchesMentionWithExplicit, recordDroppedChannelInboundHistory, + resolveInboundMentionDecision, resolveEnvelopeFormatOptions, resolveUnmentionedGroupInboundPolicy, toInboundMediaFacts, @@ -68,9 +69,17 @@ import { import { resolveConversationLabel } from "../conversation.runtime.js"; import { authorizeSlackDirectMessage } from "../dm-auth.js"; import type { SlackEventScope } from "../event-scope.js"; +import type { SlackMediaResult } from "../media-types.js"; import { resolveSlackRoomContextHints } from "../room-context.js"; import { sendMessageSlack } from "../send.runtime.js"; import { resolveSlackThreadStarter, type SlackThreadStarter } from "../thread.js"; +import { + discardSlackPreflightMedia, + findCaptionlessSlackAudioFile, + formatSlackAudioTranscriptForAgent, + resolveSlackPreflightAudioTranscript, + sendSlackPreflightAudioTranscriptEcho, +} from "./preflight-audio.js"; import { resolveSlackMessageContent } from "./prepare-content.js"; import { resolveSlackDmHistoryContext, resolveSlackDmHistoryLimit } from "./prepare-dm-history.js"; import { resolveSlackRoutingContext } from "./prepare-routing.js"; @@ -770,6 +779,7 @@ export async function prepareSlackMessage(params: { eventScope: opts.eventScope, }); + let mentionCheckTranscript: string | undefined; const resolveWasMentioned = (mentionRegexes: RegExp[]) => opts.wasMentioned ?? (!isDirectMessage && @@ -781,6 +791,7 @@ export async function prepareSlackMessage(params: { isExplicitlyMentioned: explicitlyMentioned, canResolveExplicit: Boolean(ctx.botUserId), }, + transcript: mentionCheckTranscript, })); const buildPolicyMentionRegexes = (agentId: string | undefined) => resolveCachedMentionRegexes(ctx, agentId, { @@ -793,36 +804,9 @@ export async function prepareSlackMessage(params: { const hasBoundSession = Boolean( routing.runtimeBoundSessionKey || routing.configuredBindingSessionKey, ); - // Runtime bindings already pin the root and later thread replies to the same - // target session, so only unbound regex mentions need a seeded thread reroute. - if ( - !seedTopLevelRoomThreadBySource && - wasMentioned && - isRoom && - !routing.isThreadReply && - !hasBoundSession - ) { - routing = resolveSlackRoutingContext({ - ctx, - account, - message, - isDirectMessage, - isGroupDm, - isRoom, - isRoomish, - channelConfig, - seedTopLevelRoomThread: true, - assistantThreadTs: assistantThreadContext?.threadTs, - eventScope: opts.eventScope, - }); - mentionRegexes = buildPolicyMentionRegexes(routing.route.agentId); - wasMentioned = resolveWasMentioned(mentionRegexes); - } - const { + let { route, runtimeBinding, - configuredBinding, - configuredBindingSessionKey, replyToMode, threadContext, threadTs, @@ -831,6 +815,7 @@ export async function prepareSlackMessage(params: { sessionKey, historyKey, } = routing; + const { configuredBinding, configuredBindingSessionKey } = routing; const isAssistantThreadMessage = Boolean(isDirectMessage && messageAssistantThreadContext); const shouldForceAssistantReplyThread = Boolean( assistantThreadContext?.threadTs && @@ -878,12 +863,6 @@ export async function prepareSlackMessage(params: { (error: unknown) => ({ ok: false, error }), ) : Promise.resolve({ ok: true, name: undefined }); - const directThreadRoutedToDmSession = - !assistantThreadContext && - isDirectMessage && - isThreadReply && - threadTs && - runtimeBinding?.conversation.conversationId !== threadTs; let implicitMentionKinds: ReturnType = []; if ( !isDirectMessage && @@ -970,6 +949,42 @@ export async function prepareSlackMessage(params: { }, }); }; + let threadStarterPromise: Promise | undefined; + const getThreadStarter = () => { + threadStarterPromise ??= + isThreadReply && threadTs + ? resolveSlackThreadStarter({ + channelId: message.channel, + threadTs, + client: slackClient, + workspaceScope: threadStarterWorkspaceScope, + }) + : Promise.resolve(null); + return threadStarterPromise; + }; + const resolveMessageContent = ( + contentMessage: SlackMessageEvent, + preloadedMedia?: ReadonlyMap, + ) => + getThreadStarter().then((threadStarter) => + resolveSlackMessageContent({ + message: contentMessage, + isThreadReply, + threadStarter, + isBotMessage, + botToken: ctx.botToken, + client: slackClient, + mediaMaxBytes: ctx.mediaMaxBytes, + resolveUserName: (userId) => ctx.resolveUserName(userId, opts.eventScope), + preloadedMedia, + }), + ); + let preloadedDirectMedia: ReadonlyMap | undefined; + let messageContentPromise: ReturnType | undefined; + const getMessageContent = () => { + messageContentPromise ??= resolveMessageContent(message, preloadedDirectMedia); + return messageContentPromise; + }; const senderNameForAuthResult = await senderNameForAuthPromise; if (!senderNameForAuthResult.ok) { throw senderNameForAuthResult.error; @@ -994,7 +1009,7 @@ export async function prepareSlackMessage(params: { ); return null; } - const canDetectMention = Boolean(ctx.botUserId) || mentionRegexes.length > 0; + let canDetectMention = Boolean(ctx.botUserId) || mentionRegexes.length > 0; // Strip Slack mentions (<@U123>) before command detection so "@Labrador /new" is recognized const textForCommandDetection = stripSlackMentionsForCommandDetection(message.text ?? ""); const hasControlCommandInMessage = hasControlCommand(textForCommandDetection, cfg); @@ -1023,16 +1038,6 @@ export async function prepareSlackMessage(params: { ...(ctx.threadRequireExplicitMention ? { allowedImplicitMentionKinds: [] } : {}), }, }); - const effectiveWasMentioned = messageIngress.activationAccess.effectiveWasMentioned ?? false; - const shouldBypassMention = messageIngress.activationAccess.shouldBypassMention ?? false; - const matchedImplicitMentionKinds = implicitMentionKinds; - const mentionSource = resolveSlackMentionSource({ - explicitBotMention: explicitlyMentionedBotUser || opts.source === "app_mention", - explicitSubteamMention: explicitlyMentionedBotSubteam, - matchedImplicitMentionKinds, - shouldBypassMention, - wasMentioned, - }); const senderGate = messageIngress.senderAccess.gate; if (isRoom && senderGate?.allowed === false) { logVerbose(`Blocked unauthorized slack sender ${senderId} (not in channel users)`); @@ -1055,14 +1060,6 @@ export async function prepareSlackMessage(params: { return null; } - if (isBotMessage && allowBotsMode === "mentions") { - const botMentioned = isDirectMessage || effectiveWasMentioned || shouldBypassMention; - if (!botMentioned) { - logVerbose("slack: drop bot message (allowBots=mentions, missing mention)"); - return null; - } - } - const threadContextAllowFromLower = isRoom ? channelUsersAllowlistConfigured ? normalizeAllowListLower(channelConfig?.users) @@ -1087,6 +1084,146 @@ export async function prepareSlackMessage(params: { return null; } + const canSeedMentionedRoomThread = + !seedTopLevelRoomThreadBySource && isRoom && !routing.isThreadReply && !hasBoundSession; + let seededMentionRouting: typeof routing | undefined; + const getSeededMentionRouting = () => { + seededMentionRouting ??= resolveSlackRoutingContext({ + ctx, + account, + message, + isDirectMessage, + isGroupDm, + isRoom, + isRoomish, + channelConfig, + seedTopLevelRoomThread: true, + assistantThreadTs: assistantThreadContext?.threadTs, + eventScope: opts.eventScope, + }); + return seededMentionRouting; + }; + + let preflightAudioTranscript: string | undefined; + let preflightAudioMedia: SlackMediaResult | undefined; + const preflightAudioFile = findCaptionlessSlackAudioFile(message); + const shouldPreflightAudioMention = + isRoom && + !isBotMessage && + shouldRequireMention && + cfg.tools?.media?.audio?.enabled !== false && + messageIngress.activationAccess.shouldSkip && + mentionRegexes.length > 0 && + Boolean(preflightAudioFile); + if (shouldPreflightAudioMention && preflightAudioFile) { + // Scope the provider call to the session that will own an admitted root, + // not the provisional channel session used before its spoken mention exists. + const preflightRouting = canSeedMentionedRoomThread ? getSeededMentionRouting() : routing; + const preflightContent = await resolveMessageContent({ + ...message, + files: [preflightAudioFile], + attachments: undefined, + blocks: undefined, + }); + const preflightMedia = preflightContent?.effectiveDirectMedia; + const downloadedAudioMedia = preflightMedia?.[0]; + if (downloadedAudioMedia) { + preloadedDirectMedia = new Map([[preflightAudioFile, downloadedAudioMedia]]); + } + const preflightResult = preflightMedia + ? await resolveSlackPreflightAudioTranscript({ + media: preflightMedia, + cfg, + accountId: account.accountId, + originatingTo: `channel:${message.channel}`, + sessionKey: preflightRouting.sessionKey, + messageThreadId: preflightRouting.threadContext.messageThreadId, + }) + : null; + if (preflightResult) { + mentionCheckTranscript = preflightResult.transcript; + wasMentioned = resolveWasMentioned(mentionRegexes); + if (wasMentioned) { + preflightAudioTranscript = preflightResult.transcript; + preflightAudioMedia = preflightMedia?.[preflightResult.mediaIndex]; + } + } + if (!preflightAudioTranscript) { + await discardSlackPreflightMedia(preflightMedia); + preloadedDirectMedia = undefined; + } + } + + // Runtime bindings already pin the root and later thread replies to the same + // target session. A spoken regex mention needs the same seeded root routing + // as a typed mention, or its later thread replies would use another session. + if (canSeedMentionedRoomThread && wasMentioned) { + routing = getSeededMentionRouting(); + mentionRegexes = buildPolicyMentionRegexes(routing.route.agentId); + wasMentioned = resolveWasMentioned(mentionRegexes); + ({ + route, + runtimeBinding, + replyToMode, + threadContext, + threadTs, + isThreadReply, + threadKeys, + sessionKey, + historyKey, + } = routing); + canDetectMention = Boolean(ctx.botUserId) || mentionRegexes.length > 0; + } + if (preflightAudioTranscript && !wasMentioned) { + await discardSlackPreflightMedia( + preloadedDirectMedia ? [...preloadedDirectMedia.values()] : undefined, + ); + preloadedDirectMedia = undefined; + preflightAudioTranscript = undefined; + preflightAudioMedia = undefined; + } + const directThreadRoutedToDmSession = + !assistantThreadContext && + isDirectMessage && + isThreadReply && + threadTs && + runtimeBinding?.conversation.conversationId !== threadTs; + + const mentionDecision = resolveInboundMentionDecision({ + facts: { + canDetectMention, + wasMentioned, + hasAnyMention, + implicitMentionKinds, + }, + policy: { + isGroup: isRoom, + requireMention: shouldRequireMention, + allowTextCommands, + hasControlCommand: hasControlCommandInMessage, + commandAuthorized, + ...(ctx.threadRequireExplicitMention ? { allowedImplicitMentionKinds: [] } : {}), + }, + }); + const effectiveWasMentioned = mentionDecision.effectiveWasMentioned; + const shouldBypassMention = mentionDecision.shouldBypassMention; + const matchedImplicitMentionKinds = mentionDecision.matchedImplicitMentionKinds; + const mentionSource = resolveSlackMentionSource({ + explicitBotMention: explicitlyMentionedBotUser || opts.source === "app_mention", + explicitSubteamMention: explicitlyMentionedBotSubteam, + matchedImplicitMentionKinds, + shouldBypassMention, + wasMentioned, + }); + + if (isBotMessage && allowBotsMode === "mentions") { + const botMentioned = isDirectMessage || effectiveWasMentioned || shouldBypassMention; + if (!botMentioned) { + logVerbose("slack: drop bot message (allowBots=mentions, missing mention)"); + return null; + } + } + if (isRoom && shouldRequireMention && !canDetectMention && !effectiveWasMentioned) { ctx.logger.info( { channel: message.channel, reason: "mention-detection-unavailable" }, @@ -1110,21 +1247,12 @@ export async function prepareSlackMessage(params: { return null; } - if (isRoom && shouldRequireMention && messageIngress.activationAccess.shouldSkip) { + if (isRoom && shouldRequireMention && mentionDecision.shouldSkip) { ctx.logger.info({ channel: message.channel, reason: "no-mention" }, "skipping channel message"); await recordDroppedHistory("slack-no-mention"); return null; } - const threadStarterPromise = - isThreadReply && threadTs - ? resolveSlackThreadStarter({ - channelId: message.channel, - threadTs, - client: slackClient, - workspaceScope: threadStarterWorkspaceScope, - }) - : Promise.resolve(null); const chatType = resolveSlackChatType(conversation.resolvedChannelType); const inboundEventKind = classifyChannelInboundEvent({ conversation: { kind: chatType }, @@ -1136,21 +1264,18 @@ export async function prepareSlackMessage(params: { hasControlCommand: hasControlCommandInMessage, hasAbortRequest, }); - const threadStarter = await threadStarterPromise; - const resolvedMessageContent = await resolveSlackMessageContent({ - message, - isThreadReply, - threadStarter, - isBotMessage, - botToken: ctx.botToken, - client: slackClient, - mediaMaxBytes: ctx.mediaMaxBytes, - resolveUserName: (userId) => ctx.resolveUserName(userId, opts.eventScope), - }); + const threadStarter = await getThreadStarter(); + const resolvedMessageContent = await getMessageContent(); if (!resolvedMessageContent) { return null; } const { rawBody, effectiveDirectMedia } = resolvedMessageContent; + const bodyForAgent = preflightAudioTranscript + ? formatSlackAudioTranscriptForAgent({ + transcript: preflightAudioTranscript, + rawBody, + }) + : rawBody; const ackReaction = resolveAckReaction(cfg, route.agentId, { channel: "slack", accountId: account.accountId, @@ -1211,7 +1336,7 @@ export async function prepareSlackMessage(params: { const roomLabel = channelName ? `#${channelName}` : `#${message.channel}`; const senderName = await resolveSenderName(); - const preview = truncateUtf16Safe(rawBody.replace(/\s+/g, " "), 160); + const preview = truncateUtf16Safe(bodyForAgent.replace(/\s+/g, " "), 160); const inboundLabel = isDirectMessage ? `Slack DM from ${senderName}` : `Slack message in ${roomLabel} from ${senderName}`; @@ -1237,7 +1362,7 @@ export async function prepareSlackMessage(params: { isThreadReply && threadTs ? ` thread_ts: ${threadTs}${message.parent_user_id ? ` parent_user_id: ${message.parent_user_id}` : ""}` : ""; - const textWithId = `${rawBody}\n[slack message id: ${message.ts} channel: ${message.channel}${threadInfo}]`; + const textWithId = `${bodyForAgent}\n[slack message id: ${message.ts} channel: ${message.channel}${threadInfo}]`; const storePath = resolveStorePath(ctx.cfg.session?.store, { agentId: route.agentId, }); @@ -1346,6 +1471,10 @@ export async function prepareSlackMessage(params: { // Use direct media (including forwarded attachment media) if available, else thread starter media const effectiveMedia = effectiveDirectMedia ?? threadStarterMedia; + const inboundMedia = toInboundMediaFacts(effectiveMedia, { + transcribed: (entry) => + effectiveMedia === effectiveDirectMedia && entry === preflightAudioMedia, + }); const inboundHistory = isRoomish && ctx.historyLimit > 0 ? channelHistory.buildInboundHistory({ @@ -1394,7 +1523,7 @@ export async function prepareSlackMessage(params: { message: { inboundEventKind, body: combinedBody, - bodyForAgent: rawBody, + bodyForAgent, rawBody, commandBody, inboundHistory, @@ -1414,7 +1543,7 @@ export async function prepareSlackMessage(params: { authorized: commandAuthorized, }, }, - media: toInboundMediaFacts(effectiveMedia), + media: inboundMedia, supplemental: { thread: { // Only include thread starter body for NEW sessions (existing sessions already have it in their transcript) @@ -1435,6 +1564,7 @@ export async function prepareSlackMessage(params: { SlackAssistantThreadContextChannelId: assistantThreadContext?.channelId, SlackAssistantThreadContextTeamId: assistantThreadContext?.teamId, SlackAssistantThreadContextEnterpriseId: assistantThreadContext?.enterpriseId ?? undefined, + Transcript: preflightAudioTranscript, IsFirstThreadTurn: isThreadReply && threadTs && @@ -1485,6 +1615,16 @@ export async function prepareSlackMessage(params: { return null; } + if (preflightAudioTranscript) { + await sendSlackPreflightAudioTranscriptEcho({ + transcript: preflightAudioTranscript, + cfg, + accountId: account.accountId, + originatingTo: `channel:${message.channel}`, + messageThreadId: threadContext.messageThreadId, + }); + } + if (shouldLogVerbose()) { logVerbose( `slack inbound: account=${route.accountId} agent=${route.agentId} channel=${message.channel} message_ts=${message.ts ?? "unknown"} thread_ts=${effectiveMessageThreadId ?? "none"} from=${slackFrom} chat=${chatType} chars=${rawBody.length}`,