From ee762ad761270714254c49e523a4fa5cb12c10fb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 17 Jul 2026 13:23:39 +0100 Subject: [PATCH] feat(teams-meetings): join Microsoft Teams meetings via the meeting-bot core (#109964) * feat(teams-meetings): join Microsoft Teams meetings via the meeting-bot core * fix(teams-meetings): satisfy lint and package-boundary contracts --- .github/labeler.yml | 6 + docs/docs.json | 1 + docs/docs_map.md | 18 + docs/plugins/plugin-inventory.md | 4 +- docs/plugins/reference.md | 2 +- docs/plugins/reference/teams-meetings.md | 23 + docs/plugins/teams-meetings.md | 67 ++ extensions/google-meet/src/runtime.ts | 18 +- .../google-meet/src/transports/chrome.test.ts | 100 +++ .../google-meet/src/transports/chrome.ts | 12 +- extensions/teams-meetings/index.test.ts | 256 ++++++++ extensions/teams-meetings/index.ts | 453 +++++++++++++ .../teams-meetings/openclaw.plugin.json | 110 ++++ extensions/teams-meetings/package.json | 33 + .../teams-meetings/src/agent-consult.ts | 93 +++ extensions/teams-meetings/src/cli.ts | 167 +++++ extensions/teams-meetings/src/config.test.ts | 47 ++ extensions/teams-meetings/src/config.ts | 234 +++++++ extensions/teams-meetings/src/node-host.ts | 117 ++++ .../src/node-invoke-policy.test.ts | 39 ++ .../teams-meetings/src/node-invoke-policy.ts | 43 ++ .../teams-meetings/src/runtime-node.test.ts | 147 +++++ .../teams-meetings/src/runtime-probes.test.ts | 46 ++ .../teams-meetings/src/runtime-probes.ts | 130 ++++ .../teams-meetings/src/runtime-session.ts | 42 ++ .../teams-meetings/src/runtime-setup.test.ts | 80 +++ .../teams-meetings/src/runtime-setup.ts | 166 +++++ extensions/teams-meetings/src/runtime.test.ts | 212 ++++++ extensions/teams-meetings/src/runtime.ts | 505 +++++++++++++++ .../src/transports/chrome-audio-device.ts | 5 + .../src/transports/chrome.test.ts | 181 ++++++ .../teams-meetings/src/transports/chrome.ts | 601 +++++++++++++++++ .../transports/teams-meetings-page-scripts.ts | 591 +++++++++++++++++ .../teams-meetings-platform-adapter.test.ts | 607 ++++++++++++++++++ .../teams-meetings-platform-adapter.ts | 212 ++++++ .../teams-meetings-platform-constants.ts | 7 + .../transports/teams-meetings-selectors.ts | 70 ++ .../transports/teams-meetings-urls.test.ts | 67 ++ .../src/transports/teams-meetings-urls.ts | 95 +++ .../teams-meetings/src/transports/types.ts | 90 +++ extensions/teams-meetings/tsconfig.json | 16 + pnpm-lock.yaml | 13 + scripts/generate-plugin-inventory-doc.mjs | 3 + src/meeting-bot/browser-controller.ts | 54 +- 44 files changed, 5764 insertions(+), 19 deletions(-) create mode 100644 docs/plugins/reference/teams-meetings.md create mode 100644 docs/plugins/teams-meetings.md create mode 100644 extensions/teams-meetings/index.test.ts create mode 100644 extensions/teams-meetings/index.ts create mode 100644 extensions/teams-meetings/openclaw.plugin.json create mode 100644 extensions/teams-meetings/package.json create mode 100644 extensions/teams-meetings/src/agent-consult.ts create mode 100644 extensions/teams-meetings/src/cli.ts create mode 100644 extensions/teams-meetings/src/config.test.ts create mode 100644 extensions/teams-meetings/src/config.ts create mode 100644 extensions/teams-meetings/src/node-host.ts create mode 100644 extensions/teams-meetings/src/node-invoke-policy.test.ts create mode 100644 extensions/teams-meetings/src/node-invoke-policy.ts create mode 100644 extensions/teams-meetings/src/runtime-node.test.ts create mode 100644 extensions/teams-meetings/src/runtime-probes.test.ts create mode 100644 extensions/teams-meetings/src/runtime-probes.ts create mode 100644 extensions/teams-meetings/src/runtime-session.ts create mode 100644 extensions/teams-meetings/src/runtime-setup.test.ts create mode 100644 extensions/teams-meetings/src/runtime-setup.ts create mode 100644 extensions/teams-meetings/src/runtime.test.ts create mode 100644 extensions/teams-meetings/src/runtime.ts create mode 100644 extensions/teams-meetings/src/transports/chrome-audio-device.ts create mode 100644 extensions/teams-meetings/src/transports/chrome.test.ts create mode 100644 extensions/teams-meetings/src/transports/chrome.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-platform-constants.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-selectors.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-urls.test.ts create mode 100644 extensions/teams-meetings/src/transports/teams-meetings-urls.ts create mode 100644 extensions/teams-meetings/src/transports/types.ts create mode 100644 extensions/teams-meetings/tsconfig.json diff --git a/.github/labeler.yml b/.github/labeler.yml index 9ce52e7b1d83..a448dc25c9b5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -58,6 +58,12 @@ - any-glob-to-any-file: - "extensions/google-meet/**" - "docs/plugins/google-meet.md" +"plugin: teams-meetings": + - changed-files: + - any-glob-to-any-file: + - "extensions/teams-meetings/**" + - "docs/plugins/teams-meetings.md" + - "docs/plugins/reference/teams-meetings.md" "plugin: workboard": - changed-files: - any-glob-to-any-file: diff --git a/docs/docs.json b/docs/docs.json index 962ba7f0864e..d2bb9ba076ad 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1300,6 +1300,7 @@ "plugins/codex-computer-use", "plugins/logbook", "plugins/google-meet", + "plugins/teams-meetings", "plugins/workboard", "plugins/webhooks", "plugins/admin-http-rpc", diff --git a/docs/docs_map.md b/docs/docs_map.md index 132d25486700..99361fdcd5c1 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -7073,6 +7073,15 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Surface - H2: Related docs +## plugins/reference/teams-meetings.md + +- Route: /plugins/reference/teams-meetings +- Headings: + - H1: Microsoft Teams meetings plugin + - H2: Distribution + - H2: Surface + - H2: Related docs + ## plugins/reference/telegram.md - Route: /plugins/reference/telegram @@ -7513,6 +7522,15 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Test configuration - H2: Related +## plugins/teams-meetings.md + +- Route: /plugins/teams-meetings +- Headings: + - H2: Setup + - H2: Modes + - H2: Guest join limits + - H2: Tool and gateway surface + ## plugins/tool-plugins.md - Route: /plugins/tool-plugins diff --git a/docs/plugins/plugin-inventory.md b/docs/plugins/plugin-inventory.md index 4729d1dd50d7..ebdc06d7e49d 100644 --- a/docs/plugins/plugin-inventory.md +++ b/docs/plugins/plugin-inventory.md @@ -51,7 +51,7 @@ Each entry lists the package, distribution route, and description. ## Core npm package -68 plugins +69 plugins - **[admin-http-rpc](/plugins/reference/admin-http-rpc)** (`@openclaw/admin-http-rpc`) - included in OpenClaw. OpenClaw admin HTTP RPC endpoint. @@ -161,6 +161,8 @@ Each entry lists the package, distribution route, and description. - **[synthetic](/plugins/reference/synthetic)** (`@openclaw/synthetic-provider`) - included in OpenClaw. Adds Synthetic model provider support to OpenClaw. +- **[teams-meetings](/plugins/reference/teams-meetings)** (`@openclaw/teams-meetings`) - included in OpenClaw. Join Microsoft Teams meetings as a Chrome browser guest. + - **[telegram](/plugins/reference/telegram)** (`@openclaw/telegram`) - included in OpenClaw. Adds the Telegram channel surface for sending and receiving OpenClaw messages. - **[together](/plugins/reference/together)** (`@openclaw/together-provider`) - included in OpenClaw. Adds Together model provider support to OpenClaw. diff --git a/docs/plugins/reference.md b/docs/plugins/reference.md index 4d4ccfadaf35..11258cad202a 100644 --- a/docs/plugins/reference.md +++ b/docs/plugins/reference.md @@ -15,5 +15,5 @@ This page is generated from `extensions/*/package.json` and pnpm plugins:inventory:gen ``` -Use [Plugin inventory](/plugins/plugin-inventory) to browse all 141 +Use [Plugin inventory](/plugins/plugin-inventory) to browse all 142 generated plugin reference pages by distribution, package, and description. diff --git a/docs/plugins/reference/teams-meetings.md b/docs/plugins/reference/teams-meetings.md new file mode 100644 index 000000000000..2c7cfd318184 --- /dev/null +++ b/docs/plugins/reference/teams-meetings.md @@ -0,0 +1,23 @@ +--- +summary: "Join Microsoft Teams meetings as a Chrome browser guest." +read_when: + - You are installing, configuring, or auditing the teams-meetings plugin +title: "Microsoft Teams meetings plugin" +--- + +# Microsoft Teams meetings plugin + +Join Microsoft Teams meetings as a Chrome browser guest. + +## Distribution + +- Package: `@openclaw/teams-meetings` +- Install route: included in OpenClaw + +## Surface + +contracts: `tools` + +## Related docs + +- [teams-meetings](/plugins/teams-meetings) diff --git a/docs/plugins/teams-meetings.md b/docs/plugins/teams-meetings.md new file mode 100644 index 000000000000..488d1a3fd371 --- /dev/null +++ b/docs/plugins/teams-meetings.md @@ -0,0 +1,67 @@ +--- +summary: "Microsoft Teams meetings plugin: join work or consumer meetings as a Chrome browser guest" +read_when: + - You want an OpenClaw agent to join a Microsoft Teams meeting + - You are configuring Chrome, BlackHole, or SoX for Teams meeting talk-back +title: "Microsoft Teams meetings plugin" +--- + +The `teams-meetings` plugin joins Microsoft Teams links as a guest in the OpenClaw Chrome profile. It accepts work links under `teams.microsoft.com/l/meetup-join/...` and consumer links under `teams.live.com/meet/...`. It does not create meetings, dial in, call Microsoft Graph, or record meetings. + +## Setup + +Talk-back uses the same local audio prerequisites as the [Google Meet plugin](/plugins/google-meet): macOS, the `BlackHole 2ch` virtual audio device, and SoX. + +```bash +brew install blackhole-2ch sox +sudo reboot +system_profiler SPAudioDataType | grep -i BlackHole +command -v sox +``` + +Enable the plugin, then check setup: + +```json5 +{ + plugins: { + entries: { + "teams-meetings": { + enabled: true, + config: { + defaultMode: "agent", + chrome: { guestName: "OpenClaw Agent" }, + }, + }, + }, + }, +} +``` + +```bash +openclaw teamsmeetings setup +openclaw teamsmeetings join 'https://teams.microsoft.com/l/meetup-join/...' +``` + +Use `chromeNode.node` to run Chrome, BlackHole, and SoX on a paired macOS node. The node must allow `teamsmeetings.chrome` and `browser.proxy`. + +## Modes + +| Mode | Behavior | +| ------------ | --------------------------------------------------------------------------- | +| `agent` | Realtime transcription consults the configured OpenClaw agent; TTS replies. | +| `bidi` | A realtime voice model listens and replies directly. | +| `transcribe` | Observe-only join. Caption snapshots are currently empty. | + +Caption scraping is disabled by default. Teams live-caption DOM selectors need validation across live work and consumer tenants before the plugin can promise transcript capture. + +## Guest join limits + +The browser adapter dismisses the app interstitial, fills the guest name, turns the camera off, configures the microphone for the selected mode, and clicks the join button. In-call state uses the hang-up control; lobby, tenant sign-in, and device-permission states return explicit manual-action reasons. + +Teams tenant policy can require sign-in, email verification, or organizer admission. Complete that step in the OpenClaw Chrome profile, then retry status or speech. The plugin does not bypass tenant policy. + +All in-page selectors are best-effort pending live validation. Work and consumer Teams UI changes can require selector updates. Validate these flows before unattended use: app interstitial, guest-name entry, prejoin microphone/camera toggles, join, lobby admission, tenant sign-in/email verification, media permissions, in-call detection, BlackHole output routing, leave confirmation, and consumer-meeting behavior. + +## Tool and gateway surface + +The `teams_meetings` agent tool supports `join`, `leave`, `status`, `transcript`, and `speak`. Gateway methods use the `teamsmeetings.*` prefix. The node command is `teamsmeetings.chrome`. diff --git a/extensions/google-meet/src/runtime.ts b/extensions/google-meet/src/runtime.ts index a20e8691f4ef..168841459193 100644 --- a/extensions/google-meet/src/runtime.ts +++ b/extensions/google-meet/src/runtime.ts @@ -519,6 +519,8 @@ export class GoogleMeetRuntime { config: this.params.config, mode: session.mode, readOnly: options.readOnly, + trackedMeetingUrl: session.url, + trackedTargetId: session.chrome?.browserTab?.targetId, url: session.url, }) : await recoverCurrentMeetTab({ @@ -526,10 +528,22 @@ export class GoogleMeetRuntime { config: this.params.config, mode: session.mode, readOnly: options.readOnly, + trackedMeetingUrl: session.url, + trackedTargetId: session.chrome?.browserTab?.targetId, url: session.url, }); - if (result.found && result.browser && session.chrome) { - session.chrome.health = { ...session.chrome.health, ...result.browser }; + if (result.found && session.chrome) { + if (result.targetId) { + const currentTab = session.chrome.browserTab; + session.chrome.browserTab = { + targetId: result.targetId, + openedByPlugin: + result.targetId === currentTab?.targetId ? currentTab.openedByPlugin : false, + }; + } + if (result.browser) { + session.chrome.health = { ...session.chrome.health, ...result.browser }; + } session.updatedAt = nowIso(); } } catch (error) { diff --git a/extensions/google-meet/src/transports/chrome.test.ts b/extensions/google-meet/src/transports/chrome.test.ts index 3248a292b9d1..12817fd6ca40 100644 --- a/extensions/google-meet/src/transports/chrome.test.ts +++ b/extensions/google-meet/src/transports/chrome.test.ts @@ -31,6 +31,106 @@ function browserRuntime(request: TestGatewayRequest): PluginRuntime { } describe("google meet chrome transport", () => { + it("prefers the tracked target for an unchanged Google Meet URL", async () => { + const gatewayRequest = vi.fn(async (_method, params) => { + if (params.path === "/tabs") { + return { + tabs: [ + { + targetId: "other-meet-tab", + title: "Meet", + url: "https://meet.google.com/abc-defg-hij?hl=en", + }, + { + targetId: "tracked-meet-tab", + title: "Meet", + url: "https://meet.google.com/abc-defg-hij?hl=en", + }, + ], + }; + } + if (params.path === "/tabs/focus") { + return { ok: true }; + } + if (params.path === "/act") { + return { + result: JSON.stringify({ + inCall: true, + micMuted: true, + url: "https://meet.google.com/abc-defg-hij?hl=en", + }), + }; + } + throw new Error(`unexpected browser request path ${String(params.path)}`); + }); + + const recovered = await recoverCurrentMeetTab({ + runtime: browserRuntime(gatewayRequest), + config: resolveGoogleMeetConfig({}), + mode: "transcribe", + readOnly: true, + trackedMeetingUrl: "https://meet.google.com/abc-defg-hij?authuser=0", + trackedTargetId: "tracked-meet-tab", + url: "https://meet.google.com/abc-defg-hij?hl=en", + }); + + expect(recovered).toMatchObject({ found: true, targetId: "tracked-meet-tab" }); + expect(gatewayRequest).toHaveBeenCalledWith( + "browser.request", + expect.objectContaining({ + path: "/act", + body: expect.objectContaining({ targetId: "tracked-meet-tab" }), + }), + expect.objectContaining({ scopes: ["operator.admin"] }), + ); + }); + + it("falls back from a tracked target that identifies another meeting", async () => { + const gatewayRequest = vi.fn(async (_method, params) => { + if (params.path === "/tabs") { + return { + tabs: [ + { + targetId: "matching-meet-tab", + title: "Meet", + url: "https://meet.google.com/abc-defg-hij?hl=en", + }, + { + targetId: "tracked-meet-tab", + title: "Meet", + url: "https://meet.google.com/xyz-abcd-efg?hl=en", + }, + ], + }; + } + if (params.path === "/tabs/focus") { + return { ok: true }; + } + if (params.path === "/act") { + return { + result: JSON.stringify({ + inCall: true, + micMuted: true, + url: "https://meet.google.com/abc-defg-hij?hl=en", + }), + }; + } + throw new Error(`unexpected browser request path ${String(params.path)}`); + }); + + const recovered = await recoverCurrentMeetTab({ + runtime: browserRuntime(gatewayRequest), + config: resolveGoogleMeetConfig({}), + mode: "transcribe", + readOnly: true, + trackedMeetingUrl: "https://meet.google.com/abc-defg-hij?authuser=0", + trackedTargetId: "tracked-meet-tab", + url: "https://meet.google.com/abc-defg-hij?hl=en", + }); + + expect(recovered).toMatchObject({ found: true, targetId: "matching-meet-tab" }); + }); + it("wraps malformed browser status JSON through tab recovery", async () => { const runtime = browserRuntime( vi.fn(async (_method, params) => { diff --git a/extensions/google-meet/src/transports/chrome.ts b/extensions/google-meet/src/transports/chrome.ts index a0dcb60a989e..1dae5a9ebf8e 100644 --- a/extensions/google-meet/src/transports/chrome.ts +++ b/extensions/google-meet/src/transports/chrome.ts @@ -414,6 +414,8 @@ export async function recoverCurrentMeetTab(params: { config: GoogleMeetConfig; mode?: GoogleMeetMode; readOnly?: boolean; + trackedMeetingUrl?: string; + trackedTargetId?: string; url?: string; }): Promise<{ transport: "chrome"; @@ -433,7 +435,9 @@ export async function recoverCurrentMeetTab(params: { locationLabel: "in local Chrome", mode: params.mode ?? "bidi", readOnly: params.readOnly, - url: params.url, + requestedMeetingUrl: params.url, + trackedMeetingUrl: params.trackedMeetingUrl, + trackedTargetId: params.trackedTargetId, })), }; } @@ -443,6 +447,8 @@ export async function recoverCurrentMeetTabOnNode(params: { config: GoogleMeetConfig; mode?: GoogleMeetMode; readOnly?: boolean; + trackedMeetingUrl?: string; + trackedTargetId?: string; url?: string; }): Promise<{ transport: "chrome-node"; @@ -475,7 +481,9 @@ export async function recoverCurrentMeetTabOnNode(params: { locationLabel: "on the selected Chrome node", mode: params.mode ?? "bidi", readOnly: params.readOnly, - url: params.url, + requestedMeetingUrl: params.url, + trackedMeetingUrl: params.trackedMeetingUrl, + trackedTargetId: params.trackedTargetId, })), }; } diff --git a/extensions/teams-meetings/index.test.ts b/extensions/teams-meetings/index.test.ts new file mode 100644 index 000000000000..0f0adb9d3c9d --- /dev/null +++ b/extensions/teams-meetings/index.test.ts @@ -0,0 +1,256 @@ +import { ErrorCodes } from "openclaw/plugin-sdk/gateway-runtime"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import { describe, expect, it, vi } from "vitest"; +import plugin from "./index.js"; + +const MEETING_URL = + "https://teams.microsoft.com/l/meetup-join/19%3ameeting_owned%40thread.v2/0?context=%7b%7d"; + +type GatewayHandler = (options: { + client?: { internal?: { pluginRuntimeOwnerId?: string } }; + params?: Record; + respond(ok: boolean, payload?: unknown, error?: unknown): void; +}) => Promise; + +function authorizationHarness(options?: { browserError?: Error }) { + const methods = new Map(); + let tabOpen = false; + const gatewayRequest = vi.fn(async (_method: string, params: Record) => { + if (options?.browserError) { + throw options.browserError; + } + if (params.path === "/tabs") { + return { tabs: tabOpen ? [{ targetId: "teams-tab", title: "Teams", url: MEETING_URL }] : [] }; + } + if (params.path === "/tabs/open") { + tabOpen = true; + return { targetId: "teams-tab", title: "Teams", url: MEETING_URL }; + } + if (params.path === "/tabs/focus") { + return { ok: true }; + } + if (params.method === "DELETE" && params.path === "/tabs/teams-tab") { + tabOpen = false; + return { ok: true }; + } + if (params.path === "/act") { + const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn; + const script = typeof scriptValue === "string" ? scriptValue : ""; + return script.includes("leaveAction") + ? { result: JSON.stringify({ departed: true, urlMatched: true }) } + : script.includes("expectedSessionId") + ? { + result: JSON.stringify({ + droppedLines: 0, + lines: [], + sessionMatched: true, + urlMatched: true, + }), + } + : { + result: JSON.stringify({ + cameraOff: true, + inCall: true, + micMuted: true, + title: "Teams", + url: MEETING_URL, + }), + }; + } + throw new Error(`unexpected browser request ${String(params.path)}`); + }); + const api = createTestPluginApi({ + id: "teams-meetings", + name: "Microsoft Teams meetings", + description: "test", + version: "0", + source: "test", + config: {}, + pluginConfig: { defaultMode: "transcribe", chrome: { waitForInCallMs: 1 } }, + runtime: { + gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest }, + } as unknown as OpenClawPluginApi["runtime"], + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + registerGatewayMethod: (method: string, handler: unknown) => + methods.set(method, handler as GatewayHandler), + }); + plugin.register(api); + + const invoke = async ( + method: string, + params: Record, + pluginRuntimeOwnerId?: string, + ) => { + const handler = methods.get(method); + if (!handler) { + throw new Error(`missing handler ${method}`); + } + let response: { ok: boolean; payload?: unknown; error?: unknown } | undefined; + await handler({ + params, + ...(pluginRuntimeOwnerId ? { client: { internal: { pluginRuntimeOwnerId } } } : {}), + respond: (ok, payload, error) => { + response = { ok, payload, error }; + }, + }); + return response; + }; + const call = async ( + method: string, + params: Record, + pluginRuntimeOwnerId?: string, + ) => { + const response = await invoke(method, params, pluginRuntimeOwnerId); + if (!response?.ok) { + throw new Error(`gateway call failed: ${JSON.stringify(response)}`); + } + return response.payload as Record; + }; + return { call, invoke }; +} + +describe("Microsoft Teams meetings plugin surface", () => { + it("registers the bounded gateway, tool, CLI, and node surfaces", () => { + const methods = new Map(); + const tools: Array> = []; + const cli: unknown[] = []; + const nodeCommands: unknown[] = []; + const policies: unknown[] = []; + const api = createTestPluginApi({ + id: "teams-meetings", + name: "Microsoft Teams meetings", + description: "test", + version: "0", + source: "test", + config: {}, + pluginConfig: {}, + runtime: { + gateway: { isAvailable: vi.fn(async () => false), request: vi.fn() }, + } as unknown as OpenClawPluginApi["runtime"], + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, + registerGatewayMethod: (method: string, handler: unknown) => methods.set(method, handler), + registerTool: (tool: unknown) => { + tools.push( + (typeof tool === "function" + ? (tool as (context: Record) => Record)({}) + : tool) as Record, + ); + }, + registerCli: (_registrar: unknown, options: unknown) => cli.push(options), + registerNodeHostCommand: (command: unknown) => nodeCommands.push(command), + registerNodeInvokePolicy: (policy: unknown) => policies.push(policy), + }); + + plugin.register(api); + + expect([...methods.keys()].toSorted()).toEqual( + [ + "teamsmeetings.join", + "teamsmeetings.leave", + "teamsmeetings.setup", + "teamsmeetings.speak", + "teamsmeetings.status", + "teamsmeetings.testListen", + "teamsmeetings.testSpeech", + "teamsmeetings.transcript", + ].toSorted(), + ); + expect(tools.map((tool) => tool.name)).toEqual(["teams_meetings"]); + expect(cli).toEqual([expect.objectContaining({ commands: ["teamsmeetings"] })]); + expect(nodeCommands).toEqual([ + expect.objectContaining({ command: "teamsmeetings.chrome", cap: "teams-meetings" }), + ]); + expect(policies).toHaveLength(1); + }); + + it("scopes trusted tool session operations to the invoking agent", async () => { + const { call } = authorizationHarness(); + const joined = await call( + "teamsmeetings.join", + { agentId: "support", mode: "transcribe", url: MEETING_URL }, + "teams-meetings", + ); + const sessionId = (joined.session as { id: string }).id; + + expect( + await call("teamsmeetings.status", { agentId: "other" }, "teams-meetings"), + ).toMatchObject({ found: true, sessions: [] }); + for (const method of ["status", "leave", "transcript", "speak"] as const) { + expect( + await call( + `teamsmeetings.${method}`, + { agentId: "other", message: "hello", sessionId }, + "teams-meetings", + ), + ).toMatchObject({ found: false }); + } + + expect(await call("teamsmeetings.status", { agentId: "spoofed", sessionId })).toMatchObject({ + found: true, + session: { agentId: "support", id: sessionId }, + }); + expect( + await call("teamsmeetings.leave", { agentId: "support", sessionId }, "teams-meetings"), + ).toMatchObject({ found: true, session: { id: sessionId, state: "ended" } }); + }); + + it.each([ + ["mode", "observe-only", "mode must be agent, bidi, or transcribe"], + ["transport", "desktop", "transport must be chrome or chrome-node"], + ])("rejects an explicit invalid %s", async (field, value, message) => { + const { invoke } = authorizationHarness(); + const response = await invoke("teamsmeetings.join", { + [field]: value, + url: MEETING_URL, + }); + + expect(response).toMatchObject({ + error: { code: ErrorCodes.INVALID_REQUEST }, + ok: false, + payload: { error: message }, + }); + }); + + it("rejects timeoutMs on normal join instead of silently ignoring it", async () => { + const { invoke } = authorizationHarness(); + const response = await invoke("teamsmeetings.join", { + mode: "transcribe", + timeoutMs: 1, + url: MEETING_URL, + }); + + expect(response).toMatchObject({ + ok: false, + payload: { error: "timeoutMs is supported only by testSpeech" }, + }); + }); + + it("rejects timeoutMs on testListen while captions are disabled", async () => { + const { invoke } = authorizationHarness(); + const response = await invoke("teamsmeetings.testListen", { + mode: "transcribe", + timeoutMs: 1, + url: MEETING_URL, + }); + + expect(response).toMatchObject({ + ok: false, + payload: { error: "timeoutMs is supported only by testSpeech" }, + }); + }); + + it("classifies browser failures as unavailable, not invalid requests", async () => { + const { invoke } = authorizationHarness({ browserError: new Error("browser unavailable") }); + const response = await invoke("teamsmeetings.join", { + mode: "transcribe", + url: MEETING_URL, + }); + + expect(response).toMatchObject({ + error: { code: ErrorCodes.UNAVAILABLE }, + ok: false, + payload: { error: "browser unavailable" }, + }); + }); +}); diff --git a/extensions/teams-meetings/index.ts b/extensions/teams-meetings/index.ts new file mode 100644 index 000000000000..8df86fd3293e --- /dev/null +++ b/extensions/teams-meetings/index.ts @@ -0,0 +1,453 @@ +import { + readNonNegativeIntegerParam, + readPositiveIntegerParam, +} from "openclaw/plugin-sdk/channel-actions"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + callGatewayFromCli, + ErrorCodes, + errorShape, + type GatewayRequestHandlerOptions, +} from "openclaw/plugin-sdk/gateway-runtime"; +import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; +import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; +import { normalizeAgentId, parseAgentSessionKey } from "openclaw/plugin-sdk/routing"; +import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { jsonResult as json } from "openclaw/plugin-sdk/tool-results"; +import { Type } from "typebox"; +import { + resolveTeamsMeetingsConfig, + resolveTeamsMeetingsGatewayOperationTimeoutMs, + type TeamsMeetingsConfig, + type TeamsMeetingsMode, + type TeamsMeetingsTransport, +} from "./src/config.js"; +import { handleTeamsMeetingsNodeHostCommand } from "./src/node-host.js"; +import { createTeamsMeetingsNodeInvokePolicy } from "./src/node-invoke-policy.js"; +import { TeamsMeetingsRuntime } from "./src/runtime.js"; +import { TEAMS_MEETINGS_NODE_COMMAND } from "./src/transports/teams-meetings-platform-constants.js"; +import { normalizeTeamsMeetingUrl } from "./src/transports/teams-meetings-urls.js"; + +const loadTeamsMeetingsCli = createLazyRuntimeModule(() => import("./src/cli.js")); + +const teamsMeetingsConfigSchema = { + parse(value: unknown) { + return resolveTeamsMeetingsConfig(value); + }, + uiHints: { + defaultMode: { + label: "Default Mode", + help: "Agent consults OpenClaw, bidi uses direct realtime voice, and transcribe observes only.", + }, + "chrome.browserProfile": { label: "Chrome Profile", advanced: true }, + "chrome.guestName": { label: "Guest Name" }, + "chrome.waitForInCallMs": { label: "Wait For In-Call (ms)", advanced: true }, + "chrome.audioInputCommand": { label: "Audio Input Command", advanced: true }, + "chrome.audioOutputCommand": { label: "Audio Output Command", advanced: true }, + "chromeNode.node": { + label: "Chrome Node", + help: "Node id/name/IP that owns Chrome, BlackHole, and SoX.", + advanced: true, + }, + "realtime.transcriptionProvider": { label: "Realtime Transcription Provider" }, + "realtime.voiceProvider": { label: "Bidi Voice Provider" }, + "realtime.model": { label: "Bidi Realtime Model", advanced: true }, + "realtime.instructions": { label: "Realtime Instructions", advanced: true }, + "realtime.introMessage": { label: "Realtime Intro Message" }, + "realtime.agentId": { label: "Realtime Consult Agent", advanced: true }, + "realtime.toolPolicy": { label: "Realtime Tool Policy", advanced: true }, + }, +}; + +const TeamsMeetingsToolSchema = Type.Object({ + action: Type.String({ enum: ["join", "leave", "status", "transcript", "speak"] }), + url: Type.Optional(Type.String({ description: "Microsoft Teams meeting URL" })), + transport: Type.Optional(Type.String({ enum: ["chrome", "chrome-node"] })), + mode: Type.Optional(Type.String({ enum: ["agent", "bidi", "transcribe"] })), + sessionId: Type.Optional(Type.String({ description: "Teams meeting session ID" })), + sinceIndex: Type.Optional( + Type.Integer({ minimum: 0, description: "Resume transcript from this index" }), + ), + message: Type.Optional(Type.String({ description: "Instructions to speak" })), +}); + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +class TeamsMeetingsInvalidRequestError extends Error {} + +function invalidRequest(message: string): TeamsMeetingsInvalidRequestError { + return new TeamsMeetingsInvalidRequestError(message); +} + +function normalizeTransport(value: unknown): TeamsMeetingsTransport | undefined { + if (value === undefined) { + return undefined; + } + if (value === "chrome" || value === "chrome-node") { + return value; + } + throw invalidRequest("transport must be chrome or chrome-node"); +} + +function normalizeMode(value: unknown): TeamsMeetingsMode | undefined { + if (value === undefined) { + return undefined; + } + if (value === "agent" || value === "bidi" || value === "transcribe") { + return value; + } + throw invalidRequest("mode must be agent, bidi, or transcribe"); +} + +function requireString(value: unknown, name: string): string { + const normalized = normalizeOptionalString(value); + if (!normalized) { + throw invalidRequest(`${name} required`); + } + return normalized; +} + +function readSinceIndex(raw: Record): number | undefined { + try { + return readNonNegativeIntegerParam(raw, "sinceIndex"); + } catch (error) { + throw invalidRequest(formatErrorMessage(error)); + } +} + +function keepTrustedToolAgentId( + raw: Record, + client: GatewayRequestHandlerOptions["client"], +): Record { + const { agentId: rawAgentId, ...rest } = raw; + if (client?.internal?.pluginRuntimeOwnerId !== "teams-meetings") { + return rest; + } + const agentId = normalizeOptionalString(rawAgentId); + return agentId ? { ...rest, agentId } : rest; +} + +function trustedToolAgentId( + raw: Record, + client: GatewayRequestHandlerOptions["client"], +): string | undefined { + return normalizeOptionalString(keepTrustedToolAgentId(raw, client).agentId); +} + +function joinRequest(raw: Record, options?: { allowTimeout?: boolean }) { + if (!options?.allowTimeout && raw.timeoutMs !== undefined) { + throw invalidRequest("timeoutMs is supported only by testSpeech"); + } + let url: string; + let timeoutMs: number | undefined; + try { + url = normalizeTeamsMeetingUrl(requireString(raw.url, "url")); + timeoutMs = readPositiveIntegerParam(raw, "timeoutMs"); + } catch (error) { + if (error instanceof TeamsMeetingsInvalidRequestError) { + throw error; + } + throw invalidRequest(formatErrorMessage(error)); + } + return { + url, + transport: normalizeTransport(raw.transport), + mode: normalizeMode(raw.mode), + message: normalizeOptionalString(raw.message), + requesterSessionKey: normalizeOptionalString(raw.requesterSessionKey), + agentId: normalizeOptionalString(raw.agentId), + timeoutMs, + }; +} + +type ToolAction = "join" | "leave" | "status" | "transcript" | "speak"; + +function gatewayMethod(action: ToolAction): string { + return `teamsmeetings.${action}`; +} + +function readErrorDetails(error: unknown): unknown { + return error && typeof error === "object" && "details" in error + ? (error as { details?: unknown }).details + : undefined; +} + +async function callGatewayFromTool(params: { + action: ToolAction; + config: TeamsMeetingsConfig; + raw: Record; + runtime?: OpenClawPluginApi["runtime"]; +}) { + try { + if (params.runtime) { + return await params.runtime.gateway.request(gatewayMethod(params.action), params.raw, { + timeoutMs: resolveTeamsMeetingsGatewayOperationTimeoutMs(params.config), + scopes: ["operator.admin"], + }); + } + return await callGatewayFromCli( + gatewayMethod(params.action), + { + json: true, + timeout: String(resolveTeamsMeetingsGatewayOperationTimeoutMs(params.config)), + }, + params.raw, + { progress: false, scopes: ["operator.admin"] }, + ); + } catch (error) { + const details = readErrorDetails(error); + if (details && typeof details === "object") { + return details; + } + throw error; + } +} + +export default definePluginEntry({ + id: "teams-meetings", + name: "Microsoft Teams meetings", + description: "Join Microsoft Teams meetings as a Chrome browser guest", + configSchema: teamsMeetingsConfigSchema, + register(api: OpenClawPluginApi) { + const config = teamsMeetingsConfigSchema.parse(api.pluginConfig); + let runtime: TeamsMeetingsRuntime | undefined; + + const ensureRuntime = async () => { + if (!config.enabled) { + throw new Error("Microsoft Teams meetings plugin disabled in plugin config"); + } + runtime ??= new TeamsMeetingsRuntime({ + config, + fullConfig: api.config, + runtime: api.runtime, + logger: api.logger, + }); + return runtime; + }; + + const sendError = ( + respond: GatewayRequestHandlerOptions["respond"], + error: unknown, + code: Parameters[0] = ErrorCodes.UNAVAILABLE, + ) => { + const payload = { error: formatErrorMessage(error) }; + respond(false, payload, errorShape(code, payload.error, { details: payload })); + }; + const sendRequestError = (respond: GatewayRequestHandlerOptions["respond"], error: unknown) => + sendError( + respond, + error, + error instanceof TeamsMeetingsInvalidRequestError + ? ErrorCodes.INVALID_REQUEST + : ErrorCodes.UNAVAILABLE, + ); + + api.registerGatewayMethod( + "teamsmeetings.join", + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = keepTrustedToolAgentId(asRecord(params), client); + respond(true, await (await ensureRuntime()).join(joinRequest(raw))); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + api.registerGatewayMethod( + "teamsmeetings.leave", + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = asRecord(params); + const agentId = trustedToolAgentId(raw, client); + const sessionId = requireString(raw.sessionId, "sessionId"); + const rt = await ensureRuntime(); + respond( + true, + agentId && !rt.ownsSession(agentId, sessionId) + ? { found: false } + : await rt.leave(sessionId), + ); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + api.registerGatewayMethod( + "teamsmeetings.status", + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = asRecord(params); + const agentId = trustedToolAgentId(raw, client); + const rt = await ensureRuntime(); + respond( + true, + agentId + ? await rt.statusForAgent(agentId, normalizeOptionalString(raw.sessionId)) + : await rt.status(normalizeOptionalString(raw.sessionId)), + ); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + api.registerGatewayMethod( + "teamsmeetings.transcript", + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = asRecord(params); + const sessionId = requireString(raw.sessionId, "sessionId"); + const sinceIndex = readSinceIndex(raw); + const agentId = trustedToolAgentId(raw, client); + const rt = await ensureRuntime(); + respond( + true, + agentId && !rt.ownsSession(agentId, sessionId) + ? { found: false } + : await rt.transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex }), + ); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + api.registerGatewayMethod( + "teamsmeetings.speak", + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = asRecord(params); + const sessionId = requireString(raw.sessionId, "sessionId"); + const agentId = trustedToolAgentId(raw, client); + const rt = await ensureRuntime(); + respond( + true, + agentId && !rt.ownsSession(agentId, sessionId) + ? { found: false, spoken: false } + : await rt.speak(sessionId, normalizeOptionalString(raw.message)), + ); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + api.registerGatewayMethod( + "teamsmeetings.setup", + async ({ params, respond }: GatewayRequestHandlerOptions) => { + try { + respond( + true, + await ( + await ensureRuntime() + ).setupStatus({ + mode: normalizeMode(params?.mode), + transport: normalizeTransport(params?.transport), + }), + ); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + + for (const [method, run] of [ + [ + "teamsmeetings.testSpeech", + (rt: TeamsMeetingsRuntime, raw: Record) => + rt.testSpeech(joinRequest(raw, { allowTimeout: true })), + ], + [ + "teamsmeetings.testListen", + (rt: TeamsMeetingsRuntime, raw: Record) => rt.testListen(joinRequest(raw)), + ], + ] as const) { + api.registerGatewayMethod( + method, + async ({ params, client, respond }: GatewayRequestHandlerOptions) => { + try { + const raw = keepTrustedToolAgentId(asRecord(params), client); + respond(true, await run(await ensureRuntime(), raw)); + } catch (error) { + sendRequestError(respond, error); + } + }, + ); + } + + api.registerTool( + (toolContext) => ({ + name: "teams_meetings", + label: "Microsoft Teams meetings", + description: + "Join and manage Microsoft Teams meeting browser guests. Guest admission, tenant sign-in, and media permissions may require manual action in the OpenClaw Chrome profile.", + parameters: TeamsMeetingsToolSchema, + async execute(_toolCallId, params) { + const raw = asRecord(params); + const action = raw.action as ToolAction; + const requesterSessionKey = normalizeOptionalString(toolContext.sessionKey); + const contextAgentId = + toolContext.agentId ?? parseAgentSessionKey(requesterSessionKey)?.agentId; + const agentId = contextAgentId ? normalizeAgentId(contextAgentId) : undefined; + try { + if (!(["join", "leave", "status", "transcript", "speak"] as const).includes(action)) { + throw new Error("unknown teams_meetings action"); + } + const trustedRouting = Boolean(agentId && agentId !== "main"); + const useRuntime = trustedRouting ? await api.runtime.gateway.isAvailable() : false; + if (trustedRouting && !useRuntime) { + throw new Error( + "Per-agent Microsoft Teams meeting routing requires a Gateway-hosted agent run.", + ); + } + return json( + await callGatewayFromTool({ + action, + config, + raw: { + ...raw, + ...(requesterSessionKey ? { requesterSessionKey } : {}), + ...(useRuntime ? { agentId } : {}), + }, + runtime: useRuntime ? api.runtime : undefined, + }), + ); + } catch (error) { + return json({ error: formatErrorMessage(error) }); + } + }, + }), + { name: "teams_meetings" }, + ); + + api.registerNodeHostCommand({ + command: TEAMS_MEETINGS_NODE_COMMAND, + cap: "teams-meetings", + dangerous: true, + handle: handleTeamsMeetingsNodeHostCommand, + }); + api.registerNodeInvokePolicy(createTeamsMeetingsNodeInvokePolicy(config)); + api.registerCli( + async ({ program }) => { + const cli = await loadTeamsMeetingsCli(); + cli.registerTeamsMeetingsCli({ program, config }); + }, + { + commands: ["teamsmeetings"], + descriptors: [ + { + name: "teamsmeetings", + description: "Join and manage Microsoft Teams meeting guests", + hasSubcommands: true, + }, + ], + }, + ); + }, +}); diff --git a/extensions/teams-meetings/openclaw.plugin.json b/extensions/teams-meetings/openclaw.plugin.json new file mode 100644 index 000000000000..83bb82cd51ba --- /dev/null +++ b/extensions/teams-meetings/openclaw.plugin.json @@ -0,0 +1,110 @@ +{ + "id": "teams-meetings", + "name": "Microsoft Teams meetings", + "description": "Join Microsoft Teams meetings as a Chrome browser guest.", + "icon": "https://cdn.simpleicons.org/microsoftteams", + "enabledByDefault": false, + "commandAliases": [{ "name": "teamsmeetings" }], + "activation": { + "onStartup": true, + "onCommands": ["teamsmeetings"], + "onCapabilities": ["tool"] + }, + "contracts": { + "tools": ["teams_meetings"] + }, + "uiHints": { + "defaultMode": { + "label": "Default Mode", + "help": "Agent consults OpenClaw, bidi uses direct realtime voice, and transcribe observes only." + }, + "chrome.browserProfile": { "label": "Chrome Profile", "advanced": true }, + "chrome.guestName": { "label": "Guest Name" }, + "chrome.waitForInCallMs": { "label": "Wait For In-Call (ms)", "advanced": true }, + "chrome.audioInputCommand": { "label": "Audio Input Command", "advanced": true }, + "chrome.audioOutputCommand": { "label": "Audio Output Command", "advanced": true }, + "chromeNode.node": { + "label": "Chrome Node", + "help": "Node id/name/IP that owns Chrome, BlackHole, and SoX.", + "advanced": true + }, + "realtime.transcriptionProvider": { "label": "Realtime Transcription Provider" }, + "realtime.voiceProvider": { "label": "Bidi Voice Provider" }, + "realtime.model": { "label": "Bidi Realtime Model", "advanced": true }, + "realtime.instructions": { "label": "Realtime Instructions", "advanced": true }, + "realtime.introMessage": { "label": "Realtime Intro Message" }, + "realtime.agentId": { "label": "Realtime Consult Agent", "advanced": true }, + "realtime.toolPolicy": { "label": "Realtime Tool Policy", "advanced": true } + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "defaultMode": { + "type": "string", + "enum": ["agent", "bidi", "transcribe"], + "default": "agent" + }, + "chrome": { + "type": "object", + "additionalProperties": false, + "properties": { + "audioBackend": { + "type": "string", + "enum": ["blackhole-2ch"], + "default": "blackhole-2ch" + }, + "audioFormat": { + "type": "string", + "enum": ["pcm16-24khz", "g711-ulaw-8khz"], + "default": "pcm16-24khz" + }, + "audioBufferBytes": { "type": "number", "minimum": 17 }, + "launch": { "type": "boolean" }, + "browserProfile": { "type": "string" }, + "guestName": { "type": "string" }, + "reuseExistingTab": { "type": "boolean" }, + "autoJoin": { "type": "boolean" }, + "joinTimeoutMs": { "type": "number", "minimum": 1 }, + "waitForInCallMs": { "type": "number", "minimum": 1 }, + "audioInputCommand": { "type": "array", "items": { "type": "string" } }, + "audioOutputCommand": { "type": "array", "items": { "type": "string" } }, + "bargeInInputCommand": { "type": "array", "items": { "type": "string" } }, + "bargeInRmsThreshold": { "type": "number", "exclusiveMinimum": 0 }, + "bargeInPeakThreshold": { "type": "number", "exclusiveMinimum": 0 }, + "bargeInCooldownMs": { "type": "number", "minimum": 1 } + } + }, + "chromeNode": { + "type": "object", + "additionalProperties": false, + "properties": { + "node": { "type": "string" } + } + }, + "realtime": { + "type": "object", + "additionalProperties": false, + "properties": { + "strategy": { "type": "string", "enum": ["agent", "bidi"] }, + "provider": { "type": "string" }, + "transcriptionProvider": { "type": "string" }, + "voiceProvider": { "type": "string" }, + "model": { "type": "string" }, + "instructions": { "type": "string" }, + "introMessage": { "type": "string" }, + "agentId": { "type": "string" }, + "toolPolicy": { + "type": "string", + "enum": ["safe-read-only", "owner", "none"] + }, + "providers": { + "type": "object", + "additionalProperties": { "type": "object" } + } + } + } + } + } +} diff --git a/extensions/teams-meetings/package.json b/extensions/teams-meetings/package.json new file mode 100644 index 000000000000..fb3715ed89b5 --- /dev/null +++ b/extensions/teams-meetings/package.json @@ -0,0 +1,33 @@ +{ + "name": "@openclaw/teams-meetings", + "version": "2026.7.2", + "description": "OpenClaw Microsoft Teams browser meeting participant plugin.", + "private": true, + "type": "module", + "dependencies": { + "typebox": "1.3.3" + }, + "devDependencies": { + "@openclaw/plugin-sdk": "workspace:*", + "openclaw": "workspace:*" + }, + "peerDependencies": { + "openclaw": ">=2026.7.2" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ], + "compat": { + "pluginApi": ">=2026.7.2" + }, + "build": { + "openclawVersion": "2026.7.2" + } + } +} diff --git a/extensions/teams-meetings/src/agent-consult.ts b/extensions/teams-meetings/src/agent-consult.ts new file mode 100644 index 000000000000..b7ed65e48646 --- /dev/null +++ b/extensions/teams-meetings/src/agent-consult.ts @@ -0,0 +1,93 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { + consultMeetingAgent, + handleMeetingRealtimeConsultToolCall, + resolveMeetingRealtimeTools, + type MeetingAgentConsultSurface, +} from "openclaw/plugin-sdk/meeting-runtime"; +import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import type { + RealtimeVoiceBridgeSession, + RealtimeVoiceTool, + RealtimeVoiceToolCallEvent, + TalkEventInput, +} from "openclaw/plugin-sdk/realtime-voice"; +import type { TeamsMeetingsConfig, TeamsMeetingsToolPolicy } from "./config.js"; + +const TEAMS_MEETINGS_CONSULT_SURFACE: MeetingAgentConsultSurface = { + id: "teams-meetings", + provider: "teams-meetings", + lane: "teams-meetings", + surface: "a private Microsoft Teams meeting", + userLabel: "Participant", + assistantLabel: "Agent", + questionSourceLabel: "participant", + workingResponseLabel: "participant", + extraSystemPrompt: [ + "You are a behind-the-scenes consultant for a live meeting voice agent.", + "Prioritize a fast, speakable answer over exhaustive investigation.", + "Use only bounded, task-relevant tool calls.", + "Never print secrets or dump environment variables.", + "Be accurate, brief, and speakable.", + ].join(" "), +}; + +export function resolveTeamsMeetingsRealtimeTools( + policy: TeamsMeetingsToolPolicy, +): RealtimeVoiceTool[] { + return resolveMeetingRealtimeTools(policy); +} + +export async function consultOpenClawAgentForTeamsMeeting(params: { + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + runtime: PluginRuntime; + logger: RuntimeLogger; + meetingSessionId: string; + requesterSessionKey?: string; + args: unknown; + transcript: Array<{ role: "user" | "assistant"; text: string }>; +}): Promise<{ text: string }> { + return await consultMeetingAgent({ + surface: TEAMS_MEETINGS_CONSULT_SURFACE, + config: params.fullConfig, + runtime: params.runtime, + logger: params.logger, + agentId: params.config.realtime.agentId, + toolPolicy: params.config.realtime.toolPolicy, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + args: params.args, + transcript: params.transcript, + }); +} + +export async function handleTeamsMeetingsRealtimeConsultToolCall(params: { + strategy: string; + session: RealtimeVoiceBridgeSession; + event: RealtimeVoiceToolCallEvent; + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + runtime: PluginRuntime; + logger: RuntimeLogger; + meetingSessionId: string; + requesterSessionKey?: string; + transcript: Array<{ role: "user" | "assistant"; text: string }>; + onTalkEvent?: (event: TalkEventInput) => void; +}): Promise { + await handleMeetingRealtimeConsultToolCall({ + surface: TEAMS_MEETINGS_CONSULT_SURFACE, + strategy: params.strategy, + session: params.session, + event: params.event, + config: params.fullConfig, + runtime: params.runtime, + logger: params.logger, + agentId: params.config.realtime.agentId, + toolPolicy: params.config.realtime.toolPolicy, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + transcript: params.transcript, + onTalkEvent: params.onTalkEvent, + }); +} diff --git a/extensions/teams-meetings/src/cli.ts b/extensions/teams-meetings/src/cli.ts new file mode 100644 index 000000000000..4339f31999ab --- /dev/null +++ b/extensions/teams-meetings/src/cli.ts @@ -0,0 +1,167 @@ +import type { Command } from "commander"; +import { callGatewayFromCli } from "openclaw/plugin-sdk/gateway-runtime"; +import { + addTimerTimeoutGraceMs, + parseStrictNonNegativeInteger, +} from "openclaw/plugin-sdk/number-runtime"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; +import { resolveTeamsMeetingsGatewayOperationTimeoutMs } from "./config.js"; + +type JoinOptions = { + transport?: TeamsMeetingsTransport; + mode?: TeamsMeetingsMode; + message?: string; + timeoutMs?: string; +}; + +function print(value: unknown): void { + process.stdout.write(`${JSON.stringify(value, null, 2)}\n`); +} + +function parseTimeout(value: string | undefined): number | undefined { + if (value === undefined) { + return undefined; + } + const parsed = parseStrictNonNegativeInteger(value); + if (parsed === undefined || parsed === 0) { + throw new Error("timeout-ms must be a positive integer"); + } + return parsed; +} + +async function call(params: { + config: TeamsMeetingsConfig; + method: string; + payload?: Record; +}): Promise { + const requestedTimeout = + typeof params.payload?.timeoutMs === "number" ? params.payload.timeoutMs : undefined; + const timeoutMs = Math.max( + resolveTeamsMeetingsGatewayOperationTimeoutMs(params.config), + requestedTimeout === undefined ? 0 : (addTimerTimeoutGraceMs(requestedTimeout, 30_000) ?? 1), + ); + print( + await callGatewayFromCli( + params.method, + { + json: true, + timeout: String(timeoutMs), + }, + params.payload, + { progress: false, scopes: ["operator.admin"] }, + ), + ); +} + +function joinPayload(url: string, options: JoinOptions): Record { + return { + url, + ...(options.transport ? { transport: options.transport } : {}), + ...(options.mode ? { mode: options.mode } : {}), + ...(options.message ? { message: options.message } : {}), + ...(options.timeoutMs ? { timeoutMs: parseTimeout(options.timeoutMs) } : {}), + }; +} + +function addJoinOptions(command: Command): Command { + return command + .option("--transport ", "chrome or chrome-node") + .option("--mode ", "agent, bidi, or transcribe") + .option("--message ", "instructions to speak after joining"); +} + +function addProbeOptions(command: Command): Command { + return addJoinOptions(command).option("--timeout-ms ", "probe timeout in milliseconds"); +} + +export function registerTeamsMeetingsCli(params: { + program: Command; + config: TeamsMeetingsConfig; +}): void { + const root = params.program + .command("teamsmeetings") + .description("Join and manage Microsoft Teams meeting guests"); + + addJoinOptions(root.command("join ").description("join a Teams meeting as a guest")).action( + async (url: string, options: JoinOptions) => { + await call({ + config: params.config, + method: "teamsmeetings.join", + payload: joinPayload(url, options), + }); + }, + ); + + root + .command("leave ") + .description("leave a Teams meeting") + .action(async (sessionId: string) => { + await call({ config: params.config, method: "teamsmeetings.leave", payload: { sessionId } }); + }); + + root + .command("status [session-id]") + .description("show Teams meeting session status") + .action(async (sessionId?: string) => { + await call({ + config: params.config, + method: "teamsmeetings.status", + payload: sessionId ? { sessionId } : {}, + }); + }); + + root + .command("transcript ") + .description("read the current transcript snapshot") + .option("--since-index ", "resume from a prior transcript index") + .action(async (sessionId: string, options: { sinceIndex?: string }) => { + const sinceIndex = + options.sinceIndex === undefined + ? undefined + : parseStrictNonNegativeInteger(options.sinceIndex); + if (options.sinceIndex !== undefined && sinceIndex === undefined) { + throw new Error("since-index must be a non-negative integer"); + } + await call({ + config: params.config, + method: "teamsmeetings.transcript", + payload: { sessionId, ...(sinceIndex === undefined ? {} : { sinceIndex }) }, + }); + }); + + root + .command("speak [message]") + .description("speak through an active talk-back session") + .action(async (sessionId: string, message?: string) => { + await call({ + config: params.config, + method: "teamsmeetings.speak", + payload: { sessionId, ...(message ? { message } : {}) }, + }); + }); + + root + .command("setup") + .description("check Teams meeting prerequisites") + .option("--transport ", "chrome or chrome-node") + .option("--mode ", "agent, bidi, or transcribe") + .action(async (options: { transport?: TeamsMeetingsTransport; mode?: TeamsMeetingsMode }) => { + await call({ config: params.config, method: "teamsmeetings.setup", payload: options }); + }); + + for (const [name, method, description] of [ + ["test-speech", "teamsmeetings.testSpeech", "join and verify talk-back output"], + [ + "test-listen", + "teamsmeetings.testListen", + "join in transcribe mode and report caption support", + ], + ] as const) { + const command = root.command(`${name} `).description(description); + (name === "test-speech" ? addProbeOptions(command) : addJoinOptions(command)).action( + async (url: string, options: JoinOptions) => { + await call({ config: params.config, method, payload: joinPayload(url, options) }); + }, + ); + } +} diff --git a/extensions/teams-meetings/src/config.test.ts b/extensions/teams-meetings/src/config.test.ts new file mode 100644 index 000000000000..b4b269f721b5 --- /dev/null +++ b/extensions/teams-meetings/src/config.test.ts @@ -0,0 +1,47 @@ +import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; +import { describe, expect, it } from "vitest"; +import { + resolveTeamsMeetingsConfig, + resolveTeamsMeetingsGatewayOperationTimeoutMs, +} from "./config.js"; + +describe("Microsoft Teams meetings config", () => { + it("builds matching SoX commands from the selected audio format", () => { + const config = resolveTeamsMeetingsConfig({ chrome: { audioBufferBytes: 2048 } }); + expect(config.chrome.audioInputCommand).toContain("sox"); + expect(config.chrome.audioInputCommand).toContain("2048"); + expect(config.chrome.audioOutputCommand).toContain("BlackHole 2ch"); + }); + + it("preserves explicit command overrides and realtime passthrough", () => { + const config = resolveTeamsMeetingsConfig({ + defaultMode: "bidi", + chrome: { audioInputCommand: ["capture"], audioOutputCommand: ["play"] }, + chromeNode: { node: "mac-node" }, + realtime: { + voiceProvider: "google", + model: "voice-model", + providers: { google: { apiKey: "ref" } }, + }, + }); + expect(config).toMatchObject({ + defaultMode: "bidi", + chrome: { audioInputCommand: ["capture"], audioOutputCommand: ["play"] }, + chromeNode: { node: "mac-node" }, + realtime: { + voiceProvider: "google", + model: "voice-model", + providers: { google: { apiKey: "ref" } }, + }, + }); + }); + + it("caps timer values and gateway grace", () => { + const config = resolveTeamsMeetingsConfig({ + chrome: { joinTimeoutMs: Number.MAX_VALUE, waitForInCallMs: Number.MAX_VALUE }, + }); + expect(config.chrome.joinTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS); + expect(config.chrome.waitForInCallMs).toBe(MAX_TIMER_TIMEOUT_MS); + expect(resolveTeamsMeetingsGatewayOperationTimeoutMs(config)).toBe(MAX_TIMER_TIMEOUT_MS); + }); +}); diff --git a/extensions/teams-meetings/src/config.ts b/extensions/teams-meetings/src/config.ts new file mode 100644 index 000000000000..ac7edee189a4 --- /dev/null +++ b/extensions/teams-meetings/src/config.ts @@ -0,0 +1,234 @@ +import { buildMeetingSoxAudioCommands } from "openclaw/plugin-sdk/meeting-runtime"; +import { + addTimerTimeoutGraceMs, + resolvePositiveTimerTimeoutMs, +} from "openclaw/plugin-sdk/number-runtime"; +import { + REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + resolveRealtimeVoiceAgentConsultToolPolicy, + type RealtimeVoiceAgentConsultToolPolicy, +} from "openclaw/plugin-sdk/realtime-voice"; +import { + asRecord, + normalizeOptionalLowercaseString, + normalizeOptionalString, + normalizeOptionalTrimmedStringList, +} from "openclaw/plugin-sdk/string-coerce-runtime"; + +export type TeamsMeetingsMode = "agent" | "bidi" | "transcribe"; +export type TeamsMeetingsTransport = "chrome" | "chrome-node"; +export type TeamsMeetingsToolPolicy = RealtimeVoiceAgentConsultToolPolicy; +type TeamsMeetingsRealtimeStrategy = "agent" | "bidi"; +type TeamsMeetingsAudioFormat = "pcm16-24khz" | "g711-ulaw-8khz"; + +export type TeamsMeetingsConfig = { + enabled: boolean; + defaultMode: TeamsMeetingsMode; + chrome: { + audioBackend: "blackhole-2ch"; + audioFormat: TeamsMeetingsAudioFormat; + audioBufferBytes: number; + launch: boolean; + browserProfile?: string; + guestName: string; + reuseExistingTab: boolean; + autoJoin: boolean; + joinTimeoutMs: number; + waitForInCallMs: number; + audioInputCommand: string[]; + audioOutputCommand: string[]; + bargeInInputCommand?: string[]; + bargeInRmsThreshold: number; + bargeInPeakThreshold: number; + bargeInCooldownMs: number; + }; + chromeNode: { + node?: string; + }; + realtime: { + strategy: TeamsMeetingsRealtimeStrategy; + provider?: string; + transcriptionProvider?: string; + voiceProvider?: string; + model?: string; + instructions?: string; + introMessage?: string; + agentId?: string; + toolPolicy: TeamsMeetingsToolPolicy; + providers: Record>; + }; +}; + +export function resolveTeamsMeetingsGatewayOperationTimeoutMs(config: TeamsMeetingsConfig): number { + return Math.max(60_000, addTimerTimeoutGraceMs(config.chrome.joinTimeoutMs, 30_000) ?? 1); +} + +const DEFAULT_AUDIO_BUFFER_BYTES = 4_096; +const DEFAULT_AUDIO_FORMAT: TeamsMeetingsAudioFormat = "pcm16-24khz"; + +function buildSoxCommands(format: TeamsMeetingsAudioFormat, bufferBytes: number) { + return format === "g711-ulaw-8khz" + ? buildMeetingSoxAudioCommands({ + bufferBytes, + format: { sampleRate: 8_000, channels: 1, encoding: "mu-law", bits: 8 }, + }) + : buildMeetingSoxAudioCommands({ + bufferBytes, + device: "BlackHole 2ch", + deviceType: "coreaudio", + format: { + sampleRate: 24_000, + channels: 1, + encoding: "signed-integer", + bits: 16, + endian: "little", + }, + }); +} + +const DEFAULT_SOX_COMMANDS = buildSoxCommands(DEFAULT_AUDIO_FORMAT, DEFAULT_AUDIO_BUFFER_BYTES); + +export const DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND = DEFAULT_SOX_COMMANDS.inputCommand; +export const DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND = DEFAULT_SOX_COMMANDS.outputCommand; + +const DEFAULT_REALTIME_INSTRUCTIONS = `You are joining a private Microsoft Teams meeting as an OpenClaw voice transport. Keep spoken replies brief and natural. In agent mode, wait for OpenClaw consult results and speak them exactly. In bidi mode, answer directly and call ${REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME} for deeper reasoning, current information, or tools.`; + +const DEFAULT_CONFIG: TeamsMeetingsConfig = { + enabled: true, + defaultMode: "agent", + chrome: { + audioBackend: "blackhole-2ch", + audioFormat: DEFAULT_AUDIO_FORMAT, + audioBufferBytes: DEFAULT_AUDIO_BUFFER_BYTES, + launch: true, + guestName: "OpenClaw Agent", + reuseExistingTab: true, + autoJoin: true, + joinTimeoutMs: 30_000, + waitForInCallMs: 20_000, + audioInputCommand: [...DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND], + audioOutputCommand: [...DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND], + bargeInRmsThreshold: 650, + bargeInPeakThreshold: 2_500, + bargeInCooldownMs: 900, + }, + chromeNode: {}, + realtime: { + strategy: "agent", + provider: "openai", + transcriptionProvider: "openai", + instructions: DEFAULT_REALTIME_INSTRUCTIONS, + introMessage: "Say exactly: I'm here and listening.", + toolPolicy: "safe-read-only", + providers: {}, + }, +}; + +function resolveBoolean(value: unknown, fallback: boolean): boolean { + return typeof value === "boolean" ? value : fallback; +} + +function resolvePositiveNumber(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback; +} + +function resolveTimer(value: unknown, fallback: number): number { + return resolvePositiveTimerTimeoutMs(resolvePositiveNumber(value, fallback), fallback); +} + +function resolveMode(value: unknown): TeamsMeetingsMode { + const normalized = normalizeOptionalLowercaseString(value); + return normalized === "agent" || normalized === "bidi" || normalized === "transcribe" + ? normalized + : DEFAULT_CONFIG.defaultMode; +} + +function resolveAudioFormat(value: unknown): TeamsMeetingsAudioFormat { + const normalized = normalizeOptionalLowercaseString(value)?.replaceAll("_", "-"); + return normalized === "g711-ulaw-8khz" ? normalized : DEFAULT_AUDIO_FORMAT; +} + +function resolveProviders(value: unknown): Record> { + const providers: Record> = {}; + for (const [key, entry] of Object.entries(asRecord(value))) { + const id = normalizeOptionalLowercaseString(key); + if (id) { + providers[id] = asRecord(entry); + } + } + return providers; +} + +export function resolveTeamsMeetingsConfig(input: unknown): TeamsMeetingsConfig { + const raw = asRecord(input); + const chrome = asRecord(raw.chrome); + const chromeNode = asRecord(raw.chromeNode); + const realtime = asRecord(raw.realtime); + const audioFormat = resolveAudioFormat(chrome.audioFormat); + const audioBufferBytes = Math.max( + 17, + Math.trunc(resolvePositiveNumber(chrome.audioBufferBytes, DEFAULT_AUDIO_BUFFER_BYTES)), + ); + const generatedCommands = buildSoxCommands(audioFormat, audioBufferBytes); + const provider = normalizeOptionalString(realtime.provider) ?? DEFAULT_CONFIG.realtime.provider; + return { + enabled: resolveBoolean(raw.enabled, DEFAULT_CONFIG.enabled), + defaultMode: resolveMode(raw.defaultMode), + chrome: { + audioBackend: "blackhole-2ch", + audioFormat, + audioBufferBytes, + launch: resolveBoolean(chrome.launch, DEFAULT_CONFIG.chrome.launch), + browserProfile: normalizeOptionalString(chrome.browserProfile), + guestName: normalizeOptionalString(chrome.guestName) ?? DEFAULT_CONFIG.chrome.guestName, + reuseExistingTab: resolveBoolean( + chrome.reuseExistingTab, + DEFAULT_CONFIG.chrome.reuseExistingTab, + ), + autoJoin: resolveBoolean(chrome.autoJoin, DEFAULT_CONFIG.chrome.autoJoin), + joinTimeoutMs: resolveTimer(chrome.joinTimeoutMs, DEFAULT_CONFIG.chrome.joinTimeoutMs), + waitForInCallMs: resolveTimer(chrome.waitForInCallMs, DEFAULT_CONFIG.chrome.waitForInCallMs), + audioInputCommand: + normalizeOptionalTrimmedStringList(chrome.audioInputCommand) ?? + generatedCommands.inputCommand, + audioOutputCommand: + normalizeOptionalTrimmedStringList(chrome.audioOutputCommand) ?? + generatedCommands.outputCommand, + bargeInInputCommand: normalizeOptionalTrimmedStringList(chrome.bargeInInputCommand), + bargeInRmsThreshold: resolvePositiveNumber( + chrome.bargeInRmsThreshold, + DEFAULT_CONFIG.chrome.bargeInRmsThreshold, + ), + bargeInPeakThreshold: resolvePositiveNumber( + chrome.bargeInPeakThreshold, + DEFAULT_CONFIG.chrome.bargeInPeakThreshold, + ), + bargeInCooldownMs: resolveTimer( + chrome.bargeInCooldownMs, + DEFAULT_CONFIG.chrome.bargeInCooldownMs, + ), + }, + chromeNode: { node: normalizeOptionalString(chromeNode.node) }, + realtime: { + strategy: normalizeOptionalLowercaseString(realtime.strategy) === "bidi" ? "bidi" : "agent", + provider, + transcriptionProvider: + normalizeOptionalString(realtime.transcriptionProvider) ?? + DEFAULT_CONFIG.realtime.transcriptionProvider, + voiceProvider: normalizeOptionalString(realtime.voiceProvider), + model: normalizeOptionalString(realtime.model), + instructions: + normalizeOptionalString(realtime.instructions) ?? DEFAULT_CONFIG.realtime.instructions, + introMessage: + typeof realtime.introMessage === "string" + ? realtime.introMessage.trim() + : DEFAULT_CONFIG.realtime.introMessage, + agentId: normalizeOptionalString(realtime.agentId), + toolPolicy: resolveRealtimeVoiceAgentConsultToolPolicy( + realtime.toolPolicy, + DEFAULT_CONFIG.realtime.toolPolicy, + ), + providers: resolveProviders(realtime.providers), + }, + }; +} diff --git a/extensions/teams-meetings/src/node-host.ts b/extensions/teams-meetings/src/node-host.ts new file mode 100644 index 000000000000..0b242f8ff9ed --- /dev/null +++ b/extensions/teams-meetings/src/node-host.ts @@ -0,0 +1,117 @@ +import { spawnSync } from "node:child_process"; +import { createMeetingNodeHost } from "openclaw/plugin-sdk/meeting-runtime"; +import { + DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND, + DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND, +} from "./config.js"; +import { + TEAMS_MEETINGS_SYSTEM_PROFILER_COMMAND, + outputMentionsBlackHole2ch, +} from "./transports/chrome-audio-device.js"; +import { TEAMS_MEETINGS_PLATFORM_ADAPTER } from "./transports/teams-meetings-platform-adapter.js"; +import { TEAMS_MEETINGS_NODE_COMMAND } from "./transports/teams-meetings-platform-constants.js"; + +function commandExists(command: string, timeoutMs: number): boolean { + const result = spawnSync("/bin/sh", ["-lc", 'command -v "$1" >/dev/null 2>&1', "sh", command], { + encoding: "utf8", + timeout: timeoutMs, + }); + return result.status === 0; +} + +function assertTalkBackPrerequisites( + timeoutMs: number, + commands: readonly (readonly string[])[] = [ + DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND, + DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND, + ], +) { + if (process.platform !== "darwin") { + throw new Error("Microsoft Teams meeting talk-back with BlackHole 2ch is macOS-only"); + } + const result = spawnSync(TEAMS_MEETINGS_SYSTEM_PROFILER_COMMAND, ["SPAudioDataType"], { + encoding: "utf8", + timeout: timeoutMs, + }); + const stderr = + result.stderr ?? + (result.error + ? result.error instanceof Error + ? result.error.message + : String(result.error) + : ""); + const output = `${result.stdout ?? ""}\n${stderr}`; + if ( + (typeof result.status === "number" ? result.status : result.error ? 1 : 0) !== 0 || + !outputMentionsBlackHole2ch(output) + ) { + throw new Error("BlackHole 2ch audio device not found on the node."); + } + for (const argv of commands) { + const command = argv[0]; + if (!command || !commandExists(command, timeoutMs)) { + throw new Error(`Configured audio command not found on the node: ${command || ""}`); + } + } +} + +function readCommand(params: Record, name: string): string[] { + const value = params[name]; + if ( + !Array.isArray(value) || + value.length === 0 || + value.some((entry) => typeof entry !== "string") + ) { + throw new Error(`${name} must be a non-empty string array.`); + } + return value as string[]; +} + +const teamsMeetingsNodeHost = createMeetingNodeHost({ + commandName: TEAMS_MEETINGS_NODE_COMMAND, + displayName: "Microsoft Teams meetings", + browserLabel: "Teams meeting", + bridgeIdPrefix: "teams_meeting_node_", + defaultAudioInputCommand: DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND, + defaultAudioOutputCommand: DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND, + talkBackModes: new Set(["agent", "bidi"]), + agentMode: "agent", + normalizeUrl: (url) => TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(url), + normalizeMeetingKey: (url) => TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.normalizeForReuse(url), + assertAudioAvailable: assertTalkBackPrerequisites, + browser: { + application: "Google Chrome", + buildProfileArgs: (profile) => ["--args", `--profile-directory=${profile}`], + openedStatus: "chrome-opened", + openedNotes: [ + "Teams page control is handled by OpenClaw browser automation when using chrome-node.", + ], + }, +}); + +export async function handleTeamsMeetingsNodeHostCommand( + paramsJSON?: string | null, +): Promise { + if (paramsJSON) { + let raw: unknown; + try { + raw = JSON.parse(paramsJSON) as unknown; + } catch { + throw new Error("Microsoft Teams meetings node host received malformed params JSON."); + } + const params = + raw && typeof raw === "object" && !Array.isArray(raw) ? (raw as Record) : {}; + if (params.action === "setup") { + const commands = [ + readCommand(params, "audioInputCommand"), + readCommand(params, "audioOutputCommand"), + ]; + if (params.bargeInInputCommand !== undefined) { + commands.push(readCommand(params, "bargeInInputCommand")); + } + assertTalkBackPrerequisites(10_000, commands); + return JSON.stringify({ ok: true }); + } + } + return await teamsMeetingsNodeHost.handleCommand(paramsJSON); +} diff --git a/extensions/teams-meetings/src/node-invoke-policy.test.ts b/extensions/teams-meetings/src/node-invoke-policy.test.ts new file mode 100644 index 000000000000..1368d44d2d36 --- /dev/null +++ b/extensions/teams-meetings/src/node-invoke-policy.test.ts @@ -0,0 +1,39 @@ +import type { OpenClawPluginNodeInvokePolicyContext } from "openclaw/plugin-sdk/plugin-entry"; +import { describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "./config.js"; +import { createTeamsMeetingsNodeInvokePolicy } from "./node-invoke-policy.js"; + +describe("Microsoft Teams meetings node invoke policy", () => { + it("replaces setup probe commands with trusted configured commands", async () => { + const config = resolveTeamsMeetingsConfig({ + chrome: { + audioInputCommand: ["trusted-input", "--read"], + audioOutputCommand: ["trusted-output", "--write"], + bargeInInputCommand: ["trusted-barge-in"], + }, + }); + const invokeNode = vi.fn(async () => ({ ok: true as const })); + const policy = createTeamsMeetingsNodeInvokePolicy(config); + + await policy.handle({ + command: "teamsmeetings.chrome", + config: {}, + invokeNode, + nodeId: "node-1", + params: { + action: "setup", + audioInputCommand: ["untrusted-input"], + audioOutputCommand: ["untrusted-output"], + }, + } as OpenClawPluginNodeInvokePolicyContext); + + expect(invokeNode).toHaveBeenCalledWith({ + params: { + action: "setup", + audioInputCommand: ["trusted-input", "--read"], + audioOutputCommand: ["trusted-output", "--write"], + bargeInInputCommand: ["trusted-barge-in"], + }, + }); + }); +}); diff --git a/extensions/teams-meetings/src/node-invoke-policy.ts b/extensions/teams-meetings/src/node-invoke-policy.ts new file mode 100644 index 000000000000..9eb5f1424581 --- /dev/null +++ b/extensions/teams-meetings/src/node-invoke-policy.ts @@ -0,0 +1,43 @@ +import { createMeetingBrowserNodeInvokePolicy } from "openclaw/plugin-sdk/meeting-runtime"; +import type { + OpenClawPluginNodeInvokePolicy, + OpenClawPluginNodeInvokePolicyContext, +} from "openclaw/plugin-sdk/plugin-entry"; +import type { TeamsMeetingsConfig } from "./config.js"; +import { TEAMS_MEETINGS_PLATFORM_ADAPTER } from "./transports/teams-meetings-platform-adapter.js"; +import { TEAMS_MEETINGS_NODE_COMMAND } from "./transports/teams-meetings-platform-constants.js"; + +export function createTeamsMeetingsNodeInvokePolicy( + config: TeamsMeetingsConfig, +): OpenClawPluginNodeInvokePolicy { + const base = createMeetingBrowserNodeInvokePolicy({ + commandName: TEAMS_MEETINGS_NODE_COMMAND, + displayName: "Microsoft Teams meetings", + deniedCode: "TEAMS_MEETINGS_NODE_POLICY_DENIED", + supportedModes: new Set(["agent", "bidi", "transcribe"]), + normalizeUrl: (url) => TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(url), + start: config.chrome, + }); + return { + ...base, + async handle(ctx: OpenClawPluginNodeInvokePolicyContext) { + const params = + ctx.params && typeof ctx.params === "object" && !Array.isArray(ctx.params) + ? (ctx.params as Record) + : {}; + if (ctx.command !== TEAMS_MEETINGS_NODE_COMMAND || params.action !== "setup") { + return await base.handle(ctx); + } + return await ctx.invokeNode({ + params: { + action: "setup", + audioInputCommand: [...config.chrome.audioInputCommand], + audioOutputCommand: [...config.chrome.audioOutputCommand], + ...(config.chrome.bargeInInputCommand + ? { bargeInInputCommand: [...config.chrome.bargeInInputCommand] } + : {}), + }, + }); + }, + }; +} diff --git a/extensions/teams-meetings/src/runtime-node.test.ts b/extensions/teams-meetings/src/runtime-node.test.ts new file mode 100644 index 000000000000..bcd22582bbfb --- /dev/null +++ b/extensions/teams-meetings/src/runtime-node.test.ts @@ -0,0 +1,147 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "./config.js"; + +const realtimeMocks = vi.hoisted(() => ({ + speak: vi.fn(), + startAgent: vi.fn(async () => ({ + getHealth: () => ({}), + providerId: "test", + speak: realtimeMocks.speak, + stop: vi.fn(async () => {}), + })), +})); + +vi.mock("openclaw/plugin-sdk/meeting-runtime", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + createNodeMeetingRealtimeAudioTransport: () => ({ + clearOutput: vi.fn(async () => {}), + dispose: vi.fn(async () => {}), + onFatal: vi.fn(), + startInput: vi.fn(), + stop: vi.fn(async () => {}), + writeOutput: vi.fn(async () => {}), + }), + startMeetingAgentRealtimeEngine: realtimeMocks.startAgent, + }; +}); + +import { TeamsMeetingsRuntime } from "./runtime.js"; + +const URL = "https://teams.microsoft.com/l/meetup-join/19%3ameeting_node_resume%40thread.v2/0"; + +describe("Microsoft Teams meetings node realtime recovery", () => { + it("starts the node bridge after manual admission becomes route-ready", async () => { + let routeReady = false; + let tabOpen = false; + const invoke = vi.fn(async (request: Record) => { + const params = (request.params as Record) ?? {}; + if (request.command === "browser.proxy") { + if (params.path === "/tabs") { + return { + payload: { + result: { + tabs: tabOpen ? [{ targetId: "teams-tab", title: "Teams", url: URL }] : [], + }, + }, + }; + } + if (params.path === "/tabs/open") { + tabOpen = true; + return { payload: { result: { targetId: "teams-tab", title: "Teams", url: URL } } }; + } + if (params.path === "/tabs/focus") { + return { payload: { result: { ok: true } } }; + } + if (params.path === "/act") { + const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn; + const script = typeof scriptValue === "string" ? scriptValue : ""; + if (script.includes("leaveAction")) { + return { + payload: { + result: { result: JSON.stringify({ departed: true, urlMatched: true }) }, + }, + }; + } + return { + payload: { + result: { + result: JSON.stringify( + routeReady + ? { + audioInputRouted: true, + audioOutputRouted: true, + inCall: true, + micMuted: false, + url: URL, + } + : { + inCall: false, + manualActionMessage: "Waiting for admission", + manualActionReason: "teams-admission-required", + manualActionRequired: true, + url: URL, + }, + ), + }, + }, + }; + } + } + if (params.action === "start") { + return { + payload: { + audioBridge: { type: "node-command-pair" }, + bridgeId: "bridge-1", + }, + }; + } + return { payload: { ok: true } }; + }); + const runtime = new TeamsMeetingsRuntime({ + config: resolveTeamsMeetingsConfig({ + chrome: { waitForInCallMs: 1 }, + chromeNode: { node: "node-1" }, + }), + fullConfig: {}, + logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() }, + runtime: { + nodes: { + invoke, + list: vi.fn(async () => ({ + nodes: [ + { + caps: ["browser"], + commands: ["browser.proxy", "teamsmeetings.chrome"], + connected: true, + nodeId: "node-1", + }, + ], + })), + }, + } as unknown as PluginRuntime, + }); + + const joined = await runtime.join({ + message: undefined, + mode: "agent", + requesterSessionKey: "agent:support:session:caller", + transport: "chrome-node", + url: URL, + }); + expect(joined.session.chrome?.audioBridge).toBeUndefined(); + routeReady = true; + + const spoken = await runtime.speak(joined.session.id, "hello"); + + expect(spoken.spoken).toBe(true); + expect(realtimeMocks.startAgent).toHaveBeenCalledTimes(1); + expect(realtimeMocks.startAgent).toHaveBeenCalledWith( + expect.objectContaining({ requesterSessionKey: "agent:support:session:caller" }), + ); + expect(realtimeMocks.speak).toHaveBeenCalledWith("hello"); + expect(joined.session.chrome?.audioBridge).toMatchObject({ type: "node-command-pair" }); + }); +}); diff --git a/extensions/teams-meetings/src/runtime-probes.test.ts b/extensions/teams-meetings/src/runtime-probes.test.ts new file mode 100644 index 000000000000..5c92c293a42d --- /dev/null +++ b/extensions/teams-meetings/src/runtime-probes.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "./config.js"; +import { testTeamsMeetingSpeech, type TeamsMeetingsProbeContext } from "./runtime-probes.js"; +import type { TeamsMeetingsSession } from "./transports/types.js"; + +const URL = "https://teams.microsoft.com/l/meetup-join/19%3ameeting_probe%40thread.v2/0"; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("Microsoft Teams meeting runtime probes", () => { + it("uses the per-request speech verification timeout", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + const session = { + agentId: "main", + chrome: { health: { inCall: true, lastOutputBytes: 0 } }, + id: "teams-1", + mode: "agent", + transport: "chrome", + } as TeamsMeetingsSession; + const refreshHealth = vi.fn(); + const context = { + config: resolveTeamsMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }), + hasHealthHandle: () => true, + isReusable: () => false, + join: vi.fn(async () => ({ session, spoken: true })), + list: () => [], + refreshHealth, + resolveAgentId: () => "main", + } satisfies TeamsMeetingsProbeContext; + + const pending = testTeamsMeetingSpeech(context, { + mode: "agent", + timeoutMs: 150, + url: URL, + }); + await vi.advanceTimersByTimeAsync(200); + const result = await pending; + + expect(result.speechOutputTimedOut).toBe(true); + expect(refreshHealth).toHaveBeenCalledTimes(2); + expect(Date.now()).toBe(200); + }); +}); diff --git a/extensions/teams-meetings/src/runtime-probes.ts b/extensions/teams-meetings/src/runtime-probes.ts new file mode 100644 index 000000000000..01bfed83757f --- /dev/null +++ b/extensions/teams-meetings/src/runtime-probes.ts @@ -0,0 +1,130 @@ +import { sleep } from "openclaw/plugin-sdk/runtime-env"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; +import type { + TeamsMeetingsJoinRequest, + TeamsMeetingsJoinResult, + TeamsMeetingsSession, +} from "./transports/types.js"; + +export type TeamsMeetingsProbeContext = { + config: TeamsMeetingsConfig; + resolveAgentId(request: TeamsMeetingsJoinRequest): string; + list(): TeamsMeetingsSession[]; + join(request: TeamsMeetingsJoinRequest): Promise; + isReusable( + session: TeamsMeetingsSession, + resolved: { + url: string; + transport: TeamsMeetingsTransport; + mode: TeamsMeetingsMode; + agentId: string; + }, + ): boolean; + hasHealthHandle(sessionId: string): boolean; + refreshHealth(sessionId: string): void; +}; + +function talkBackMode(mode: TeamsMeetingsMode): boolean { + return mode === "agent" || mode === "bidi"; +} + +function resolveProbeTimeoutMs(input: number | undefined, fallback: number): number { + if (input === undefined) { + return Math.min(Math.max(fallback, 1), 120_000); + } + if (!Number.isFinite(input) || input <= 0) { + throw new Error("timeoutMs must be a positive number"); + } + return Math.min(Math.trunc(input), 120_000); +} + +export async function testTeamsMeetingSpeech( + context: TeamsMeetingsProbeContext, + request: TeamsMeetingsJoinRequest, +) { + if (request.mode === "transcribe") { + throw new Error("test_speech requires mode: agent or bidi"); + } + const mode = talkBackMode(request.mode ?? context.config.defaultMode) + ? (request.mode ?? context.config.defaultMode) + : "agent"; + const resolved = { + url: request.url, + transport: request.transport ?? (context.config.chromeNode.node ? "chrome-node" : "chrome"), + mode, + agentId: context.resolveAgentId(request), + } satisfies { + url: string; + transport: TeamsMeetingsTransport; + mode: TeamsMeetingsMode; + agentId: string; + }; + const beforeSessions = context.list(); + const before = new Set(beforeSessions.map((session) => session.id)); + const existing = beforeSessions.find((session) => context.isReusable(session, resolved)); + const existingOutputBytes = existing?.chrome?.health?.lastOutputBytes ?? 0; + const result = await context.join({ + ...request, + ...resolved, + message: request.message ?? "Say exactly: Microsoft Teams speech test complete.", + }); + const startOutputBytes = existing?.id === result.session.id ? existingOutputBytes : 0; + let health = result.session.chrome?.health; + const shouldWait = + result.spoken === true && + health?.manualActionRequired !== true && + context.hasHealthHandle(result.session.id); + if (shouldWait && (health?.lastOutputBytes ?? 0) <= startOutputBytes) { + const deadline = + Date.now() + resolveProbeTimeoutMs(request.timeoutMs, context.config.chrome.joinTimeoutMs); + while (Date.now() < deadline && (health?.lastOutputBytes ?? 0) <= startOutputBytes) { + await sleep(100); + context.refreshHealth(result.session.id); + health = result.session.chrome?.health; + } + } + const speechOutputVerified = (health?.lastOutputBytes ?? 0) > startOutputBytes; + return { + createdSession: !before.has(result.session.id), + inCall: health?.inCall, + manualActionRequired: health?.manualActionRequired, + manualActionReason: health?.manualActionReason, + manualActionMessage: health?.manualActionMessage, + spoken: result.spoken ?? false, + speechOutputVerified, + speechOutputTimedOut: shouldWait && !speechOutputVerified, + speechReady: health?.speechReady, + speechBlockedReason: health?.speechBlockedReason, + speechBlockedMessage: health?.speechBlockedMessage, + audioOutputActive: health?.audioOutputActive, + lastOutputBytes: health?.lastOutputBytes, + session: result.session, + }; +} + +export async function testTeamsMeetingListening( + context: TeamsMeetingsProbeContext, + request: TeamsMeetingsJoinRequest, +) { + if (request.timeoutMs !== undefined) { + throw new Error("timeoutMs is not supported while Teams caption scraping is disabled"); + } + if (request.mode && request.mode !== "transcribe") { + throw new Error("test_listen requires mode: transcribe"); + } + const before = new Set(context.list().map((session) => session.id)); + const result = await context.join({ ...request, mode: "transcribe", message: undefined }); + const health = result.session.chrome?.health; + return { + createdSession: !before.has(result.session.id), + inCall: health?.inCall, + manualActionRequired: health?.manualActionRequired, + manualActionReason: health?.manualActionReason, + manualActionMessage: health?.manualActionMessage, + listenVerified: false, + listenTimedOut: false, + captionScrapingEnabled: false, + note: "Teams caption scraping is disabled pending live selector validation.", + session: result.session, + }; +} diff --git a/extensions/teams-meetings/src/runtime-session.ts b/extensions/teams-meetings/src/runtime-session.ts new file mode 100644 index 000000000000..5930f2fc6a30 --- /dev/null +++ b/extensions/teams-meetings/src/runtime-session.ts @@ -0,0 +1,42 @@ +import { randomUUID } from "node:crypto"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; +import type { TeamsMeetingsSession } from "./transports/types.js"; + +export function createTeamsMeetingsSession(params: { + config: TeamsMeetingsConfig; + resolved: { + url: string; + transport: TeamsMeetingsTransport; + mode: TeamsMeetingsMode; + agentId: string; + }; + createdAt: string; +}): TeamsMeetingsSession { + const { config, createdAt, resolved } = params; + return { + id: `teams_meeting_${randomUUID()}`, + ...resolved, + state: "active", + createdAt, + updatedAt: createdAt, + participantIdentity: + resolved.transport === "chrome-node" + ? "Microsoft Teams guest in Chrome on a paired node" + : "Microsoft Teams guest in the OpenClaw Chrome profile", + realtime: { + enabled: resolved.mode === "agent" || resolved.mode === "bidi", + strategy: resolved.mode === "bidi" ? "bidi" : "agent", + provider: + resolved.mode === "bidi" + ? (config.realtime.voiceProvider ?? config.realtime.provider) + : undefined, + model: resolved.mode === "bidi" ? config.realtime.model : undefined, + transcriptionProvider: + resolved.mode === "agent" + ? (config.realtime.transcriptionProvider ?? config.realtime.provider) + : undefined, + toolPolicy: config.realtime.toolPolicy, + }, + notes: [], + }; +} diff --git a/extensions/teams-meetings/src/runtime-setup.test.ts b/extensions/teams-meetings/src/runtime-setup.test.ts new file mode 100644 index 000000000000..036de74de3f6 --- /dev/null +++ b/extensions/teams-meetings/src/runtime-setup.test.ts @@ -0,0 +1,80 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "./config.js"; +import { getTeamsMeetingsSetupStatus } from "./runtime-setup.js"; + +function runtimeWithNode(invoke: (params: Record) => Promise) { + return { + nodes: { + invoke: vi.fn(invoke), + list: vi.fn(async () => ({ + nodes: [ + { + caps: ["browser"], + commands: ["browser.proxy", "teamsmeetings.chrome"], + connected: true, + displayName: "teams-node", + nodeId: "node-1", + }, + ], + })), + }, + } as unknown as PluginRuntime; +} + +describe("Microsoft Teams meetings runtime setup", () => { + it("probes remote talk-back prerequisites through the selected Chrome node", async () => { + const runtime = runtimeWithNode(async () => ({ ok: true })); + const config = resolveTeamsMeetingsConfig({ + chrome: { + audioInputCommand: ["custom-input", "--read"], + audioOutputCommand: ["custom-output", "--write"], + bargeInInputCommand: ["custom-barge-in"], + }, + chromeNode: { node: "teams-node" }, + }); + const status = await getTeamsMeetingsSetupStatus({ + config, + fullConfig: {}, + runtime, + options: { mode: "agent", transport: "chrome-node" }, + }); + + expect(runtime.nodes.invoke).toHaveBeenCalledWith({ + command: "teamsmeetings.chrome", + nodeId: "node-1", + params: { + action: "setup", + audioInputCommand: ["custom-input", "--read"], + audioOutputCommand: ["custom-output", "--write"], + bargeInInputCommand: ["custom-barge-in"], + }, + timeoutMs: 12_000, + }); + expect(status.checks).toContainEqual({ + id: "chrome-node-audio-prerequisites", + message: "Remote macOS, BlackHole 2ch, and SoX prerequisites are ready", + ok: true, + }); + expect(status.ok).toBe(true); + }); + + it("fails setup when the remote prerequisite probe fails", async () => { + const runtime = runtimeWithNode(async () => { + throw new Error("SoX audio command not found on the node."); + }); + const status = await getTeamsMeetingsSetupStatus({ + config: resolveTeamsMeetingsConfig({ chromeNode: { node: "teams-node" } }), + fullConfig: {}, + runtime, + options: { mode: "bidi", transport: "chrome-node" }, + }); + + expect(status.ok).toBe(false); + expect(status.checks).toContainEqual({ + id: "chrome-node-audio-prerequisites", + message: "SoX audio command not found on the node.", + ok: false, + }); + }); +}); diff --git a/extensions/teams-meetings/src/runtime-setup.ts b/extensions/teams-meetings/src/runtime-setup.ts new file mode 100644 index 000000000000..b0ab8f538e37 --- /dev/null +++ b/extensions/teams-meetings/src/runtime-setup.ts @@ -0,0 +1,166 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + addMeetingSetupCheck, + createMeetingSetupStatus, + resolveMeetingBrowserNodeInfo, + type MeetingSetupStatus, +} from "openclaw/plugin-sdk/meeting-runtime"; +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; +import { assertBlackHole2chAvailable } from "./transports/chrome.js"; +import { TEAMS_MEETINGS_BROWSER_NODE_ADAPTER } from "./transports/teams-meetings-platform-constants.js"; + +function audioCommands(config: TeamsMeetingsConfig): string[] { + return uniqueStrings( + [ + config.chrome.audioInputCommand[0], + config.chrome.audioOutputCommand[0], + config.chrome.bargeInInputCommand?.[0], + ].filter((value): value is string => Boolean(value?.trim())), + ); +} + +async function commandExists(runtime: PluginRuntime, command: string): Promise { + const result = await runtime.system.runCommandWithTimeout( + ["/bin/sh", "-lc", 'command -v "$1" >/dev/null 2>&1', "sh", command], + { timeoutMs: 5_000 }, + ); + return result.code === 0; +} + +export async function getTeamsMeetingsSetupStatus(params: { + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + runtime: PluginRuntime; + options?: { mode?: TeamsMeetingsMode; transport?: TeamsMeetingsTransport }; +}): Promise { + const mode = params.options?.mode ?? params.config.defaultMode; + const transport = + params.options?.transport ?? (params.config.chromeNode.node ? "chrome-node" : "chrome"); + const talkBack = mode === "agent" || mode === "bidi"; + let status = createMeetingSetupStatus([ + { + id: "chrome-profile", + ok: true, + message: params.config.chrome.browserProfile + ? `Chrome node profile configured: ${params.config.chrome.browserProfile}` + : "Local Chrome uses the configured OpenClaw browser profile", + }, + { + id: "guest-join", + ok: Boolean( + params.config.chrome.guestName && + params.config.chrome.autoJoin && + params.config.chrome.reuseExistingTab, + ), + message: + params.config.chrome.guestName && + params.config.chrome.autoJoin && + params.config.chrome.reuseExistingTab + ? "Guest name, auto-join, and tab reuse are configured" + : "Set chrome.guestName, chrome.autoJoin, and chrome.reuseExistingTab for unattended guest joins", + }, + { + id: "captions", + ok: true, + message: + mode === "transcribe" + ? "Teams caption scraping is disabled pending live selector validation; transcript snapshots are empty" + : "Caption scraping is not used by talk-back modes", + }, + ]); + + if (transport === "chrome-node") { + try { + const node = await resolveMeetingBrowserNodeInfo({ + runtime: params.runtime, + requestedNode: params.config.chromeNode.node, + adapter: TEAMS_MEETINGS_BROWSER_NODE_ADAPTER, + }); + status = addMeetingSetupCheck(status, { + id: "chrome-node-connected", + ok: true, + message: `Connected Teams meeting node ready: ${node.displayName ?? node.remoteIp ?? node.nodeId}`, + }); + if (talkBack) { + if (!node.nodeId) { + throw new Error("Connected Microsoft Teams meetings node did not include a node id."); + } + await params.runtime.nodes.invoke({ + nodeId: node.nodeId, + command: TEAMS_MEETINGS_BROWSER_NODE_ADAPTER.nodeCommandName, + params: { + action: "setup", + audioInputCommand: params.config.chrome.audioInputCommand, + audioOutputCommand: params.config.chrome.audioOutputCommand, + ...(params.config.chrome.bargeInInputCommand + ? { bargeInInputCommand: params.config.chrome.bargeInInputCommand } + : {}), + }, + timeoutMs: 12_000, + }); + status = addMeetingSetupCheck(status, { + id: "chrome-node-audio-prerequisites", + ok: true, + message: "Remote macOS, BlackHole 2ch, and SoX prerequisites are ready", + }); + } + } catch (error) { + const connected = status.checks.some( + (check) => check.id === "chrome-node-connected" && check.ok, + ); + status = addMeetingSetupCheck(status, { + id: connected ? "chrome-node-audio-prerequisites" : "chrome-node-connected", + ok: false, + message: formatErrorMessage(error), + }); + } + } + + if (!talkBack) { + return status; + } + status = addMeetingSetupCheck(status, { + id: "audio-bridge", + ok: + params.config.chrome.audioInputCommand.length > 0 && + params.config.chrome.audioOutputCommand.length > 0, + message: `SoX command-pair audio bridge configured (${params.config.chrome.audioFormat})`, + }); + if (transport === "chrome-node") { + return status; + } + try { + await assertBlackHole2chAvailable({ + runtime: params.runtime, + timeoutMs: Math.min(params.config.chrome.joinTimeoutMs, 10_000), + }); + status = addMeetingSetupCheck(status, { + id: "chrome-local-audio-device", + ok: true, + message: "BlackHole 2ch audio device found", + }); + } catch (error) { + status = addMeetingSetupCheck(status, { + id: "chrome-local-audio-device", + ok: false, + message: formatErrorMessage(error), + }); + } + const missing: string[] = []; + for (const command of audioCommands(params.config)) { + if (!(await commandExists(params.runtime, command).catch(() => false))) { + missing.push(command); + } + } + return addMeetingSetupCheck(status, { + id: "chrome-local-audio-commands", + ok: missing.length === 0, + message: + missing.length === 0 + ? "Configured Chrome audio commands are available" + : `Chrome audio commands missing: ${missing.join(", ")}`, + }); +} diff --git a/extensions/teams-meetings/src/runtime.test.ts b/extensions/teams-meetings/src/runtime.test.ts new file mode 100644 index 000000000000..d1724903ff26 --- /dev/null +++ b/extensions/teams-meetings/src/runtime.test.ts @@ -0,0 +1,212 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "./config.js"; +import { TeamsMeetingsRuntime } from "./runtime.js"; + +const URL = + "https://teams.microsoft.com/l/meetup-join/19%3ameeting_runtime%40thread.v2/0?context=%7b%22Tid%22%3a%22one%22%7d"; + +const logger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), +}; + +function runtimeHarness(options?: { tabOpen?: boolean }) { + let tabOpen = options?.tabOpen ?? false; + let targetId = "teams-tab"; + let tabUrl = URL; + const gatewayRequest = vi.fn(async (_method: string, params: Record) => { + if (params.path === "/tabs") { + return { + tabs: tabOpen ? [{ targetId, title: "Teams call", url: tabUrl }] : [], + }; + } + if (params.path === "/tabs/open") { + tabOpen = true; + tabUrl = URL; + return { targetId, title: "Teams call", url: tabUrl }; + } + if (params.path === "/tabs/focus") { + return { ok: true }; + } + if (params.path === "/act") { + const rawFn = (params.body as { fn?: unknown } | undefined)?.fn; + const fn = typeof rawFn === "string" ? rawFn : ""; + if (fn.includes("expectedSessionId")) { + return { + result: JSON.stringify({ + urlMatched: true, + sessionMatched: true, + droppedLines: 0, + lines: [], + }), + }; + } + if (fn.includes("leaveAction")) { + return { result: JSON.stringify({ departed: true, urlMatched: true }) }; + } + return { + result: JSON.stringify({ + inCall: true, + micMuted: true, + cameraOff: true, + url: tabUrl, + title: "Teams call", + }), + }; + } + if (params.method === "DELETE" && params.path === `/tabs/${targetId}`) { + tabOpen = false; + return { ok: true }; + } + throw new Error(`unexpected browser request ${String(params.method)} ${String(params.path)}`); + }); + const runtime = { + gateway: { + isAvailable: vi.fn(async () => true), + request: gatewayRequest, + }, + system: { + runCommandWithTimeout: vi.fn(async () => ({ code: 0, stdout: "", stderr: "" })), + }, + } as unknown as PluginRuntime; + return { + runtime, + gatewayRequest, + setTargetId(value: string) { + targetId = value; + }, + setTabUrl(value: string) { + tabUrl = value; + }, + }; +} + +describe("Microsoft Teams meeting session flow", () => { + it("joins, reuses, reports, snapshots, speaks safely, and leaves through core", async () => { + const harness = runtimeHarness(); + const runtime = new TeamsMeetingsRuntime({ + config: resolveTeamsMeetingsConfig({ + defaultMode: "transcribe", + chrome: { waitForInCallMs: 1 }, + }), + fullConfig: {}, + runtime: harness.runtime, + logger, + }); + + const first = await runtime.join({ url: URL, mode: "transcribe" }); + expect(first.session.chrome?.health).toMatchObject({ inCall: true, cameraOff: true }); + + const reused = await runtime.join({ + url: `${URL.split("?")[0]}?context=%7b%22Tid%22%3a%22two%22%7d`, + mode: "transcribe", + }); + expect(reused.session.id).toBe(first.session.id); + expect(runtime.list()).toHaveLength(1); + + expect(await runtime.status(first.session.id)).toMatchObject({ + found: true, + session: { id: first.session.id }, + }); + expect(await runtime.transcript(first.session.id)).toMatchObject({ + found: true, + lines: [], + nextIndex: 0, + }); + expect(await runtime.speak(first.session.id, "hello")).toMatchObject({ + found: true, + spoken: false, + }); + expect(await runtime.leave(first.session.id)).toMatchObject({ + found: true, + browserLeft: true, + session: { state: "ended" }, + }); + expect(harness.gatewayRequest).toHaveBeenCalledWith( + "browser.request", + expect.objectContaining({ path: "/tabs/open" }), + expect.objectContaining({ scopes: ["operator.admin"] }), + ); + }); + + it("recovers and leaves a manually opened tab when Chrome launching is disabled", async () => { + const harness = runtimeHarness({ tabOpen: true }); + const runtime = new TeamsMeetingsRuntime({ + config: resolveTeamsMeetingsConfig({ + defaultMode: "transcribe", + chrome: { launch: false, waitForInCallMs: 1 }, + }), + fullConfig: {}, + runtime: harness.runtime, + logger, + }); + + const joined = await runtime.join({ url: URL, mode: "transcribe" }); + expect(joined.session.chrome).toMatchObject({ + browserTab: { openedByPlugin: false, targetId: "teams-tab" }, + launched: false, + }); + expect(harness.gatewayRequest).not.toHaveBeenCalledWith( + "browser.request", + expect.objectContaining({ path: "/tabs/open" }), + expect.anything(), + ); + expect(await runtime.leave(joined.session.id)).toMatchObject({ + browserLeft: true, + session: { state: "ended" }, + }); + }); + + it("refreshes a recovered browser tab target", async () => { + const harness = runtimeHarness(); + const runtime = new TeamsMeetingsRuntime({ + config: resolveTeamsMeetingsConfig({ + defaultMode: "transcribe", + chrome: { waitForInCallMs: 1 }, + }), + fullConfig: {}, + runtime: harness.runtime, + logger, + }); + const joined = await runtime.join({ url: URL, mode: "transcribe" }); + harness.setTargetId("teams-tab-replaced"); + + await runtime.status(joined.session.id); + + expect(joined.session.chrome?.browserTab).toEqual({ + openedByPlugin: false, + targetId: "teams-tab-replaced", + }); + }); + + it("recovers the tracked tab after Teams rewrites the in-call URL", async () => { + const harness = runtimeHarness(); + const runtime = new TeamsMeetingsRuntime({ + config: resolveTeamsMeetingsConfig({ + defaultMode: "transcribe", + chrome: { waitForInCallMs: 1 }, + }), + fullConfig: {}, + runtime: harness.runtime, + logger, + }); + const joined = await runtime.join({ url: URL, mode: "transcribe" }); + harness.setTabUrl("https://teams.microsoft.com/v2/"); + harness.gatewayRequest.mockClear(); + + const status = await runtime.status(joined.session.id); + + expect(status.session?.chrome?.health?.browserUrl).toBe("https://teams.microsoft.com/v2/"); + expect(harness.gatewayRequest).toHaveBeenCalledWith( + "browser.request", + expect.objectContaining({ + path: "/act", + body: expect.objectContaining({ targetId: "teams-tab" }), + }), + expect.objectContaining({ scopes: ["operator.admin"] }), + ); + }); +}); diff --git a/extensions/teams-meetings/src/runtime.ts b/extensions/teams-meetings/src/runtime.ts new file mode 100644 index 000000000000..679cc89da7d2 --- /dev/null +++ b/extensions/teams-meetings/src/runtime.ts @@ -0,0 +1,505 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { + MeetingSessionRuntime, + type MeetingSessionRuntimeHandles, + type MeetingSessionRuntimeJoinContext, +} from "openclaw/plugin-sdk/meeting-runtime"; +import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode, TeamsMeetingsTransport } from "./config.js"; +import { + testTeamsMeetingListening, + testTeamsMeetingSpeech, + type TeamsMeetingsProbeContext, +} from "./runtime-probes.js"; +import { createTeamsMeetingsSession } from "./runtime-session.js"; +import { getTeamsMeetingsSetupStatus } from "./runtime-setup.js"; +import { + launchTeamsMeetingInChrome, + launchTeamsMeetingOnNode, + leaveTeamsMeetingInBrowser, + readTeamsMeetingTranscript, + recoverCurrentTeamsMeetingTab, +} from "./transports/chrome.js"; +import { + TEAMS_MEETINGS_PLATFORM_ADAPTER, + isTeamsMeetingsRealtimeRouteReady, + isTeamsMeetingsTalkBackMode, +} from "./transports/teams-meetings-platform-adapter.js"; +import type { + TeamsMeetingsBrowserTab, + TeamsMeetingsChromeHealth, + TeamsMeetingsJoinRequest, + TeamsMeetingsJoinResult, + TeamsMeetingsSession, +} from "./transports/types.js"; + +type ManualActionReason = NonNullable; +type SpeechBlockedReason = NonNullable; +type SessionRuntime = MeetingSessionRuntime< + TeamsMeetingsSession, + TeamsMeetingsJoinRequest, + TeamsMeetingsTransport, + TeamsMeetingsMode, + TeamsMeetingsChromeHealth, + TeamsMeetingsBrowserTab, + ManualActionReason, + SpeechBlockedReason +>; +type JoinContext = MeetingSessionRuntimeJoinContext< + TeamsMeetingsSession, + TeamsMeetingsTransport, + TeamsMeetingsMode, + TeamsMeetingsChromeHealth, + TeamsMeetingsBrowserTab +>; +type LaunchResult = + | Awaited> + | Awaited>; +type AudioBridge = NonNullable; + +function nowIso(): string { + return new Date().toISOString(); +} + +function resolveTransport( + request: TeamsMeetingsJoinRequest, + config: TeamsMeetingsConfig, +): TeamsMeetingsTransport { + return request.transport ?? (config.chromeNode.node ? "chrome-node" : "chrome"); +} + +function withSessionAgentConfig(config: TeamsMeetingsConfig, agentId: string): TeamsMeetingsConfig { + return config.realtime.agentId === agentId + ? config + : { ...config, realtime: { ...config.realtime, agentId } }; +} + +function noteSession(session: TeamsMeetingsSession, note: string): void { + session.notes = [...session.notes.filter((item) => item !== note), note]; +} + +export class TeamsMeetingsRuntime { + readonly #sessions: SessionRuntime; + readonly #requesterSessionKeys = new Map(); + + constructor( + private readonly params: { + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + runtime: PluginRuntime; + logger: RuntimeLogger; + }, + ) { + this.#sessions = new MeetingSessionRuntime({ + logger: params.logger, + logScope: TEAMS_MEETINGS_PLATFORM_ADAPTER.logScope, + formatError: formatErrorMessage, + reuseExistingBrowserTab: params.config.chrome.reuseExistingTab, + waitForInCallMs: params.config.chrome.waitForInCallMs, + joinTimeoutMs: params.config.chrome.joinTimeoutMs, + defaultSpeechInstructions: params.config.realtime.introMessage, + transientSpeechBlockedReasons: new Set([ + "not-in-call", + "browser-unverified", + "teams-microphone-muted", + ]), + messages: { + previousBrowserLeaveFailed: + "Could not leave the previous Teams meeting tab before reassignment.", + reassignedSessionNote: + "Ended before the same Teams meeting tab was reassigned to another agent.", + reusedSessionNote: "Reused existing active Microsoft Teams meeting session.", + replacementBrowserLeaveFailed: + "Could not leave the previous Teams meeting tab before reassignment.", + speechBlockedFallback: "Realtime speech blocked until Microsoft Teams is ready.", + speech: { + audioBridgeUnavailable: "Realtime speech requires an active Chrome audio bridge.", + browserUnverified: "Microsoft Teams browser state has not been verified yet.", + manualActionFallback: + "Resolve the Microsoft Teams browser prompt before asking OpenClaw to speak.", + microphoneMuted: "Turn on the OpenClaw Teams microphone before asking OpenClaw to speak.", + microphoneMutedReason: "teams-microphone-muted", + notInCall: "Microsoft Teams has not reported that the browser guest is in the call.", + notInCallReason: "not-in-call", + browserUnverifiedReason: "browser-unverified", + audioBridgeUnavailableReason: "audio-bridge-unavailable", + }, + }, + resolveJoin: (request) => ({ + url: TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(request.url), + transport: resolveTransport(request, params.config), + mode: request.mode ?? params.config.defaultMode, + agentId: normalizeAgentId(request.agentId ?? params.config.realtime.agentId), + }), + createSession: ({ request, resolved, createdAt }) => { + const session = createTeamsMeetingsSession({ config: params.config, resolved, createdAt }); + if (request.requesterSessionKey) { + this.#requesterSessionKeys.set(session.id, request.requesterSessionKey); + } + return session; + }, + resolveSpeechInstructions: (request) => + request.message ?? params.config.realtime.introMessage, + isBrowserTransport: () => true, + isTalkBackMode: isTeamsMeetingsTalkBackMode, + isTranscribeMode: (mode) => mode === "transcribe", + sameMeetingUrl: (left, right) => + TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.isSameMeeting(left, right), + normalizeMeetingUrlForReuse: (url) => + TEAMS_MEETINGS_PLATFORM_ADAPTER.urls.normalizeForReuse(url), + getBrowser: (session) => + session.chrome + ? { + launched: session.chrome.launched, + nodeId: session.chrome.nodeId, + tab: session.chrome.browserTab, + health: session.chrome.health, + hasAudioBridge: Boolean(session.chrome.audioBridge), + } + : undefined, + setBrowserTab: (session, tab) => { + if (session.chrome) { + session.chrome.browserTab = tab; + } + }, + setBrowserHealth: (session, health) => { + if (session.chrome) { + session.chrome.health = health; + } + }, + joinTransport: async ({ request, session, context }) => + await this.#joinTransport(request, session, context), + releaseBrowserTab: async (session) => await this.#releaseBrowserTab(session), + refreshBrowserHealth: async (session, options) => + await this.#refreshBrowserHealth(session, options), + refreshStatus: async (session) => + await this.#sessions.refreshBrowserHealth(session, { force: true, readOnly: true }), + refreshReusableSession: async () => {}, + ensureRealtimeBridge: async (session) => await this.#ensureRealtimeBridge(session), + captureTranscript: async (session, options) => + await this.#captureTranscript(session, options), + speakViaTransport: async () => undefined, + }); + } + + list(): TeamsMeetingsSession[] { + return this.#sessions.list(); + } + + ownsSession(agentId: string, sessionId: string): boolean { + return this.list().some((session) => session.id === sessionId && session.agentId === agentId); + } + + async join(request: TeamsMeetingsJoinRequest): Promise { + try { + return await this.#sessions.join(request); + } catch (error) { + const activeIds = new Set(this.list().map((session) => session.id)); + for (const sessionId of this.#requesterSessionKeys.keys()) { + if (!activeIds.has(sessionId)) { + this.#requesterSessionKeys.delete(sessionId); + } + } + throw error; + } + } + + async leave(sessionId: string) { + try { + return await this.#sessions.leave(sessionId); + } finally { + this.#requesterSessionKeys.delete(sessionId); + } + } + + async status(sessionId?: string) { + return await this.#sessions.status(sessionId); + } + + async statusForAgent(agentId: string, sessionId?: string) { + if (sessionId) { + return this.ownsSession(agentId, sessionId) + ? await this.#sessions.status(sessionId) + : { found: false }; + } + const sessions = this.list().filter((session) => session.agentId === agentId); + await Promise.all(sessions.map((session) => this.#sessions.status(session.id))); + return { found: true, sessions }; + } + + async transcript(sessionId: string, options: { sinceIndex?: number } = {}) { + return await this.#sessions.transcript(sessionId, options); + } + + async speak(sessionId: string, instructions?: string) { + return await this.#sessions.speak(sessionId, instructions); + } + + async setupStatus(options?: { mode?: TeamsMeetingsMode; transport?: TeamsMeetingsTransport }) { + return await getTeamsMeetingsSetupStatus({ + config: this.params.config, + fullConfig: this.params.fullConfig, + runtime: this.params.runtime, + options, + }); + } + + async testSpeech(request: TeamsMeetingsJoinRequest) { + return await testTeamsMeetingSpeech(this.#probeContext(), request); + } + + async testListen(request: TeamsMeetingsJoinRequest) { + return await testTeamsMeetingListening(this.#probeContext(), request); + } + + #probeContext(): TeamsMeetingsProbeContext { + return { + config: this.params.config, + resolveAgentId: (request) => + normalizeAgentId(request.agentId ?? this.params.config.realtime.agentId), + list: () => this.list(), + join: async (request) => await this.join(request), + isReusable: (session, resolved) => this.#sessions.isReusableSession(session, resolved), + hasHealthHandle: (sessionId) => this.#sessions.hasHealthHandle(sessionId), + refreshHealth: (sessionId) => this.#sessions.refreshHealth(sessionId), + }; + } + + async #joinTransport( + request: TeamsMeetingsJoinRequest, + session: TeamsMeetingsSession, + context: JoinContext, + ): Promise<{ delegatedSpoken?: boolean }> { + const config = withSessionAgentConfig(this.params.config, session.agentId); + const result: LaunchResult = + session.transport === "chrome-node" + ? await launchTeamsMeetingOnNode({ + runtime: this.params.runtime, + config, + fullConfig: this.params.fullConfig, + meetingSessionId: session.id, + requesterSessionKey: request.requesterSessionKey, + mode: session.mode, + url: session.url, + logger: this.params.logger, + }) + : await launchTeamsMeetingInChrome({ + runtime: this.params.runtime, + config, + fullConfig: this.params.fullConfig, + meetingSessionId: session.id, + requesterSessionKey: request.requesterSessionKey, + mode: session.mode, + url: session.url, + logger: this.params.logger, + }); + const nodeId = "nodeId" in result ? result.nodeId : undefined; + const tab = context.inheritedBrowserTab({ + session, + transport: session.transport, + nodeId, + meetingUrl: session.url, + tab: result.tab, + }); + session.chrome = { + audioBackend: "blackhole-2ch", + launched: result.launched, + nodeId, + browserProfile: this.params.config.chrome.browserProfile, + browserTab: tab, + health: result.browser, + }; + const handles = this.#attachAudioBridge(session, result.audioBridge); + if (handles) { + context.attachRuntimeHandles(session, handles); + } + session.notes.push( + result.audioBridge + ? session.transport === "chrome-node" + ? "Teams guest joined in Chrome on the selected node with realtime audio through the node bridge." + : "Teams guest joined in local Chrome with realtime audio through BlackHole 2ch and SoX." + : session.mode === "transcribe" + ? "Teams guest joined observe-only; caption snapshots remain empty pending live selector validation." + : "Teams guest join is waiting for the browser to become ready before starting realtime audio.", + ); + this.#sessions.refreshSpeechReadiness(session); + return {}; + } + + #attachAudioBridge( + session: TeamsMeetingsSession, + audioBridge: AudioBridge | undefined, + ): MeetingSessionRuntimeHandles | undefined { + if (!session.chrome || !audioBridge) { + return undefined; + } + session.chrome.audioBridge = { + type: audioBridge.type, + provider: audioBridge.providerId, + }; + return { + stop: audioBridge.stop, + speak: audioBridge.speak, + getHealth: audioBridge.getHealth, + }; + } + + async #ensureRealtimeBridge( + session: TeamsMeetingsSession, + ): Promise | undefined> { + if ( + !isTeamsMeetingsTalkBackMode(session.mode) || + session.state !== "active" || + !session.chrome || + session.chrome.audioBridge || + !isTeamsMeetingsRealtimeRouteReady(session.mode, session.chrome.health) + ) { + return undefined; + } + const config = withSessionAgentConfig(this.params.config, session.agentId); + const recoveryConfig = { + ...config, + chrome: { ...config.chrome, launch: false }, + chromeNode: { node: session.chrome.nodeId ?? config.chromeNode.node }, + }; + const result = + session.transport === "chrome-node" + ? await launchTeamsMeetingOnNode({ + runtime: this.params.runtime, + config: recoveryConfig, + fullConfig: this.params.fullConfig, + meetingSessionId: session.id, + requesterSessionKey: this.#requesterSessionKeys.get(session.id), + mode: session.mode, + trackedTargetId: session.chrome.browserTab?.targetId, + url: session.url, + logger: this.params.logger, + }) + : await launchTeamsMeetingInChrome({ + runtime: this.params.runtime, + config: recoveryConfig, + fullConfig: this.params.fullConfig, + meetingSessionId: session.id, + requesterSessionKey: this.#requesterSessionKeys.get(session.id), + mode: session.mode, + trackedTargetId: session.chrome.browserTab?.targetId, + url: session.url, + logger: this.params.logger, + }); + if (result.tab) { + const currentTab = session.chrome.browserTab; + session.chrome.browserTab = { + ...result.tab, + openedByPlugin: + result.tab.targetId === currentTab?.targetId + ? currentTab.openedByPlugin + : result.tab.openedByPlugin, + }; + } + if (result.browser) { + session.chrome.health = { ...session.chrome.health, ...result.browser }; + } + session.updatedAt = nowIso(); + return this.#attachAudioBridge(session, result.audioBridge); + } + + async #refreshBrowserHealth( + session: TeamsMeetingsSession, + options: { readOnly?: boolean } = {}, + ): Promise { + try { + const result = await recoverCurrentTeamsMeetingTab({ + runtime: this.params.runtime, + config: this.params.config, + mode: session.mode, + nodeId: session.chrome?.nodeId, + readOnly: options.readOnly, + trackedMeetingUrl: session.url, + trackedTargetId: session.chrome?.browserTab?.targetId, + transport: session.transport, + url: session.url, + }); + if (result.found && session.chrome) { + if (result.tab?.targetId) { + const currentTab = session.chrome.browserTab; + session.chrome.browserTab = { + targetId: result.tab.targetId, + openedByPlugin: + result.tab.targetId === currentTab?.targetId ? currentTab.openedByPlugin : false, + }; + } + if (result.browser) { + session.chrome.health = { ...session.chrome.health, ...result.browser }; + } + session.updatedAt = nowIso(); + } + } catch (error) { + this.params.logger.debug?.( + `${TEAMS_MEETINGS_PLATFORM_ADAPTER.logScope} browser readiness refresh ignored: ${formatErrorMessage(error)}`, + ); + } + } + + async #captureTranscript(session: TeamsMeetingsSession, options: { finalize?: boolean } = {}) { + const tab = session.chrome?.browserTab; + if (!tab) { + return undefined; + } + return await readTeamsMeetingTranscript({ + runtime: this.params.runtime, + config: this.params.config, + finalize: options.finalize, + meetingUrl: session.url, + meetingSessionId: session.id, + nodeId: session.chrome?.nodeId, + tab, + }); + } + + async #releaseBrowserTab(session: TeamsMeetingsSession): Promise { + const tab = session.chrome?.browserTab; + if (!tab) { + noteSession( + session, + "No tracked Teams meeting tab; leave the browser meeting manually if it is still active.", + ); + session.browserLeft = false; + return false; + } + const shared = this.list().some( + (other) => + other.id !== session.id && + other.state === "active" && + other.chrome?.browserTab?.targetId === tab.targetId && + other.chrome?.nodeId === session.chrome?.nodeId, + ); + if (shared) { + noteSession(session, "Kept the shared Teams meeting tab open for another active session."); + return undefined; + } + try { + const result = await leaveTeamsMeetingInBrowser({ + runtime: this.params.runtime, + config: this.params.config, + meetingUrl: session.url, + nodeId: session.chrome?.nodeId, + tab, + }); + noteSession(session, result.note); + if (result.left && session.chrome) { + session.chrome.browserTab = undefined; + } + session.browserLeft = result.left; + return result.left; + } catch (error) { + noteSession( + session, + `Browser control could not leave the Teams meeting tab: ${formatErrorMessage(error)}`, + ); + session.browserLeft = false; + return false; + } + } +} diff --git a/extensions/teams-meetings/src/transports/chrome-audio-device.ts b/extensions/teams-meetings/src/transports/chrome-audio-device.ts new file mode 100644 index 000000000000..3b5150f62f34 --- /dev/null +++ b/extensions/teams-meetings/src/transports/chrome-audio-device.ts @@ -0,0 +1,5 @@ +export const TEAMS_MEETINGS_SYSTEM_PROFILER_COMMAND = "/usr/sbin/system_profiler"; + +export function outputMentionsBlackHole2ch(output: string): boolean { + return /\bBlackHole\s+2ch\b/i.test(output); +} diff --git a/extensions/teams-meetings/src/transports/chrome.test.ts b/extensions/teams-meetings/src/transports/chrome.test.ts new file mode 100644 index 000000000000..93d129b3ecfe --- /dev/null +++ b/extensions/teams-meetings/src/transports/chrome.test.ts @@ -0,0 +1,181 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveTeamsMeetingsConfig } from "../config.js"; + +const engineMocks = vi.hoisted(() => ({ + localDispose: vi.fn(async () => {}), + nodeDispose: vi.fn(async () => {}), + startAgent: vi.fn(), +})); + +vi.mock("openclaw/plugin-sdk/meeting-runtime", async (importOriginal) => { + const original = await importOriginal(); + const transport = (dispose: () => Promise) => ({ + clearOutput: vi.fn(async () => {}), + dispose, + onFatal: vi.fn(), + startInput: vi.fn(), + stop: dispose, + writeOutput: vi.fn(async () => {}), + }); + return { + ...original, + createLocalMeetingRealtimeAudioTransport: () => transport(engineMocks.localDispose), + createNodeMeetingRealtimeAudioTransport: () => transport(engineMocks.nodeDispose), + startMeetingAgentRealtimeEngine: engineMocks.startAgent, + }; +}); + +import { launchTeamsMeetingInChrome, launchTeamsMeetingOnNode } from "./chrome.js"; + +const URL = "https://teams.microsoft.com/l/meetup-join/19%3ameeting_rollback%40thread.v2/0"; + +const logger = { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), +}; + +function browserResult(params: Record, state: { tabOpen: boolean }) { + if (params.path === "/tabs") { + return { + tabs: state.tabOpen ? [{ targetId: "teams-tab", title: "Teams", url: URL }] : [], + }; + } + if (params.path === "/tabs/open") { + state.tabOpen = true; + return { targetId: "teams-tab", title: "Teams", url: URL }; + } + if (params.path === "/tabs/focus") { + return { ok: true }; + } + if (params.path === "/act") { + const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn; + const script = typeof scriptValue === "string" ? scriptValue : ""; + return script.includes("leaveAction") + ? { result: JSON.stringify({ departed: true, urlMatched: true }) } + : { + result: JSON.stringify({ + audioInputRouted: true, + audioOutputRouted: true, + inCall: true, + micMuted: false, + url: URL, + }), + }; + } + if (params.method === "DELETE" && params.path === "/tabs/teams-tab") { + state.tabOpen = false; + return { ok: true }; + } + throw new Error( + ["Unexpected browser request:", String(params.method), String(params.path)].join(" "), + ); +} + +beforeEach(() => { + vi.clearAllMocks(); + engineMocks.startAgent.mockRejectedValue(new Error("realtime startup failed")); +}); + +describe("Microsoft Teams meeting Chrome startup cleanup", () => { + it("disposes local audio and leaves the browser when realtime startup fails", async () => { + const state = { tabOpen: false }; + const gatewayRequest = vi.fn(async (_method: string, params: Record) => + browserResult(params, state), + ); + const runtime = { + gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest }, + system: { + runCommandWithTimeout: vi.fn(async () => ({ + code: 0, + stderr: "", + stdout: "BlackHole 2ch", + })), + }, + } as unknown as PluginRuntime; + + await expect( + launchTeamsMeetingInChrome({ + config: resolveTeamsMeetingsConfig({ chrome: { waitForInCallMs: 1 } }), + fullConfig: {}, + logger, + meetingSessionId: "session-1", + mode: "agent", + runtime, + url: URL, + }), + ).rejects.toThrow("realtime startup failed"); + + expect(engineMocks.localDispose).toHaveBeenCalled(); + expect(gatewayRequest).toHaveBeenCalledWith( + "browser.request", + expect.objectContaining({ method: "DELETE", path: "/tabs/teams-tab" }), + expect.anything(), + ); + expect(state.tabOpen).toBe(false); + }); + + it("stops node audio and leaves the remote browser when realtime startup fails", async () => { + const state = { tabOpen: false }; + const invoke = vi.fn(async (request: Record) => { + if (request.command === "browser.proxy") { + return { + payload: { + result: browserResult((request.params as Record) ?? {}, state), + }, + }; + } + const params = (request.params as Record) ?? {}; + if (params.action === "start") { + return { payload: { audioBridge: { type: "node-command-pair" }, bridgeId: "bridge-1" } }; + } + return { payload: { ok: true } }; + }); + const runtime = { + nodes: { + invoke, + list: vi.fn(async () => ({ + nodes: [ + { + caps: ["browser"], + commands: ["browser.proxy", "teamsmeetings.chrome"], + connected: true, + nodeId: "node-1", + }, + ], + })), + }, + } as unknown as PluginRuntime; + + await expect( + launchTeamsMeetingOnNode({ + config: resolveTeamsMeetingsConfig({ chrome: { waitForInCallMs: 1 } }), + fullConfig: {}, + logger, + meetingSessionId: "session-1", + mode: "agent", + runtime, + url: URL, + }), + ).rejects.toThrow("realtime startup failed"); + + expect(engineMocks.nodeDispose).toHaveBeenCalled(); + expect( + invoke.mock.calls.filter( + ([request]) => + request.command === "teamsmeetings.chrome" && + (request.params as Record)?.action === "stopByUrl", + ), + ).toHaveLength(2); + expect( + invoke.mock.calls.some( + ([request]) => + request.command === "browser.proxy" && + (request.params as Record)?.method === "DELETE", + ), + ).toBe(true); + expect(state.tabOpen).toBe(false); + }); +}); diff --git a/extensions/teams-meetings/src/transports/chrome.ts b/extensions/teams-meetings/src/transports/chrome.ts new file mode 100644 index 000000000000..a20494565ecd --- /dev/null +++ b/extensions/teams-meetings/src/transports/chrome.ts @@ -0,0 +1,601 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { + callMeetingBrowserProxyOnNode, + createLocalMeetingRealtimeAudioTransport, + createNodeMeetingRealtimeAudioTransport, + leaveMeetingWithBrowser, + openMeetingWithBrowser, + readMeetingTranscriptWithBrowser, + recoverMeetingBrowserTab, + resolveLocalMeetingBrowserRequest, + resolveMeetingBrowserNode, + startMeetingAgentRealtimeEngine, + startMeetingRealtimeEngine, + type MeetingBrowserRequestCaller, + type MeetingAgentConsultParams, + type MeetingRealtimeAudioEngineHandle, + type MeetingRealtimeToolCallParams, + type MeetingRuntimePlatform, +} from "openclaw/plugin-sdk/meeting-runtime"; +import { addTimerTimeoutGraceMs } from "openclaw/plugin-sdk/number-runtime"; +import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime"; +import { + consultOpenClawAgentForTeamsMeeting, + handleTeamsMeetingsRealtimeConsultToolCall, + resolveTeamsMeetingsRealtimeTools, +} from "../agent-consult.js"; +import type { TeamsMeetingsConfig, TeamsMeetingsMode } from "../config.js"; +import { + TEAMS_MEETINGS_SYSTEM_PROFILER_COMMAND, + outputMentionsBlackHole2ch, +} from "./chrome-audio-device.js"; +import { + TEAMS_MEETINGS_PLATFORM_ADAPTER, + isTeamsMeetingsRealtimeRouteReady, + isTeamsMeetingsTalkBackMode, +} from "./teams-meetings-platform-adapter.js"; +import { + TEAMS_MEETINGS_BROWSER_NODE_ADAPTER, + TEAMS_MEETINGS_NODE_COMMAND, +} from "./teams-meetings-platform-constants.js"; +import type { + TeamsMeetingsBrowserTab, + TeamsMeetingsChromeHealth, + TeamsMeetingsTranscriptSnapshot, +} from "./types.js"; + +const TEAMS_MEETINGS_RUNTIME_PLATFORM = { + displayName: TEAMS_MEETINGS_PLATFORM_ADAPTER.displayName, + logScope: TEAMS_MEETINGS_PLATFORM_ADAPTER.logScope, + sessionIdPrefix: TEAMS_MEETINGS_PLATFORM_ADAPTER.id, +} satisfies MeetingRuntimePlatform; + +type LocalAudioBridge = MeetingRealtimeAudioEngineHandle & { + type: "command-pair"; +}; + +type NodeAudioBridge = MeetingRealtimeAudioEngineHandle & { + type: "node-command-pair"; + nodeId: string; + bridgeId: string; +}; + +async function openOrRecoverTeamsMeeting(params: { + callBrowser: MeetingBrowserRequestCaller; + config: TeamsMeetingsConfig; + meetingSessionId: string; + mode: TeamsMeetingsMode; + trackedTargetId?: string; + url: string; + locationLabel: string; +}) { + if (params.config.chrome.launch) { + return await openMeetingWithBrowser({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: params.callBrowser, + config: params.config.chrome, + session: { + meetingSessionId: params.meetingSessionId, + mode: params.mode, + url: params.url, + }, + }); + } + const recovered = await recoverMeetingBrowserTab({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: params.callBrowser, + config: params.config.chrome, + locationLabel: params.locationLabel, + mode: params.mode, + requestedMeetingUrl: params.url, + trackedMeetingUrl: params.url, + trackedTargetId: params.trackedTargetId, + }); + return { + launched: false, + browser: recovered.browser, + tab: recovered.targetId ? { targetId: recovered.targetId, openedByPlugin: false } : undefined, + }; +} + +async function rollbackTeamsBrowserJoin(params: { + callBrowser: MeetingBrowserRequestCaller; + config: TeamsMeetingsConfig; + logger: RuntimeLogger; + tab?: TeamsMeetingsBrowserTab; + url: string; +}) { + if (!params.tab) { + return; + } + const result = await leaveMeetingWithBrowser({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: params.callBrowser, + launch: true, + meetingUrl: params.url, + tab: params.tab, + timeoutMs: params.config.chrome.joinTimeoutMs, + }).catch((error: unknown) => ({ + left: false, + note: error instanceof Error ? error.message : String(error), + })); + if (!result.left) { + params.logger.warn( + `${TEAMS_MEETINGS_RUNTIME_PLATFORM.logScope} browser rollback after realtime startup failure did not complete: ${result.note}`, + ); + } +} + +function realtimeBindings(params: { + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + runtime: PluginRuntime; + logger: RuntimeLogger; +}) { + return { + platform: TEAMS_MEETINGS_RUNTIME_PLATFORM, + consultAgent: (consult: MeetingAgentConsultParams) => + consultOpenClawAgentForTeamsMeeting({ + config: params.config, + fullConfig: params.fullConfig, + runtime: params.runtime, + logger: params.logger, + ...consult, + }), + tools: resolveTeamsMeetingsRealtimeTools(params.config.realtime.toolPolicy), + handleToolCall: (call: MeetingRealtimeToolCallParams) => + handleTeamsMeetingsRealtimeConsultToolCall({ + config: params.config, + fullConfig: params.fullConfig, + runtime: params.runtime, + logger: params.logger, + ...call, + }), + }; +} + +export async function assertBlackHole2chAvailable(params: { + runtime: PluginRuntime; + timeoutMs: number; +}): Promise { + if (process.platform !== "darwin") { + throw new Error("Microsoft Teams meeting talk-back with BlackHole 2ch is macOS-only"); + } + const result = await params.runtime.system.runCommandWithTimeout( + [TEAMS_MEETINGS_SYSTEM_PROFILER_COMMAND, "SPAudioDataType"], + { timeoutMs: params.timeoutMs }, + ); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + if (result.code !== 0 || !outputMentionsBlackHole2ch(output)) { + const hint = + params.runtime.system.formatNativeDependencyHint?.({ + packageName: "BlackHole 2ch", + downloadCommand: "brew install blackhole-2ch", + }) ?? ""; + throw new Error( + ["BlackHole 2ch audio device not found.", "Install BlackHole 2ch and SoX.", hint] + .filter(Boolean) + .join(" "), + ); + } +} + +async function startLocalAudioBridge(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + meetingSessionId: string; + requesterSessionKey?: string; + mode: TeamsMeetingsMode; + logger: RuntimeLogger; +}): Promise { + if (!isTeamsMeetingsTalkBackMode(params.mode)) { + return undefined; + } + const transport = createLocalMeetingRealtimeAudioTransport({ + inputCommand: params.config.chrome.audioInputCommand, + outputCommand: params.config.chrome.audioOutputCommand, + bargeInInputCommand: params.config.chrome.bargeInInputCommand, + bargeInRmsThreshold: params.config.chrome.bargeInRmsThreshold, + bargeInPeakThreshold: params.config.chrome.bargeInPeakThreshold, + bargeInCooldownMs: params.config.chrome.bargeInCooldownMs, + logger: params.logger, + logScope: TEAMS_MEETINGS_RUNTIME_PLATFORM.logScope, + }); + const bindings = realtimeBindings(params); + try { + const engine = + params.mode === "agent" + ? await startMeetingAgentRealtimeEngine({ + config: params.config, + fullConfig: params.fullConfig, + runtime: params.runtime, + platform: bindings.platform, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + transport, + logger: params.logger, + consultAgent: bindings.consultAgent, + }) + : await startMeetingRealtimeEngine({ + config: { + ...params.config, + realtime: { ...params.config.realtime, strategy: "bidi" }, + }, + fullConfig: params.fullConfig, + runtime: params.runtime, + ...bindings, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + transport, + logger: params.logger, + }); + return { type: "command-pair", ...engine }; + } catch (error) { + await transport.dispose().catch(() => {}); + throw error; + } +} + +export async function launchTeamsMeetingInChrome(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + meetingSessionId: string; + requesterSessionKey?: string; + mode: TeamsMeetingsMode; + trackedTargetId?: string; + url: string; + logger: RuntimeLogger; +}): Promise<{ + launched: boolean; + audioBridge?: LocalAudioBridge; + browser?: TeamsMeetingsChromeHealth; + tab?: TeamsMeetingsBrowserTab; +}> { + if (isTeamsMeetingsTalkBackMode(params.mode)) { + await assertBlackHole2chAvailable({ + runtime: params.runtime, + timeoutMs: Math.min(params.config.chrome.joinTimeoutMs, 10_000), + }); + } + const callBrowser = await resolveLocalMeetingBrowserRequest(params.runtime); + const result = await openOrRecoverTeamsMeeting({ + callBrowser, + config: params.config, + locationLabel: "in local Chrome", + meetingSessionId: params.meetingSessionId, + mode: params.mode, + trackedTargetId: params.trackedTargetId, + url: params.url, + }); + if (!isTeamsMeetingsRealtimeRouteReady(params.mode, result.browser)) { + return result; + } + try { + return { ...result, audioBridge: await startLocalAudioBridge(params) }; + } catch (error) { + await rollbackTeamsBrowserJoin({ + callBrowser, + config: params.config, + logger: params.logger, + tab: result.tab, + url: params.url, + }); + throw error; + } +} + +async function resolveChromeNode(params: { + runtime: PluginRuntime; + requestedNode?: string; +}): Promise { + return await resolveMeetingBrowserNode({ + ...params, + adapter: TEAMS_MEETINGS_BROWSER_NODE_ADAPTER, + }); +} + +async function callNodeBrowser(params: { + runtime: PluginRuntime; + nodeId: string; + method: "GET" | "POST" | "DELETE"; + path: string; + body?: unknown; + timeoutMs: number; +}) { + return await callMeetingBrowserProxyOnNode({ + ...params, + adapter: TEAMS_MEETINGS_BROWSER_NODE_ADAPTER, + }); +} + +type TeamsMeetingsNodeStartResult = { + launched?: boolean; + bridgeId?: string; + audioBridge?: { type?: string }; + browser?: TeamsMeetingsChromeHealth; +}; + +function parseNodeStartResult(raw: unknown): TeamsMeetingsNodeStartResult { + const value = + raw && typeof raw === "object" && "payload" in raw + ? (raw as { payload?: unknown }).payload + : raw; + if (!value || typeof value !== "object") { + throw new Error("Microsoft Teams meeting node returned an invalid start result."); + } + return value as TeamsMeetingsNodeStartResult; +} + +export async function launchTeamsMeetingOnNode(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + fullConfig: OpenClawConfig; + meetingSessionId: string; + requesterSessionKey?: string; + mode: TeamsMeetingsMode; + trackedTargetId?: string; + url: string; + logger: RuntimeLogger; +}): Promise<{ + nodeId: string; + launched: boolean; + audioBridge?: NodeAudioBridge; + browser?: TeamsMeetingsChromeHealth; + tab?: TeamsMeetingsBrowserTab; +}> { + const nodeId = await resolveChromeNode({ + runtime: params.runtime, + requestedNode: params.config.chromeNode.node, + }); + try { + await params.runtime.nodes.invoke({ + nodeId, + command: TEAMS_MEETINGS_NODE_COMMAND, + params: { action: "stopByUrl", url: params.url, mode: params.mode }, + timeoutMs: 5_000, + }); + } catch (error) { + params.logger.debug?.( + `${TEAMS_MEETINGS_RUNTIME_PLATFORM.logScope} node bridge cleanup ignored: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const callBrowser: MeetingBrowserRequestCaller = async (request) => + await callNodeBrowser({ + runtime: params.runtime, + nodeId, + method: request.method, + path: request.path, + body: request.body, + timeoutMs: request.timeoutMs, + }); + const browser = await openOrRecoverTeamsMeeting({ + callBrowser, + config: params.config, + locationLabel: "on the selected Chrome node", + meetingSessionId: params.meetingSessionId, + mode: params.mode, + trackedTargetId: params.trackedTargetId, + url: params.url, + }); + if (!isTeamsMeetingsRealtimeRouteReady(params.mode, browser.browser)) { + return { + nodeId, + launched: browser.launched, + browser: browser.browser, + tab: browser.tab, + }; + } + try { + const raw = await params.runtime.nodes.invoke({ + nodeId, + command: TEAMS_MEETINGS_NODE_COMMAND, + params: { + action: "start", + url: params.url, + mode: params.mode, + launch: false, + browserProfile: params.config.chrome.browserProfile, + joinTimeoutMs: params.config.chrome.joinTimeoutMs, + audioInputCommand: params.config.chrome.audioInputCommand, + audioOutputCommand: params.config.chrome.audioOutputCommand, + }, + timeoutMs: addTimerTimeoutGraceMs(params.config.chrome.joinTimeoutMs) ?? 1, + }); + const result = parseNodeStartResult(raw); + if (result.audioBridge?.type !== "node-command-pair") { + return { + nodeId, + launched: browser.launched || result.launched === true, + browser: browser.browser ?? result.browser, + tab: browser.tab, + }; + } + if (!result.bridgeId) { + throw new Error("Microsoft Teams meeting node did not return an audio bridge id."); + } + const transport = createNodeMeetingRealtimeAudioTransport({ + runtime: params.runtime, + nodeId, + bridgeId: result.bridgeId, + logger: params.logger, + commandName: TEAMS_MEETINGS_NODE_COMMAND, + logScope: TEAMS_MEETINGS_RUNTIME_PLATFORM.logScope, + logPrefix: params.mode === "agent" ? "node agent" : "node", + }); + const bindings = realtimeBindings(params); + let engine: MeetingRealtimeAudioEngineHandle; + try { + engine = + params.mode === "agent" + ? await startMeetingAgentRealtimeEngine({ + config: params.config, + fullConfig: params.fullConfig, + runtime: params.runtime, + platform: bindings.platform, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + logPrefix: "node", + transport, + logger: params.logger, + consultAgent: bindings.consultAgent, + }) + : await startMeetingRealtimeEngine({ + config: { + ...params.config, + realtime: { ...params.config.realtime, strategy: "bidi" }, + }, + fullConfig: params.fullConfig, + runtime: params.runtime, + ...bindings, + meetingSessionId: params.meetingSessionId, + requesterSessionKey: params.requesterSessionKey, + logPrefix: "node", + talkSessionId: `teams-meetings:${params.meetingSessionId}:${result.bridgeId}:node-realtime`, + talkContext: { nodeId, bridgeId: result.bridgeId }, + transport, + logger: params.logger, + }); + } catch (error) { + await transport.dispose().catch(() => {}); + throw error; + } + return { + nodeId, + launched: browser.launched || result.launched === true, + audioBridge: { + type: "node-command-pair", + nodeId, + bridgeId: result.bridgeId, + ...engine, + }, + browser: browser.browser ?? result.browser, + tab: browser.tab, + }; + } catch (error) { + await params.runtime.nodes + .invoke({ + nodeId, + command: TEAMS_MEETINGS_NODE_COMMAND, + params: { action: "stopByUrl", url: params.url, mode: params.mode }, + timeoutMs: 5_000, + }) + .catch(() => {}); + await rollbackTeamsBrowserJoin({ + callBrowser, + config: params.config, + logger: params.logger, + tab: browser.tab, + url: params.url, + }); + throw error; + } +} + +export async function recoverCurrentTeamsMeetingTab(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + mode: TeamsMeetingsMode; + nodeId?: string; + readOnly?: boolean; + trackedMeetingUrl?: string; + trackedTargetId?: string; + transport: "chrome" | "chrome-node"; + url?: string; +}) { + const nodeId = + params.transport === "chrome-node" + ? (params.nodeId ?? + (await resolveChromeNode({ + runtime: params.runtime, + requestedNode: params.config.chromeNode.node, + }))) + : undefined; + return { + transport: params.transport, + ...(nodeId ? { nodeId } : {}), + ...(await recoverMeetingBrowserTab({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: nodeId + ? async (request) => + await callNodeBrowser({ + runtime: params.runtime, + nodeId, + method: request.method, + path: request.path, + body: request.body, + timeoutMs: request.timeoutMs, + }) + : await resolveLocalMeetingBrowserRequest(params.runtime), + config: params.config.chrome, + locationLabel: nodeId ? "on the selected Chrome node" : "in local Chrome", + mode: params.mode, + readOnly: params.readOnly, + requestedMeetingUrl: params.url, + trackedMeetingUrl: params.trackedMeetingUrl, + trackedTargetId: params.trackedTargetId, + })), + }; +} + +export async function leaveTeamsMeetingInBrowser(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + meetingUrl: string; + nodeId?: string; + tab: TeamsMeetingsBrowserTab; +}) { + const nodeId = params.nodeId; + return await leaveMeetingWithBrowser({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: nodeId + ? async (request) => + await callNodeBrowser({ + runtime: params.runtime, + nodeId, + method: request.method, + path: request.path, + body: request.body, + timeoutMs: request.timeoutMs, + }) + : await resolveLocalMeetingBrowserRequest(params.runtime), + launch: params.config.chrome.launch || !params.tab.openedByPlugin, + meetingUrl: params.meetingUrl, + tab: params.tab, + timeoutMs: params.config.chrome.joinTimeoutMs, + }); +} + +export async function readTeamsMeetingTranscript(params: { + runtime: PluginRuntime; + config: TeamsMeetingsConfig; + finalize?: boolean; + meetingUrl: string; + meetingSessionId: string; + nodeId?: string; + tab: TeamsMeetingsBrowserTab; +}): Promise { + const nodeId = params.nodeId; + return await readMeetingTranscriptWithBrowser({ + adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER, + callBrowser: nodeId + ? async (request) => + await callNodeBrowser({ + runtime: params.runtime, + nodeId, + method: request.method, + path: request.path, + body: request.body, + timeoutMs: request.timeoutMs, + }) + : await resolveLocalMeetingBrowserRequest(params.runtime), + finalize: params.finalize === true, + meetingUrl: params.meetingUrl, + meetingSessionId: params.meetingSessionId, + tab: params.tab, + timeoutMs: Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000), + }); +} diff --git a/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts b/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts new file mode 100644 index 000000000000..3051b40c0f8b --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts @@ -0,0 +1,591 @@ +import { TEAMS_MEETING_SELECTORS } from "./teams-meetings-selectors.js"; +import { normalizeTeamsMeetingUrlForReuse } from "./teams-meetings-urls.js"; + +function pageIdentityFunctionSource(): string { + return `const meetingIdentity = (rawUrl) => { + try { + const parsed = new URL(rawUrl); + const host = parsed.hostname.toLowerCase(); + if (parsed.protocol !== "https:") return undefined; + if (host === "teams.microsoft.com") { + const match = parsed.pathname.match(/^\\/l\\/meetup-join\\/([^/]+)(?:\\/0)?\\/?$/i); + if (!match?.[1]) return undefined; + const threadId = decodeURIComponent(match[1]); + return /^19:[^/]+@thread\\.(?:v2|tacv2)$/i.test(threadId) + ? "teams-work:" + threadId + : undefined; + } + if (host === "teams.live.com") { + const match = parsed.pathname.match(/^\\/meet\\/([^/]+)\\/?$/i); + if (!match?.[1]) return undefined; + const code = decodeURIComponent(match[1]); + const password = parsed.searchParams.get("p"); + return /^[a-z0-9_-]+$/i.test(code) + ? "teams-consumer:" + code.toLowerCase() + ":p:" + encodeURIComponent(password || "") + : undefined; + } + } catch {} + return undefined; + };`; +} + +function teamsMeetingToggleStateFunctionSource(): string { + return `(input) => { + const pressed = String(input?.ariaPressed || "").toLowerCase(); + if (pressed === "true") return "on"; + if (pressed === "false") return "off"; + const value = String(input?.label || "").toLowerCase().replace(/\\s+/g, " ").trim(); + if (!value) return undefined; + if (input?.kind === "camera") { + if (/\\bturn (?:your )?camera off\\b|\\bturn off (?:your )?camera\\b|\\bstop video\\b|\\bdisable (?:your )?(?:camera|video)\\b/.test(value)) return "on"; + if (/\\bturn (?:your )?camera on\\b|\\bturn on (?:your )?camera\\b|\\bstart video\\b|\\benable (?:your )?(?:camera|video)\\b/.test(value)) return "off"; + if (/\\b(?:camera|video) (?:is |currently )?(?:off|disabled)\\b/.test(value)) return "off"; + if (/\\b(?:camera|video) (?:is |currently )?(?:on|enabled)\\b/.test(value)) return "on"; + return undefined; + } + if (/^mute$|\\bturn (?:your )?(?:microphone|mic) off\\b|\\bturn off (?:your )?(?:microphone|mic)\\b|\\bmute (?:your )?(?:microphone|mic)\\b|\\bdisable (?:your )?(?:microphone|mic)\\b/.test(value)) return "on"; + if (/^unmute$|\\bturn (?:your )?(?:microphone|mic) on\\b|\\bturn on (?:your )?(?:microphone|mic)\\b|\\bunmute (?:your )?(?:microphone|mic)\\b|\\benable (?:your )?(?:microphone|mic)\\b/.test(value)) return "off"; + if (/\\b(?:microphone|mic) (?:is |currently )?(?:off|muted|disabled)\\b/.test(value)) return "off"; + if (/\\b(?:microphone|mic) (?:is |currently )?(?:on|unmuted|enabled)\\b/.test(value)) return "on"; + return undefined; + }`; +} + +export function teamsMeetingStatusScript(params: { + allowMicrophone: boolean; + autoJoin: boolean; + guestName: string; + meetingSessionId?: string; + meetingUrl: string; + readOnly?: boolean; +}) { + const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS); + const expectedIdentity = normalizeTeamsMeetingUrlForReuse(params.meetingUrl); + const toggleStateFunction = teamsMeetingToggleStateFunctionSource(); + return `async () => { + ${pageIdentityFunctionSource()} + const parseToggleState = ${toggleStateFunction}; + const selectors = ${selectors}; + const expectedIdentity = ${JSON.stringify(expectedIdentity)}; + const allowMicrophone = ${JSON.stringify(params.allowMicrophone)}; + const autoJoin = ${JSON.stringify(params.autoJoin)}; + const readOnly = ${JSON.stringify(Boolean(params.readOnly))}; + const sessionId = ${JSON.stringify(params.meetingSessionId)}; + const text = (node) => (node?.innerText || node?.textContent || "").trim(); + const label = (node) => [ + node?.getAttribute?.("aria-label"), + node?.getAttribute?.("title"), + node?.getAttribute?.("data-tid"), + text(node), + ].filter(Boolean).join(" "); + const clickable = (node) => node?.matches?.("button") + ? node + : node?.querySelector?.("button") || node?.closest?.("button") || node; + const first = (list) => { + for (const selector of list) { + const node = document.querySelector(selector); + if (node) return clickable(node); + } + return undefined; + }; + const firstRaw = (list) => { + for (const selector of list) { + const node = document.querySelector(selector); + if (node) return node; + } + return undefined; + }; + const firstWithin = (root, list) => { + if (!root) return undefined; + for (const selector of list) { + if (root.matches?.(selector)) return root; + const node = root.querySelector?.(selector); + if (node) return node; + } + return undefined; + }; + const buttons = [...document.querySelectorAll("button")]; + const findTextButton = (pattern) => buttons.find((button) => !button.disabled && pattern.test(label(button))); + const waitForUi = () => new Promise((resolve) => setTimeout(resolve, 120)); + const toggleState = (node, kind) => parseToggleState({ + kind, + ariaPressed: node?.getAttribute?.("aria-pressed"), + label: label(node), + }); + const notes = []; + const currentIdentity = meetingIdentity(location.href); + const priorMeeting = window.__openclawTeamsMeeting; + if (expectedIdentity && currentIdentity && currentIdentity !== expectedIdentity) { + delete window.__openclawTeamsMeeting; + return JSON.stringify({ + inCall: false, + manualActionRequired: true, + manualActionReason: "teams-session-conflict", + manualActionMessage: "The tracked Teams tab now shows a different meeting. Return to the requested meeting link, then retry.", + title: document.title, + url: location.href, + notes, + }); + } + const identityMatchedUrl = Boolean(expectedIdentity && currentIdentity === expectedIdentity); + const continueInBrowser = first(selectors.continueInBrowser) || + findTextButton(/continue on this browser|join on the web|use the web app|continue without the app/i); + if (!readOnly && identityMatchedUrl && continueInBrowser) { + continueInBrowser.click(); + notes.push("Continued to the Teams web client."); + await waitForUi(); + } + const guestInput = first(selectors.guestName) || [...document.querySelectorAll("input")].find((input) => + /enter your name|type your name|your name|display name/i.test(label(input) + " " + (input.placeholder || "")) + ); + if (!readOnly && identityMatchedUrl && autoJoin && guestInput && !guestInput.value) { + const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set; + guestInput.focus(); + if (setter) setter.call(guestInput, ${JSON.stringify(params.guestName)}); + else guestInput.value = ${JSON.stringify(params.guestName)}; + guestInput.dispatchEvent(new Event("input", { bubbles: true })); + guestInput.dispatchEvent(new Event("change", { bubbles: true })); + } + const leave = first(selectors.leave); + // Teams replaces the meeting URL after admission. Preserve identity only + // while adopting the first in-call control or retaining that exact control. + const markerAgeMs = Date.now() - (priorMeeting?.verifiedAt || 0); + const identityAdoptedInCall = Boolean( + !currentIdentity && + priorMeeting?.identity === expectedIdentity && + !priorMeeting?.inCallControl && + markerAgeMs >= 0 && + markerAgeMs < 30_000 && + leave && + leave.isConnected !== false + ); + const identityRerenderedInCall = Boolean( + !currentIdentity && + priorMeeting?.identity === expectedIdentity && + priorMeeting?.inCallControl && + priorMeeting.inCallControl !== leave && + priorMeeting.inCallControl.isConnected === false && + priorMeeting?.inCallUrl === location.href && + markerAgeMs >= 0 && + markerAgeMs < 5_000 && + leave && + leave.isConnected !== false + ); + const identityAwaitingRerender = Boolean( + !currentIdentity && + priorMeeting?.identity === expectedIdentity && + priorMeeting?.inCallControl && + priorMeeting.inCallControl.isConnected === false && + priorMeeting?.inCallUrl === location.href && + markerAgeMs >= 0 && + markerAgeMs < 5_000 && + !leave + ); + const identityPreservedInCall = Boolean( + !currentIdentity && + priorMeeting?.identity === expectedIdentity && + leave && + leave.isConnected !== false && + ( + identityAdoptedInCall || + identityRerenderedInCall || + ( + priorMeeting?.inCallControl === leave && + priorMeeting?.inCallUrl === location.href + ) + ) + ); + const identityVerified = identityMatchedUrl || identityPreservedInCall; + const inCall = Boolean(identityVerified && leave); + if (identityMatchedUrl || identityPreservedInCall) { + window.__openclawTeamsMeeting = { + ...(priorMeeting?.identity === expectedIdentity ? priorMeeting : {}), + identity: expectedIdentity, + sessionId: sessionId || priorMeeting?.sessionId, + verifiedAt: Date.now(), + ...(inCall ? { inCallControl: leave, inCallUrl: location.href } : {}), + }; + } else if ( + !currentIdentity && + priorMeeting && + !identityAwaitingRerender && + (priorMeeting.inCallControl || markerAgeMs >= 30_000) + ) { + delete window.__openclawTeamsMeeting; + } + const microphone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + let microphoneState = identityVerified ? toggleState(microphone, "microphone") : undefined; + const camera = first(selectors.camera) || findTextButton(/camera|video/i); + let cameraState = identityVerified ? toggleState(camera, "camera") : undefined; + let controlManualActionReason; + let controlManualActionMessage; + if (!readOnly && identityVerified && !inCall && camera && cameraState === "on") { + camera.click(); + await waitForUi(); + const currentCamera = first(selectors.camera) || findTextButton(/camera|video/i); + cameraState = toggleState(currentCamera, "camera"); + if (cameraState === "off") { + notes.push("Turned the Teams camera off before joining."); + } + } + const join = first(selectors.join) || findTextButton(/^\\s*(join now|ask to join|join meeting)\\s*$/i); + if (identityVerified && !inCall && join && cameraState !== "off") { + controlManualActionReason = "teams-camera-required"; + controlManualActionMessage = "Turn the Teams camera off and verify the camera control shows it is off, then retry joining."; + } + const isBlackHole = (value) => + String(value || "").replace(/\\s+/g, " ").trim().toLowerCase() === "blackhole 2ch"; + const isBlackHoleNode = (node) => [ + node?.getAttribute?.("aria-label"), + node?.getAttribute?.("title"), + node?.label, + node?.value, + text(node), + ].some(isBlackHole); + const microphoneDeviceRoots = () => { + const control = firstRaw(selectors.microphoneDevice); + if (!control) return { control, roots: [] }; + const roots = [control]; + const scope = control.closest?.('[data-tid="device-settings-microphone"]'); + if (scope && !roots.includes(scope)) roots.push(scope); + const listboxId = control.getAttribute?.("aria-controls"); + const listbox = listboxId ? document.getElementById?.(listboxId) : undefined; + if (listbox && !roots.includes(listbox)) roots.push(listbox); + return { control, roots }; + }; + const selectedMicrophoneLabel = () => { + const { control, roots } = microphoneDeviceRoots(); + const selectedOption = control?.selectedOptions?.[0]; + if (selectedOption && isBlackHoleNode(selectedOption)) { + return label(selectedOption) || selectedOption.value; + } + if (control && isBlackHoleNode(control)) return label(control) || control.value; + for (const root of roots) { + const selected = firstWithin(root, selectors.selectedMicrophoneDevice); + if (selected && isBlackHoleNode(selected)) { + return label(selected) || selected.value; + } + } + return undefined; + }; + let audioInputRouted; + let audioInputDeviceLabel; + let audioInputRouteError; + const ensureVirtualAudioInput = async () => { + if (!navigator.mediaDevices?.enumerateDevices) return false; + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const input = devices.find((device) => device.kind === "audioinput" && isBlackHole(device.label)); + if (!input?.deviceId) return false; + audioInputDeviceLabel = input.label || "BlackHole 2ch"; + let selected = Boolean(selectedMicrophoneLabel()); + if (!selected && !readOnly) { + const settings = first(selectors.deviceSettings); + if (settings) { + settings.click(); + await waitForUi(); + } + const { control } = microphoneDeviceRoots(); + if (control?.tagName?.toLowerCase() === "select") { + const options = [...control.options]; + const option = options.find(isBlackHoleNode); + if (option) { + control.value = option.value; + control.dispatchEvent(new Event("change", { bubbles: true })); + await waitForUi(); + } + } else if (control) { + clickable(control)?.click?.(); + await waitForUi(); + } + const choices = microphoneDeviceRoots().roots.flatMap((root) => + selectors.audioDeviceOptions.flatMap((selector) => [ + ...(root.querySelectorAll?.(selector) || []), + ]) + ); + const choice = choices.find(isBlackHoleNode); + if (choice && choice.getAttribute?.("aria-selected") !== "true") { + clickable(choice)?.click?.(); + await waitForUi(); + } + selected = Boolean(selectedMicrophoneLabel()); + } + if (selected && window.__openclawTeamsMeeting?.identity === expectedIdentity) { + window.__openclawTeamsMeeting.audioInputDeviceId = input.deviceId; + } + return selected; + } catch (error) { + audioInputRouteError = error?.message || String(error); + return false; + } + }; + if (identityVerified && !inCall && allowMicrophone && microphone) { + audioInputRouted = await ensureVirtualAudioInput(); + if (!audioInputRouted) { + if (!readOnly && microphoneState === "on") { + microphone.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + } + controlManualActionReason = "teams-audio-choice-required"; + controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."; + } else if (!readOnly && microphoneState === "off") { + microphone.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + if (microphoneState === "on") { + notes.push("Unmuted the Teams microphone after verifying BlackHole 2ch input."); + } + } + if (audioInputRouted && microphoneState !== "on") { + controlManualActionReason = "teams-microphone-required"; + controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."; + } + } else if (!readOnly && identityVerified && !inCall && !allowMicrophone && microphoneState === "on") { + microphone.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + if (microphoneState === "off") { + notes.push("Muted the Teams microphone for observe-only mode."); + } + } + if (identityVerified && inCall && allowMicrophone) { + if (!selectedMicrophoneLabel() && !readOnly && microphoneState === "on") { + microphone?.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + } + audioInputRouted = await ensureVirtualAudioInput(); + if (audioInputRouted && !readOnly && microphoneState === "off") { + microphone?.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + } else if (!audioInputRouted && !readOnly && microphoneState === "on") { + microphone?.click(); + await waitForUi(); + const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i); + microphoneState = toggleState(currentMicrophone, "microphone"); + if (microphoneState === "off") { + notes.push("Muted the Teams microphone because BlackHole 2ch input could not be reverified."); + } + } + } + if (identityVerified && !inCall && join && !allowMicrophone && microphoneState !== "off") { + controlManualActionReason = "teams-microphone-required"; + controlManualActionMessage = "Mute the Teams microphone and verify the microphone control shows it is off, then retry joining."; + } + if (identityVerified && !inCall && join && allowMicrophone && !controlManualActionReason) { + if (!microphone) { + controlManualActionReason = "teams-microphone-required"; + controlManualActionMessage = "Open Teams device settings and verify the microphone control before enabling talk-back."; + } else if (audioInputRouted !== true) { + controlManualActionReason = "teams-audio-choice-required"; + controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back."; + } else if (microphoneState !== "on") { + controlManualActionReason = "teams-microphone-required"; + controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining."; + } + } + const micMuted = microphoneState === "off" ? true : microphoneState === "on" ? false : undefined; + const cameraOff = cameraState === "off" ? true : cameraState === "on" ? false : undefined; + const pageText = text(document.body); + const pageTextLower = pageText.toLowerCase(); + const lobbyWaiting = Boolean(first(selectors.lobby)) || + /someone will let you in shortly|waiting for someone to let you in|when someone admits you|you.?re in the lobby|we.?ve let people in the meeting know you.?re waiting/i.test(pageTextLower); + const signInControl = first(selectors.signIn); + const hostname = location.hostname.toLowerCase(); + const tenantLoginRequired = + /only people with a work or school account|sign in with an account from this organization|anonymous users (?:can.?t|cannot) join|verify your email|enter the code sent to/i.test(pageTextLower); + const loginRequired = hostname === "login.microsoftonline.com" || + hostname.endsWith(".microsoftonline.com") || + tenantLoginRequired || + (Boolean(signInControl) && !guestInput && !join && /sign in to (?:join|continue)|sign in to your account/i.test(pageTextLower)); + const permissionRequired = Boolean(first(selectors.permissionPrompt)); + let manualActionReason; + let manualActionMessage; + if (!inCall && loginRequired) { + manualActionReason = "teams-login-required"; + manualActionMessage = tenantLoginRequired + ? "This Teams tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry." + : "Sign in to Microsoft Teams in the OpenClaw browser profile, then retry the meeting join."; + } else if (!inCall && lobbyWaiting) { + manualActionReason = "teams-admission-required"; + manualActionMessage = "Admit the OpenClaw guest from the Microsoft Teams lobby, then retry speech."; + } else if (!inCall && permissionRequired) { + manualActionReason = "teams-permission-required"; + manualActionMessage = allowMicrophone + ? "Allow microphone and camera permissions for Teams in the OpenClaw browser profile, then retry." + : "Dismiss the Teams device-permission prompt or continue without devices, then retry."; + } else if (!inCall && controlManualActionReason) { + manualActionReason = controlManualActionReason; + manualActionMessage = controlManualActionMessage; + } + let clickedJoin = false; + if (!readOnly && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) { + join.click(); + clickedJoin = true; + notes.push("Clicked the Teams guest join button."); + } + let audioOutputRouted; + let audioOutputDeviceLabel; + let audioOutputRouteError; + if (inCall && allowMicrophone && navigator.mediaDevices?.enumerateDevices) { + const media = [...document.querySelectorAll("audio, video")].filter((element) => + typeof element.setSinkId === "function" + ); + if (media.length > 0) { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const output = devices.find((device) => device.kind === "audiooutput" && isBlackHole(device.label)); + if (output?.deviceId) { + for (const element of media) { + if (!readOnly && element.sinkId !== output.deviceId) await element.setSinkId(output.deviceId); + } + audioOutputRouted = media.some((element) => element.sinkId === output.deviceId); + audioOutputDeviceLabel = output.label || "BlackHole 2ch"; + } else { + audioOutputRouted = false; + notes.push("BlackHole 2ch speaker output was not visible to Teams."); + } + } catch (error) { + audioOutputRouted = false; + audioOutputRouteError = error?.message || String(error); + notes.push("Could not route Teams speaker output to BlackHole 2ch: " + audioOutputRouteError); + } + } else { + audioOutputRouted = false; + } + } + if (inCall && allowMicrophone) { + if (audioInputRouted !== true || audioOutputRouted !== true) { + manualActionReason = "teams-audio-choice-required"; + manualActionMessage = "Verify BlackHole 2ch is selected as both the Teams microphone and speaker before starting talk-back."; + } else if (micMuted !== false) { + manualActionReason = "teams-microphone-required"; + manualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on before starting talk-back."; + } + } + return JSON.stringify({ + clickedContinueInBrowser: Boolean(continueInBrowser), + clickedJoin, + inCall, + micMuted, + cameraOff, + lobbyWaiting, + audioInputRouted, + audioInputDeviceLabel, + audioInputRouteError, + audioOutputRouted, + audioOutputDeviceLabel, + audioOutputRouteError, + manualActionRequired: Boolean(manualActionReason), + manualActionReason, + manualActionMessage, + title: document.title, + url: location.href, + notes, + }); +}`; +} + +export function teamsMeetingTranscriptScript(meetingUrl: string, meetingSessionId: string) { + const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl); + return `() => { + ${pageIdentityFunctionSource()} + const expectedIdentity = ${JSON.stringify(expectedIdentity)}; + const expectedSessionId = ${JSON.stringify(meetingSessionId)}; + const currentIdentity = meetingIdentity(location.href); + const state = window.__openclawTeamsMeeting; + const effectiveIdentity = currentIdentity || state?.identity; + if (!expectedIdentity || effectiveIdentity !== expectedIdentity) { + return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] }); + } + if (state?.sessionId && state.sessionId !== expectedSessionId) { + return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); + } + return JSON.stringify({ urlMatched: true, sessionMatched: true, droppedLines: 0, lines: [] }); +}`; +} + +export function teamsMeetingLeaveScript(meetingUrl: string) { + const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS); + const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl); + return `() => { + ${pageIdentityFunctionSource()} + const selectors = ${selectors}; + const expectedIdentity = ${JSON.stringify(expectedIdentity)}; + const currentIdentity = meetingIdentity(location.href); + const state = window.__openclawTeamsMeeting; + const first = (list) => { + for (const selector of list) { + const node = document.querySelector(selector); + if (!node) continue; + return node.matches?.("button") ? node : node.querySelector?.("button") || node.closest?.("button") || node; + } + return undefined; + }; + const leave = first(selectors.leave); + const confirmation = first(selectors.leaveConfirmation); + const postCall = first(selectors.postCall); + const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity); + const preservedCallMatches = Boolean( + expectedIdentity && + !currentIdentity && + state?.identity === expectedIdentity && + state?.inCallControl === leave && + state?.inCallUrl === location.href && + leave && + leave.isConnected !== false + ); + const pendingLeaveMatches = Boolean( + expectedIdentity && + state?.identity === expectedIdentity && + state?.leavePending === true && + state?.inCallUrl === location.href && + Date.now() - state?.leavePendingAt < 10_000 + ); + const rerenderPendingMatches = Boolean( + expectedIdentity && + !currentIdentity && + state?.identity === expectedIdentity && + state?.inCallControl?.isConnected === false && + state?.inCallUrl === location.href && + Date.now() - state?.verifiedAt < 5_000 && + !leave + ); + if ( + !currentUrlMatches && + !preservedCallMatches && + !pendingLeaveMatches && + !rerenderPendingMatches + ) { + return JSON.stringify({ departed: false, urlMatched: false }); + } + if (postCall) { + delete window.__openclawTeamsMeeting; + return JSON.stringify({ departed: true, urlMatched: true }); + } + if (confirmation) { + confirmation.click(); + return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }); + } + if (leave) { + window.__openclawTeamsMeeting = { + ...state, + identity: expectedIdentity, + inCallControl: leave, + inCallUrl: location.href, + leavePending: true, + leavePendingAt: Date.now(), + }; + leave.click(); + return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true }); + } + return JSON.stringify({ departed: false, urlMatched: true }); +}`; +} diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts new file mode 100644 index 000000000000..57839f74ab4e --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts @@ -0,0 +1,607 @@ +import { runInNewContext } from "node:vm"; +import { describe, expect, it } from "vitest"; +import { + teamsMeetingLeaveScript, + teamsMeetingStatusScript, +} from "./teams-meetings-page-scripts.js"; +import { + TEAMS_MEETINGS_PLATFORM_ADAPTER, + isTeamsMeetingsRealtimeRouteReady, +} from "./teams-meetings-platform-adapter.js"; + +const URL = + "https://teams.microsoft.com/l/meetup-join/19%3ameeting_test%40thread.v2/0?context=%7b%7d"; +const MEETING_STATE_KEY = "__openclawTeamsMeeting"; + +function status(manualActionReason: string, manualActionMessage = "manual action") { + const health = TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ + result: JSON.stringify({ + inCall: false, + manualActionRequired: true, + manualActionReason, + manualActionMessage, + url: URL, + }), + }); + if (!health) { + throw new Error("expected parsed health"); + } + return health; +} + +type PageControl = { + disabled?: boolean; + clicks: number; + isConnected: boolean; + click(): void; + closest(selector?: string): PageControl; + getAttribute(name: string): string | null; + matches(selector: string): boolean; + querySelector(): undefined; + querySelectorAll?(selector: string): PageControl[]; + setPressed(pressed: boolean): void; + textContent: string; +}; + +function control(params: { + label: string; + pressed?: boolean; + onClick?: (control: PageControl) => void; +}): PageControl { + const attributes = new Map([["aria-label", params.label]]); + if (params.pressed !== undefined) { + attributes.set("aria-pressed", String(params.pressed)); + } + const node: PageControl = { + clicks: 0, + isConnected: true, + textContent: "", + click() { + node.clicks += 1; + params.onClick?.(node); + }, + closest: () => node, + getAttribute: (name) => attributes.get(name) ?? null, + matches: (selector) => selector === "button", + querySelector: () => undefined, + setPressed: (pressed) => attributes.set("aria-pressed", String(pressed)), + }; + return node; +} + +async function runStatusScript(params: { + allowMicrophone: boolean; + bodyText?: string; + currentUrl?: string; + microphone?: PageControl; + camera?: PageControl; + join?: PageControl; + leave?: PageControl; + microphoneDevice?: PageControl; + permissionPrompt?: PageControl; + priorMeeting?: Record; + readOnly?: boolean; + globalSelectedOption?: PageControl; + media?: Array<{ sinkId: string; setSinkId(value: string): Promise }>; + devices?: Array<{ deviceId: string; kind: string; label: string }>; +}) { + const currentUrl = params.currentUrl ?? URL; + const location = new globalThis.URL(currentUrl); + const controls = [params.microphone, params.camera, params.join, params.leave].filter( + (entry): entry is PageControl => Boolean(entry), + ); + const document = { + body: { textContent: params.bodyText ?? "" }, + title: "Teams", + getElementById() { + return undefined; + }, + querySelector(selector: string) { + if (selector.includes("toggle-mute")) { + return params.microphone; + } + if (selector.includes("toggle-video")) { + return params.camera; + } + if (selector.includes("prejoin-join-button")) { + return params.join; + } + if (selector.includes("call-hangup")) { + return params.leave; + } + if ( + selector.includes("microphone-select") || + selector.includes("audio-device-input") || + selector.includes("device-settings-microphone") + ) { + return params.microphoneDevice; + } + if (selector.includes("permission-prompt") || selector.includes("permission-error")) { + return params.permissionPrompt; + } + if (selector === '[role="option"][aria-selected="true"]') { + return params.globalSelectedOption; + } + return undefined; + }, + querySelectorAll(selector: string) { + if (selector === "button") { + return controls; + } + if (selector === "audio, video") { + return params.media ?? []; + } + if (selector.includes('[role="option"]')) { + return params.globalSelectedOption ? [params.globalSelectedOption] : []; + } + return []; + }, + }; + const window: Record = {}; + if (params.priorMeeting) { + window[MEETING_STATE_KEY] = params.priorMeeting; + } + const script = teamsMeetingStatusScript({ + allowMicrophone: params.allowMicrophone, + autoJoin: true, + guestName: "OpenClaw Guest", + meetingSessionId: "session-1", + meetingUrl: URL, + readOnly: params.readOnly, + }); + const run = runInNewContext(`(${script})`, { + Event: globalThis.Event, + HTMLInputElement: function HTMLInputElement() {}, + URL: globalThis.URL, + document, + location, + navigator: { + mediaDevices: { + enumerateDevices: async () => params.devices ?? [], + }, + }, + setTimeout, + window, + }) as () => Promise; + return { result: JSON.parse(await run()) as Record, window }; +} + +function runLeaveScript(params: { + bodyText?: string; + currentUrl?: string; + leave?: PageControl; + postCall?: PageControl; + priorMeeting?: Record; +}) { + const currentUrl = params.currentUrl ?? URL; + const location = new globalThis.URL(currentUrl); + const document = { + body: { textContent: params.bodyText ?? "" }, + querySelector(selector: string) { + if (selector.includes("call-hangup")) { + return params.leave; + } + if ( + selector.includes("call-ended-screen") || + selector.includes("post-call-screen") || + selector.includes("prejoin-rejoin-button") + ) { + return params.postCall; + } + return undefined; + }, + }; + const window: Record = {}; + if (params.priorMeeting) { + window[MEETING_STATE_KEY] = params.priorMeeting; + } + const run = runInNewContext(`(${teamsMeetingLeaveScript(URL)})`, { + URL: globalThis.URL, + document, + location, + window, + }) as () => string; + return { result: JSON.parse(run()) as Record, window }; +} + +describe("Microsoft Teams meeting platform adapter", () => { + it.each([ + ["teams-login-required", "login-required"], + ["teams-admission-required", "admission-required"], + ["teams-permission-required", "permission-required"], + ["teams-audio-choice-required", "audio-choice-required"], + ["teams-session-conflict", "session-conflict"], + ["browser-control-unavailable", "browser-control-unavailable"], + ])("classifies %s as %s", (reason, category) => { + expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.classifyManualAction(status(reason))).toEqual({ + category, + reason, + message: "manual action", + }); + }); + + it.each([ + ["camera", "Turn camera off", undefined, "on"], + ["camera", "Turn camera on", undefined, "off"], + ["camera", "Stop video", undefined, "on"], + ["camera", "Start video", undefined, "off"], + ["camera", "Turn camera on", "true", "on"], + ["microphone", "Mute", undefined, "on"], + ["microphone", "Unmute", undefined, "off"], + ["microphone", "Turn microphone off", undefined, "on"], + ["microphone", "Turn microphone on", undefined, "off"], + ["microphone", "Microphone is muted", undefined, "off"], + ["microphone", "Turn microphone off", "false", "off"], + ])( + "parses %s control %j with aria-pressed %j as %s", + async (kind, label, ariaPressed, expected) => { + const target = control({ + label, + ...(ariaPressed === undefined ? {} : { pressed: ariaPressed === "true" }), + }); + const { result } = await runStatusScript({ + allowMicrophone: false, + ...(kind === "camera" ? { camera: target } : { microphone: target }), + readOnly: true, + }); + expect(kind === "camera" ? result.cameraOff : result.micMuted).toBe(expected === "off"); + }, + ); + + it("re-reads camera and microphone state after toggling before joining", async () => { + const camera = control({ + label: "Turn camera off", + pressed: true, + onClick: (node) => node.setPressed(false), + }); + const microphone = control({ + label: "Turn microphone off", + pressed: true, + onClick: (node) => node.setPressed(false), + }); + const join = control({ label: "Join now" }); + + const { result } = await runStatusScript({ + allowMicrophone: false, + camera, + join, + microphone, + }); + + expect(result).toMatchObject({ cameraOff: true, clickedJoin: true, micMuted: true }); + expect(camera.clicks).toBe(1); + expect(microphone.clicks).toBe(1); + expect(join.clicks).toBe(1); + }); + + it("does not unmute or join until BlackHole is visibly selected as the Teams input", async () => { + const camera = control({ label: "Turn camera on", pressed: false }); + const microphone = control({ label: "Turn microphone on", pressed: false }); + const join = control({ label: "Join now" }); + + const { result } = await runStatusScript({ + allowMicrophone: true, + camera, + devices: [{ deviceId: "blackhole", kind: "audioinput", label: "BlackHole 2ch" }], + join, + microphone, + }); + + expect(result).toMatchObject({ + audioInputRouted: false, + clickedJoin: false, + manualActionReason: "teams-audio-choice-required", + micMuted: true, + }); + expect(microphone.clicks).toBe(0); + expect(join.clicks).toBe(0); + }); + + it("does not auto-join talk-back when the microphone control is missing", async () => { + const camera = control({ label: "Turn camera on", pressed: false }); + const join = control({ label: "Join now" }); + + const { result } = await runStatusScript({ + allowMicrophone: true, + camera, + join, + }); + + expect(result).toMatchObject({ + manualActionReason: "teams-microphone-required", + manualActionRequired: true, + }); + expect(join.clicks).toBe(0); + }); + + it("does not accept a selected BlackHole speaker option as the microphone", async () => { + const camera = control({ label: "Turn camera on", pressed: false }); + const microphone = control({ label: "Turn microphone on", pressed: false }); + const microphoneDevice = control({ label: "MacBook Pro Microphone" }); + const selectedSpeaker = control({ label: "BlackHole 2ch" }); + const join = control({ label: "Join now" }); + + const { result } = await runStatusScript({ + allowMicrophone: true, + camera, + devices: [{ deviceId: "blackhole", kind: "audioinput", label: "BlackHole 2ch" }], + globalSelectedOption: selectedSpeaker, + join, + microphone, + microphoneDevice, + }); + + expect(result).toMatchObject({ + audioInputRouted: false, + clickedJoin: false, + manualActionReason: "teams-audio-choice-required", + }); + expect(microphone.clicks).toBe(0); + }); + + it("does not stamp meeting identity onto unrelated Teams pages", async () => { + const leave = control({ label: "Leave" }); + const { result, window } = await runStatusScript({ + allowMicrophone: false, + currentUrl: "https://teams.microsoft.com/v2/", + leave, + }); + + expect(result.inCall).toBe(false); + expect(window).not.toHaveProperty("__openclawTeamsMeeting"); + }); + + it("preserves a verified identity only across an in-call URL transition", async () => { + const leave = control({ label: "Leave" }); + const inCallUrl = "https://teams.microsoft.com/v2/"; + const priorMeeting = { + identity: "teams-work:19:meeting_test@thread.v2", + inCallControl: leave, + inCallUrl, + sessionId: "session-1", + }; + const { result, window } = await runStatusScript({ + allowMicrophone: false, + currentUrl: inCallUrl, + leave, + priorMeeting, + }); + + expect(result.inCall).toBe(true); + expect(window[MEETING_STATE_KEY]).toMatchObject(priorMeeting); + }); + + it("adopts the first live hang-up control during the verified join transition", async () => { + const prejoin = await runStatusScript({ allowMicrophone: false }); + const leave = control({ label: "Leave" }); + const admitted = await runStatusScript({ + allowMicrophone: false, + currentUrl: "https://teams.microsoft.com/v2/", + leave, + priorMeeting: prejoin.window[MEETING_STATE_KEY] as Record, + }); + + expect(admitted.result.inCall).toBe(true); + expect(admitted.window[MEETING_STATE_KEY]).toMatchObject({ + identity: "teams-work:19:meeting_test@thread.v2", + inCallControl: leave, + inCallUrl: "https://teams.microsoft.com/v2/", + }); + }); + + it("re-adopts a replaced hang-up control only within the bounded rerender window", async () => { + const previousLeave = control({ label: "Leave" }); + previousLeave.isConnected = false; + const currentLeave = control({ label: "Leave" }); + const inCallUrl = "https://teams.microsoft.com/v2/"; + const { result, window } = await runStatusScript({ + allowMicrophone: false, + currentUrl: inCallUrl, + leave: currentLeave, + priorMeeting: { + identity: "teams-work:19:meeting_test@thread.v2", + inCallControl: previousLeave, + inCallUrl, + verifiedAt: Date.now(), + }, + }); + + expect(result.inCall).toBe(true); + expect(window[MEETING_STATE_KEY]).toMatchObject({ + inCallControl: currentLeave, + inCallUrl, + }); + }); + + it("does not trust a stale identity marker to leave a different SPA call", () => { + const staleLeave = control({ label: "Leave old call" }); + const currentLeave = control({ label: "Leave current call" }); + const { result } = runLeaveScript({ + currentUrl: "https://teams.microsoft.com/v2/", + leave: currentLeave, + priorMeeting: { + identity: "teams-work:19:meeting_test@thread.v2", + inCallControl: staleLeave, + inCallUrl: "https://teams.microsoft.com/v2/", + }, + }); + + expect(result).toEqual({ departed: false, urlMatched: false }); + expect(currentLeave.clicks).toBe(0); + }); + + it.each([ + "Alice: meeting ended — rejoin after lunch", + "Bob: allow Teams to use your microphone; device permissions are blocked", + ])("ignores participant-controlled in-call text: %s", async (bodyText) => { + const leave = control({ label: "Leave" }); + const { result } = await runStatusScript({ + allowMicrophone: false, + bodyText, + leave, + }); + + expect(result).toMatchObject({ + inCall: true, + manualActionRequired: false, + }); + }); + + it("classifies a stable device permission prompt outside the call", async () => { + const { result } = await runStatusScript({ + allowMicrophone: false, + permissionPrompt: control({ label: "Device permission prompt" }), + }); + + expect(result).toMatchObject({ + inCall: false, + manualActionReason: "teams-permission-required", + manualActionRequired: true, + }); + }); + + it.each(["meeting ended", "call ended — rejoin"])( + "does not infer departure from page-wide text: %s", + (bodyText) => { + const { result } = runLeaveScript({ bodyText }); + expect(result).toEqual({ departed: false, urlMatched: true }); + }, + ); + + it("requires positive input and output route evidence before realtime", () => { + expect( + isTeamsMeetingsRealtimeRouteReady("agent", { + audioInputRouted: true, + audioOutputRouted: true, + inCall: true, + micMuted: false, + }), + ).toBe(true); + for (const health of [ + { audioOutputRouted: true, inCall: true, micMuted: false }, + { audioInputRouted: true, inCall: true, micMuted: false }, + { audioInputRouted: true, audioOutputRouted: true, inCall: true }, + ]) { + expect(isTeamsMeetingsRealtimeRouteReady("agent", health)).toBe(false); + } + expect( + isTeamsMeetingsRealtimeRouteReady("transcribe", { + audioInputRouted: true, + audioOutputRouted: true, + inCall: true, + micMuted: false, + }), + ).toBe(false); + }); + + it("reports verified routes only after the exact input marker and output sink agree", async () => { + const leave = control({ label: "Leave" }); + const microphone = control({ label: "Turn microphone off", pressed: true }); + const media = { + sinkId: "", + async setSinkId(value: string) { + media.sinkId = value; + }, + }; + const { result } = await runStatusScript({ + allowMicrophone: true, + devices: [ + { deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }, + { deviceId: "blackhole-output", kind: "audiooutput", label: "BlackHole 2ch" }, + ], + leave, + media: [media], + microphone, + microphoneDevice: control({ label: "BlackHole 2ch" }), + priorMeeting: { + audioInputDeviceId: "blackhole-input", + identity: "teams-work:19:meeting_test@thread.v2", + }, + }); + + expect(result).toMatchObject({ + audioInputRouted: true, + audioOutputRouted: true, + inCall: true, + manualActionRequired: false, + micMuted: false, + }); + expect(media.sinkId).toBe("blackhole-output"); + }); + + it("mutes an in-call physical microphone when Teams resets the prejoin selection", async () => { + const microphone = control({ + label: "Turn microphone off", + pressed: true, + onClick: (node) => node.setPressed(false), + }); + const { result } = await runStatusScript({ + allowMicrophone: true, + devices: [{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }], + leave: control({ label: "Leave" }), + microphone, + microphoneDevice: control({ label: "MacBook Pro Microphone" }), + priorMeeting: { + audioInputDeviceId: "blackhole-input", + identity: "teams-work:19:meeting_test@thread.v2", + }, + }); + + expect(result).toMatchObject({ + audioInputRouted: false, + manualActionReason: "teams-audio-choice-required", + micMuted: true, + }); + expect(microphone.clicks).toBe(1); + }); + + it("builds the guest join script from centralized stable selectors and text fallbacks", () => { + const script = teamsMeetingStatusScript({ + allowMicrophone: true, + autoJoin: true, + guestName: "OpenClaw Guest", + meetingSessionId: "session-1", + meetingUrl: URL, + }); + expect(script).toContain('data-tid=\\"prejoin-display-name-input\\"'); + expect(script).toContain('data-tid=\\"call-hangup\\"'); + expect(script).toContain("continue on this browser"); + expect(script).toContain("someone will let you in shortly"); + expect(script).toContain("setSinkId"); + expect(script).toContain("BlackHole"); + }); + + it("keeps caption capture disabled and returns empty snapshots", () => { + expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(false); + expect( + TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.parseTranscript({ + result: JSON.stringify({ urlMatched: true, sessionMatched: true, lines: ["ignored"] }), + }), + ).toEqual({ + droppedLines: 0, + lines: [], + urlMatched: true, + sessionMatched: true, + }); + }); + + it("does not guess a permission origin across work and consumer Teams", () => { + expect( + TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({ allowMicrophone: true }), + ).toBeUndefined(); + expect( + TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissionNotes({ allowMicrophone: true }), + ).toContain("Teams media permissions are handled in the browser when prompted."); + }); + + it("parses leave steps and malformed status", () => { + expect( + TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseLeaveResult({ + result: JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }), + }), + ).toEqual({ departed: false, leaveAction: "confirm", urlMatched: true }); + expect(() => + TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ result: "not-json" }), + ).toThrow("Microsoft Teams browser status JSON is malformed."); + }); +}); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts new file mode 100644 index 000000000000..e4fdc3434b4d --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts @@ -0,0 +1,212 @@ +import type { + MeetingBrowserJoinSession, + MeetingManualActionCategory, + MeetingPlatformAdapter, +} from "openclaw/plugin-sdk/meeting-runtime"; +import type { TeamsMeetingsMode } from "../config.js"; +import { + teamsMeetingLeaveScript, + teamsMeetingStatusScript, + teamsMeetingTranscriptScript, +} from "./teams-meetings-page-scripts.js"; +import { TEAMS_MEETINGS_NODE_COMMAND } from "./teams-meetings-platform-constants.js"; +import { + isRecoverableTeamsMeetingTab, + isSameTeamsMeetingUrl, + normalizeTeamsMeetingUrl, + normalizeTeamsMeetingUrlForReuse, +} from "./teams-meetings-urls.js"; +import type { TeamsMeetingsChromeHealth, TeamsMeetingsTranscriptSnapshot } from "./types.js"; + +export function isTeamsMeetingsTalkBackMode(mode: TeamsMeetingsMode): boolean { + return mode === "agent" || mode === "bidi"; +} + +export function isTeamsMeetingsRealtimeRouteReady( + mode: TeamsMeetingsMode, + health: TeamsMeetingsChromeHealth | undefined, +): boolean { + return ( + isTeamsMeetingsTalkBackMode(mode) && + health?.inCall === true && + health.micMuted === false && + health.audioInputRouted === true && + health.audioOutputRouted === true && + health.manualActionRequired !== true + ); +} + +function parseBrowserStatus(result: unknown): TeamsMeetingsChromeHealth | undefined { + const record = result && typeof result === "object" ? (result as Record) : {}; + if (typeof record.result !== "string" || !record.result.trim()) { + return undefined; + } + let parsed: Record; + try { + parsed = JSON.parse(record.result) as Record; + } catch { + throw new Error("Microsoft Teams browser status JSON is malformed."); + } + return { + inCall: typeof parsed.inCall === "boolean" ? parsed.inCall : undefined, + micMuted: typeof parsed.micMuted === "boolean" ? parsed.micMuted : undefined, + cameraOff: typeof parsed.cameraOff === "boolean" ? parsed.cameraOff : undefined, + lobbyWaiting: typeof parsed.lobbyWaiting === "boolean" ? parsed.lobbyWaiting : undefined, + audioInputRouted: + typeof parsed.audioInputRouted === "boolean" ? parsed.audioInputRouted : undefined, + audioInputDeviceLabel: + typeof parsed.audioInputDeviceLabel === "string" ? parsed.audioInputDeviceLabel : undefined, + audioInputRouteError: + typeof parsed.audioInputRouteError === "string" ? parsed.audioInputRouteError : undefined, + audioOutputRouted: + typeof parsed.audioOutputRouted === "boolean" ? parsed.audioOutputRouted : undefined, + audioOutputDeviceLabel: + typeof parsed.audioOutputDeviceLabel === "string" ? parsed.audioOutputDeviceLabel : undefined, + audioOutputRouteError: + typeof parsed.audioOutputRouteError === "string" ? parsed.audioOutputRouteError : undefined, + manualActionRequired: + typeof parsed.manualActionRequired === "boolean" ? parsed.manualActionRequired : undefined, + manualActionReason: + typeof parsed.manualActionReason === "string" + ? (parsed.manualActionReason as TeamsMeetingsChromeHealth["manualActionReason"]) + : undefined, + manualActionMessage: + typeof parsed.manualActionMessage === "string" ? parsed.manualActionMessage : undefined, + browserUrl: typeof parsed.url === "string" ? parsed.url : undefined, + browserTitle: typeof parsed.title === "string" ? parsed.title : undefined, + status: "browser-control", + notes: Array.isArray(parsed.notes) + ? parsed.notes.filter((note): note is string => typeof note === "string") + : undefined, + }; +} + +function classifyManualAction(health: TeamsMeetingsChromeHealth) { + if (!health.manualActionRequired || !health.manualActionReason || !health.manualActionMessage) { + return undefined; + } + const category: MeetingManualActionCategory = + health.manualActionReason === "teams-login-required" + ? "login-required" + : health.manualActionReason === "teams-admission-required" + ? "admission-required" + : health.manualActionReason === "teams-permission-required" + ? "permission-required" + : health.manualActionReason === "teams-audio-choice-required" + ? "audio-choice-required" + : health.manualActionReason === "teams-session-conflict" + ? "session-conflict" + : health.manualActionReason === "browser-control-unavailable" + ? "browser-control-unavailable" + : "custom"; + return { + category, + reason: health.manualActionReason, + message: health.manualActionMessage, + }; +} + +function parseLeaveResult(result: unknown): { + departed: boolean; + leaveAction?: "leave" | "confirm"; + urlMatched?: boolean; +} { + const record = result && typeof result === "object" ? (result as Record) : {}; + if (typeof record.result !== "string" || !record.result.trim()) { + return { departed: false }; + } + try { + const parsed = JSON.parse(record.result) as Record; + const leaveAction = + parsed.leaveAction === "leave" || parsed.leaveAction === "confirm" + ? parsed.leaveAction + : undefined; + return { + departed: parsed.departed === true, + ...(leaveAction ? { leaveAction } : {}), + ...(typeof parsed.urlMatched === "boolean" ? { urlMatched: parsed.urlMatched } : {}), + }; + } catch { + return { departed: false }; + } +} + +function parseTranscript(result: unknown) { + const record = result && typeof result === "object" ? (result as Record) : {}; + if (typeof record.result !== "string" || !record.result.trim()) { + return { droppedLines: 0, lines: [] }; + } + try { + const parsed = JSON.parse(record.result) as Record; + return { + droppedLines: 0, + lines: [], + ...(typeof parsed.urlMatched === "boolean" ? { urlMatched: parsed.urlMatched } : {}), + ...(typeof parsed.sessionMatched === "boolean" + ? { sessionMatched: parsed.sessionMatched } + : {}), + }; + } catch { + throw new Error("Microsoft Teams transcript JSON is malformed."); + } +} + +export const TEAMS_MEETINGS_PLATFORM_ADAPTER: MeetingPlatformAdapter< + MeetingBrowserJoinSession, + TeamsMeetingsMode, + TeamsMeetingsChromeHealth, + TeamsMeetingsTranscriptSnapshot +> = { + id: "teams-meetings", + displayName: "Microsoft Teams meetings", + browserLabel: "Teams meeting", + logScope: "[teams-meetings]", + nodeCommandName: TEAMS_MEETINGS_NODE_COMMAND, + nodeConfigPath: "plugins.entries.teams-meetings.config.chromeNode.node", + urls: { + validateAndNormalize: normalizeTeamsMeetingUrl, + normalizeForReuse: normalizeTeamsMeetingUrlForReuse, + isSameMeeting: isSameTeamsMeetingUrl, + buildJoinUrl: (session) => session.url, + accountHint: () => undefined, + isPreferredJoinUrl: (url) => Boolean(normalizeTeamsMeetingUrlForReuse(url)), + isRecoverableTab: isRecoverableTeamsMeetingTab, + localeAction: () => undefined, + }, + browser: { + allowsMicrophone: isTeamsMeetingsTalkBackMode, + buildStatusJoinScript: (params) => + teamsMeetingStatusScript({ + allowMicrophone: isTeamsMeetingsTalkBackMode(params.mode), + autoJoin: params.autoJoin, + guestName: params.guestName, + meetingSessionId: params.meetingSessionId || undefined, + meetingUrl: params.url, + readOnly: params.readOnly, + }), + parseStatus: parseBrowserStatus, + classifyManualAction, + browserControlUnavailable: () => ({ + category: "browser-control-unavailable", + reason: "browser-control-unavailable", + message: + "Open the OpenClaw browser profile, finish the Teams sign-in, admission, or permission prompt, then retry.", + }), + buildLeaveScript: teamsMeetingLeaveScript, + parseLeaveResult, + captions: { + // Teams caption DOM is not enabled until a live tenant flow validates stable selectors. + enabled: () => false, + buildTranscriptScript: ({ meetingSessionId, meetingUrl }) => + teamsMeetingTranscriptScript(meetingUrl, meetingSessionId), + parseTranscript, + }, + // The core permission hook has no meeting URL, so it cannot select between + // teams.microsoft.com and teams.live.com without guessing an origin. + permissions: () => undefined, + permissionNotes: ({ allowMicrophone }) => + allowMicrophone + ? ["Teams media permissions are handled in the browser when prompted."] + : ["Observe-only mode does not request Teams microphone access."], + }, +}; diff --git a/extensions/teams-meetings/src/transports/teams-meetings-platform-constants.ts b/extensions/teams-meetings/src/transports/teams-meetings-platform-constants.ts new file mode 100644 index 000000000000..58d99e988a29 --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-platform-constants.ts @@ -0,0 +1,7 @@ +export const TEAMS_MEETINGS_NODE_COMMAND = "teamsmeetings.chrome"; + +export const TEAMS_MEETINGS_BROWSER_NODE_ADAPTER = { + displayName: "Microsoft Teams meetings", + nodeCommandName: TEAMS_MEETINGS_NODE_COMMAND, + nodeConfigPath: "plugins.entries.teams-meetings.config.chromeNode.node", +}; diff --git a/extensions/teams-meetings/src/transports/teams-meetings-selectors.ts b/extensions/teams-meetings/src/transports/teams-meetings-selectors.ts new file mode 100644 index 000000000000..61502f61da4b --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-selectors.ts @@ -0,0 +1,70 @@ +// Best-effort Teams web selectors. data-tid values are preferred because Teams +// uses them extensively; every flow still needs validation against a live tenant. +export const TEAMS_MEETING_SELECTORS = { + continueInBrowser: [ + '[data-tid="joinOnWeb"]', + '[data-tid="joinOnWebButton"]', + 'button[data-tid="continue-on-browser"]', + ], + guestName: [ + 'input[data-tid="prejoin-display-name-input"]', + '[data-tid="prejoin-display-name-input"] input', + 'input[data-tid="guest-name-input"]', + ], + join: [ + 'button[data-tid="prejoin-join-button"]', + '[data-tid="prejoin-join-button"] button', + 'button[data-tid="join-now"]', + 'button[data-tid="join-button"]', + ], + microphone: [ + 'button[data-tid="toggle-mute"]', + '[data-tid="toggle-mute"] button', + 'button[data-tid="microphone-button"]', + ], + camera: [ + 'button[data-tid="toggle-video"]', + '[data-tid="toggle-video"] button', + 'button[data-tid="camera-button"]', + ], + deviceSettings: [ + 'button[data-tid="prejoin-device-settings-button"]', + 'button[data-tid="device-settings-button"]', + 'button[data-tid="audio-device-settings-button"]', + ], + microphoneDevice: [ + '[data-tid="microphone-select"]', + '[data-tid="audio-device-input"]', + '[data-tid="device-settings-microphone"] [role="combobox"]', + 'select[data-tid="microphone-select"]', + ], + selectedMicrophoneDevice: ["option:checked", '[role="option"][aria-selected="true"]'], + audioDeviceOptions: ["option", '[role="option"]'], + leave: [ + 'button[data-tid="call-hangup"]', + '[data-tid="call-hangup"] button', + 'button[data-tid="hangup-button"]', + 'button[data-tid="call-hangup-button"]', + ], + leaveConfirmation: [ + 'button[data-tid="confirm-leave-button"]', + 'button[data-tid="leave-meeting-confirm"]', + 'button[data-tid="leave-call-confirm"]', + ], + postCall: [ + '[data-tid="call-ended-screen"]', + '[data-tid="post-call-screen"]', + 'button[data-tid="prejoin-rejoin-button"]', + ], + lobby: ['[data-tid="lobby-screen"]', '[data-tid="lobby-waiting-screen"]'], + signIn: [ + 'button[data-tid="signin-button"]', + 'button[data-tid="sign-in-button"]', + '[data-tid="auth-signin"]', + ], + permissionPrompt: [ + '[data-tid="device-permission-prompt"]', + '[data-tid="media-permission-prompt"]', + '[data-tid="browser-permission-error"]', + ], +} as const; diff --git a/extensions/teams-meetings/src/transports/teams-meetings-urls.test.ts b/extensions/teams-meetings/src/transports/teams-meetings-urls.test.ts new file mode 100644 index 000000000000..bf50900fef9e --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-urls.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { + isSameTeamsMeetingUrl, + normalizeTeamsMeetingUrl, + normalizeTeamsMeetingUrlForReuse, +} from "./teams-meetings-urls.js"; + +const WORK_URL = + "https://teams.microsoft.com/l/meetup-join/19%3ameeting_NzQ1ZDBjYzItZDAxNS00N2YxLTg4Y2EtYjQ1N2I4NDg2Njli%40thread.v2/0?context=%7b%22Tid%22%3a%22tenant%22%7d"; + +describe("Microsoft Teams meeting URL normalization", () => { + it.each([ + [WORK_URL, "teams-work:19:meeting_NzQ1ZDBjYzItZDAxNS00N2YxLTg4Y2EtYjQ1N2I4NDg2Njli@thread.v2"], + ["https://teams.live.com/meet/9326458712345?p=abc", "teams-consumer:9326458712345:p:abc"], + ])("extracts a stable identity from %s", (url, expected) => { + expect(normalizeTeamsMeetingUrlForReuse(url)).toBe(expected); + }); + + it("ignores query parameters when comparing meeting identity", () => { + expect( + isSameTeamsMeetingUrl( + WORK_URL, + `${WORK_URL.split("?")[0]}?context=%7b%22Tid%22%3a%22other%22%7d&anon=true`, + ), + ).toBe(true); + expect( + isSameTeamsMeetingUrl( + "https://teams.live.com/meet/abc-123?invite=one", + "https://teams.live.com/meet/ABC-123?invite=two", + ), + ).toBe(true); + }); + + it.each([ + [ + "https://teams.live.com/meet/abc-123?p=one", + "https://teams.live.com/meet/ABC-123?p=one&invite=x", + true, + ], + [ + "https://teams.live.com/meet/abc-123?p=one", + "https://teams.live.com/meet/abc-123?p=two", + false, + ], + ["https://teams.live.com/meet/abc-123?p=one", "https://teams.live.com/meet/abc-123", false], + ])("compares consumer meeting password identity: %s / %s", (left, right, expected) => { + expect(isSameTeamsMeetingUrl(left, right)).toBe(expected); + }); + + it.each([ + "https://teams.microsoft.com/", + "https://teams.microsoft.com/v2/", + "https://teams.microsoft.com/l/channel/19%3Achannel%40thread.tacv2", + "https://teams.live.com/", + "http://teams.live.com/meet/abc", + "https://example.com/l/meetup-join/19%3ameeting_x%40thread.v2/0", + ])("rejects non-meeting Teams input: %s", (url) => { + expect(normalizeTeamsMeetingUrlForReuse(url)).toBeUndefined(); + expect(() => normalizeTeamsMeetingUrl(url)).toThrow("Microsoft Teams meeting URL must use"); + }); + + it("preserves the join query while removing fragments", () => { + const normalized = normalizeTeamsMeetingUrl(`${WORK_URL}#ignored`); + expect(normalized).toContain("context="); + expect(normalized).not.toContain("#ignored"); + }); +}); diff --git a/extensions/teams-meetings/src/transports/teams-meetings-urls.ts b/extensions/teams-meetings/src/transports/teams-meetings-urls.ts new file mode 100644 index 000000000000..b397128e69b6 --- /dev/null +++ b/extensions/teams-meetings/src/transports/teams-meetings-urls.ts @@ -0,0 +1,95 @@ +import type { MeetingBrowserCandidateTab } from "openclaw/plugin-sdk/meeting-runtime"; + +type TeamsMeetingIdentity = { kind: "work"; key: string } | { kind: "consumer"; key: string }; + +function parseTeamsMeetingIdentity(url: string | undefined): TeamsMeetingIdentity | undefined { + if (!url) { + return undefined; + } + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:" || parsed.port || parsed.username || parsed.password) { + return undefined; + } + const hostname = parsed.hostname.toLowerCase(); + if (hostname === "teams.microsoft.com") { + const match = parsed.pathname.match(/^\/l\/meetup-join\/([^/]+)(?:\/0)?\/?$/i); + if (!match?.[1]) { + return undefined; + } + const threadId = decodeURIComponent(match[1]); + if (!/^19:[^/]+@thread\.(?:v2|tacv2)$/i.test(threadId)) { + return undefined; + } + return { kind: "work", key: threadId }; + } + if (hostname === "teams.live.com") { + const match = parsed.pathname.match(/^\/meet\/([^/]+)\/?$/i); + if (!match?.[1]) { + return undefined; + } + const meetCode = decodeURIComponent(match[1]); + if (!/^[a-z0-9_-]+$/i.test(meetCode)) { + return undefined; + } + const password = parsed.searchParams.get("p"); + return { + kind: "consumer", + key: `${meetCode.toLowerCase()}:p:${encodeURIComponent(password ?? "")}`, + }; + } + } catch { + return undefined; + } + return undefined; +} + +export function normalizeTeamsMeetingUrl(input: unknown): string { + if (typeof input !== "string" || !input.trim()) { + throw new Error("Microsoft Teams meeting URL is required"); + } + const value = input.trim(); + if (!parseTeamsMeetingIdentity(value)) { + throw new Error( + "Microsoft Teams meeting URL must use https://teams.microsoft.com/l/meetup-join/... or https://teams.live.com/meet/", + ); + } + const parsed = new URL(value); + parsed.hash = ""; + return parsed.toString(); +} + +export function normalizeTeamsMeetingUrlForReuse(url: string | undefined): string | undefined { + const identity = parseTeamsMeetingIdentity(url); + return identity ? `teams-${identity.kind}:${identity.key}` : undefined; +} + +export function isSameTeamsMeetingUrl( + left: string | undefined, + right: string | undefined, +): boolean { + const normalizedLeft = normalizeTeamsMeetingUrlForReuse(left); + const normalizedRight = normalizeTeamsMeetingUrlForReuse(right); + return Boolean(normalizedLeft && normalizedRight && normalizedLeft === normalizedRight); +} + +export function isRecoverableTeamsMeetingTab( + tab: MeetingBrowserCandidateTab, + url?: string, +): boolean { + if (url) { + return isSameTeamsMeetingUrl(tab.url, url); + } + if (normalizeTeamsMeetingUrlForReuse(tab.url)) { + return true; + } + try { + const hostname = new URL(tab.url ?? "").hostname.toLowerCase(); + return ( + (hostname === "login.microsoftonline.com" || hostname.endsWith(".microsoftonline.com")) && + /sign in|microsoft|teams/i.test(tab.title ?? "") + ); + } catch { + return false; + } +} diff --git a/extensions/teams-meetings/src/transports/types.ts b/extensions/teams-meetings/src/transports/types.ts new file mode 100644 index 000000000000..9cc65c3fd74d --- /dev/null +++ b/extensions/teams-meetings/src/transports/types.ts @@ -0,0 +1,90 @@ +import type { + MeetingBrowserHealth, + MeetingBrowserTab, + MeetingSessionRecord, + MeetingTranscriptSnapshot, +} from "openclaw/plugin-sdk/meeting-runtime"; +import type { TeamsMeetingsMode, TeamsMeetingsTransport } from "../config.js"; + +export type TeamsMeetingsTranscriptSnapshot = MeetingTranscriptSnapshot; + +export type TeamsMeetingsJoinRequest = { + url: string; + transport?: TeamsMeetingsTransport; + mode?: TeamsMeetingsMode; + message?: string; + requesterSessionKey?: string; + agentId?: string; + timeoutMs?: number; +}; + +type TeamsMeetingsManualActionReason = + | "teams-login-required" + | "teams-admission-required" + | "teams-permission-required" + | "teams-audio-choice-required" + | "teams-camera-required" + | "teams-microphone-required" + | "teams-session-conflict" + | "browser-control-unavailable"; + +type TeamsMeetingsSpeechBlockedReason = + | TeamsMeetingsManualActionReason + | "not-in-call" + | "browser-unverified" + | "audio-bridge-unavailable" + | "teams-microphone-muted"; + +export type TeamsMeetingsChromeHealth = MeetingBrowserHealth< + TeamsMeetingsManualActionReason, + TeamsMeetingsSpeechBlockedReason +> & { + inCall?: boolean; + micMuted?: boolean; + cameraOff?: boolean; + lobbyWaiting?: boolean; + audioInputRouted?: boolean; + audioInputDeviceLabel?: string; + audioInputRouteError?: string; + audioOutputRouted?: boolean; + audioOutputDeviceLabel?: string; + audioOutputRouteError?: string; + providerConnected?: boolean; + realtimeReady?: boolean; + audioInputActive?: boolean; + audioOutputActive?: boolean; + lastInputAt?: string; + lastOutputAt?: string; + lastInputBytes?: number; + lastOutputBytes?: number; + bridgeClosed?: boolean; + browserUrl?: string; + browserTitle?: string; + status?: string; + notes?: string[]; +}; + +export type TeamsMeetingsBrowserTab = MeetingBrowserTab; + +export type TeamsMeetingsSession = MeetingSessionRecord< + TeamsMeetingsTransport, + TeamsMeetingsMode +> & { + chrome?: { + audioBackend: "blackhole-2ch"; + launched: boolean; + nodeId?: string; + browserProfile?: string; + browserTab?: TeamsMeetingsBrowserTab; + audioBridge?: { + type: "command-pair" | "node-command-pair"; + provider?: string; + }; + health?: TeamsMeetingsChromeHealth; + }; +}; + +export type TeamsMeetingsJoinResult = { + session: TeamsMeetingsSession; + spoken?: boolean; +}; diff --git a/extensions/teams-meetings/tsconfig.json b/extensions/teams-meetings/tsconfig.json new file mode 100644 index 000000000000..b8a85a99ac3d --- /dev/null +++ b/extensions/teams-meetings/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../tsconfig.package-boundary.base.json", + "compilerOptions": { + "rootDir": "." + }, + "include": ["./*.ts", "./src/**/*.ts"], + "exclude": [ + "./**/*.test.ts", + "./dist/**", + "./node_modules/**", + "./src/test-support/**", + "./src/**/*test-helpers.ts", + "./src/**/*test-harness.ts", + "./src/**/*test-support.ts" + ] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0c160743cd3d..55a2a903c7a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1729,6 +1729,19 @@ importers: specifier: workspace:* version: link:../../packages/plugin-sdk + extensions/teams-meetings: + dependencies: + typebox: + specifier: 1.3.3 + version: 1.3.3 + devDependencies: + '@openclaw/plugin-sdk': + specifier: workspace:* + version: link:../../packages/plugin-sdk + openclaw: + specifier: workspace:* + version: link:../.. + extensions/telegram: dependencies: '@grammyjs/runner': diff --git a/scripts/generate-plugin-inventory-doc.mjs b/scripts/generate-plugin-inventory-doc.mjs index 5782513f03d0..4ed57f5a8d4b 100644 --- a/scripts/generate-plugin-inventory-doc.mjs +++ b/scripts/generate-plugin-inventory-doc.mjs @@ -125,6 +125,9 @@ function pluginReferenceLabel(record) { } function humanizeId(value) { + if (value === "teams-meetings") { + return "Microsoft Teams meetings"; + } const names = new Map([ ["acpx", "ACPx"], ["ai", "AI"], diff --git a/src/meeting-bot/browser-controller.ts b/src/meeting-bot/browser-controller.ts index ecdd4e9cb5f1..8a2d320f98a0 100644 --- a/src/meeting-bot/browser-controller.ts +++ b/src/meeting-bot/browser-controller.ts @@ -288,15 +288,15 @@ function findRecoverableTab< >(params: { adapter: BrowserAdapter; tabs: MeetingBrowserCandidateTab[]; - url?: string; + requestedMeetingUrl: string | undefined; }): MeetingBrowserCandidateTab | undefined { const candidates = params.tabs.filter((tab) => - params.adapter.urls.isRecoverableTab(tab, params.url), + params.adapter.urls.isRecoverableTab(tab, params.requestedMeetingUrl), ); - if (!params.url) { + if (!params.requestedMeetingUrl) { return candidates[0]; } - const accountHint = params.adapter.urls.accountHint(params.url); + const accountHint = params.adapter.urls.accountHint(params.requestedMeetingUrl); const accountCandidates = accountHint ? candidates.filter((tab) => params.adapter.urls.accountHint(tab.url) === accountHint) : candidates; @@ -321,6 +321,7 @@ async function inspectRecoverableTab< config: MeetingBrowserControllerConfig; mode: Mode; readOnly?: boolean; + requestedMeetingUrl: string | undefined; tab: MeetingBrowserCandidateTab; targetId: string; timeoutMs: number; @@ -367,7 +368,7 @@ async function inspectRecoverableTab< fn: params.adapter.browser.buildStatusJoinScript({ meetingSessionId: "", mode: params.mode, - url: params.tab.url ?? "", + url: params.requestedMeetingUrl ?? params.tab.url ?? "", autoJoin: false, captureCaptions: params.adapter.browser.captions.enabled(params.mode), guestName: params.config.guestName, @@ -418,8 +419,10 @@ export async function recoverMeetingBrowserTab< config: MeetingBrowserControllerConfig; locationLabel: string; mode: Mode; - url?: string; + requestedMeetingUrl: string | undefined; readOnly?: boolean; + trackedMeetingUrl: string | undefined; + trackedTargetId: string | undefined; }): Promise<{ found: boolean; targetId?: string; @@ -435,18 +438,42 @@ export async function recoverMeetingBrowserTab< timeoutMs: Math.min(timeoutMs, 5_000), }), ); - const tab = findRecoverableTab({ - adapter: params.adapter, - tabs, - url: params.url, - }); + const trackedCandidate = params.trackedTargetId + ? tabs.find((tab) => tab.targetId === params.trackedTargetId) + : undefined; + const trackedUrlHasMeetingIdentity = Boolean( + params.adapter.urls.normalizeForReuse(trackedCandidate?.url), + ); + const trackedIdentityMatches = params.adapter.urls.isSameMeeting( + params.trackedMeetingUrl, + params.requestedMeetingUrl, + ); + const trackedUrlMatches = params.adapter.urls.isSameMeeting( + trackedCandidate?.url, + params.requestedMeetingUrl, + ); + // Meeting SPAs may replace the join URL after admission. Keep the persisted + // target for non-identifying URLs, but never follow it into another meeting. + const trackedTab = + trackedCandidate && + trackedIdentityMatches && + (!trackedUrlHasMeetingIdentity || trackedUrlMatches) + ? trackedCandidate + : undefined; + const tab = + trackedTab ?? + findRecoverableTab({ + adapter: params.adapter, + tabs, + requestedMeetingUrl: params.requestedMeetingUrl, + }); const targetId = tab?.targetId; if (!tab || !targetId) { return { found: false, tab, - message: params.url - ? `No existing ${params.adapter.browserLabel} tab matched ${params.url}.` + message: params.requestedMeetingUrl + ? `No existing ${params.adapter.browserLabel} tab matched ${params.requestedMeetingUrl}.` : `No existing ${params.adapter.browserLabel} tab found ${params.locationLabel}.`, }; } @@ -456,6 +483,7 @@ export async function recoverMeetingBrowserTab< config: params.config, mode: params.mode, readOnly: params.readOnly, + requestedMeetingUrl: params.requestedMeetingUrl, timeoutMs, tab, targetId,