From c97b8ffdfc83fec7ebabf22ab7b9ec04777eae97 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 18 Aug 2026 19:11:13 -0700 Subject: [PATCH] refactor: consolidate meeting and media provider families (#126053) * refactor(plugins): consolidate provider family helpers * fix(plugin-sdk): keep meeting script helpers private * fix(plugins): sync meeting boundary paths --- docs/plugins/sdk-subpaths.md | 1 + .../openrouter/generation-request-context.ts | 45 +++ .../openrouter/image-generation-provider.ts | 43 +-- .../openrouter/music-generation-provider.ts | 35 +-- .../openrouter/video-generation-provider.ts | 35 +-- .../transports/teams-meetings-page-scripts.ts | 249 +++------------- .../tsconfig.package-boundary.paths.json | 3 + extensions/vydra/image-generation-provider.ts | 87 +----- extensions/vydra/shared.ts | 105 ++++++- extensions/vydra/video-generation-provider.ts | 101 +------ extensions/xai/tsconfig.json | 3 + .../transports/zoom-meetings-page-scripts.ts | 266 +++-------------- package.json | 4 + scripts/lib/plugin-sdk-entrypoints.json | 1 + ...lugin-sdk-private-local-only-subpaths.json | 1 + src/meeting-bot/page-script-source.ts | 270 ++++++++++++++++++ src/plugin-sdk/meeting-page-script-runtime.ts | 5 + 17 files changed, 566 insertions(+), 688 deletions(-) create mode 100644 extensions/openrouter/generation-request-context.ts create mode 100644 src/meeting-bot/page-script-source.ts create mode 100644 src/plugin-sdk/meeting-page-script-runtime.ts diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 798715c09975..06ac92f3d277 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -340,6 +340,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It | `plugin-sdk/realtime-voice-audio-queue` | Private-local JavaScript-only host runtime for bundled or separately published official plugins; narrow bounded audio queue seam for lazy realtime voice provider facades without importing the broader realtime voice runtime; not for third-party plugins | | `plugin-sdk/realtime-voice-activation` | Private-local; dependency-light realtime-voice activation-name helpers (normalize, match, word-count, sort) for doctor contract closures and other control-plane paths that must not load the realtime voice runtime | | `plugin-sdk/realtime-voice` | Private-local after July 2026; Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including the transport-independent session harness and output activity tracking. For official runtime consumers, sender-auth contract revision 1 forwards ingress-authenticated `senderId` and `senderIsOwner` unchanged; ingress owns authentication, and consumers requiring the handoff must fail closed on other revisions. | + | `plugin-sdk/meeting-page-script-runtime` | Private-local JavaScript-only host runtime for official browser-meeting plugins; shared transcript and leave page-script source builders; not a third-party plugin API | | `plugin-sdk/meeting-runtime` | Browser-meeting session runtime, realtime audio engines/transports, `MeetingPlatformAdapter`, browser/node control, agent-consult, voice-call delegation, setup checks, and SoX command helpers | | `plugin-sdk/image-generation` | Private-local after July 2026; Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder | | `plugin-sdk/image-generation-core` | Private-local after July 2026; Shared image-generation types, failover, auth, and registry helpers | diff --git a/extensions/openrouter/generation-request-context.ts b/extensions/openrouter/generation-request-context.ts new file mode 100644 index 000000000000..14623314a332 --- /dev/null +++ b/extensions/openrouter/generation-request-context.ts @@ -0,0 +1,45 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; +import { + resolveProviderHttpRequestConfig, + sanitizeConfiguredModelProviderRequest, +} from "openclaw/plugin-sdk/provider-http"; +import { OPENROUTER_BASE_URL } from "./provider-catalog.js"; + +type OpenRouterAuthStore = Parameters[0]["store"]; + +export async function resolveOpenRouterGenerationRequestContext(params: { + cfg: OpenClawConfig; + agentDir?: string; + authStore?: OpenRouterAuthStore; + capability: "audio" | "image" | "video"; + jsonContentType: boolean; +}) { + const auth = await resolveApiKeyForProvider({ + provider: "openrouter", + cfg: params.cfg, + agentDir: params.agentDir, + store: params.authStore, + }); + if (!auth.apiKey) { + throw new Error("OpenRouter API key missing"); + } + + return resolveProviderHttpRequestConfig({ + baseUrl: params.cfg.models?.providers?.openrouter?.baseUrl, + defaultBaseUrl: OPENROUTER_BASE_URL, + allowPrivateNetwork: false, + defaultHeaders: { + Authorization: `Bearer ${auth.apiKey}`, + ...(params.jsonContentType ? { "Content-Type": "application/json" } : {}), + "HTTP-Referer": "https://openclaw.ai", + "X-OpenRouter-Title": "OpenClaw", + }, + request: sanitizeConfiguredModelProviderRequest( + params.cfg.models?.providers?.openrouter?.request, + ), + provider: "openrouter", + capability: params.capability, + transport: "http", + }); +} diff --git a/extensions/openrouter/image-generation-provider.ts b/extensions/openrouter/image-generation-provider.ts index 051655ca9473..80ac14d21a05 100644 --- a/extensions/openrouter/image-generation-provider.ts +++ b/extensions/openrouter/image-generation-provider.ts @@ -14,16 +14,14 @@ import { import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; import { resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, postJsonRequest, readProviderJsonResponse, - resolveProviderHttpRequestConfig, - sanitizeConfiguredModelProviderRequest, } from "openclaw/plugin-sdk/provider-http"; import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { normalizeOpenRouterBaseUrl, OPENROUTER_BASE_URL } from "./provider-catalog.js"; +import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js"; +import { normalizeOpenRouterBaseUrl } from "./provider-catalog.js"; const DEFAULT_MODEL = "google/gemini-3.1-flash-image-preview"; const DEFAULT_TIMEOUT_MS = 180_000; @@ -291,36 +289,19 @@ export function buildOpenRouterImageGenerationProvider(): ImageGenerationProvide }, }, async generateImage(req) { - const auth = await resolveApiKeyForProvider({ - provider: "openrouter", - cfg: req.cfg, - agentDir: req.agentDir, - store: req.authStore, - }); - if (!auth.apiKey) { - throw new Error("OpenRouter API key missing"); - } + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + await resolveOpenRouterGenerationRequestContext({ + cfg: req.cfg, + agentDir: req.agentDir, + authStore: req.authStore, + capability: "image", + // Preserve the existing resolved header contract; postJsonRequest supplies + // the JSON content type for both chat-completion and dedicated image requests. + jsonContentType: false, + }); const model = normalizeOptionalString(req.model) ?? DEFAULT_MODEL; const imageConfig = buildImageConfig(req, model); - const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = - resolveProviderHttpRequestConfig({ - baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl, - defaultBaseUrl: OPENROUTER_BASE_URL, - allowPrivateNetwork: false, - defaultHeaders: { - Authorization: `Bearer ${auth.apiKey}`, - "HTTP-Referer": "https://openclaw.ai", - "X-OpenRouter-Title": "OpenClaw", - }, - request: sanitizeConfiguredModelProviderRequest( - req.cfg?.models?.providers?.openrouter?.request, - ), - provider: "openrouter", - capability: "image", - transport: "http", - }); - const count = resolveImageCount(req.count); const canonicalBaseUrl = normalizeOpenRouterBaseUrl(baseUrl); if (canonicalBaseUrl) { diff --git a/extensions/openrouter/music-generation-provider.ts b/extensions/openrouter/music-generation-provider.ts index 648d67567a76..76fe808c12ac 100644 --- a/extensions/openrouter/music-generation-provider.ts +++ b/extensions/openrouter/music-generation-provider.ts @@ -9,18 +9,15 @@ import type { } from "openclaw/plugin-sdk/music-generation"; import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, createProviderOperationDeadline, postJsonRequest, - resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, - sanitizeConfiguredModelProviderRequest, type ProviderOperationDeadline, } from "openclaw/plugin-sdk/provider-http"; import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { OPENROUTER_BASE_URL } from "./provider-catalog.js"; +import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js"; const DEFAULT_OPENROUTER_MUSIC_MODEL = "google/lyria-3-pro-preview"; const OPENROUTER_CLIP_MUSIC_MODEL = "google/lyria-3-clip-preview"; @@ -371,33 +368,13 @@ export function buildOpenRouterMusicGenerationProvider(): MusicGenerationProvide if ((req.inputImages?.length ?? 0) > 1) { throw new Error("OpenRouter music generation supports at most one reference image."); } - const auth = await resolveApiKeyForProvider({ - provider: "openrouter", - cfg: req.cfg, - agentDir: req.agentDir, - store: req.authStore, - }); - if (!auth.apiKey) { - throw new Error("OpenRouter API key missing"); - } - const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = - resolveProviderHttpRequestConfig({ - baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl, - defaultBaseUrl: OPENROUTER_BASE_URL, - allowPrivateNetwork: false, - defaultHeaders: { - Authorization: `Bearer ${auth.apiKey}`, - "Content-Type": "application/json", - "HTTP-Referer": "https://openclaw.ai", - "X-OpenRouter-Title": "OpenClaw", - }, - request: sanitizeConfiguredModelProviderRequest( - req.cfg?.models?.providers?.openrouter?.request, - ), - provider: "openrouter", + await resolveOpenRouterGenerationRequestContext({ + cfg: req.cfg, + agentDir: req.agentDir, + authStore: req.authStore, capability: "audio", - transport: "http", + jsonContentType: true, }); const model = resolveOpenRouterMusicModel(req.model); const format = req.format ?? "wav"; diff --git a/extensions/openrouter/video-generation-provider.ts b/extensions/openrouter/video-generation-provider.ts index 06d9c2f09cf2..47dca8bb6e83 100644 --- a/extensions/openrouter/video-generation-provider.ts +++ b/extensions/openrouter/video-generation-provider.ts @@ -3,15 +3,12 @@ import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation"; import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; import { extensionForMime } from "openclaw/plugin-sdk/media-mime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { assertOkOrThrowHttpError, createProviderOperationDeadline, postJsonRequest, readProviderJsonResponse, - resolveProviderHttpRequestConfig, resolveProviderOperationTimeoutMs, - sanitizeConfiguredModelProviderRequest, waitProviderOperationPollInterval, } from "openclaw/plugin-sdk/provider-http"; import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime"; @@ -22,7 +19,7 @@ import type { VideoGenerationRequest, VideoGenerationSourceAsset, } from "openclaw/plugin-sdk/video-generation"; -import { OPENROUTER_BASE_URL } from "./provider-catalog.js"; +import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js"; import { fetchOpenRouterVideoGet, resolveOpenRouterVideoUrl, @@ -463,34 +460,14 @@ export function buildOpenRouterVideoGenerationProvider(): VideoGenerationProvide throw new Error("OpenRouter video generation does not support video reference inputs."); } - const auth = await resolveApiKeyForProvider({ - provider: "openrouter", - cfg: req.cfg, - agentDir: req.agentDir, - store: req.authStore, - }); - if (!auth.apiKey) { - throw new Error("OpenRouter API key missing"); - } - const model = normalizeOptionalString(req.model) ?? DEFAULT_MODEL; const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = - resolveProviderHttpRequestConfig({ - baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl, - defaultBaseUrl: OPENROUTER_BASE_URL, - allowPrivateNetwork: false, - defaultHeaders: { - Authorization: `Bearer ${auth.apiKey}`, - "Content-Type": "application/json", - "HTTP-Referer": "https://openclaw.ai", - "X-OpenRouter-Title": "OpenClaw", - }, - request: sanitizeConfiguredModelProviderRequest( - req.cfg?.models?.providers?.openrouter?.request, - ), - provider: "openrouter", + await resolveOpenRouterGenerationRequestContext({ + cfg: req.cfg, + agentDir: req.agentDir, + authStore: req.authStore, capability: "video", - transport: "http", + jsonContentType: true, }); const deadline = createProviderOperationDeadline({ timeoutMs: req.timeoutMs, diff --git a/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts b/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts index fb92a49fb4d4..c6bc5f5ac657 100644 --- a/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts +++ b/extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts @@ -1,8 +1,11 @@ +import { + createMeetingLeaveSource, + createMeetingTranscriptSource, +} from "openclaw/plugin-sdk/meeting-page-script-runtime"; import { TEAMS_MEETING_SELECTORS } from "./teams-meetings-selectors.js"; import { teamsMeetingStatusCallSource } from "./teams-meetings-status-call-source.js"; import { teamsMeetingStatusPreludeSource } from "./teams-meetings-status-prejoin-source.js"; import { normalizeTeamsMeetingUrlForReuse } from "./teams-meetings-urls.js"; -const TEAMS_MEETING_TRANSCRIPT_MAX_LINES = 500; function pageIdentityFunctionSource(): string { return `const meetingIdentity = (rawUrl) => { @@ -109,84 +112,18 @@ export function teamsMeetingTranscriptScript( finalize: boolean, ) { const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl); - return `() => { - ${pageIdentityFunctionSource()} - const expectedIdentity = ${JSON.stringify(expectedIdentity)}; - const expectedSessionId = ${JSON.stringify(meetingSessionId)}; - const currentIdentity = meetingIdentity(location.href); - const state = window.__openclawTeamsMeeting; - const activeCaptions = window.__openclawTeamsCaptions; - const archivedCaptions = window.__openclawTeamsCaptionArchive?.[expectedSessionId]; - const captions = activeCaptions && - (!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId) - ? activeCaptions - : archivedCaptions; - // A same-session finalized buffer belongs to the departed call even if Teams - // immediately navigated this tab into another meeting before transcript pickup. - const useFinalizedCaptions = Boolean( - captions?.finalized === true && - captions?.identity === expectedIdentity && - (!captions?.sessionId || captions.sessionId === expectedSessionId) - ); - const effectiveIdentity = useFinalizedCaptions - ? captions.identity - : currentIdentity || state?.identity || captions?.identity; - if (!expectedIdentity || effectiveIdentity !== expectedIdentity) { - return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] }); - } - if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) { - return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); - } - if (captions?.sessionId && captions.sessionId !== expectedSessionId) { - return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); - } - if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) { - if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); - captions.settleTimer = undefined; - captions.lines = Array.isArray(captions.lines) ? captions.lines : []; - captions.lines.push(...captions.visible.map((entry) => ({ - at: entry.at, - speaker: entry.speaker, - text: entry.text, - }))); - captions.visible = []; - const excess = captions.lines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES}; - if (excess > 0) { - captions.lines.splice(0, excess); - captions.droppedLines = (captions.droppedLines || 0) + excess; - } - } - if (${JSON.stringify(finalize)} && captions) { - if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); - captions.settleTimer = undefined; - captions.observer?.disconnect?.(); - captions.observer = undefined; - captions.observerInstalled = false; - captions.identity = expectedIdentity; - captions.finalized = true; - captions.finalizedAt = Date.now(); - } - const allLines = [ - ...(Array.isArray(captions?.lines) ? captions.lines : []), - ...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible), - ]; - const visibleOverflow = Math.max(0, allLines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES}); - const lines = allLines.slice(-${TEAMS_MEETING_TRANSCRIPT_MAX_LINES}); - const result = { - urlMatched: true, - sessionMatched: true, - epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined, - droppedLines: (Number.isFinite(captions?.droppedLines) - ? Math.max(0, Math.trunc(captions.droppedLines)) - : 0) + visibleOverflow, - lines: lines.map((line) => ({ - at: typeof line?.at === "string" ? line.at : undefined, - speaker: typeof line?.speaker === "string" ? line.speaker : undefined, - text: typeof line?.text === "string" ? line.text : "", - })).filter((line) => line.text), - }; - return JSON.stringify(result); -}`; + return createMeetingTranscriptSource({ + expectedIdentity, + finalize, + globals: { + captionArchive: "__openclawTeamsCaptionArchive", + captions: "__openclawTeamsCaptions", + meeting: "__openclawTeamsMeeting", + }, + meetingSessionId, + pageIdentitySource: pageIdentityFunctionSource(), + platformDisplayName: "Teams", + }); } export function teamsMeetingLeaveScript(params: { @@ -196,75 +133,8 @@ export function teamsMeetingLeaveScript(params: { }) { const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS); const expectedIdentity = normalizeTeamsMeetingUrlForReuse(params.meetingUrl); - return `() => { - ${pageIdentityFunctionSource()} - const selectors = ${selectors}; - const expectedIdentity = ${JSON.stringify(expectedIdentity)}; - const expectedSessionId = ${JSON.stringify(params.meetingSessionId)}; - const leaveInitiated = ${JSON.stringify(params.leaveInitiated)}; - const currentIdentity = meetingIdentity(location.href); - const state = window.__openclawTeamsMeeting; - const enforceSessionOwnership = Boolean(expectedSessionId); - if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) { - return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true }); - } - const sessionMatched = !enforceSessionOwnership || state?.sessionId === expectedSessionId; - const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated); - if (!sessionMatched && !retainedLeaveOwnership) { - return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true }); - } - const retireOwnedAudioBridges = () => { - const entries = Array.isArray(window.__openclawTeamsAudioOutputs) - ? window.__openclawTeamsAudioOutputs - : []; - const retained = []; - const activeSessionId = expectedSessionId || state?.sessionId; - for (const entry of entries) { - const ownedByActiveSession = Boolean( - !entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId) - ); - if (!ownedByActiveSession) { - retained.push(entry); - continue; - } - const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || ""); - const sources = Array.isArray(entry?.sources) - ? entry.sources - : entry?.source - ? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }] - : []; - for (const source of sources) { - const element = source?.element; - const sourceMatches = source?.stream || element?.srcObject - ? element?.srcObject === source?.stream - : Boolean(source?.url && mediaSourceUrl(element) === source.url); - const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element)); - if (!element) continue; - if (sourceIsEmpty) { - element.muted = true; - continue; - } - if (!sourceMatches) continue; - const detachedLiveSource = Boolean( - element.isConnected === false && - element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live") - ); - if (detachedLiveSource) { - element.muted = true; - element.pause?.(); - element.srcObject = null; - } else { - element.muted = Boolean(source.muted); - } - } - entry?.bridge?.pause?.(); - if (entry?.bridge) entry.bridge.srcObject = null; - entry?.bridge?.remove?.(); - } - if (retained.length > 0) window.__openclawTeamsAudioOutputs = retained; - else delete window.__openclawTeamsAudioOutputs; - }; - const first = (list) => { + return createMeetingLeaveSource({ + controlSource: `const first = (list) => { for (const selector of list) { const node = document.querySelector(selector); if (!node) continue; @@ -275,70 +145,21 @@ export function teamsMeetingLeaveScript(params: { const leave = first(selectors.leave); const confirmation = first(selectors.leaveConfirmation); const postCall = first(selectors.postCall); - const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity); - const preservedCallMatches = Boolean( - expectedIdentity && - !currentIdentity && - state?.identity === expectedIdentity && - state?.inCallControl === leave && - state?.inCallUrl === location.href && - leave && - leave.isConnected !== false - ); - const pendingLeaveMatches = Boolean( - expectedIdentity && - state?.identity === expectedIdentity && - state?.leavePending === true && - state?.inCallUrl === location.href && - Date.now() - state?.leavePendingAt < 10_000 - ); - const rerenderPendingMatches = Boolean( - expectedIdentity && - !currentIdentity && - state?.identity === expectedIdentity && - state?.inCallControl?.isConnected === false && - state?.inCallUrl === location.href && - Date.now() - state?.verifiedAt < 5_000 && - !leave - ); - const meetingIdentityMatches = Boolean( - currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches - ); - // Teams can replace the document between our Leave click and its post-call marker. - // Retain request ownership only while no identity or live-call control contradicts it. - const initiatedLeaveTransitionMatches = Boolean( - leaveInitiated && - !currentIdentity && - !leave && - (!state?.identity || state.identity === expectedIdentity) - ); - if (postCall && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) { - retireOwnedAudioBridges(); - if (sessionMatched) delete window.__openclawTeamsMeeting; - return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true }); - } - if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) { - return JSON.stringify({ departed: false, urlMatched: false }); - } - if (!sessionMatched) { - return JSON.stringify({ departed: false, urlMatched: true }); - } - if (confirmation) { - confirmation.click(); - return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }); - } - if (leave) { - window.__openclawTeamsMeeting = { - ...state, - identity: expectedIdentity, - inCallControl: leave, - inCallUrl: location.href, - leavePending: true, - leavePendingAt: Date.now(), - }; - leave.click(); - return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true }); - } - return JSON.stringify({ departed: false, urlMatched: true }); -}`; + const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity);`, + departedMarkerSource: "postCall", + expectedIdentity, + leaveInitiated: params.leaveInitiated, + meetingSessionId: params.meetingSessionId, + pageIdentitySource: pageIdentityFunctionSource(), + platform: { + displayName: "Teams", + globals: { + audioOutputs: "__openclawTeamsAudioOutputs", + meeting: "__openclawTeamsMeeting", + }, + }, + selectors, + sessionMatchSource: + "const sessionMatched = !enforceSessionOwnership || state?.sessionId === expectedSessionId;", + }); } diff --git a/extensions/tsconfig.package-boundary.paths.json b/extensions/tsconfig.package-boundary.paths.json index 733b8098e94b..b55665ea5df3 100644 --- a/extensions/tsconfig.package-boundary.paths.json +++ b/extensions/tsconfig.package-boundary.paths.json @@ -95,6 +95,9 @@ "openclaw/plugin-sdk/media-generation-runtime": [ "../packages/plugin-sdk/dist/src/plugin-sdk/media-generation-runtime.d.ts" ], + "openclaw/plugin-sdk/meeting-page-script-runtime": [ + "../packages/plugin-sdk/dist/src/plugin-sdk/meeting-page-script-runtime.d.ts" + ], "openclaw/plugin-sdk/conversation-binding-runtime": [ "../packages/plugin-sdk/dist/src/plugin-sdk/conversation-binding-runtime.d.ts" ], diff --git a/extensions/vydra/image-generation-provider.ts b/extensions/vydra/image-generation-provider.ts index bdd43b494075..861cca3cb2ec 100644 --- a/extensions/vydra/image-generation-provider.ts +++ b/extensions/vydra/image-generation-provider.ts @@ -1,21 +1,7 @@ // Vydra provider module implements model/runtime integration. import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation"; -import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { - assertOkOrThrowHttpError, - postJsonRequest, - readProviderJsonResponse, -} from "openclaw/plugin-sdk/provider-http"; -import { - DEFAULT_VYDRA_IMAGE_MODEL, - downloadVydraAsset, - extractVydraResultUrls, - resolveCompletedVydraPayload, - resolveVydraResponseJobId, - resolveVydraResponseStatus, - resolveVydraRequestContext, -} from "./shared.js"; +import { DEFAULT_VYDRA_IMAGE_MODEL, runVydraGeneration } from "./shared.js"; export function buildVydraImageGenerationProvider(): ImageGenerationProvider { return { @@ -50,72 +36,29 @@ export function buildVydraImageGenerationProvider(): ImageGenerationProvider { throw new Error("Vydra image generation supports at most one image per request."); } - const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({ + const model = req.model?.trim() || DEFAULT_VYDRA_IMAGE_MODEL; + const generated = await runVydraGeneration({ cfg: req.cfg, agentDir: req.agentDir, authStore: req.authStore, - capability: "image", - ssrfPolicy: req.ssrfPolicy, - }); - - const model = req.model?.trim() || DEFAULT_VYDRA_IMAGE_MODEL; - const { response, release } = await postJsonRequest({ - url: `${baseUrl}/models/${model}`, - headers: requestPolicy.headers, + kind: "image", + model, body: { prompt: req.prompt, model: "text-to-image", }, timeoutMs: req.timeoutMs, - fetchFn, - allowPrivateNetwork: requestPolicy.allowPrivateNetwork, - ssrfPolicy: requestPolicy.ssrfPolicy, - dispatcherPolicy: requestPolicy.dispatcherPolicy, + ssrfPolicy: req.ssrfPolicy, }); - - try { - await assertOkOrThrowHttpError(response, "Vydra image generation failed"); - const submitted = await readProviderJsonResponse(response, "vydra.image-generation"); - const completedPayload = await resolveCompletedVydraPayload({ - submitted, - baseUrl, - timeoutMs: req.timeoutMs, - fetchFn, - kind: "image", - missingJobIdMessage: "Vydra image generation response missing job id", - requestPolicy, - }); - const imageUrl = extractVydraResultUrls(completedPayload, "image")[0]; - if (!imageUrl) { - throw new Error("Vydra image generation completed without an image URL"); - } - const image = await downloadVydraAsset({ - url: imageUrl, - kind: "image", - timeoutMs: req.timeoutMs, - fetchFn, - maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "image"), - requestPolicy, - }); - return { - images: [ - { - buffer: image.buffer, - mimeType: image.mimeType, - fileName: image.fileName, - }, - ], - model, - metadata: { - jobId: - resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted), - imageUrl, - status: resolveVydraResponseStatus(completedPayload) ?? "completed", - }, - }; - } finally { - await release(); - } + return { + images: [generated.asset], + model, + metadata: { + jobId: generated.jobId, + imageUrl: generated.resultUrl, + status: generated.status, + }, + }; }, }; } diff --git a/extensions/vydra/shared.ts b/extensions/vydra/shared.ts index b740bf624e61..c51c5ce483da 100644 --- a/extensions/vydra/shared.ts +++ b/extensions/vydra/shared.ts @@ -1,5 +1,6 @@ // Vydra plugin module implements shared behavior. import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime"; import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; import { @@ -8,7 +9,10 @@ import { createProviderOperationTimeoutResolver, fetchWithTimeoutGuarded, pollProviderOperationJson, + postJsonRequest, + readProviderJsonResponse, resolveProviderHttpRequestConfig, + resolveProviderOperationTimeoutMs, sanitizeConfiguredModelProviderRequest, type ProviderOperationDeadline, type ProviderOperationTimeoutMs, @@ -94,7 +98,7 @@ function resolveVydraBaseUrlFromConfig(cfg: unknown): string { return normalizeVydraBaseUrl(normalizeOptionalString(vydra?.baseUrl)); } -export async function resolveVydraRequestContext(params: { +async function resolveVydraRequestContext(params: { cfg: OpenClawConfig; agentDir?: string; authStore?: VydraAuthStore; @@ -142,12 +146,12 @@ export async function resolveVydraRequestContext(params: { }; } -export function resolveVydraResponseJobId(payload: unknown): string | undefined { +function resolveVydraResponseJobId(payload: unknown): string | undefined { const object = asOptionalRecord(payload) as VydraJobPayload | undefined; return normalizeOptionalString(object?.jobId) ?? normalizeOptionalString(object?.id); } -export function resolveVydraResponseStatus(payload: unknown): string | undefined { +function resolveVydraResponseStatus(payload: unknown): string | undefined { return normalizeOptionalLowercaseString( normalizeOptionalString(asOptionalRecord(payload)?.status), ); @@ -362,7 +366,7 @@ async function waitForVydraJob(params: { }); } -export async function resolveCompletedVydraPayload(params: { +async function resolveCompletedVydraPayload(params: { submitted: unknown; baseUrl: string; timeoutMs?: number; @@ -392,3 +396,96 @@ export async function resolveCompletedVydraPayload(params: { requestPolicy: params.requestPolicy, }); } + +export async function runVydraGeneration(params: { + cfg: OpenClawConfig; + agentDir?: string; + authStore?: VydraAuthStore; + body: unknown; + deadlineTimeoutMs?: number; + kind: Extract; + model: string; + ssrfPolicy?: SsrFPolicy; + timeoutMs?: number; +}): Promise<{ + asset: { buffer: Buffer; mimeType: string; fileName: string }; + jobId?: string; + resultUrl: string; + status: string; +}> { + const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({ + cfg: params.cfg, + agentDir: params.agentDir, + authStore: params.authStore, + capability: params.kind, + ...(params.ssrfPolicy ? { ssrfPolicy: params.ssrfPolicy } : {}), + }); + const operationLabel = `Vydra ${params.kind} generation`; + const deadline = + params.deadlineTimeoutMs === undefined + ? undefined + : createProviderOperationDeadline({ + timeoutMs: params.deadlineTimeoutMs, + label: operationLabel, + }); + const timeoutMs = deadline + ? resolveProviderOperationTimeoutMs({ + deadline, + defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS, + }) + : params.timeoutMs; + const { response, release } = await postJsonRequest({ + url: `${baseUrl}/models/${params.model}`, + headers: requestPolicy.headers, + body: params.body, + timeoutMs, + fetchFn, + allowPrivateNetwork: requestPolicy.allowPrivateNetwork, + ...(requestPolicy.ssrfPolicy ? { ssrfPolicy: requestPolicy.ssrfPolicy } : {}), + dispatcherPolicy: requestPolicy.dispatcherPolicy, + }); + + try { + await assertOkOrThrowHttpError(response, `${operationLabel} failed`); + const submitted = await readProviderJsonResponse( + response, + params.kind === "image" ? "vydra.image-generation" : operationLabel, + ); + const completedPayload = await resolveCompletedVydraPayload({ + submitted, + baseUrl, + ...(deadline ? { deadline } : { timeoutMs: params.timeoutMs }), + fetchFn, + kind: params.kind, + missingJobIdMessage: `${operationLabel} response missing job id`, + requestPolicy, + }); + const resultUrl = extractVydraResultUrls(completedPayload, params.kind)[0]; + if (!resultUrl) { + throw new Error(`${operationLabel} completed without a ${params.kind} URL`); + } + const asset = await downloadVydraAsset({ + url: resultUrl, + kind: params.kind, + timeoutMs: deadline + ? createProviderOperationTimeoutResolver({ + deadline, + defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS, + }) + : params.timeoutMs, + fetchFn, + maxBytes: resolveGeneratedMediaMaxBytes(params.cfg, params.kind), + requestPolicy, + }); + const jobId = + resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted); + return { + asset, + ...(jobId ? { jobId } : {}), + resultUrl, + status: resolveVydraResponseStatus(completedPayload) ?? "completed", + }; + } finally { + await release(); + } +} diff --git a/extensions/vydra/video-generation-provider.ts b/extensions/vydra/video-generation-provider.ts index 8928aee17a2e..496410612e81 100644 --- a/extensions/vydra/video-generation-provider.ts +++ b/extensions/vydra/video-generation-provider.ts @@ -1,24 +1,7 @@ // Vydra provider module implements model/runtime integration. -import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime"; import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; -import { - assertOkOrThrowHttpError, - createProviderOperationDeadline, - createProviderOperationTimeoutResolver, - postJsonRequest, - readProviderJsonResponse, - resolveProviderOperationTimeoutMs, -} from "openclaw/plugin-sdk/provider-http"; import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation"; -import { - DEFAULT_VYDRA_VIDEO_MODEL, - downloadVydraAsset, - extractVydraResultUrls, - resolveCompletedVydraPayload, - resolveVydraResponseJobId, - resolveVydraResponseStatus, - resolveVydraRequestContext, -} from "./shared.js"; +import { DEFAULT_VYDRA_VIDEO_MODEL, runVydraGeneration } from "./shared.js"; const VYDRA_KLING_MODEL = "kling"; const DEFAULT_VYDRA_VIDEO_TIMEOUT_MS = 120_000; @@ -79,79 +62,25 @@ export function buildVydraVideoGenerationProvider(): VideoGenerationProvider { throw new Error("Vydra video generation does not support video reference inputs."); } - const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({ + const { model, body } = resolveVydraVideoRequestBody(req); + const generated = await runVydraGeneration({ cfg: req.cfg, agentDir: req.agentDir, authStore: req.authStore, - capability: "video", - }); - const deadline = createProviderOperationDeadline({ - timeoutMs: req.timeoutMs ?? DEFAULT_VYDRA_VIDEO_TIMEOUT_MS, - label: "Vydra video generation", - }); - const { model, body } = resolveVydraVideoRequestBody(req); - const { response, release } = await postJsonRequest({ - url: `${baseUrl}/models/${model}`, - headers: requestPolicy.headers, + kind: "video", + model, body, - timeoutMs: resolveProviderOperationTimeoutMs({ - deadline, - defaultTimeoutMs: DEFAULT_VYDRA_VIDEO_TIMEOUT_MS, - }), - fetchFn, - allowPrivateNetwork: requestPolicy.allowPrivateNetwork, - dispatcherPolicy: requestPolicy.dispatcherPolicy, + deadlineTimeoutMs: req.timeoutMs ?? DEFAULT_VYDRA_VIDEO_TIMEOUT_MS, }); - - try { - await assertOkOrThrowHttpError(response, "Vydra video generation failed"); - const submitted = await readProviderJsonResponse( - response, - "Vydra video generation", - ); - const completedPayload = await resolveCompletedVydraPayload({ - submitted, - baseUrl, - deadline, - fetchFn, - kind: "video", - missingJobIdMessage: "Vydra video generation response missing job id", - requestPolicy, - }); - const videoUrl = extractVydraResultUrls(completedPayload, "video")[0]; - if (!videoUrl) { - throw new Error("Vydra video generation completed without a video URL"); - } - const video = await downloadVydraAsset({ - url: videoUrl, - kind: "video", - timeoutMs: createProviderOperationTimeoutResolver({ - deadline, - defaultTimeoutMs: DEFAULT_VYDRA_VIDEO_TIMEOUT_MS, - }), - fetchFn, - maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"), - requestPolicy, - }); - return { - videos: [ - { - buffer: video.buffer, - mimeType: video.mimeType, - fileName: video.fileName, - }, - ], - model, - metadata: { - jobId: - resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted), - videoUrl, - status: resolveVydraResponseStatus(completedPayload) ?? "completed", - }, - }; - } finally { - await release(); - } + return { + videos: [generated.asset], + model, + metadata: { + jobId: generated.jobId, + videoUrl: generated.resultUrl, + status: generated.status, + }, + }; }, }; } diff --git a/extensions/xai/tsconfig.json b/extensions/xai/tsconfig.json index cc7f2b0b7ed8..b0d93f8516d1 100644 --- a/extensions/xai/tsconfig.json +++ b/extensions/xai/tsconfig.json @@ -95,6 +95,9 @@ "openclaw/plugin-sdk/media-generation-runtime": [ "../../packages/plugin-sdk/dist/src/plugin-sdk/media-generation-runtime.d.ts" ], + "openclaw/plugin-sdk/meeting-page-script-runtime": [ + "../../packages/plugin-sdk/dist/src/plugin-sdk/meeting-page-script-runtime.d.ts" + ], "openclaw/plugin-sdk/conversation-binding-runtime": [ "../../packages/plugin-sdk/dist/src/plugin-sdk/conversation-binding-runtime.d.ts" ], diff --git a/extensions/zoom-meetings/src/transports/zoom-meetings-page-scripts.ts b/extensions/zoom-meetings/src/transports/zoom-meetings-page-scripts.ts index 07a11fa0e4ff..a01e414034c5 100644 --- a/extensions/zoom-meetings/src/transports/zoom-meetings-page-scripts.ts +++ b/extensions/zoom-meetings/src/transports/zoom-meetings-page-scripts.ts @@ -1,8 +1,11 @@ +import { + createMeetingLeaveSource, + createMeetingTranscriptSource, +} from "openclaw/plugin-sdk/meeting-page-script-runtime"; import { ZOOM_MEETING_SELECTORS } from "./zoom-meetings-selectors.js"; import { zoomMeetingStatusCallSource } from "./zoom-meetings-status-call-source.js"; import { zoomMeetingStatusPreludeSource } from "./zoom-meetings-status-prejoin-source.js"; import { normalizeZoomMeetingUrlForReuse } from "./zoom-meetings-urls.js"; -const ZOOM_MEETING_TRANSCRIPT_MAX_LINES = 500; function pageIdentityFunctionSource(): string { return `const meetingIdentity = (rawUrl) => { @@ -81,84 +84,18 @@ export function zoomMeetingTranscriptScript( finalize: boolean, ) { const expectedIdentity = normalizeZoomMeetingUrlForReuse(meetingUrl); - return `() => { - ${pageIdentityFunctionSource()} - const expectedIdentity = ${JSON.stringify(expectedIdentity)}; - const expectedSessionId = ${JSON.stringify(meetingSessionId)}; - const currentIdentity = meetingIdentity(location.href); - const state = window.__openclawZoomMeeting; - const activeCaptions = window.__openclawZoomCaptions; - const archivedCaptions = window.__openclawZoomCaptionArchive?.[expectedSessionId]; - const captions = activeCaptions && - (!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId) - ? activeCaptions - : archivedCaptions; - // A same-session finalized buffer belongs to the departed call even if Zoom - // immediately navigated this tab into another meeting before transcript pickup. - const useFinalizedCaptions = Boolean( - captions?.finalized === true && - captions?.identity === expectedIdentity && - (!captions?.sessionId || captions.sessionId === expectedSessionId) - ); - const effectiveIdentity = useFinalizedCaptions - ? captions.identity - : currentIdentity || state?.identity || captions?.identity; - if (!expectedIdentity || effectiveIdentity !== expectedIdentity) { - return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] }); - } - if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) { - return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); - } - if (captions?.sessionId && captions.sessionId !== expectedSessionId) { - return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); - } - if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) { - if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); - captions.settleTimer = undefined; - captions.lines = Array.isArray(captions.lines) ? captions.lines : []; - captions.lines.push(...captions.visible.map((entry) => ({ - at: entry.at, - speaker: entry.speaker, - text: entry.text, - }))); - captions.visible = []; - const excess = captions.lines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES}; - if (excess > 0) { - captions.lines.splice(0, excess); - captions.droppedLines = (captions.droppedLines || 0) + excess; - } - } - if (${JSON.stringify(finalize)} && captions) { - if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); - captions.settleTimer = undefined; - captions.observer?.disconnect?.(); - captions.observer = undefined; - captions.observerInstalled = false; - captions.identity = expectedIdentity; - captions.finalized = true; - captions.finalizedAt = Date.now(); - } - const allLines = [ - ...(Array.isArray(captions?.lines) ? captions.lines : []), - ...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible), - ]; - const visibleOverflow = Math.max(0, allLines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES}); - const lines = allLines.slice(-${ZOOM_MEETING_TRANSCRIPT_MAX_LINES}); - const result = { - urlMatched: true, - sessionMatched: true, - epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined, - droppedLines: (Number.isFinite(captions?.droppedLines) - ? Math.max(0, Math.trunc(captions.droppedLines)) - : 0) + visibleOverflow, - lines: lines.map((line) => ({ - at: typeof line?.at === "string" ? line.at : undefined, - speaker: typeof line?.speaker === "string" ? line.speaker : undefined, - text: typeof line?.text === "string" ? line.text : "", - })).filter((line) => line.text), - }; - return JSON.stringify(result); -}`; + return createMeetingTranscriptSource({ + expectedIdentity, + finalize, + globals: { + captionArchive: "__openclawZoomCaptionArchive", + captions: "__openclawZoomCaptions", + meeting: "__openclawZoomMeeting", + }, + meetingSessionId, + pageIdentitySource: pageIdentityFunctionSource(), + platformDisplayName: "Zoom", + }); } export function zoomMeetingLeaveScript(params: { @@ -168,85 +105,8 @@ export function zoomMeetingLeaveScript(params: { }) { const selectors = JSON.stringify(ZOOM_MEETING_SELECTORS); const expectedIdentity = normalizeZoomMeetingUrlForReuse(params.meetingUrl); - return `() => { - ${pageIdentityFunctionSource()} - const topDocument = globalThis.document; - const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument; - const selectors = ${selectors}; - const expectedIdentity = ${JSON.stringify(expectedIdentity)}; - const expectedSessionId = ${JSON.stringify(params.meetingSessionId)}; - const leaveInitiated = ${JSON.stringify(params.leaveInitiated)}; - const currentIdentity = meetingIdentity(location.href); - const state = window.__openclawZoomMeeting; - const enforceSessionOwnership = Boolean(expectedSessionId); - if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) { - return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true }); - } - const sessionAdoptedFromUrl = Boolean( - enforceSessionOwnership && - !state?.sessionId && - currentIdentity === expectedIdentity && - (!state?.identity || state.identity === expectedIdentity) - ); - const sessionMatched = !enforceSessionOwnership || - state?.sessionId === expectedSessionId || - sessionAdoptedFromUrl; - const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated); - if (!sessionMatched && !retainedLeaveOwnership) { - return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true }); - } - const retireOwnedAudioBridges = () => { - const entries = Array.isArray(window.__openclawZoomAudioOutputs) - ? window.__openclawZoomAudioOutputs - : []; - const retained = []; - const activeSessionId = expectedSessionId || state?.sessionId; - for (const entry of entries) { - const ownedByActiveSession = Boolean( - !entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId) - ); - if (!ownedByActiveSession) { - retained.push(entry); - continue; - } - const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || ""); - const sources = Array.isArray(entry?.sources) - ? entry.sources - : entry?.source - ? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }] - : []; - for (const source of sources) { - const element = source?.element; - const sourceMatches = source?.stream || element?.srcObject - ? element?.srcObject === source?.stream - : Boolean(source?.url && mediaSourceUrl(element) === source.url); - const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element)); - if (!element) continue; - if (sourceIsEmpty) { - element.muted = true; - continue; - } - if (!sourceMatches) continue; - const detachedLiveSource = Boolean( - element.isConnected === false && - element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live") - ); - if (detachedLiveSource) { - element.muted = true; - element.pause?.(); - element.srcObject = null; - } else { - element.muted = Boolean(source.muted); - } - } - entry?.bridge?.pause?.(); - if (entry?.bridge) entry.bridge.srcObject = null; - entry?.bridge?.remove?.(); - } - if (retained.length > 0) window.__openclawZoomAudioOutputs = retained; - else delete window.__openclawZoomAudioOutputs; - }; - const first = (list) => { + return createMeetingLeaveSource({ + controlSource: `const first = (list) => { for (const selector of list) { const node = document.querySelector(selector); if (!node) continue; @@ -270,71 +130,31 @@ export function zoomMeetingLeaveScript(params: { try { const currentUrl = new URL(location.href); webClientHome = !leave && currentUrl.hostname === "app.zoom.us" && /^\\/wc\\/?$/.test(currentUrl.pathname); - } catch {} - const preservedCallMatches = Boolean( - expectedIdentity && - !currentIdentity && - state?.identity === expectedIdentity && - state?.inCallControl === leave && - state?.inCallUrl === location.href && - leave && - leave.isConnected !== false - ); - const pendingLeaveMatches = Boolean( - expectedIdentity && - state?.identity === expectedIdentity && - state?.leavePending === true && - state?.inCallUrl === location.href && - Date.now() - state?.leavePendingAt < 10_000 - ); - const rerenderPendingMatches = Boolean( - expectedIdentity && - !currentIdentity && - state?.identity === expectedIdentity && - state?.inCallControl?.isConnected === false && - state?.inCallUrl === location.href && - Date.now() - state?.verifiedAt < 5_000 && - !leave - ); - const meetingIdentityMatches = Boolean( - currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches - ); - // Zoom can replace the document between our Leave click and its post-call marker. - // Retain request ownership only while no identity or live-call control contradicts it. - const initiatedLeaveTransitionMatches = Boolean( - leaveInitiated && - !currentIdentity && - !leave && + } catch {}`, + departedMarkerSource: "(postCall || webClientHome)", + documentSetupSource: `const topDocument = globalThis.document; + const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument;`, + expectedIdentity, + leaveInitiated: params.leaveInitiated, + meetingSessionId: params.meetingSessionId, + meetingStateSource: "sessionId: expectedSessionId || state?.sessionId,", + pageIdentitySource: pageIdentityFunctionSource(), + platform: { + displayName: "Zoom", + globals: { + audioOutputs: "__openclawZoomAudioOutputs", + meeting: "__openclawZoomMeeting", + }, + }, + selectors, + sessionMatchSource: `const sessionAdoptedFromUrl = Boolean( + enforceSessionOwnership && + !state?.sessionId && + currentIdentity === expectedIdentity && (!state?.identity || state.identity === expectedIdentity) ); - if ((postCall || webClientHome) && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) { - retireOwnedAudioBridges(); - if (sessionMatched) delete window.__openclawZoomMeeting; - return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true }); - } - if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) { - return JSON.stringify({ departed: false, urlMatched: false }); - } - if (!sessionMatched) { - return JSON.stringify({ departed: false, urlMatched: true }); - } - if (confirmation) { - confirmation.click(); - return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }); - } - if (leave) { - window.__openclawZoomMeeting = { - ...state, - identity: expectedIdentity, - sessionId: expectedSessionId || state?.sessionId, - inCallControl: leave, - inCallUrl: location.href, - leavePending: true, - leavePendingAt: Date.now(), - }; - leave.click(); - return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true }); - } - return JSON.stringify({ departed: false, urlMatched: true }); -}`; + const sessionMatched = !enforceSessionOwnership || + state?.sessionId === expectedSessionId || + sessionAdoptedFromUrl;`, + }); } diff --git a/package.json b/package.json index c3e1249db5f5..6aa086690256 100644 --- a/package.json +++ b/package.json @@ -108,6 +108,7 @@ "!dist/plugin-sdk/llm.d.ts", "!dist/plugin-sdk/markdown-table-runtime.d.ts", "!dist/plugin-sdk/media-generation-runtime.d.ts", + "!dist/plugin-sdk/meeting-page-script-runtime.d.ts", "!dist/plugin-sdk/memory-core-host-embedding-registry.d.ts", "!dist/plugin-sdk/memory-core-host-engine-curated.d.ts", "!dist/plugin-sdk/memory-core-host-engine-embeddings.d.ts", @@ -1162,6 +1163,9 @@ "./plugin-sdk/realtime-voice": { "default": "./dist/plugin-sdk/realtime-voice.js" }, + "./plugin-sdk/meeting-page-script-runtime": { + "default": "./dist/plugin-sdk/meeting-page-script-runtime.js" + }, "./plugin-sdk/meeting-runtime": { "types": "./dist/plugin-sdk/meeting-runtime.d.ts", "default": "./dist/plugin-sdk/meeting-runtime.js" diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 8243b649a022..89040aed666c 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -240,6 +240,7 @@ "realtime-voice-audio-queue", "realtime-voice-activation", "realtime-voice", + "meeting-page-script-runtime", "meeting-runtime", "transcripts", "media-understanding", diff --git a/scripts/lib/plugin-sdk-private-local-only-subpaths.json b/scripts/lib/plugin-sdk-private-local-only-subpaths.json index 718c8bd23572..f960254dbd99 100644 --- a/scripts/lib/plugin-sdk-private-local-only-subpaths.json +++ b/scripts/lib/plugin-sdk-private-local-only-subpaths.json @@ -65,6 +65,7 @@ "llm", "markdown-table-runtime", "media-generation-runtime", + "meeting-page-script-runtime", "memory-core-host-embedding-registry", "memory-core-host-engine-curated", "memory-core-host-engine-embeddings", diff --git a/src/meeting-bot/page-script-source.ts b/src/meeting-bot/page-script-source.ts new file mode 100644 index 000000000000..065d53e87b2f --- /dev/null +++ b/src/meeting-bot/page-script-source.ts @@ -0,0 +1,270 @@ +type MeetingPageScriptGlobals = { + audioOutputs: string; + captionArchive: string; + captions: string; + meeting: string; +}; + +function pageGlobalSource(name: string): string { + if (!/^[$A-Z_a-z][$\w]*$/u.test(name)) { + throw new Error(`Invalid meeting page global: ${name}`); + } + return `window.${name}`; +} + +export function createMeetingTranscriptSource(params: { + expectedIdentity?: string; + finalize: boolean; + globals: Pick; + meetingSessionId: string; + pageIdentitySource: string; + platformDisplayName: string; + transcriptMaxLines?: number; +}): string { + const transcriptMaxLines = params.transcriptMaxLines ?? 500; + const meetingGlobal = pageGlobalSource(params.globals.meeting); + const captionsGlobal = pageGlobalSource(params.globals.captions); + const captionArchiveGlobal = pageGlobalSource(params.globals.captionArchive); + return `() => { + ${params.pageIdentitySource} + const expectedIdentity = ${JSON.stringify(params.expectedIdentity)}; + const expectedSessionId = ${JSON.stringify(params.meetingSessionId)}; + const currentIdentity = meetingIdentity(location.href); + const state = ${meetingGlobal}; + const activeCaptions = ${captionsGlobal}; + const archivedCaptions = ${captionArchiveGlobal}?.[expectedSessionId]; + const captions = activeCaptions && + (!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId) + ? activeCaptions + : archivedCaptions; + // A same-session finalized buffer belongs to the departed call even if ${params.platformDisplayName} + // immediately navigated this tab into another meeting before transcript pickup. + const useFinalizedCaptions = Boolean( + captions?.finalized === true && + captions?.identity === expectedIdentity && + (!captions?.sessionId || captions.sessionId === expectedSessionId) + ); + const effectiveIdentity = useFinalizedCaptions + ? captions.identity + : currentIdentity || state?.identity || captions?.identity; + if (!expectedIdentity || effectiveIdentity !== expectedIdentity) { + return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] }); + } + if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) { + return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); + } + if (captions?.sessionId && captions.sessionId !== expectedSessionId) { + return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] }); + } + if (${JSON.stringify(params.finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) { + if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); + captions.settleTimer = undefined; + captions.lines = Array.isArray(captions.lines) ? captions.lines : []; + captions.lines.push(...captions.visible.map((entry) => ({ + at: entry.at, + speaker: entry.speaker, + text: entry.text, + }))); + captions.visible = []; + const excess = captions.lines.length - ${transcriptMaxLines}; + if (excess > 0) { + captions.lines.splice(0, excess); + captions.droppedLines = (captions.droppedLines || 0) + excess; + } + } + if (${JSON.stringify(params.finalize)} && captions) { + if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer); + captions.settleTimer = undefined; + captions.observer?.disconnect?.(); + captions.observer = undefined; + captions.observerInstalled = false; + captions.identity = expectedIdentity; + captions.finalized = true; + captions.finalizedAt = Date.now(); + } + const allLines = [ + ...(Array.isArray(captions?.lines) ? captions.lines : []), + ...(${JSON.stringify(params.finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible), + ]; + const visibleOverflow = Math.max(0, allLines.length - ${transcriptMaxLines}); + const lines = allLines.slice(-${transcriptMaxLines}); + const result = { + urlMatched: true, + sessionMatched: true, + epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined, + droppedLines: (Number.isFinite(captions?.droppedLines) + ? Math.max(0, Math.trunc(captions.droppedLines)) + : 0) + visibleOverflow, + lines: lines.map((line) => ({ + at: typeof line?.at === "string" ? line.at : undefined, + speaker: typeof line?.speaker === "string" ? line.speaker : undefined, + text: typeof line?.text === "string" ? line.text : "", + })).filter((line) => line.text), + }; + return JSON.stringify(result); +}`; +} + +function createMeetingOwnedAudioLeaveSource(params: { audioOutputsGlobal: string }): string { + const audioOutputsGlobal = pageGlobalSource(params.audioOutputsGlobal); + return `const retireOwnedAudioBridges = () => { + const entries = Array.isArray(${audioOutputsGlobal}) + ? ${audioOutputsGlobal} + : []; + const retained = []; + const activeSessionId = expectedSessionId || state?.sessionId; + for (const entry of entries) { + const ownedByActiveSession = Boolean( + !entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId) + ); + if (!ownedByActiveSession) { + retained.push(entry); + continue; + } + const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || ""); + const sources = Array.isArray(entry?.sources) + ? entry.sources + : entry?.source + ? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }] + : []; + for (const source of sources) { + const element = source?.element; + const sourceMatches = source?.stream || element?.srcObject + ? element?.srcObject === source?.stream + : Boolean(source?.url && mediaSourceUrl(element) === source.url); + const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element)); + if (!element) continue; + if (sourceIsEmpty) { + element.muted = true; + continue; + } + if (!sourceMatches) continue; + const detachedLiveSource = Boolean( + element.isConnected === false && + element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live") + ); + if (detachedLiveSource) { + element.muted = true; + element.pause?.(); + element.srcObject = null; + } else { + element.muted = Boolean(source.muted); + } + } + entry?.bridge?.pause?.(); + if (entry?.bridge) entry.bridge.srcObject = null; + entry?.bridge?.remove?.(); + } + if (retained.length > 0) ${audioOutputsGlobal} = retained; + else delete ${audioOutputsGlobal}; + };`; +} + +export function createMeetingLeaveSource(params: { + controlSource: string; + departedMarkerSource: string; + documentSetupSource?: string; + expectedIdentity?: string; + leaveInitiated: boolean; + meetingSessionId: string; + meetingStateSource?: string; + pageIdentitySource: string; + platform: { + displayName: string; + globals: Pick; + }; + selectors: string; + sessionMatchSource: string; +}): string { + const documentSetupSource = params.documentSetupSource ? `${params.documentSetupSource}\n ` : ""; + const meetingStateSource = params.meetingStateSource + ? ` ${params.meetingStateSource}\n` + : ""; + const meetingGlobal = pageGlobalSource(params.platform.globals.meeting); + return `() => { + ${params.pageIdentitySource} + ${documentSetupSource}const selectors = ${params.selectors}; + const expectedIdentity = ${JSON.stringify(params.expectedIdentity)}; + const expectedSessionId = ${JSON.stringify(params.meetingSessionId)}; + const leaveInitiated = ${JSON.stringify(params.leaveInitiated)}; + const currentIdentity = meetingIdentity(location.href); + const state = ${meetingGlobal}; + const enforceSessionOwnership = Boolean(expectedSessionId); + if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) { + return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true }); + } + ${params.sessionMatchSource} + const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated); + if (!sessionMatched && !retainedLeaveOwnership) { + return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true }); + } + ${createMeetingOwnedAudioLeaveSource({ + audioOutputsGlobal: params.platform.globals.audioOutputs, + })} + ${params.controlSource} + const preservedCallMatches = Boolean( + expectedIdentity && + !currentIdentity && + state?.identity === expectedIdentity && + state?.inCallControl === leave && + state?.inCallUrl === location.href && + leave && + leave.isConnected !== false + ); + const pendingLeaveMatches = Boolean( + expectedIdentity && + state?.identity === expectedIdentity && + state?.leavePending === true && + state?.inCallUrl === location.href && + Date.now() - state?.leavePendingAt < 10_000 + ); + const rerenderPendingMatches = Boolean( + expectedIdentity && + !currentIdentity && + state?.identity === expectedIdentity && + state?.inCallControl?.isConnected === false && + state?.inCallUrl === location.href && + Date.now() - state?.verifiedAt < 5_000 && + !leave + ); + const meetingIdentityMatches = Boolean( + currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches + ); + // ${params.platform.displayName} can replace the document between our Leave click and its post-call marker. + // Retain request ownership only while no identity or live-call control contradicts it. + const initiatedLeaveTransitionMatches = Boolean( + leaveInitiated && + !currentIdentity && + !leave && + (!state?.identity || state.identity === expectedIdentity) + ); + if (${params.departedMarkerSource} && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) { + retireOwnedAudioBridges(); + if (sessionMatched) delete ${meetingGlobal}; + return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true }); + } + if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) { + return JSON.stringify({ departed: false, urlMatched: false }); + } + if (!sessionMatched) { + return JSON.stringify({ departed: false, urlMatched: true }); + } + if (confirmation) { + confirmation.click(); + return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }); + } + if (leave) { + ${meetingGlobal} = { + ...state, + identity: expectedIdentity, +${meetingStateSource} inCallControl: leave, + inCallUrl: location.href, + leavePending: true, + leavePendingAt: Date.now(), + }; + leave.click(); + return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true }); + } + return JSON.stringify({ departed: false, urlMatched: true }); +}`; +} diff --git a/src/plugin-sdk/meeting-page-script-runtime.ts b/src/plugin-sdk/meeting-page-script-runtime.ts new file mode 100644 index 000000000000..4734d7e44bdc --- /dev/null +++ b/src/plugin-sdk/meeting-page-script-runtime.ts @@ -0,0 +1,5 @@ +/** Production-private page-script seam for official browser-meeting plugins. */ +export { + createMeetingLeaveSource, + createMeetingTranscriptSource, +} from "../meeting-bot/page-script-source.js";