refactor: consolidate meeting and media provider families (#126053)

* refactor(plugins): consolidate provider family helpers

* fix(plugin-sdk): keep meeting script helpers private

* fix(plugins): sync meeting boundary paths
This commit is contained in:
Peter Steinberger
2026-08-18 19:11:13 -07:00
committed by GitHub
parent 15fb00eb6b
commit c97b8ffdfc
17 changed files with 566 additions and 688 deletions
+1
View File
@@ -340,6 +340,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
| `plugin-sdk/realtime-voice-audio-queue` | Private-local JavaScript-only host runtime for bundled or separately published official plugins; narrow bounded audio queue seam for lazy realtime voice provider facades without importing the broader realtime voice runtime; not for third-party plugins |
| `plugin-sdk/realtime-voice-activation` | Private-local; dependency-light realtime-voice activation-name helpers (normalize, match, word-count, sort) for doctor contract closures and other control-plane paths that must not load the realtime voice runtime |
| `plugin-sdk/realtime-voice` | Private-local after July 2026; Realtime voice provider types, registry helpers, shared audio-energy/speech-onset gates, and realtime voice behavior helpers, including the transport-independent session harness and output activity tracking. For official runtime consumers, sender-auth contract revision 1 forwards ingress-authenticated `senderId` and `senderIsOwner` unchanged; ingress owns authentication, and consumers requiring the handoff must fail closed on other revisions. |
| `plugin-sdk/meeting-page-script-runtime` | Private-local JavaScript-only host runtime for official browser-meeting plugins; shared transcript and leave page-script source builders; not a third-party plugin API |
| `plugin-sdk/meeting-runtime` | Browser-meeting session runtime, realtime audio engines/transports, `MeetingPlatformAdapter`, browser/node control, agent-consult, voice-call delegation, setup checks, and SoX command helpers |
| `plugin-sdk/image-generation` | Private-local after July 2026; Image generation provider types plus image asset/data URL helpers and the OpenAI-compatible image provider builder |
| `plugin-sdk/image-generation-core` | Private-local after July 2026; Shared image-generation types, failover, auth, and registry helpers |
@@ -0,0 +1,45 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
resolveProviderHttpRequestConfig,
sanitizeConfiguredModelProviderRequest,
} from "openclaw/plugin-sdk/provider-http";
import { OPENROUTER_BASE_URL } from "./provider-catalog.js";
type OpenRouterAuthStore = Parameters<typeof resolveApiKeyForProvider>[0]["store"];
export async function resolveOpenRouterGenerationRequestContext(params: {
cfg: OpenClawConfig;
agentDir?: string;
authStore?: OpenRouterAuthStore;
capability: "audio" | "image" | "video";
jsonContentType: boolean;
}) {
const auth = await resolveApiKeyForProvider({
provider: "openrouter",
cfg: params.cfg,
agentDir: params.agentDir,
store: params.authStore,
});
if (!auth.apiKey) {
throw new Error("OpenRouter API key missing");
}
return resolveProviderHttpRequestConfig({
baseUrl: params.cfg.models?.providers?.openrouter?.baseUrl,
defaultBaseUrl: OPENROUTER_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
...(params.jsonContentType ? { "Content-Type": "application/json" } : {}),
"HTTP-Referer": "https://openclaw.ai",
"X-OpenRouter-Title": "OpenClaw",
},
request: sanitizeConfiguredModelProviderRequest(
params.cfg.models?.providers?.openrouter?.request,
),
provider: "openrouter",
capability: params.capability,
transport: "http",
});
}
@@ -14,16 +14,14 @@ import {
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
import { resolveIntegerOption } from "openclaw/plugin-sdk/number-runtime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
postJsonRequest,
readProviderJsonResponse,
resolveProviderHttpRequestConfig,
sanitizeConfiguredModelProviderRequest,
} from "openclaw/plugin-sdk/provider-http";
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import { normalizeOpenRouterBaseUrl, OPENROUTER_BASE_URL } from "./provider-catalog.js";
import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js";
import { normalizeOpenRouterBaseUrl } from "./provider-catalog.js";
const DEFAULT_MODEL = "google/gemini-3.1-flash-image-preview";
const DEFAULT_TIMEOUT_MS = 180_000;
@@ -291,36 +289,19 @@ export function buildOpenRouterImageGenerationProvider(): ImageGenerationProvide
},
},
async generateImage(req) {
const auth = await resolveApiKeyForProvider({
provider: "openrouter",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("OpenRouter API key missing");
}
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
await resolveOpenRouterGenerationRequestContext({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
capability: "image",
// Preserve the existing resolved header contract; postJsonRequest supplies
// the JSON content type for both chat-completion and dedicated image requests.
jsonContentType: false,
});
const model = normalizeOptionalString(req.model) ?? DEFAULT_MODEL;
const imageConfig = buildImageConfig(req, model);
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl,
defaultBaseUrl: OPENROUTER_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"HTTP-Referer": "https://openclaw.ai",
"X-OpenRouter-Title": "OpenClaw",
},
request: sanitizeConfiguredModelProviderRequest(
req.cfg?.models?.providers?.openrouter?.request,
),
provider: "openrouter",
capability: "image",
transport: "http",
});
const count = resolveImageCount(req.count);
const canonicalBaseUrl = normalizeOpenRouterBaseUrl(baseUrl);
if (canonicalBaseUrl) {
@@ -9,18 +9,15 @@ import type {
} from "openclaw/plugin-sdk/music-generation";
import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
createProviderOperationDeadline,
postJsonRequest,
resolveProviderHttpRequestConfig,
resolveProviderOperationTimeoutMs,
sanitizeConfiguredModelProviderRequest,
type ProviderOperationDeadline,
} from "openclaw/plugin-sdk/provider-http";
import { isRecord, normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import { OPENROUTER_BASE_URL } from "./provider-catalog.js";
import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js";
const DEFAULT_OPENROUTER_MUSIC_MODEL = "google/lyria-3-pro-preview";
const OPENROUTER_CLIP_MUSIC_MODEL = "google/lyria-3-clip-preview";
@@ -371,33 +368,13 @@ export function buildOpenRouterMusicGenerationProvider(): MusicGenerationProvide
if ((req.inputImages?.length ?? 0) > 1) {
throw new Error("OpenRouter music generation supports at most one reference image.");
}
const auth = await resolveApiKeyForProvider({
provider: "openrouter",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("OpenRouter API key missing");
}
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl,
defaultBaseUrl: OPENROUTER_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"Content-Type": "application/json",
"HTTP-Referer": "https://openclaw.ai",
"X-OpenRouter-Title": "OpenClaw",
},
request: sanitizeConfiguredModelProviderRequest(
req.cfg?.models?.providers?.openrouter?.request,
),
provider: "openrouter",
await resolveOpenRouterGenerationRequestContext({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
capability: "audio",
transport: "http",
jsonContentType: true,
});
const model = resolveOpenRouterMusicModel(req.model);
const format = req.format ?? "wav";
@@ -3,15 +3,12 @@ import { toImageDataUrl } from "openclaw/plugin-sdk/image-generation";
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
import { extensionForMime } from "openclaw/plugin-sdk/media-mime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
createProviderOperationDeadline,
postJsonRequest,
readProviderJsonResponse,
resolveProviderHttpRequestConfig,
resolveProviderOperationTimeoutMs,
sanitizeConfiguredModelProviderRequest,
waitProviderOperationPollInterval,
} from "openclaw/plugin-sdk/provider-http";
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
@@ -22,7 +19,7 @@ import type {
VideoGenerationRequest,
VideoGenerationSourceAsset,
} from "openclaw/plugin-sdk/video-generation";
import { OPENROUTER_BASE_URL } from "./provider-catalog.js";
import { resolveOpenRouterGenerationRequestContext } from "./generation-request-context.js";
import {
fetchOpenRouterVideoGet,
resolveOpenRouterVideoUrl,
@@ -463,34 +460,14 @@ export function buildOpenRouterVideoGenerationProvider(): VideoGenerationProvide
throw new Error("OpenRouter video generation does not support video reference inputs.");
}
const auth = await resolveApiKeyForProvider({
provider: "openrouter",
cfg: req.cfg,
agentDir: req.agentDir,
store: req.authStore,
});
if (!auth.apiKey) {
throw new Error("OpenRouter API key missing");
}
const model = normalizeOptionalString(req.model) ?? DEFAULT_MODEL;
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: req.cfg?.models?.providers?.openrouter?.baseUrl,
defaultBaseUrl: OPENROUTER_BASE_URL,
allowPrivateNetwork: false,
defaultHeaders: {
Authorization: `Bearer ${auth.apiKey}`,
"Content-Type": "application/json",
"HTTP-Referer": "https://openclaw.ai",
"X-OpenRouter-Title": "OpenClaw",
},
request: sanitizeConfiguredModelProviderRequest(
req.cfg?.models?.providers?.openrouter?.request,
),
provider: "openrouter",
await resolveOpenRouterGenerationRequestContext({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
capability: "video",
transport: "http",
jsonContentType: true,
});
const deadline = createProviderOperationDeadline({
timeoutMs: req.timeoutMs,
@@ -1,8 +1,11 @@
import {
createMeetingLeaveSource,
createMeetingTranscriptSource,
} from "openclaw/plugin-sdk/meeting-page-script-runtime";
import { TEAMS_MEETING_SELECTORS } from "./teams-meetings-selectors.js";
import { teamsMeetingStatusCallSource } from "./teams-meetings-status-call-source.js";
import { teamsMeetingStatusPreludeSource } from "./teams-meetings-status-prejoin-source.js";
import { normalizeTeamsMeetingUrlForReuse } from "./teams-meetings-urls.js";
const TEAMS_MEETING_TRANSCRIPT_MAX_LINES = 500;
function pageIdentityFunctionSource(): string {
return `const meetingIdentity = (rawUrl) => {
@@ -109,84 +112,18 @@ export function teamsMeetingTranscriptScript(
finalize: boolean,
) {
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(meetingSessionId)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawTeamsMeeting;
const activeCaptions = window.__openclawTeamsCaptions;
const archivedCaptions = window.__openclawTeamsCaptionArchive?.[expectedSessionId];
const captions = activeCaptions &&
(!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId)
? activeCaptions
: archivedCaptions;
// A same-session finalized buffer belongs to the departed call even if Teams
// immediately navigated this tab into another meeting before transcript pickup.
const useFinalizedCaptions = Boolean(
captions?.finalized === true &&
captions?.identity === expectedIdentity &&
(!captions?.sessionId || captions.sessionId === expectedSessionId)
);
const effectiveIdentity = useFinalizedCaptions
? captions.identity
: currentIdentity || state?.identity || captions?.identity;
if (!expectedIdentity || effectiveIdentity !== expectedIdentity) {
return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] });
}
if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (captions?.sessionId && captions.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.lines = Array.isArray(captions.lines) ? captions.lines : [];
captions.lines.push(...captions.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
captions.visible = [];
const excess = captions.lines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
captions.lines.splice(0, excess);
captions.droppedLines = (captions.droppedLines || 0) + excess;
}
}
if (${JSON.stringify(finalize)} && captions) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.observer?.disconnect?.();
captions.observer = undefined;
captions.observerInstalled = false;
captions.identity = expectedIdentity;
captions.finalized = true;
captions.finalizedAt = Date.now();
}
const allLines = [
...(Array.isArray(captions?.lines) ? captions.lines : []),
...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible),
];
const visibleOverflow = Math.max(0, allLines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES});
const lines = allLines.slice(-${TEAMS_MEETING_TRANSCRIPT_MAX_LINES});
const result = {
urlMatched: true,
sessionMatched: true,
epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined,
droppedLines: (Number.isFinite(captions?.droppedLines)
? Math.max(0, Math.trunc(captions.droppedLines))
: 0) + visibleOverflow,
lines: lines.map((line) => ({
at: typeof line?.at === "string" ? line.at : undefined,
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
text: typeof line?.text === "string" ? line.text : "",
})).filter((line) => line.text),
};
return JSON.stringify(result);
}`;
return createMeetingTranscriptSource({
expectedIdentity,
finalize,
globals: {
captionArchive: "__openclawTeamsCaptionArchive",
captions: "__openclawTeamsCaptions",
meeting: "__openclawTeamsMeeting",
},
meetingSessionId,
pageIdentitySource: pageIdentityFunctionSource(),
platformDisplayName: "Teams",
});
}
export function teamsMeetingLeaveScript(params: {
@@ -196,75 +133,8 @@ export function teamsMeetingLeaveScript(params: {
}) {
const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS);
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(params.meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const leaveInitiated = ${JSON.stringify(params.leaveInitiated)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawTeamsMeeting;
const enforceSessionOwnership = Boolean(expectedSessionId);
if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true });
}
const sessionMatched = !enforceSessionOwnership || state?.sessionId === expectedSessionId;
const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated);
if (!sessionMatched && !retainedLeaveOwnership) {
return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true });
}
const retireOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const retained = [];
const activeSessionId = expectedSessionId || state?.sessionId;
for (const entry of entries) {
const ownedByActiveSession = Boolean(
!entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId)
);
if (!ownedByActiveSession) {
retained.push(entry);
continue;
}
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const sources = Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }]
: [];
for (const source of sources) {
const element = source?.element;
const sourceMatches = source?.stream || element?.srcObject
? element?.srcObject === source?.stream
: Boolean(source?.url && mediaSourceUrl(element) === source.url);
const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element));
if (!element) continue;
if (sourceIsEmpty) {
element.muted = true;
continue;
}
if (!sourceMatches) continue;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
} else {
element.muted = Boolean(source.muted);
}
}
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
}
if (retained.length > 0) window.__openclawTeamsAudioOutputs = retained;
else delete window.__openclawTeamsAudioOutputs;
};
const first = (list) => {
return createMeetingLeaveSource({
controlSource: `const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (!node) continue;
@@ -275,70 +145,21 @@ export function teamsMeetingLeaveScript(params: {
const leave = first(selectors.leave);
const confirmation = first(selectors.leaveConfirmation);
const postCall = first(selectors.postCall);
const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity);
const preservedCallMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl === leave &&
state?.inCallUrl === location.href &&
leave &&
leave.isConnected !== false
);
const pendingLeaveMatches = Boolean(
expectedIdentity &&
state?.identity === expectedIdentity &&
state?.leavePending === true &&
state?.inCallUrl === location.href &&
Date.now() - state?.leavePendingAt < 10_000
);
const rerenderPendingMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl?.isConnected === false &&
state?.inCallUrl === location.href &&
Date.now() - state?.verifiedAt < 5_000 &&
!leave
);
const meetingIdentityMatches = Boolean(
currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches
);
// Teams can replace the document between our Leave click and its post-call marker.
// Retain request ownership only while no identity or live-call control contradicts it.
const initiatedLeaveTransitionMatches = Boolean(
leaveInitiated &&
!currentIdentity &&
!leave &&
(!state?.identity || state.identity === expectedIdentity)
);
if (postCall && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) {
retireOwnedAudioBridges();
if (sessionMatched) delete window.__openclawTeamsMeeting;
return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true });
}
if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) {
return JSON.stringify({ departed: false, urlMatched: false });
}
if (!sessionMatched) {
return JSON.stringify({ departed: false, urlMatched: true });
}
if (confirmation) {
confirmation.click();
return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true });
}
if (leave) {
window.__openclawTeamsMeeting = {
...state,
identity: expectedIdentity,
inCallControl: leave,
inCallUrl: location.href,
leavePending: true,
leavePendingAt: Date.now(),
};
leave.click();
return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true });
}
return JSON.stringify({ departed: false, urlMatched: true });
}`;
const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity);`,
departedMarkerSource: "postCall",
expectedIdentity,
leaveInitiated: params.leaveInitiated,
meetingSessionId: params.meetingSessionId,
pageIdentitySource: pageIdentityFunctionSource(),
platform: {
displayName: "Teams",
globals: {
audioOutputs: "__openclawTeamsAudioOutputs",
meeting: "__openclawTeamsMeeting",
},
},
selectors,
sessionMatchSource:
"const sessionMatched = !enforceSessionOwnership || state?.sessionId === expectedSessionId;",
});
}
@@ -95,6 +95,9 @@
"openclaw/plugin-sdk/media-generation-runtime": [
"../packages/plugin-sdk/dist/src/plugin-sdk/media-generation-runtime.d.ts"
],
"openclaw/plugin-sdk/meeting-page-script-runtime": [
"../packages/plugin-sdk/dist/src/plugin-sdk/meeting-page-script-runtime.d.ts"
],
"openclaw/plugin-sdk/conversation-binding-runtime": [
"../packages/plugin-sdk/dist/src/plugin-sdk/conversation-binding-runtime.d.ts"
],
+15 -72
View File
@@ -1,21 +1,7 @@
// Vydra provider module implements model/runtime integration.
import type { ImageGenerationProvider } from "openclaw/plugin-sdk/image-generation";
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import {
assertOkOrThrowHttpError,
postJsonRequest,
readProviderJsonResponse,
} from "openclaw/plugin-sdk/provider-http";
import {
DEFAULT_VYDRA_IMAGE_MODEL,
downloadVydraAsset,
extractVydraResultUrls,
resolveCompletedVydraPayload,
resolveVydraResponseJobId,
resolveVydraResponseStatus,
resolveVydraRequestContext,
} from "./shared.js";
import { DEFAULT_VYDRA_IMAGE_MODEL, runVydraGeneration } from "./shared.js";
export function buildVydraImageGenerationProvider(): ImageGenerationProvider {
return {
@@ -50,72 +36,29 @@ export function buildVydraImageGenerationProvider(): ImageGenerationProvider {
throw new Error("Vydra image generation supports at most one image per request.");
}
const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({
const model = req.model?.trim() || DEFAULT_VYDRA_IMAGE_MODEL;
const generated = await runVydraGeneration({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
capability: "image",
ssrfPolicy: req.ssrfPolicy,
});
const model = req.model?.trim() || DEFAULT_VYDRA_IMAGE_MODEL;
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${model}`,
headers: requestPolicy.headers,
kind: "image",
model,
body: {
prompt: req.prompt,
model: "text-to-image",
},
timeoutMs: req.timeoutMs,
fetchFn,
allowPrivateNetwork: requestPolicy.allowPrivateNetwork,
ssrfPolicy: requestPolicy.ssrfPolicy,
dispatcherPolicy: requestPolicy.dispatcherPolicy,
ssrfPolicy: req.ssrfPolicy,
});
try {
await assertOkOrThrowHttpError(response, "Vydra image generation failed");
const submitted = await readProviderJsonResponse(response, "vydra.image-generation");
const completedPayload = await resolveCompletedVydraPayload({
submitted,
baseUrl,
timeoutMs: req.timeoutMs,
fetchFn,
kind: "image",
missingJobIdMessage: "Vydra image generation response missing job id",
requestPolicy,
});
const imageUrl = extractVydraResultUrls(completedPayload, "image")[0];
if (!imageUrl) {
throw new Error("Vydra image generation completed without an image URL");
}
const image = await downloadVydraAsset({
url: imageUrl,
kind: "image",
timeoutMs: req.timeoutMs,
fetchFn,
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "image"),
requestPolicy,
});
return {
images: [
{
buffer: image.buffer,
mimeType: image.mimeType,
fileName: image.fileName,
},
],
model,
metadata: {
jobId:
resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted),
imageUrl,
status: resolveVydraResponseStatus(completedPayload) ?? "completed",
},
};
} finally {
await release();
}
return {
images: [generated.asset],
model,
metadata: {
jobId: generated.jobId,
imageUrl: generated.resultUrl,
status: generated.status,
},
};
},
};
}
+101 -4
View File
@@ -1,5 +1,6 @@
// Vydra plugin module implements shared behavior.
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
import { extensionForMime, type MediaKind } from "openclaw/plugin-sdk/media-mime";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
@@ -8,7 +9,10 @@ import {
createProviderOperationTimeoutResolver,
fetchWithTimeoutGuarded,
pollProviderOperationJson,
postJsonRequest,
readProviderJsonResponse,
resolveProviderHttpRequestConfig,
resolveProviderOperationTimeoutMs,
sanitizeConfiguredModelProviderRequest,
type ProviderOperationDeadline,
type ProviderOperationTimeoutMs,
@@ -94,7 +98,7 @@ function resolveVydraBaseUrlFromConfig(cfg: unknown): string {
return normalizeVydraBaseUrl(normalizeOptionalString(vydra?.baseUrl));
}
export async function resolveVydraRequestContext(params: {
async function resolveVydraRequestContext(params: {
cfg: OpenClawConfig;
agentDir?: string;
authStore?: VydraAuthStore;
@@ -142,12 +146,12 @@ export async function resolveVydraRequestContext(params: {
};
}
export function resolveVydraResponseJobId(payload: unknown): string | undefined {
function resolveVydraResponseJobId(payload: unknown): string | undefined {
const object = asOptionalRecord(payload) as VydraJobPayload | undefined;
return normalizeOptionalString(object?.jobId) ?? normalizeOptionalString(object?.id);
}
export function resolveVydraResponseStatus(payload: unknown): string | undefined {
function resolveVydraResponseStatus(payload: unknown): string | undefined {
return normalizeOptionalLowercaseString(
normalizeOptionalString(asOptionalRecord(payload)?.status),
);
@@ -362,7 +366,7 @@ async function waitForVydraJob(params: {
});
}
export async function resolveCompletedVydraPayload(params: {
async function resolveCompletedVydraPayload(params: {
submitted: unknown;
baseUrl: string;
timeoutMs?: number;
@@ -392,3 +396,96 @@ export async function resolveCompletedVydraPayload(params: {
requestPolicy: params.requestPolicy,
});
}
export async function runVydraGeneration(params: {
cfg: OpenClawConfig;
agentDir?: string;
authStore?: VydraAuthStore;
body: unknown;
deadlineTimeoutMs?: number;
kind: Extract<VydraMediaKind, "image" | "video">;
model: string;
ssrfPolicy?: SsrFPolicy;
timeoutMs?: number;
}): Promise<{
asset: { buffer: Buffer; mimeType: string; fileName: string };
jobId?: string;
resultUrl: string;
status: string;
}> {
const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({
cfg: params.cfg,
agentDir: params.agentDir,
authStore: params.authStore,
capability: params.kind,
...(params.ssrfPolicy ? { ssrfPolicy: params.ssrfPolicy } : {}),
});
const operationLabel = `Vydra ${params.kind} generation`;
const deadline =
params.deadlineTimeoutMs === undefined
? undefined
: createProviderOperationDeadline({
timeoutMs: params.deadlineTimeoutMs,
label: operationLabel,
});
const timeoutMs = deadline
? resolveProviderOperationTimeoutMs({
deadline,
defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
})
: params.timeoutMs;
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${params.model}`,
headers: requestPolicy.headers,
body: params.body,
timeoutMs,
fetchFn,
allowPrivateNetwork: requestPolicy.allowPrivateNetwork,
...(requestPolicy.ssrfPolicy ? { ssrfPolicy: requestPolicy.ssrfPolicy } : {}),
dispatcherPolicy: requestPolicy.dispatcherPolicy,
});
try {
await assertOkOrThrowHttpError(response, `${operationLabel} failed`);
const submitted = await readProviderJsonResponse(
response,
params.kind === "image" ? "vydra.image-generation" : operationLabel,
);
const completedPayload = await resolveCompletedVydraPayload({
submitted,
baseUrl,
...(deadline ? { deadline } : { timeoutMs: params.timeoutMs }),
fetchFn,
kind: params.kind,
missingJobIdMessage: `${operationLabel} response missing job id`,
requestPolicy,
});
const resultUrl = extractVydraResultUrls(completedPayload, params.kind)[0];
if (!resultUrl) {
throw new Error(`${operationLabel} completed without a ${params.kind} URL`);
}
const asset = await downloadVydraAsset({
url: resultUrl,
kind: params.kind,
timeoutMs: deadline
? createProviderOperationTimeoutResolver({
deadline,
defaultTimeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
})
: params.timeoutMs,
fetchFn,
maxBytes: resolveGeneratedMediaMaxBytes(params.cfg, params.kind),
requestPolicy,
});
const jobId =
resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted);
return {
asset,
...(jobId ? { jobId } : {}),
resultUrl,
status: resolveVydraResponseStatus(completedPayload) ?? "completed",
};
} finally {
await release();
}
}
+15 -86
View File
@@ -1,24 +1,7 @@
// Vydra provider module implements model/runtime integration.
import { resolveGeneratedMediaMaxBytes } from "openclaw/plugin-sdk/media-generation-runtime";
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
import {
assertOkOrThrowHttpError,
createProviderOperationDeadline,
createProviderOperationTimeoutResolver,
postJsonRequest,
readProviderJsonResponse,
resolveProviderOperationTimeoutMs,
} from "openclaw/plugin-sdk/provider-http";
import type { VideoGenerationProvider } from "openclaw/plugin-sdk/video-generation";
import {
DEFAULT_VYDRA_VIDEO_MODEL,
downloadVydraAsset,
extractVydraResultUrls,
resolveCompletedVydraPayload,
resolveVydraResponseJobId,
resolveVydraResponseStatus,
resolveVydraRequestContext,
} from "./shared.js";
import { DEFAULT_VYDRA_VIDEO_MODEL, runVydraGeneration } from "./shared.js";
const VYDRA_KLING_MODEL = "kling";
const DEFAULT_VYDRA_VIDEO_TIMEOUT_MS = 120_000;
@@ -79,79 +62,25 @@ export function buildVydraVideoGenerationProvider(): VideoGenerationProvider {
throw new Error("Vydra video generation does not support video reference inputs.");
}
const { fetchFn, baseUrl, requestPolicy } = await resolveVydraRequestContext({
const { model, body } = resolveVydraVideoRequestBody(req);
const generated = await runVydraGeneration({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
capability: "video",
});
const deadline = createProviderOperationDeadline({
timeoutMs: req.timeoutMs ?? DEFAULT_VYDRA_VIDEO_TIMEOUT_MS,
label: "Vydra video generation",
});
const { model, body } = resolveVydraVideoRequestBody(req);
const { response, release } = await postJsonRequest({
url: `${baseUrl}/models/${model}`,
headers: requestPolicy.headers,
kind: "video",
model,
body,
timeoutMs: resolveProviderOperationTimeoutMs({
deadline,
defaultTimeoutMs: DEFAULT_VYDRA_VIDEO_TIMEOUT_MS,
}),
fetchFn,
allowPrivateNetwork: requestPolicy.allowPrivateNetwork,
dispatcherPolicy: requestPolicy.dispatcherPolicy,
deadlineTimeoutMs: req.timeoutMs ?? DEFAULT_VYDRA_VIDEO_TIMEOUT_MS,
});
try {
await assertOkOrThrowHttpError(response, "Vydra video generation failed");
const submitted = await readProviderJsonResponse<unknown>(
response,
"Vydra video generation",
);
const completedPayload = await resolveCompletedVydraPayload({
submitted,
baseUrl,
deadline,
fetchFn,
kind: "video",
missingJobIdMessage: "Vydra video generation response missing job id",
requestPolicy,
});
const videoUrl = extractVydraResultUrls(completedPayload, "video")[0];
if (!videoUrl) {
throw new Error("Vydra video generation completed without a video URL");
}
const video = await downloadVydraAsset({
url: videoUrl,
kind: "video",
timeoutMs: createProviderOperationTimeoutResolver({
deadline,
defaultTimeoutMs: DEFAULT_VYDRA_VIDEO_TIMEOUT_MS,
}),
fetchFn,
maxBytes: resolveGeneratedMediaMaxBytes(req.cfg, "video"),
requestPolicy,
});
return {
videos: [
{
buffer: video.buffer,
mimeType: video.mimeType,
fileName: video.fileName,
},
],
model,
metadata: {
jobId:
resolveVydraResponseJobId(completedPayload) ?? resolveVydraResponseJobId(submitted),
videoUrl,
status: resolveVydraResponseStatus(completedPayload) ?? "completed",
},
};
} finally {
await release();
}
return {
videos: [generated.asset],
model,
metadata: {
jobId: generated.jobId,
videoUrl: generated.resultUrl,
status: generated.status,
},
};
},
};
}
+3
View File
@@ -95,6 +95,9 @@
"openclaw/plugin-sdk/media-generation-runtime": [
"../../packages/plugin-sdk/dist/src/plugin-sdk/media-generation-runtime.d.ts"
],
"openclaw/plugin-sdk/meeting-page-script-runtime": [
"../../packages/plugin-sdk/dist/src/plugin-sdk/meeting-page-script-runtime.d.ts"
],
"openclaw/plugin-sdk/conversation-binding-runtime": [
"../../packages/plugin-sdk/dist/src/plugin-sdk/conversation-binding-runtime.d.ts"
],
@@ -1,8 +1,11 @@
import {
createMeetingLeaveSource,
createMeetingTranscriptSource,
} from "openclaw/plugin-sdk/meeting-page-script-runtime";
import { ZOOM_MEETING_SELECTORS } from "./zoom-meetings-selectors.js";
import { zoomMeetingStatusCallSource } from "./zoom-meetings-status-call-source.js";
import { zoomMeetingStatusPreludeSource } from "./zoom-meetings-status-prejoin-source.js";
import { normalizeZoomMeetingUrlForReuse } from "./zoom-meetings-urls.js";
const ZOOM_MEETING_TRANSCRIPT_MAX_LINES = 500;
function pageIdentityFunctionSource(): string {
return `const meetingIdentity = (rawUrl) => {
@@ -81,84 +84,18 @@ export function zoomMeetingTranscriptScript(
finalize: boolean,
) {
const expectedIdentity = normalizeZoomMeetingUrlForReuse(meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(meetingSessionId)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawZoomMeeting;
const activeCaptions = window.__openclawZoomCaptions;
const archivedCaptions = window.__openclawZoomCaptionArchive?.[expectedSessionId];
const captions = activeCaptions &&
(!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId)
? activeCaptions
: archivedCaptions;
// A same-session finalized buffer belongs to the departed call even if Zoom
// immediately navigated this tab into another meeting before transcript pickup.
const useFinalizedCaptions = Boolean(
captions?.finalized === true &&
captions?.identity === expectedIdentity &&
(!captions?.sessionId || captions.sessionId === expectedSessionId)
);
const effectiveIdentity = useFinalizedCaptions
? captions.identity
: currentIdentity || state?.identity || captions?.identity;
if (!expectedIdentity || effectiveIdentity !== expectedIdentity) {
return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] });
}
if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (captions?.sessionId && captions.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.lines = Array.isArray(captions.lines) ? captions.lines : [];
captions.lines.push(...captions.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
captions.visible = [];
const excess = captions.lines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
captions.lines.splice(0, excess);
captions.droppedLines = (captions.droppedLines || 0) + excess;
}
}
if (${JSON.stringify(finalize)} && captions) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.observer?.disconnect?.();
captions.observer = undefined;
captions.observerInstalled = false;
captions.identity = expectedIdentity;
captions.finalized = true;
captions.finalizedAt = Date.now();
}
const allLines = [
...(Array.isArray(captions?.lines) ? captions.lines : []),
...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible),
];
const visibleOverflow = Math.max(0, allLines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES});
const lines = allLines.slice(-${ZOOM_MEETING_TRANSCRIPT_MAX_LINES});
const result = {
urlMatched: true,
sessionMatched: true,
epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined,
droppedLines: (Number.isFinite(captions?.droppedLines)
? Math.max(0, Math.trunc(captions.droppedLines))
: 0) + visibleOverflow,
lines: lines.map((line) => ({
at: typeof line?.at === "string" ? line.at : undefined,
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
text: typeof line?.text === "string" ? line.text : "",
})).filter((line) => line.text),
};
return JSON.stringify(result);
}`;
return createMeetingTranscriptSource({
expectedIdentity,
finalize,
globals: {
captionArchive: "__openclawZoomCaptionArchive",
captions: "__openclawZoomCaptions",
meeting: "__openclawZoomMeeting",
},
meetingSessionId,
pageIdentitySource: pageIdentityFunctionSource(),
platformDisplayName: "Zoom",
});
}
export function zoomMeetingLeaveScript(params: {
@@ -168,85 +105,8 @@ export function zoomMeetingLeaveScript(params: {
}) {
const selectors = JSON.stringify(ZOOM_MEETING_SELECTORS);
const expectedIdentity = normalizeZoomMeetingUrlForReuse(params.meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const topDocument = globalThis.document;
const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument;
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const leaveInitiated = ${JSON.stringify(params.leaveInitiated)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawZoomMeeting;
const enforceSessionOwnership = Boolean(expectedSessionId);
if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true });
}
const sessionAdoptedFromUrl = Boolean(
enforceSessionOwnership &&
!state?.sessionId &&
currentIdentity === expectedIdentity &&
(!state?.identity || state.identity === expectedIdentity)
);
const sessionMatched = !enforceSessionOwnership ||
state?.sessionId === expectedSessionId ||
sessionAdoptedFromUrl;
const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated);
if (!sessionMatched && !retainedLeaveOwnership) {
return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true });
}
const retireOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const retained = [];
const activeSessionId = expectedSessionId || state?.sessionId;
for (const entry of entries) {
const ownedByActiveSession = Boolean(
!entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId)
);
if (!ownedByActiveSession) {
retained.push(entry);
continue;
}
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const sources = Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }]
: [];
for (const source of sources) {
const element = source?.element;
const sourceMatches = source?.stream || element?.srcObject
? element?.srcObject === source?.stream
: Boolean(source?.url && mediaSourceUrl(element) === source.url);
const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element));
if (!element) continue;
if (sourceIsEmpty) {
element.muted = true;
continue;
}
if (!sourceMatches) continue;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
} else {
element.muted = Boolean(source.muted);
}
}
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
}
if (retained.length > 0) window.__openclawZoomAudioOutputs = retained;
else delete window.__openclawZoomAudioOutputs;
};
const first = (list) => {
return createMeetingLeaveSource({
controlSource: `const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (!node) continue;
@@ -270,71 +130,31 @@ export function zoomMeetingLeaveScript(params: {
try {
const currentUrl = new URL(location.href);
webClientHome = !leave && currentUrl.hostname === "app.zoom.us" && /^\\/wc\\/?$/.test(currentUrl.pathname);
} catch {}
const preservedCallMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl === leave &&
state?.inCallUrl === location.href &&
leave &&
leave.isConnected !== false
);
const pendingLeaveMatches = Boolean(
expectedIdentity &&
state?.identity === expectedIdentity &&
state?.leavePending === true &&
state?.inCallUrl === location.href &&
Date.now() - state?.leavePendingAt < 10_000
);
const rerenderPendingMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl?.isConnected === false &&
state?.inCallUrl === location.href &&
Date.now() - state?.verifiedAt < 5_000 &&
!leave
);
const meetingIdentityMatches = Boolean(
currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches
);
// Zoom can replace the document between our Leave click and its post-call marker.
// Retain request ownership only while no identity or live-call control contradicts it.
const initiatedLeaveTransitionMatches = Boolean(
leaveInitiated &&
!currentIdentity &&
!leave &&
} catch {}`,
departedMarkerSource: "(postCall || webClientHome)",
documentSetupSource: `const topDocument = globalThis.document;
const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument;`,
expectedIdentity,
leaveInitiated: params.leaveInitiated,
meetingSessionId: params.meetingSessionId,
meetingStateSource: "sessionId: expectedSessionId || state?.sessionId,",
pageIdentitySource: pageIdentityFunctionSource(),
platform: {
displayName: "Zoom",
globals: {
audioOutputs: "__openclawZoomAudioOutputs",
meeting: "__openclawZoomMeeting",
},
},
selectors,
sessionMatchSource: `const sessionAdoptedFromUrl = Boolean(
enforceSessionOwnership &&
!state?.sessionId &&
currentIdentity === expectedIdentity &&
(!state?.identity || state.identity === expectedIdentity)
);
if ((postCall || webClientHome) && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) {
retireOwnedAudioBridges();
if (sessionMatched) delete window.__openclawZoomMeeting;
return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true });
}
if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) {
return JSON.stringify({ departed: false, urlMatched: false });
}
if (!sessionMatched) {
return JSON.stringify({ departed: false, urlMatched: true });
}
if (confirmation) {
confirmation.click();
return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true });
}
if (leave) {
window.__openclawZoomMeeting = {
...state,
identity: expectedIdentity,
sessionId: expectedSessionId || state?.sessionId,
inCallControl: leave,
inCallUrl: location.href,
leavePending: true,
leavePendingAt: Date.now(),
};
leave.click();
return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true });
}
return JSON.stringify({ departed: false, urlMatched: true });
}`;
const sessionMatched = !enforceSessionOwnership ||
state?.sessionId === expectedSessionId ||
sessionAdoptedFromUrl;`,
});
}
+4
View File
@@ -108,6 +108,7 @@
"!dist/plugin-sdk/llm.d.ts",
"!dist/plugin-sdk/markdown-table-runtime.d.ts",
"!dist/plugin-sdk/media-generation-runtime.d.ts",
"!dist/plugin-sdk/meeting-page-script-runtime.d.ts",
"!dist/plugin-sdk/memory-core-host-embedding-registry.d.ts",
"!dist/plugin-sdk/memory-core-host-engine-curated.d.ts",
"!dist/plugin-sdk/memory-core-host-engine-embeddings.d.ts",
@@ -1162,6 +1163,9 @@
"./plugin-sdk/realtime-voice": {
"default": "./dist/plugin-sdk/realtime-voice.js"
},
"./plugin-sdk/meeting-page-script-runtime": {
"default": "./dist/plugin-sdk/meeting-page-script-runtime.js"
},
"./plugin-sdk/meeting-runtime": {
"types": "./dist/plugin-sdk/meeting-runtime.d.ts",
"default": "./dist/plugin-sdk/meeting-runtime.js"
+1
View File
@@ -240,6 +240,7 @@
"realtime-voice-audio-queue",
"realtime-voice-activation",
"realtime-voice",
"meeting-page-script-runtime",
"meeting-runtime",
"transcripts",
"media-understanding",
@@ -65,6 +65,7 @@
"llm",
"markdown-table-runtime",
"media-generation-runtime",
"meeting-page-script-runtime",
"memory-core-host-embedding-registry",
"memory-core-host-engine-curated",
"memory-core-host-engine-embeddings",
+270
View File
@@ -0,0 +1,270 @@
type MeetingPageScriptGlobals = {
audioOutputs: string;
captionArchive: string;
captions: string;
meeting: string;
};
function pageGlobalSource(name: string): string {
if (!/^[$A-Z_a-z][$\w]*$/u.test(name)) {
throw new Error(`Invalid meeting page global: ${name}`);
}
return `window.${name}`;
}
export function createMeetingTranscriptSource(params: {
expectedIdentity?: string;
finalize: boolean;
globals: Pick<MeetingPageScriptGlobals, "captionArchive" | "captions" | "meeting">;
meetingSessionId: string;
pageIdentitySource: string;
platformDisplayName: string;
transcriptMaxLines?: number;
}): string {
const transcriptMaxLines = params.transcriptMaxLines ?? 500;
const meetingGlobal = pageGlobalSource(params.globals.meeting);
const captionsGlobal = pageGlobalSource(params.globals.captions);
const captionArchiveGlobal = pageGlobalSource(params.globals.captionArchive);
return `() => {
${params.pageIdentitySource}
const expectedIdentity = ${JSON.stringify(params.expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const currentIdentity = meetingIdentity(location.href);
const state = ${meetingGlobal};
const activeCaptions = ${captionsGlobal};
const archivedCaptions = ${captionArchiveGlobal}?.[expectedSessionId];
const captions = activeCaptions &&
(!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId)
? activeCaptions
: archivedCaptions;
// A same-session finalized buffer belongs to the departed call even if ${params.platformDisplayName}
// immediately navigated this tab into another meeting before transcript pickup.
const useFinalizedCaptions = Boolean(
captions?.finalized === true &&
captions?.identity === expectedIdentity &&
(!captions?.sessionId || captions.sessionId === expectedSessionId)
);
const effectiveIdentity = useFinalizedCaptions
? captions.identity
: currentIdentity || state?.identity || captions?.identity;
if (!expectedIdentity || effectiveIdentity !== expectedIdentity) {
return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] });
}
if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (captions?.sessionId && captions.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (${JSON.stringify(params.finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.lines = Array.isArray(captions.lines) ? captions.lines : [];
captions.lines.push(...captions.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
captions.visible = [];
const excess = captions.lines.length - ${transcriptMaxLines};
if (excess > 0) {
captions.lines.splice(0, excess);
captions.droppedLines = (captions.droppedLines || 0) + excess;
}
}
if (${JSON.stringify(params.finalize)} && captions) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.observer?.disconnect?.();
captions.observer = undefined;
captions.observerInstalled = false;
captions.identity = expectedIdentity;
captions.finalized = true;
captions.finalizedAt = Date.now();
}
const allLines = [
...(Array.isArray(captions?.lines) ? captions.lines : []),
...(${JSON.stringify(params.finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible),
];
const visibleOverflow = Math.max(0, allLines.length - ${transcriptMaxLines});
const lines = allLines.slice(-${transcriptMaxLines});
const result = {
urlMatched: true,
sessionMatched: true,
epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined,
droppedLines: (Number.isFinite(captions?.droppedLines)
? Math.max(0, Math.trunc(captions.droppedLines))
: 0) + visibleOverflow,
lines: lines.map((line) => ({
at: typeof line?.at === "string" ? line.at : undefined,
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
text: typeof line?.text === "string" ? line.text : "",
})).filter((line) => line.text),
};
return JSON.stringify(result);
}`;
}
function createMeetingOwnedAudioLeaveSource(params: { audioOutputsGlobal: string }): string {
const audioOutputsGlobal = pageGlobalSource(params.audioOutputsGlobal);
return `const retireOwnedAudioBridges = () => {
const entries = Array.isArray(${audioOutputsGlobal})
? ${audioOutputsGlobal}
: [];
const retained = [];
const activeSessionId = expectedSessionId || state?.sessionId;
for (const entry of entries) {
const ownedByActiveSession = Boolean(
!entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId)
);
if (!ownedByActiveSession) {
retained.push(entry);
continue;
}
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const sources = Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }]
: [];
for (const source of sources) {
const element = source?.element;
const sourceMatches = source?.stream || element?.srcObject
? element?.srcObject === source?.stream
: Boolean(source?.url && mediaSourceUrl(element) === source.url);
const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element));
if (!element) continue;
if (sourceIsEmpty) {
element.muted = true;
continue;
}
if (!sourceMatches) continue;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
} else {
element.muted = Boolean(source.muted);
}
}
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
}
if (retained.length > 0) ${audioOutputsGlobal} = retained;
else delete ${audioOutputsGlobal};
};`;
}
export function createMeetingLeaveSource(params: {
controlSource: string;
departedMarkerSource: string;
documentSetupSource?: string;
expectedIdentity?: string;
leaveInitiated: boolean;
meetingSessionId: string;
meetingStateSource?: string;
pageIdentitySource: string;
platform: {
displayName: string;
globals: Pick<MeetingPageScriptGlobals, "audioOutputs" | "meeting">;
};
selectors: string;
sessionMatchSource: string;
}): string {
const documentSetupSource = params.documentSetupSource ? `${params.documentSetupSource}\n ` : "";
const meetingStateSource = params.meetingStateSource
? ` ${params.meetingStateSource}\n`
: "";
const meetingGlobal = pageGlobalSource(params.platform.globals.meeting);
return `() => {
${params.pageIdentitySource}
${documentSetupSource}const selectors = ${params.selectors};
const expectedIdentity = ${JSON.stringify(params.expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const leaveInitiated = ${JSON.stringify(params.leaveInitiated)};
const currentIdentity = meetingIdentity(location.href);
const state = ${meetingGlobal};
const enforceSessionOwnership = Boolean(expectedSessionId);
if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true });
}
${params.sessionMatchSource}
const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated);
if (!sessionMatched && !retainedLeaveOwnership) {
return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true });
}
${createMeetingOwnedAudioLeaveSource({
audioOutputsGlobal: params.platform.globals.audioOutputs,
})}
${params.controlSource}
const preservedCallMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl === leave &&
state?.inCallUrl === location.href &&
leave &&
leave.isConnected !== false
);
const pendingLeaveMatches = Boolean(
expectedIdentity &&
state?.identity === expectedIdentity &&
state?.leavePending === true &&
state?.inCallUrl === location.href &&
Date.now() - state?.leavePendingAt < 10_000
);
const rerenderPendingMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl?.isConnected === false &&
state?.inCallUrl === location.href &&
Date.now() - state?.verifiedAt < 5_000 &&
!leave
);
const meetingIdentityMatches = Boolean(
currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches
);
// ${params.platform.displayName} can replace the document between our Leave click and its post-call marker.
// Retain request ownership only while no identity or live-call control contradicts it.
const initiatedLeaveTransitionMatches = Boolean(
leaveInitiated &&
!currentIdentity &&
!leave &&
(!state?.identity || state.identity === expectedIdentity)
);
if (${params.departedMarkerSource} && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) {
retireOwnedAudioBridges();
if (sessionMatched) delete ${meetingGlobal};
return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true });
}
if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) {
return JSON.stringify({ departed: false, urlMatched: false });
}
if (!sessionMatched) {
return JSON.stringify({ departed: false, urlMatched: true });
}
if (confirmation) {
confirmation.click();
return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true });
}
if (leave) {
${meetingGlobal} = {
...state,
identity: expectedIdentity,
${meetingStateSource} inCallControl: leave,
inCallUrl: location.href,
leavePending: true,
leavePendingAt: Date.now(),
};
leave.click();
return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true });
}
return JSON.stringify({ departed: false, urlMatched: true });
}`;
}
@@ -0,0 +1,5 @@
/** Production-private page-script seam for official browser-meeting plugins. */
export {
createMeetingLeaveSource,
createMeetingTranscriptSource,
} from "../meeting-bot/page-script-source.js";