diff --git a/extensions/codex/media-understanding-provider.test.ts b/extensions/codex/media-understanding-provider.test.ts index f2693983d160..71b2a4fb4b94 100644 --- a/extensions/codex/media-understanding-provider.test.ts +++ b/extensions/codex/media-understanding-provider.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { buildCodexMediaUnderstandingProvider } from "./media-understanding-provider.js"; import type { CodexAppServerClient } from "./src/app-server/client.js"; import type { CodexServerNotification, JsonValue } from "./src/app-server/protocol.js"; +import type { CodexAppServerClientFactory } from "./src/app-server/shared-client.js"; const sharedClientMocks = vi.hoisted(() => ({ createIsolatedCodexAppServerClient: vi.fn(), @@ -84,9 +85,11 @@ function turnStartResult(status = "inProgress", items: JsonValue[] = []) { function createFakeClient(options?: { inputModalities?: string[]; completeWithItems?: boolean; + deferTurnCompletion?: boolean; notifyError?: string; approvalRequestMethod?: string; responseText?: string; + onTurnStart?: () => void; }) { const notifications = new Set<(notification: CodexServerNotification) => void>(); const requestHandlers = new Set<(request: { method: string }) => JsonValue | undefined>(); @@ -104,6 +107,7 @@ function createFakeClient(options?: { return threadStartResult(); } if (method === "turn/start") { + options?.onTurnStart?.(); if (options?.approvalRequestMethod) { for (const handler of requestHandlers) { const response = handler({ method: options.approvalRequestMethod }); @@ -128,7 +132,7 @@ function createFakeClient(options?: { }, }); } - } else if (!options?.completeWithItems) { + } else if (!options?.completeWithItems && !options?.deferTurnCompletion) { for (const notify of notifications) { notify({ method: "item/agentMessage/delta", @@ -190,6 +194,63 @@ describe("codex media understanding provider", () => { sharedClientMocks.createIsolatedCodexAppServerClient.mockReset(); }); + it("does not start a bounded turn for an already-aborted media request", async () => { + const clientFactory = vi.fn(); + const provider = buildCodexMediaUnderstandingProvider({ clientFactory }); + const controller = new AbortController(); + controller.abort(new Error("caller cancelled Codex media request")); + + await expect( + provider.describeImage?.({ + buffer: Buffer.from("image-bytes"), + fileName: "image.png", + mime: "image/png", + provider: "codex", + model: "gpt-5.4", + timeoutMs: 30_000, + signal: controller.signal, + cfg: {}, + agentDir: "/tmp/openclaw-agent", + }), + ).rejects.toThrow("caller cancelled Codex media request"); + + expect(clientFactory).not.toHaveBeenCalled(); + }); + + it("abandons app-server startup when the media request aborts", async () => { + const clientFactory = vi.fn( + async (options) => + await new Promise((_, reject) => { + options?.abandonSignal?.addEventListener( + "abort", + () => { + const reason = options.abandonSignal?.reason; + reject(reason instanceof Error ? reason : new Error("Codex startup aborted")); + }, + { once: true }, + ); + }), + ); + const provider = buildCodexMediaUnderstandingProvider({ clientFactory }); + const controller = new AbortController(); + const result = provider.describeImage?.({ + buffer: Buffer.from("image-bytes"), + fileName: "image.png", + mime: "image/png", + provider: "codex", + model: "gpt-5.4", + timeoutMs: 30_000, + signal: controller.signal, + cfg: {}, + agentDir: "/tmp/openclaw-agent", + }); + + await vi.waitFor(() => expect(clientFactory).toHaveBeenCalledOnce()); + controller.abort(new Error("caller cancelled Codex startup")); + await expect(result).rejects.toThrow("caller cancelled Codex startup"); + expect(clientFactory.mock.calls[0]?.[0]?.abandonSignal).toBe(controller.signal); + }); + it("runs image understanding through a bounded Codex app-server turn", async () => { const { client, requests } = createFakeClient(); const clientFactory = vi.fn(async () => client); @@ -333,6 +394,42 @@ describe("codex media understanding provider", () => { expect(requests[2]?.params).toEqual(expect.objectContaining({ cwd: "/tmp/openclaw-agent" })); }); + it("interrupts a configured app-server turn when the media request aborts", async () => { + const controller = new AbortController(); + const { client, requests } = createFakeClient({ + deferTurnCompletion: true, + onTurnStart: () => setTimeout(() => controller.abort(new Error("media cancelled")), 0), + }); + const provider = buildCodexMediaUnderstandingProvider({ + pluginConfig: { + appServer: { + transport: "websocket", + url: "ws://127.0.0.1:4501", + }, + }, + clientFactory: async () => client, + }); + + await expect( + provider.describeImage?.({ + buffer: Buffer.from("image-bytes"), + fileName: "image.png", + mime: "image/png", + provider: "codex", + model: "gpt-5.4", + timeoutMs: 30_000, + signal: controller.signal, + cfg: {}, + agentDir: "/tmp/openclaw-agent", + }), + ).rejects.toThrow(); + + expect(requests).toContainEqual({ + method: "turn/interrupt", + params: { threadId: "thread-1", turnId: "turn-1" }, + }); + }); + it("passes the scoped auth store into isolated app-server startup", async () => { const { client } = createFakeClient(); sharedClientMocks.createIsolatedCodexAppServerClient.mockResolvedValue(client); diff --git a/extensions/codex/media-understanding-provider.ts b/extensions/codex/media-understanding-provider.ts index 30fe6a1dfffd..00586275e6dc 100644 --- a/extensions/codex/media-understanding-provider.ts +++ b/extensions/codex/media-understanding-provider.ts @@ -51,6 +51,7 @@ export function buildCodexMediaUnderstandingProvider( prompt: req.prompt, maxTokens: req.maxTokens, timeoutMs: req.timeoutMs, + ...(req.signal ? { signal: req.signal } : {}), profile: req.profile, preferredProfile: req.preferredProfile, authStore: req.authStore, @@ -72,12 +73,14 @@ async function describeCodexImages( if (!model) { throw new Error("Codex image understanding requires model id."); } + req.signal?.throwIfAborted(); const { text } = await runBoundedCodexAppServerTurn({ config: req.cfg, model: { mode: "required", id: model }, profile: req.profile, timeoutMs: req.timeoutMs, + signal: req.signal, agentDir: req.agentDir, authProfileStore: req.authStore, options, @@ -115,12 +118,14 @@ async function extractCodexStructured( if (!req.input.some((entry) => entry.type === "image")) { throw new Error("Codex structured extraction requires at least one image input."); } + req.signal?.throwIfAborted(); const { text } = await runBoundedCodexAppServerTurn({ config: req.cfg, model: { mode: "required", id: model }, profile: req.profile, timeoutMs: req.timeoutMs, + signal: req.signal, agentDir: req.agentDir, authProfileStore: req.authStore, options, diff --git a/extensions/codex/src/app-server/attempt-client-cleanup.ts b/extensions/codex/src/app-server/attempt-client-cleanup.ts index 325f5a17b07e..6c04826a3e91 100644 --- a/extensions/codex/src/app-server/attempt-client-cleanup.ts +++ b/extensions/codex/src/app-server/attempt-client-cleanup.ts @@ -93,18 +93,29 @@ export function interruptCodexTurnBestEffort( timeoutMs?: number; }, ): void { + void interruptCodexTurnAndWaitBestEffort(client, params); +} + +/** Sends a bounded turn interrupt and waits for Codex to confirm terminal abort handling. */ +export async function interruptCodexTurnAndWaitBestEffort( + client: CodexAppServerClient, + params: { + threadId: string; + turnId: string; + timeoutMs?: number; + }, +): Promise { const requestOptions = params.timeoutMs && Number.isFinite(params.timeoutMs) && params.timeoutMs > 0 ? { timeoutMs: params.timeoutMs } : undefined; const requestParams = { threadId: params.threadId, turnId: params.turnId }; try { - const interrupt = requestOptions + // Non-empty interrupts resolve after Codex emits TurnAborted; the empty + // startup form resolves after Op::Interrupt is submitted because no turn exists yet. + await (requestOptions ? client.request("turn/interrupt", requestParams, requestOptions) - : client.request("turn/interrupt", requestParams); - void Promise.resolve(interrupt).catch((error: unknown) => { - embeddedAgentLog.debug("codex app-server turn interrupt failed during abort", { error }); - }); + : client.request("turn/interrupt", requestParams)); } catch (error) { embeddedAgentLog.debug("codex app-server turn interrupt failed during abort", { error }); } diff --git a/extensions/codex/src/app-server/bounded-turn.ts b/extensions/codex/src/app-server/bounded-turn.ts index c10faf79f6d5..811bafa3acc0 100644 --- a/extensions/codex/src/app-server/bounded-turn.ts +++ b/extensions/codex/src/app-server/bounded-turn.ts @@ -4,6 +4,10 @@ import type { AuthProfileStore } from "openclaw/plugin-sdk/agent-runtime"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime"; import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; +import { + CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + interruptCodexTurnAndWaitBestEffort, +} from "./attempt-client-cleanup.js"; import { isRetryableErrorNotification, readCodexNotificationItem, @@ -154,6 +158,7 @@ async function runBoundedCodexAppServerTurnInWorkspace( agentDir, config: params.config, timeoutMs, + ...(params.signal ? { abandonSignal: params.signal } : {}), }) : await import("./shared-client.js").then(({ createIsolatedCodexAppServerClient }) => createIsolatedCodexAppServerClient({ @@ -163,10 +168,30 @@ async function runBoundedCodexAppServerTurnInWorkspace( agentDir, authProfileStore: params.authProfileStore, config: params.config, + ...(params.signal ? { abandonSignal: params.signal } : {}), }), ); const abortController = new AbortController(); - const abortFromCaller = () => abortController.abort(params.signal?.reason ?? "aborted"); + let activeThreadId: string | undefined; + let activeTurnId = ""; + let interruptPromise: Promise | undefined; + const requestInterrupt = () => { + if (!activeThreadId || interruptPromise) { + return; + } + // Codex serializes start/interrupt per thread; an empty turn id is its + // explicit startup-interrupt contract while turn/start is still resolving. + interruptPromise = interruptCodexTurnAndWaitBestEffort(client, { + threadId: activeThreadId, + turnId: activeTurnId, + timeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + }); + }; + const abortRun = (reason: unknown) => { + abortController.abort(reason); + requestInterrupt(); + }; + const abortFromCaller = () => abortRun(params.signal?.reason ?? "aborted"); if (params.signal?.aborted) { abortFromCaller(); } else { @@ -174,9 +199,9 @@ async function runBoundedCodexAppServerTurnInWorkspace( } const remainingRunMs = deadline - Date.now(); if (remainingRunMs <= 0) { - abortController.abort("timeout"); + abortRun("timeout"); } - const timeout = setTimeout(() => abortController.abort("timeout"), Math.max(1, remainingRunMs)); + const timeout = setTimeout(() => abortRun("timeout"), Math.max(1, remainingRunMs)); timeout.unref?.(); let retrySelection = false; @@ -218,6 +243,10 @@ async function runBoundedCodexAppServerTurnInWorkspace( { timeoutMs, signal: abortController.signal }, ), ); + activeThreadId = thread.thread.id; + if (abortController.signal.aborted) { + requestInterrupt(); + } if (params.requireNoExternalCapabilities) { // Attest the started thread before injecting historical tool evidence. // Otherwise inherited MCP state could act on a finalization-only turn. @@ -254,6 +283,10 @@ async function runBoundedCodexAppServerTurnInWorkspace( { timeoutMs, signal: abortController.signal }, ), ); + activeTurnId = turn.turn.id; + if (abortController.signal.aborted) { + requestInterrupt(); + } return { ...(await collector.collect(turn.turn, { timeoutMs, @@ -274,6 +307,7 @@ async function runBoundedCodexAppServerTurnInWorkspace( } finally { clearTimeout(timeout); params.signal?.removeEventListener("abort", abortFromCaller); + await interruptPromise; if (ownsClient) { client.close(); } diff --git a/extensions/deepgram/audio.ts b/extensions/deepgram/audio.ts index 126beaa83304..4994f1dfa093 100644 --- a/extensions/deepgram/audio.ts +++ b/extensions/deepgram/audio.ts @@ -85,6 +85,7 @@ export async function transcribeDeepgramAudio( headers, body, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), fetchFn, allowPrivateNetwork, dispatcherPolicy, diff --git a/extensions/elevenlabs/media-understanding-provider.ts b/extensions/elevenlabs/media-understanding-provider.ts index a91807deb46d..4c13b671db00 100644 --- a/extensions/elevenlabs/media-understanding-provider.ts +++ b/extensions/elevenlabs/media-understanding-provider.ts @@ -55,6 +55,7 @@ export async function transcribeElevenLabsAudio( headers, body: form, timeoutMs: req.timeoutMs, + ...(req.signal ? { signal: req.signal } : {}), fetchFn, allowPrivateNetwork, dispatcherPolicy, diff --git a/extensions/google/media-understanding-provider.ts b/extensions/google/media-understanding-provider.ts index 97f969df5d84..7772a6d274be 100644 --- a/extensions/google/media-understanding-provider.ts +++ b/extensions/google/media-understanding-provider.ts @@ -35,6 +35,7 @@ async function generateGeminiInlineDataText(params: { model?: string; prompt?: string; timeoutMs: number; + signal?: AbortSignal; fetchFn?: typeof fetch; defaultBaseUrl: string; defaultModel: string; @@ -90,6 +91,7 @@ async function generateGeminiInlineDataText(params: { headers, body, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), fetchFn, allowPrivateNetwork, dispatcherPolicy, diff --git a/extensions/moonshot/media-understanding-provider.ts b/extensions/moonshot/media-understanding-provider.ts index 7fbe8c83323f..de264f0fa318 100644 --- a/extensions/moonshot/media-understanding-provider.ts +++ b/extensions/moonshot/media-understanding-provider.ts @@ -58,6 +58,7 @@ async function describeMoonshotVideo( headers, body, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), fetchFn, allowPrivateNetwork, dispatcherPolicy, diff --git a/extensions/openrouter/media-understanding-provider.ts b/extensions/openrouter/media-understanding-provider.ts index 9ea128fe0750..d47e97d52048 100644 --- a/extensions/openrouter/media-understanding-provider.ts +++ b/extensions/openrouter/media-understanding-provider.ts @@ -141,6 +141,7 @@ async function transcribeOpenRouterAudio( ...(temperature !== undefined ? { temperature } : {}), }, timeoutMs: params.timeoutMs, + ...(params.signal ? { signal: params.signal } : {}), fetchFn, allowPrivateNetwork, dispatcherPolicy, diff --git a/extensions/qwen/media-understanding-provider.ts b/extensions/qwen/media-understanding-provider.ts index c40c4bd5ebd0..109cf20fe294 100644 --- a/extensions/qwen/media-understanding-provider.ts +++ b/extensions/qwen/media-understanding-provider.ts @@ -52,6 +52,7 @@ async function describeQwenVideo(params: VideoDescriptionRequest): Promise