From 323a9fbe29b4645ea6284391fd99a1504ebd9b07 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 13 Jul 2026 12:57:36 -0700 Subject: [PATCH] refactor(codex): split app server run attempt --- .../src/app-server/run-attempt-active-turn.ts | 219 + .../src/app-server/run-attempt-cleanup.ts | 105 + .../src/app-server/run-attempt-connection.ts | 386 ++ .../src/app-server/run-attempt-context.ts | 188 + .../src/app-server/run-attempt-finalize.ts | 468 ++ .../run-attempt-lifecycle-controller.ts | 262 ++ .../run-attempt-notification-controller.ts | 266 ++ .../src/app-server/run-attempt-prompt.ts | 480 +++ .../src/app-server/run-attempt-resources.ts | 251 ++ .../codex/src/app-server/run-attempt-route.ts | 104 + .../src/app-server/run-attempt-runtime.ts | 211 + .../app-server/run-attempt-server-requests.ts | 360 ++ .../codex/src/app-server/run-attempt-start.ts | 186 + .../src/app-server/run-attempt-tool-setup.ts | 187 + .../app-server/run-attempt-turn-request.ts | 198 + .../src/app-server/run-attempt-turn-start.ts | 298 ++ .../src/app-server/run-attempt-turn-state.ts | 207 + .../codex/src/app-server/run-attempt-types.ts | 29 + .../codex/src/app-server/run-attempt.ts | 3780 +---------------- 19 files changed, 4470 insertions(+), 3715 deletions(-) create mode 100644 extensions/codex/src/app-server/run-attempt-active-turn.ts create mode 100644 extensions/codex/src/app-server/run-attempt-cleanup.ts create mode 100644 extensions/codex/src/app-server/run-attempt-connection.ts create mode 100644 extensions/codex/src/app-server/run-attempt-context.ts create mode 100644 extensions/codex/src/app-server/run-attempt-finalize.ts create mode 100644 extensions/codex/src/app-server/run-attempt-lifecycle-controller.ts create mode 100644 extensions/codex/src/app-server/run-attempt-notification-controller.ts create mode 100644 extensions/codex/src/app-server/run-attempt-prompt.ts create mode 100644 extensions/codex/src/app-server/run-attempt-resources.ts create mode 100644 extensions/codex/src/app-server/run-attempt-route.ts create mode 100644 extensions/codex/src/app-server/run-attempt-runtime.ts create mode 100644 extensions/codex/src/app-server/run-attempt-server-requests.ts create mode 100644 extensions/codex/src/app-server/run-attempt-start.ts create mode 100644 extensions/codex/src/app-server/run-attempt-tool-setup.ts create mode 100644 extensions/codex/src/app-server/run-attempt-turn-request.ts create mode 100644 extensions/codex/src/app-server/run-attempt-turn-start.ts create mode 100644 extensions/codex/src/app-server/run-attempt-turn-state.ts create mode 100644 extensions/codex/src/app-server/run-attempt-types.ts diff --git a/extensions/codex/src/app-server/run-attempt-active-turn.ts b/extensions/codex/src/app-server/run-attempt-active-turn.ts new file mode 100644 index 000000000000..cbc8fd043d66 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-active-turn.ts @@ -0,0 +1,219 @@ +import { setActiveEmbeddedRun } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + interruptCodexTurnBestEffort, + retireCodexAppServerClientAfterTimedOutTurn, +} from "./attempt-client-cleanup.js"; +import { isTerminalTurnStatus } from "./attempt-notifications.js"; +import { createCodexSteeringQueue, type CodexSteeringQueueOptions } from "./attempt-steering.js"; +import { CodexAppServerEventProjector } from "./event-projector.js"; +import type { CodexTurnStartResponse, JsonObject } from "./protocol.js"; +import { readRecentCodexRateLimits } from "./rate-limit-cache.js"; +import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import type { CodexAttemptNotificationController } from "./run-attempt-notification-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { + createCodexAppServerUserMessagePersistenceNotifier, + mirrorPromptAtTurnStartBestEffort, +} from "./transcript-mirror.js"; +import { createCodexUserInputBridge } from "./user-input-bridge.js"; + +export async function activateCodexAttemptTurn( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + lifecycle: CodexAttemptLifecycleController, + notifications: CodexAttemptNotificationController, + turn: CodexTurnStartResponse, +) { + const { + prompt, + state: resourceState, + projectorRef, + trajectoryRecorder, + pendingNativePreToolUseFailures, + } = resources; + const { context, turnState } = prompt; + const { runtime, attemptTools } = context; + const { connection } = runtime; + const { + params, + runAbortController, + terminalState, + abortExplicitly, + abortFromUpstream, + bindingStore, + bindingIdentity, + sessionAgentId, + sandboxSessionKey, + effectiveCwd, + } = connection; + const { dynamicToolParams, computerContextEpoch } = attemptTools; + const { state, userInputBridgeRef, steeringQueueRef, turnWatches } = turnRuntime; + const { emitExecutionPhaseOnce, emitLifecycleStart, maybeAnnounceFastModeAutoOff } = lifecycle; + const { enqueueNotification } = notifications; + const activeTurnId = turn.turn.id; + const streamState = { eventEmitted: false, needsTerminalSnapshot: false }; + emitExecutionPhaseOnce("turn_accepted", { phase: "turn_accepted" }); + userInputBridgeRef.current = createCodexUserInputBridge({ + paramsForRun: params, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + signal: runAbortController.signal, + }); + trajectoryRecorder?.recordEvent("prompt.submitted", { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + prompt: turnState.codexTurnPromptText, + imagesCount: params.images?.length ?? 0, + }); + projectorRef.current = new CodexAppServerEventProjector( + { + ...dynamicToolParams, + onAgentEvent: (event) => { + if (event.stream === "assistant" && typeof event.data.delta === "string") { + streamState.eventEmitted = true; + streamState.needsTerminalSnapshot ||= event.data.replaceable === true; + } + return dynamicToolParams.onAgentEvent?.(event); + }, + }, + resourceState.thread.threadId, + activeTurnId, + { + nativePostToolUseRelayEnabled: + resourceState.nativeHookRelay?.allowedEvents.includes("post_tool_use") === true && + resourceState.nativeHookRelay.shouldRelayEvent("post_tool_use"), + readRecentRateLimits: () => readRecentCodexRateLimits(resourceState.client), + runAbortSignal: runAbortController.signal, + trajectoryRecorder, + onNativeToolResultRecorded: maybeAnnounceFastModeAutoOff, + onContextCompacted: () => { + computerContextEpoch.value += 1; + delete computerContextEpoch.frameToolCallId; + delete computerContextEpoch.frameImageIdentity; + }, + }, + ); + if (isTerminalTurnStatus(turn.turn.status)) { + state.terminalTurnNotificationQueued = true; + } + emitLifecycleStart(); + const activeProjector = projectorRef.current; + turnWatches.armTerminalIdleWatch(); + turnWatches.touchActivity("turn:start", { arm: true }); + turnWatches.armAttemptIdleWatch(); + turnWatches.touchActivity("turn:start", { attemptProgress: true }); + for (const failure of pendingNativePreToolUseFailures.splice(0)) { + activeProjector.recordNativeToolPreToolUseFailure(failure); + } + // The route buffers early events. Publish full turn context, then release in wire order. + if (resourceState.turnRoute) { + try { + await resourceState.turnRoute.bindTurn(activeTurnId); + } catch (error) { + if (!state.terminalTurnNotificationQueued) { + throw error; + } + await resourceState.turnRoute.drain(); + if (!state.completed) { + turnWatches.clearAllTimers(); + throw error; + } + } + } + if (!state.completed && isTerminalTurnStatus(turn.turn.status)) { + await enqueueNotification( + { + method: "turn/completed", + params: { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + turn: turn.turn as unknown as JsonObject, + }, + }, + { threadId: resourceState.thread.threadId, turnId: activeTurnId }, + ); + } + const activeSteeringQueue = createCodexSteeringQueue({ + client: resourceState.client, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + answerPendingUserInput: (text) => + userInputBridgeRef.current?.handleQueuedMessage(text) ?? false, + signal: runAbortController.signal, + }); + steeringQueueRef.current = activeSteeringQueue; + const handle = { + kind: "embedded" as const, + runId: params.runId, + queueMessage: async (text: string, optionsLocal?: CodexSteeringQueueOptions) => + activeSteeringQueue.queue(text, optionsLocal), + isStreaming: () => !state.completed && !runAbortController.signal.aborted, + isStopped: () => state.completed || state.timedOut || runAbortController.signal.aborted, + isAbortable: () => + !terminalState.terminalOutcomeFrozen || terminalState.sharedAbortAllowedAfterTerminalOutcome, + isCompacting: () => projectorRef.current?.isCompacting() ?? false, + sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, + cancel: () => abortExplicitly("cancelled"), + abort: () => abortExplicitly("aborted"), + }; + params.replyOperation?.attachBackend(handle); + setActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile); + const freezeRunTerminalOutcome = () => { + if (terminalState.terminalOutcomeFrozen) { + return; + } + terminalState.terminalOutcomeFrozen = true; + params.abortSignal?.removeEventListener("abort", abortFromUpstream); + }; + const notifyUserMessagePersisted = createCodexAppServerUserMessagePersistenceNotifier(params); + void mirrorPromptAtTurnStartBestEffort({ + params, + agentId: sessionAgentId, + notifyUserMessagePersisted, + sessionKey: sandboxSessionKey, + cwd: effectiveCwd, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + }); + const abortListener = () => { + if (state.timedOut) { + void (async () => { + // Supervised sessions stay native; clearing scope would silently move the next attempt. + if (resourceState.thread.connectionScope !== "supervision") { + await bindingStore.mutate(bindingIdentity, { + kind: "clear", + threadId: resourceState.thread.threadId, + }); + } + await retireCodexAppServerClientAfterTimedOutTurn(resourceState.client, { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + reason: String(runAbortController.signal.reason ?? "timeout"), + suspectPhysicalClient: state.turnWatchTimeoutKind === "terminal", + }); + })().finally(() => state.resolveCompletion?.()); + return; + } + interruptCodexTurnBestEffort(resourceState.client, { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + }); + state.resolveCompletion?.(); + }; + runAbortController.signal.addEventListener("abort", abortListener, { once: true }); + if (runAbortController.signal.aborted) { + abortListener(); + } + return { + activeTurnId, + activeProjector, + streamState, + handle, + freezeRunTerminalOutcome, + notifyUserMessagePersisted, + abortListener, + }; +} + +export type CodexAttemptActiveTurn = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-cleanup.ts b/extensions/codex/src/app-server/run-attempt-cleanup.ts new file mode 100644 index 000000000000..cfbb59542ca6 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-cleanup.ts @@ -0,0 +1,105 @@ +import { + clearActiveEmbeddedRun, + embeddedAgentLog, + runAgentCleanupStep, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, + unsubscribeCodexThreadBestEffort, +} from "./attempt-client-cleanup.js"; +import { scheduleCodexNativeHookRelayUnregister } from "./native-hook-relay.js"; +import type { CodexAttemptActiveTurn } from "./run-attempt-active-turn.js"; +import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; + +export async function cleanupCodexAttempt( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + lifecycle: CodexAttemptLifecycleController, + requestRuntime: Awaited>, + activeTurn: CodexAttemptActiveTurn, +) { + const { + prompt, + state: resourceState, + trajectoryRecorder, + releaseCurrentRoute, + releaseSharedClientLeaseAndRetireOneShotClient, + releaseSandboxExecEnvironment, + } = resources; + const { connection } = prompt.context.runtime; + const { params, options, runAbortController } = connection; + const { state, steeringQueueRef, userInputBridgeRef, turnWatches } = turnRuntime; + const { + maybeEmitFastModeAutoResetBestEffort, + emitLifecycleTerminal, + buildLifecycleTerminalMeta, + } = lifecycle; + const { codexModelCallDiagnostics } = requestRuntime; + const { activeTurnId, abortListener, handle, freezeRunTerminalOutcome } = activeTurn; + if (params.isFinalFallbackAttempt !== false) { + await maybeEmitFastModeAutoResetBestEffort(); + } + codexModelCallDiagnostics.emitError( + "codex app-server run completed without model-call terminal event", + ); + emitLifecycleTerminal({ + phase: "error", + error: "codex app-server run completed without lifecycle terminal event", + ...buildLifecycleTerminalMeta({ + aborted: runAbortController.signal.aborted && !state.clientClosedAbort, + timedOut: state.timedOut, + }), + }); + if (trajectoryRecorder && !resourceState.trajectoryEndRecorded) { + trajectoryRecorder.recordEvent("session.ended", { + status: + state.timedOut || (runAbortController.signal.aborted && !state.clientClosedAbort) + ? "interrupted" + : "cleanup", + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + timedOut: state.timedOut, + aborted: runAbortController.signal.aborted && !state.clientClosedAbort, + }); + } + await runAgentCleanupStep({ + runId: params.runId, + sessionId: params.sessionId, + step: "codex-trajectory-flush", + log: embeddedAgentLog, + cleanup: async () => trajectoryRecorder?.flush(), + }); + if (!state.timedOut && !runAbortController.signal.aborted) { + await steeringQueueRef.current?.flushPending(); + } + if (!state.timedOut) { + await unsubscribeCodexThreadBestEffort(resourceState.client, { + threadId: resourceState.thread.threadId, + timeoutMs: CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, + }); + } + userInputBridgeRef.current?.cancelPending(); + turnWatches.clearAllTimers(); + releaseCurrentRoute(); + await releaseSharedClientLeaseAndRetireOneShotClient(); + if (resourceState.nativeHookRelay) { + if (state.shouldDelayNativeHookRelayUnregister) { + // Native hook subprocesses can finish shortly after turn completion. + scheduleCodexNativeHookRelayUnregister({ + relay: resourceState.nativeHookRelay, + hookTimeoutSec: options.nativeHookRelay?.hookTimeoutSec, + }); + } else { + resourceState.nativeHookRelay.unregister(); + } + } + await releaseSandboxExecEnvironment(); + runAbortController.signal.removeEventListener("abort", abortListener); + steeringQueueRef.current?.cancel(); + freezeRunTerminalOutcome(); + params.replyOperation?.detachBackend(handle); + clearActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile); +} diff --git a/extensions/codex/src/app-server/run-attempt-connection.ts b/extensions/codex/src/app-server/run-attempt-connection.ts new file mode 100644 index 000000000000..4045a6bb2ece --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-connection.ts @@ -0,0 +1,386 @@ +import { + embeddedAgentLog, + getBeforeToolCallPolicyDiagnosticState, + isActiveHarnessContextEngine, + resolveSandboxContext, + resolveSessionAgentIds, + resolveUserPath, + type FastModeAutoProgressState, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime"; +import { + createDiagnosticTraceContextFromActiveScope, + freezeDiagnosticTraceContext, + resolveDiagnosticModelContentCapturePolicy, +} from "openclaw/plugin-sdk/diagnostic-runtime"; +import { loadExecApprovals } from "openclaw/plugin-sdk/exec-approvals-runtime"; +import { + resolveCodexAppServerForModelProvider, + resolveCodexAppServerForOpenClawToolPolicy, +} from "./app-server-policy.js"; +import { + resolveCodexAppServerAuthProfileId, + resolveCodexAppServerAuthProfileIdForAgent, + resolveCodexAppServerPreparedAuthHandoff, +} from "./auth-bridge.js"; +import { resolveCodexBindingAppServerConnection } from "./binding-connection.js"; +import { + isCodexAppServerApprovalPolicyAllowedByRequirements, + readCodexPluginConfig, + resolveCodexComputerUseConfig, + resolveCodexModelBackedReviewerPolicyContext, + resolveOpenClawExecPolicyForCodexAppServer, +} from "./config.js"; +import { createCodexDynamicToolBuildStageTracker } from "./dynamic-tool-build.js"; +import { resolveCodexNativeHookRelayEvents } from "./native-hook-relay.js"; +import { isCodexAppServerProfilerEnabled } from "./profiler-flag.js"; +import { ensureCodexWorkspaceDirOnce } from "./run-attempt-lifecycle.js"; +import type { CodexRunAttemptInput } from "./run-attempt-types.js"; +import { + reclaimCurrentCodexSessionGeneration, + sessionBindingIdentity, + type CodexAppServerThreadBinding, +} from "./session-binding.js"; +import { getLeasedSharedCodexAppServerClient } from "./shared-client.js"; +import { rotateOversizedCodexAppServerStartupBinding } from "./startup-binding.js"; + +export async function prepareCodexAttemptConnection({ params, options }: CodexRunAttemptInput) { + const attemptStartedAt = Date.now(); + const profilerEnabled = isCodexAppServerProfilerEnabled(params.config); + const codexModelCallTrace = freezeDiagnosticTraceContext( + createDiagnosticTraceContextFromActiveScope(), + ); + const codexModelContentCapture = resolveDiagnosticModelContentCapturePolicy(params.config); + const codexModelCallId = `${params.runId}:codex-model:1`; + const fastModeAutoStartedAtMs = + typeof params.fastModeStartedAtMs === "number" && Number.isFinite(params.fastModeStartedAtMs) + ? params.fastModeStartedAtMs + : undefined; + const fastModeAutoProgressState: FastModeAutoProgressState = params.fastModeAutoProgressState ?? { + offAnnounced: false, + resetAnnounced: false, + }; + const preDynamicStartupStages = createCodexDynamicToolBuildStageTracker({ + enabled: profilerEnabled, + }); + const attemptClientFactory = options.clientFactory ?? getLeasedSharedCodexAppServerClient; + const runtimeArtifactRequest = + params.captureRuntimeArtifact || params.expectedRuntimeArtifact + ? params.expectedRuntimeArtifact + ? { expected: params.expectedRuntimeArtifact } + : {} + : undefined; + const pluginConfig = readCodexPluginConfig(options.pluginConfig); + const computerUseConfig = resolveCodexComputerUseConfig({ pluginConfig }); + const { sessionAgentId } = resolveSessionAgentIds({ + sessionKey: params.sessionKey, + config: params.config, + agentId: params.agentId, + }); + const beforeToolCallPolicy = getBeforeToolCallPolicyDiagnosticState(); + preDynamicStartupStages.mark("config"); + const resolvedWorkspace = resolveUserPath(params.workspaceDir); + await ensureCodexWorkspaceDirOnce(resolvedWorkspace); + preDynamicStartupStages.mark("workspace"); + const sandboxSessionKey = + params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId; + const contextSessionKey = params.sessionKey?.trim() || sandboxSessionKey; + const sandbox = await resolveSandboxContext({ + config: params.config, + sessionKey: sandboxSessionKey, + workspaceDir: resolvedWorkspace, + }); + preDynamicStartupStages.mark("sandbox"); + const execPolicy = resolveOpenClawExecPolicyForCodexAppServer({ + execOverrides: params.execOverrides, + approvals: loadExecApprovals(), + config: params.config, + agentId: sessionAgentId, + }); + const agentDir = params.agentDir ?? resolveAgentDir(params.config ?? {}, sessionAgentId); + const bindingIdentity = sessionBindingIdentity({ + sessionId: params.sessionId, + sessionKey: params.sessionKey, + agentId: params.agentId, + config: params.config, + }); + const bindingStore = options.bindingStore; + preDynamicStartupStages.mark("session-agent"); + let activeContextEngine = isActiveHarnessContextEngine(params.contextEngine) + ? params.contextEngine + : undefined; + const isInactiveThreadBootstrapBinding = (binding: CodexAppServerThreadBinding | undefined) => + !activeContextEngine && binding?.contextEngine?.projection?.mode === "thread_bootstrap"; + let startupBinding = await bindingStore.read(bindingIdentity); + if (!startupBinding && bindingIdentity.kind === "session" && bindingIdentity.sessionKey) { + const reclaimed = await reclaimCurrentCodexSessionGeneration({ + bindingStore, + identity: bindingIdentity, + config: params.config, + }); + if (!reclaimed) { + throw new Error( + `Codex session generation is no longer current: ${bindingIdentity.sessionId}`, + ); + } + startupBinding = await bindingStore.read(bindingIdentity); + } + preDynamicStartupStages.mark("read-binding"); + const usesSupervisionConnection = startupBinding?.connectionScope === "supervision"; + if (usesSupervisionConnection) { + activeContextEngine = undefined; + } + if (usesSupervisionConnection && pluginConfig.supervision?.enabled !== true) { + throw new Error( + "Codex supervision is disabled; refusing to open a native user-home supervised session", + ); + } + const resolveRuntimeOptionsForBinding = (selection: { modelProvider?: string; model?: string }) => + resolveCodexBindingAppServerConnection({ + binding: startupBinding, + pluginConfig, + execPolicy, + modelProvider: selection.modelProvider, + model: selection.model, + config: params.config, + agentDir, + openClawSandboxActive: sandbox?.enabled === true, + }).appServer; + const initialStartupBindingHadInactiveThreadBootstrap = + isInactiveThreadBootstrapBinding(startupBinding); + const preparedAuthRoute = usesSupervisionConnection + ? undefined + : params.runtimePlan?.auth.modelRoute; + const startupAuthProfileCandidate = usesSupervisionConnection + ? undefined + : preparedAuthRoute + ? params.runtimePlan?.auth.forwardedAuthProfileId + : (params.runtimePlan?.auth.forwardedAuthProfileId ?? + params.authProfileId ?? + startupBinding?.authProfileId); + const resolvedStartupAuthProfileId = usesSupervisionConnection + ? undefined + : preparedAuthRoute + ? startupAuthProfileCandidate + : params.authProfileStore + ? resolveCodexAppServerAuthProfileId({ + authProfileId: startupAuthProfileCandidate, + store: params.authProfileStore, + config: params.config, + }) + : resolveCodexAppServerAuthProfileIdForAgent({ + authProfileId: startupAuthProfileCandidate, + agentDir, + config: params.config, + }); + const authHandoff = usesSupervisionConnection + ? { authProfileId: undefined, nativeAuthProfile: true, preparedAuth: undefined } + : await resolveCodexAppServerPreparedAuthHandoff({ + authRequirement: preparedAuthRoute?.authRequirement, + resolvedApiKey: params.resolvedApiKey, + authProfileId: resolvedStartupAuthProfileId, + authProfileStore: params.authProfileStore, + agentDir, + config: params.config, + subscriptionProfileRequiredError: + "Prepared Codex subscription route requires a forwarded OpenAI OAuth or token profile.", + subscriptionProfileUnusableError: "Prepared Codex subscription auth profile is unusable.", + }); + const { + authProfileId: startupAuthProfileId, + nativeAuthProfile, + preparedAuth: startupPreparedAuth, + } = authHandoff; + const startupClientAuthProfileId = + usesSupervisionConnection || startupPreparedAuth?.kind === "api-key" + ? null + : startupAuthProfileId; + const resolveReviewerPolicyContext = (binding: CodexAppServerThreadBinding | undefined) => { + const nativeModelOwned = binding?.preserveNativeModel === true; + return resolveCodexModelBackedReviewerPolicyContext({ + provider: nativeModelOwned ? "codex" : params.provider, + model: nativeModelOwned ? binding.model : params.modelId, + bindingModelProvider: binding?.modelProvider, + bindingModel: binding?.model, + nativeAuthProfile, + }); + }; + let reviewerPolicyContext = resolveReviewerPolicyContext(startupBinding); + preDynamicStartupStages.mark("auth-profile"); + let configuredAppServer = resolveRuntimeOptionsForBinding({ + modelProvider: reviewerPolicyContext.modelProvider, + model: reviewerPolicyContext.model, + }); + const effectiveWorkspace = sandbox?.enabled + ? sandbox.workspaceAccess === "rw" + ? resolvedWorkspace + : sandbox.workspaceDir + : resolvedWorkspace; + const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined; + if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) { + throw new Error( + "cwd override is not supported for sandboxed Codex app-server runs; omit cwd or use the agent workspace as cwd", + ); + } + const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace); + await ensureCodexWorkspaceDirOnce(effectiveWorkspace); + preDynamicStartupStages.mark("effective-workspace"); + const resolvePolicyAppServer = () => + resolveCodexAppServerForOpenClawToolPolicy({ + appServer: configuredAppServer, + pluginConfig, + env: process.env, + shouldPromote: + beforeToolCallPolicy.hasBeforeToolCallHook || + beforeToolCallPolicy.trustedToolPolicies.length > 0, + execPolicy, + canUseUntrustedApprovalPolicy: + configuredAppServer.start.transport !== "stdio" || + isCodexAppServerApprovalPolicyAllowedByRequirements("untrusted"), + }); + let policyAppServer = resolvePolicyAppServer(); + let appServer = resolveCodexAppServerForModelProvider({ + appServer: policyAppServer, + provider: reviewerPolicyContext.modelProvider, + model: reviewerPolicyContext.model, + config: params.config, + env: process.env, + agentDir, + }); + if (configuredAppServer.approvalPolicy === "never" && appServer.approvalPolicy === "untrusted") { + embeddedAgentLog.info("codex app-server approval policy promoted for OpenClaw tool policy", { + from: "never", + to: "untrusted", + beforeToolCallHook: beforeToolCallPolicy.hasBeforeToolCallHook, + trustedToolPolicies: beforeToolCallPolicy.trustedToolPolicies, + }); + } + preDynamicStartupStages.mark("app-server-policy"); + preDynamicStartupStages.mark("native-hook-relay"); + const terminalState = { + explicitCancellationObserved: false, + explicitCancellationReason: undefined as unknown, + terminalOutcomeFrozen: false, + sharedAbortAllowedAfterTerminalOutcome: false, + }; + const runAbortController = new AbortController(); + let attemptAbortNotified = false; + const notifyAttemptAbort = () => { + if (attemptAbortNotified) { + return; + } + attemptAbortNotified = true; + params.onAttemptAbort?.(); + }; + const abortExplicitly = (reason: unknown) => { + if (terminalState.terminalOutcomeFrozen) { + if (terminalState.sharedAbortAllowedAfterTerminalOutcome) { + notifyAttemptAbort(); + } + return; + } + notifyAttemptAbort(); + terminalState.explicitCancellationObserved = true; + terminalState.explicitCancellationReason ??= reason; + runAbortController.abort(reason); + }; + const abortFromUpstream = () => { + abortExplicitly(params.abortSignal?.reason ?? "upstream_abort"); + }; + if (params.abortSignal?.aborted) { + abortFromUpstream(); + } else { + params.abortSignal?.addEventListener("abort", abortFromUpstream, { once: true }); + } + startupBinding = await rotateOversizedCodexAppServerStartupBinding({ + binding: startupBinding, + bindingStore, + identity: bindingIdentity, + sessionFile: params.sessionFile, + agentDir, + codexHome: appServer.start.env?.CODEX_HOME, + config: params.config, + contextEngineActive: Boolean(activeContextEngine), + }); + const initialInactiveThreadBootstrapBindingForcedFreshStart = + initialStartupBindingHadInactiveThreadBootstrap && !startupBinding?.threadId; + preDynamicStartupStages.mark("rotate-binding"); + reviewerPolicyContext = resolveReviewerPolicyContext(startupBinding); + configuredAppServer = resolveRuntimeOptionsForBinding({ + modelProvider: reviewerPolicyContext.modelProvider, + model: reviewerPolicyContext.model, + }); + policyAppServer = resolvePolicyAppServer(); + appServer = resolveCodexAppServerForModelProvider({ + appServer: policyAppServer, + provider: reviewerPolicyContext.modelProvider, + model: reviewerPolicyContext.model, + config: params.config, + env: process.env, + agentDir, + }); + const nativeHookRelayEvents = resolveCodexNativeHookRelayEvents({ + configuredEvents: options.nativeHookRelay?.events, + appServer, + }); + const mutable = { startupBinding, pluginAppServer: appServer }; + const resolveRuntimeOptionsForCurrentBinding = (selection: { + modelProvider?: string; + model?: string; + }) => + resolveCodexBindingAppServerConnection({ + binding: mutable.startupBinding, + pluginConfig, + execPolicy, + modelProvider: selection.modelProvider, + model: selection.model, + config: params.config, + agentDir, + openClawSandboxActive: sandbox?.enabled === true, + }).appServer; + return { + params, + options, + attemptStartedAt, + profilerEnabled, + codexModelCallTrace, + codexModelContentCapture, + codexModelCallId, + fastModeAutoStartedAtMs, + fastModeAutoProgressState, + preDynamicStartupStages, + attemptClientFactory, + runtimeArtifactRequest, + pluginConfig, + computerUseConfig, + sessionAgentId, + resolvedWorkspace, + sandboxSessionKey, + contextSessionKey, + sandbox, + agentDir, + bindingIdentity, + bindingStore, + activeContextEngine, + isInactiveThreadBootstrapBinding, + usesSupervisionConnection, + startupAuthProfileId, + startupPreparedAuth, + startupClientAuthProfileId, + effectiveWorkspace, + effectiveCwd, + appServer, + nativeHookRelayEvents, + runAbortController, + terminalState, + abortExplicitly, + abortFromUpstream, + resolveReviewerPolicyContext, + resolveRuntimeOptionsForCurrentBinding, + mutable, + initialStartupBindingHadInactiveThreadBootstrap, + initialInactiveThreadBootstrapBindingForcedFreshStart, + }; +} + +export type CodexAttemptConnection = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-context.ts b/extensions/codex/src/app-server/run-attempt-context.ts new file mode 100644 index 000000000000..ecb95fb9e5af --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-context.ts @@ -0,0 +1,188 @@ +import { + bootstrapHarnessContextEngine, + buildHarnessContextEngineRuntimeContext, + CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + embeddedAgentLog, + getAgentHarnessHookRunner, + resolveContextEngineOwnerPluginId, + runHarnessContextEngineMaintenance, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + buildCodexOpenClawPromptContext, + buildCodexWorkspaceBootstrapContext, + getCodexWorkspaceMemoryToolNames, + readMirroredSessionHistoryMessages, + renderCodexSkillsCollaborationInstructions, +} from "./attempt-context.js"; +import { + resolveCodexContextEngineProjectionMaxChars, + resolveCodexContextEngineProjectionReserveTokens, + type CodexProjectedContextRange, +} from "./context-engine-projection.js"; +import type { CodexAttemptRuntime } from "./run-attempt-runtime.js"; +import { joinPresentSections } from "./run-attempt-state.js"; +import type { CodexAttemptTools } from "./run-attempt-tool-setup.js"; +import { + buildDeveloperInstructions, + type CodexContextEngineThreadBootstrapProjection, +} from "./thread-lifecycle.js"; + +export async function prepareCodexAttemptContext( + runtime: CodexAttemptRuntime, + attemptTools: CodexAttemptTools, +) { + const { + connection, + runtimeParams, + activeSessionId, + activeSessionFile, + buildActiveRunAttemptParams, + effectiveContextWindowInfo, + effectiveContextTokenBudget, + effectiveRuntimeProviderId, + effectiveRuntimeModelId, + hookChannelId, + } = runtime; + const { + params, + sessionAgentId, + contextSessionKey, + activeContextEngine, + initialStartupBindingHadInactiveThreadBootstrap, + sandboxSessionKey, + effectiveWorkspace, + effectiveCwd, + agentDir, + usesSupervisionConnection, + resolvedWorkspace, + initialInactiveThreadBootstrapBindingForcedFreshStart, + } = connection; + const { toolBridge } = attemptTools; + const activeTranscriptTarget = { + agentId: sessionAgentId, + sessionFile: activeSessionFile, + sessionId: activeSessionId, + sessionKey: contextSessionKey, + }; + const historyState = { + messages: + !activeContextEngine && initialStartupBindingHadInactiveThreadBootstrap + ? [] + : ((await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? []), + }; + const hadSessionTranscriptState = historyState.messages.length > 0; + const hookContextWindowFields = { + ...(effectiveContextWindowInfo?.tokens + ? { contextTokenBudget: effectiveContextWindowInfo.tokens } + : effectiveContextTokenBudget + ? { contextTokenBudget: effectiveContextTokenBudget } + : {}), + ...(effectiveContextWindowInfo?.source + ? { contextWindowSource: effectiveContextWindowInfo.source } + : {}), + ...(effectiveContextWindowInfo?.referenceTokens + ? { contextWindowReferenceTokens: effectiveContextWindowInfo.referenceTokens } + : {}), + }; + const hookContext = { + runId: params.runId, + agentId: sessionAgentId, + sessionKey: sandboxSessionKey, + sessionId: params.sessionId, + workspaceDir: params.workspaceDir, + messageProvider: params.messageProvider ?? undefined, + trigger: params.trigger, + channelId: hookChannelId, + ...hookContextWindowFields, + }; + const hookRunner = getAgentHarnessHookRunner(); + const activeContextEnginePluginId = activeContextEngine + ? resolveContextEngineOwnerPluginId(activeContextEngine) + : undefined; + const buildActiveContextEngineRuntimeContext = () => + buildHarnessContextEngineRuntimeContext({ + attempt: buildActiveRunAttemptParams(), + workspaceDir: effectiveWorkspace, + cwd: effectiveCwd, + agentDir, + activeAgentId: sessionAgentId, + contextEnginePluginId: activeContextEnginePluginId, + tokenBudget: effectiveContextTokenBudget, + }); + if (activeContextEngine) { + await bootstrapHarnessContextEngine({ + hadSessionFile: hadSessionTranscriptState, + contextEngine: activeContextEngine, + sessionId: activeSessionId, + sessionKey: contextSessionKey, + sessionFile: activeSessionFile, + sessionTarget: params.sessionTarget, + runtimeContext: buildActiveContextEngineRuntimeContext(), + contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + providerId: effectiveRuntimeProviderId, + requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, + modelId: effectiveRuntimeModelId, + fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, + degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, + runMaintenance: runHarnessContextEngineMaintenance, + config: params.config, + warn: (message) => embeddedAgentLog.warn(message), + }); + historyState.messages = + (await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? historyState.messages; + } + const memoryToolNames = getCodexWorkspaceMemoryToolNames(toolBridge.availableSpecs); + const workspaceBootstrapContext = await buildCodexWorkspaceBootstrapContext({ + params: runtimeParams, + resolvedWorkspace, + effectiveWorkspace, + sessionKey: contextSessionKey, + sessionAgentId, + memoryToolNames, + }); + const baseDeveloperInstructions = joinPresentSections( + buildDeveloperInstructions(runtimeParams, { dynamicTools: toolBridge.availableSpecs }), + workspaceBootstrapContext.developerInstructions, + ); + const openClawPromptContext = buildCodexOpenClawPromptContext({ + params: runtimeParams, + workspacePromptContext: workspaceBootstrapContext.promptContext, + }); + const skillsCollaborationInstructions = renderCodexSkillsCollaborationInstructions({ + attempt: runtimeParams, + skillsPrompt: params.skillsSnapshot?.prompt, + }); + const promptState = { + promptText: params.prompt, + promptContextRange: undefined as CodexProjectedContextRange | undefined, + developerInstructions: baseDeveloperInstructions, + prePromptMessageCount: historyState.messages.length, + contextEngineProjection: undefined as CodexContextEngineThreadBootstrapProjection | undefined, + precomputedStaleBindingContinuityProjectionApplied: false, + staleBindingContinuityForcedFreshStart: false, + inactiveThreadBootstrapBindingForcedFreshStart: + initialInactiveThreadBootstrapBindingForcedFreshStart, + }; + const codexContextProjectionMaxChars = resolveCodexContextEngineProjectionMaxChars({ + contextTokenBudget: effectiveContextTokenBudget, + reserveTokens: resolveCodexContextEngineProjectionReserveTokens({ config: params.config }), + }); + return { + runtime, + attemptTools, + activeTranscriptTarget, + historyState, + hookContext, + hookContextWindowFields, + hookRunner, + buildActiveContextEngineRuntimeContext, + workspaceBootstrapContext, + baseDeveloperInstructions, + openClawPromptContext, + skillsCollaborationInstructions, + promptState, + codexContextProjectionMaxChars, + }; +} + +export type CodexAttemptContext = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-finalize.ts b/extensions/codex/src/app-server/run-attempt-finalize.ts new file mode 100644 index 000000000000..9939af441117 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-finalize.ts @@ -0,0 +1,468 @@ +import { + buildHarnessContextEngineRuntimeContextFromUsage, + CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + embeddedAgentLog, + finalizeHarnessContextEngineTurn, + formatErrorMessage, + resolveContextEngineOwnerPluginId, + runAgentHarnessLlmOutputHook, + runHarnessContextEngineMaintenance, + type EmbeddedRunAttemptResult, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { readMirroredSessionHistoryMessages } from "./attempt-context.js"; +import { classifyCodexModelCallFailureKind } from "./attempt-diagnostics.js"; +import { + buildCodexAppServerPromptTimeoutOutcome, + collectTerminalAssistantText, + isInvalidCodexImagePayloadError, + resolveCodexAppServerReplayBlockedReason, +} from "./attempt-results.js"; +import type { CodexAttemptActiveTurn } from "./run-attempt-active-turn.js"; +import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import { + emitCodexAppServerEvent, + runCodexAgentEndHook, + shouldKeepCodexSharedAbortOpen, +} from "./run-attempt-lifecycle.js"; +import type { CodexAttemptNotificationController } from "./run-attempt-notification-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { + buildCodexAppServerTimeoutDiagnostics, + clearCodexBindingAfterInvalidImagePayload, + markCodexAppServerBindingCoveredThroughTurn, + shouldUseFreshCodexThreadAfterContextEngineOverflow, +} from "./run-attempt-state.js"; +import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { normalizeCodexTrajectoryError, recordCodexTrajectoryCompletion } from "./trajectory.js"; +import { codexTranscriptMirrorRuntime } from "./transcript-mirror.js"; +import { refreshCodexUsageLimitPromptError } from "./usage-limit-error.js"; + +export async function finalizeCodexAttempt( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + lifecycle: CodexAttemptLifecycleController, + notifications: CodexAttemptNotificationController, + requestRuntime: Awaited>, + activeTurn: CodexAttemptActiveTurn, +): Promise { + const { prompt, state: resourceState, trajectoryRecorder, markTrajectoryEndRecorded } = resources; + const { context, systemPromptReport } = prompt; + const { runtime, attemptTools, activeTranscriptTarget, historyState, hookContext } = context; + const { hookContextWindowFields, hookRunner, promptState } = context; + const { connection, preparedAuthBinding, activeSessionId, activeSessionFile } = runtime; + const { + buildActiveRunAttemptParams, + effectiveContextTokenBudget, + effectiveRuntimeProviderId, + effectiveRuntimeModelId, + } = runtime; + const { + params, + terminalState, + runAbortController, + activeContextEngine, + bindingStore, + bindingIdentity, + appServer, + usesSupervisionConnection, + sessionAgentId, + contextSessionKey, + effectiveCwd, + effectiveWorkspace, + agentDir, + attemptStartedAt, + } = connection; + const { toolBridge, toolState } = attemptTools; + const { + state, + completion, + pendingOpenClawDynamicToolCompletionIds, + activeTurnItemIds, + activeCompletionBlockerItemIds, + activeFinalizationHookRunIds, + turnWatches, + } = turnRuntime; + const { emitLifecycleTerminal, buildLifecycleTerminalMeta } = lifecycle; + const { drainNotificationQueue } = notifications; + const { codexModelCallDiagnostics } = requestRuntime; + const { + activeTurnId, + activeProjector, + streamState, + freezeRunTerminalOutcome, + notifyUserMessagePersisted, + } = activeTurn; + await completion; + // Include projection work already queued when timeout completion wins. + await drainNotificationQueue(); + const hasQuiescentCompletedAssistant = + activeProjector.hasCompletedTerminalAssistantText() && + state.activeAppServerTurnRequests === 0 && + activeTurnItemIds.size === 0 && + activeCompletionBlockerItemIds.size === 0 && + pendingOpenClawDynamicToolCompletionIds.size === 0 && + activeFinalizationHookRunIds.size === 0 && + state.unsettledFinalizationHookCount === 0 && + state.rejectedFinalizationHookAssistant === undefined; + const hasRecoverableCompletedAssistant = + !turnWatches.isCompletionIdleWatchPinnedByTerminalError() && + turnWatches.isAssistantCompletionIdleWatchArmed() && + hasQuiescentCompletedAssistant; + const recoveredTurnWatchTimeout = + state.turnCompletionIdleTimedOut && + !terminalState.explicitCancellationObserved && + !state.terminalTurnNotificationQueued && + hasRecoverableCompletedAssistant && + activeProjector.recoverCompletedTerminalAssistantAfterTurnWatchTimeout(); + if (recoveredTurnWatchTimeout) { + embeddedAgentLog.warn( + "codex app-server recovered completed assistant output after missing turn completion", + { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + timeoutKind: state.turnWatchTimeoutKind, + idleMs: state.turnWatchTimeoutIdleMs, + timeoutMs: state.turnWatchTimeoutMs, + }, + ); + trajectoryRecorder?.recordEvent("turn.watch_timeout_recovered", { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + timeoutKind: state.turnWatchTimeoutKind, + idleMs: state.turnWatchTimeoutIdleMs, + timeoutMs: state.turnWatchTimeoutMs, + }); + } + const result = activeProjector.buildResult(toolBridge.telemetry, { + yieldDetected: toolState.yieldDetected, + }); + const effectiveTimedOut = state.timedOut && !recoveredTurnWatchTimeout; + const effectiveTurnCompletionIdleTimedOut = + state.turnCompletionIdleTimedOut && !recoveredTurnWatchTimeout; + const isFinalAborted = () => + result.aborted || + terminalState.explicitCancellationObserved || + (runAbortController.signal.aborted && !state.clientClosedAbort && !recoveredTurnWatchTimeout); + const clientClosedPromptErrorForFinal = + state.clientClosedPromptError && hasRecoverableCompletedAssistant + ? undefined + : state.clientClosedPromptError; + let finalPromptError = + clientClosedPromptErrorForFinal ?? + (effectiveTurnCompletionIdleTimedOut + ? state.turnCompletionIdleTimeoutMessage + : effectiveTimedOut + ? "codex app-server attempt timed out" + : result.promptError); + const finalPromptErrorMessage = + typeof finalPromptError === "string" + ? finalPromptError + : finalPromptError + ? formatErrorMessage(finalPromptError) + : undefined; + if (isInvalidCodexImagePayloadError(finalPromptErrorMessage)) { + await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, { + phase: "turn_completed", + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + error: finalPromptErrorMessage, + }); + } + if ( + resourceState.thread.connectionScope !== "supervision" && + shouldUseFreshCodexThreadAfterContextEngineOverflow({ + error: finalPromptError, + contextEngineActive: Boolean(activeContextEngine), + thread: resourceState.thread, + }) + ) { + embeddedAgentLog.warn( + "codex app-server context-engine turn overflowed after resume; clearing thread binding for recovery", + { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + error: finalPromptErrorMessage, + }, + ); + await bindingStore.mutate(bindingIdentity, { + kind: "clear", + threadId: resourceState.thread.threadId, + }); + } + const refreshedUsageLimitPromptError = await refreshCodexUsageLimitPromptError({ + client: resourceState.client, + message: finalPromptErrorMessage, + timeoutMs: appServer.requestTimeoutMs, + signal: runAbortController.signal, + }); + if (refreshedUsageLimitPromptError) { + finalPromptError = refreshedUsageLimitPromptError; + } + const finalPromptErrorSource = + effectiveTimedOut || clientClosedPromptErrorForFinal ? "prompt" : result.promptErrorSource; + const codexAppServerFailureKind = clientClosedPromptErrorForFinal + ? "client_closed_before_turn_completed" + : effectiveTurnCompletionIdleTimedOut + ? "turn_completion_idle_timeout" + : undefined; + const replayBlockedReason = codexAppServerFailureKind + ? resolveCodexAppServerReplayBlockedReason(result) + : undefined; + const promptTimeoutOutcome = buildCodexAppServerPromptTimeoutOutcome({ + result, + turnCompletionIdleTimedOut: effectiveTurnCompletionIdleTimedOut, + turnWatchTimeoutKind: state.turnWatchTimeoutKind, + }); + const failureDiagnostics = + codexAppServerFailureKind === "turn_completion_idle_timeout" && + state.turnWatchTimeoutKind === "completion" + ? buildCodexAppServerTimeoutDiagnostics({ + idleMs: state.turnWatchTimeoutIdleMs, + timeoutMs: state.turnWatchTimeoutMs, + lastActivityReason: state.turnWatchTimeoutLastActivityReason, + details: state.turnWatchTimeoutDetails, + }) + : undefined; + const codexAppServerFailure = codexAppServerFailureKind + ? ({ + kind: codexAppServerFailureKind, + ...(codexAppServerFailureKind === "turn_completion_idle_timeout" && + state.turnWatchTimeoutKind + ? { turnWatchTimeoutKind: state.turnWatchTimeoutKind } + : {}), + transport: appServer.start.transport, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + replaySafe: replayBlockedReason === undefined, + ...(replayBlockedReason ? { replayBlockedReason } : {}), + ...(failureDiagnostics ? { diagnostics: failureDiagnostics } : {}), + } satisfies NonNullable) + : undefined; + const finalAborted = isFinalAborted(); + const completedTurnStatus = activeProjector.getCompletedTurnStatus(); + const completedWithoutTerminalNotification = + state.completed && + !state.terminalTurnNotificationQueued && + !state.timedOut && + clientClosedPromptErrorForFinal === undefined; + const attemptSucceeded = + !finalAborted && + !effectiveTimedOut && + (finalPromptError === null || finalPromptError === undefined) && + result.agentHarnessResultClassification === undefined && + (completedTurnStatus === "completed" || + recoveredTurnWatchTimeout || + completedWithoutTerminalNotification); + terminalState.sharedAbortAllowedAfterTerminalOutcome = shouldKeepCodexSharedAbortOpen({ + trigger: params.trigger, + result, + attemptSucceeded, + explicitCancellationObserved: terminalState.explicitCancellationObserved, + }); + // Every terminal observer must see the same immutable outcome. + freezeRunTerminalOutcome(); + const modelCallFailureKind = + classifyCodexModelCallFailureKind({ + error: finalPromptError, + timedOut: effectiveTimedOut, + turnCompletionIdleTimedOut: effectiveTurnCompletionIdleTimedOut, + runAborted: finalAborted, + abortReason: terminalState.explicitCancellationReason ?? runAbortController.signal.reason, + clientClosedAbort: state.clientClosedAbort, + formatError: formatErrorMessage, + }) ?? (finalAborted ? "aborted" : undefined); + if (modelCallFailureKind) { + codexModelCallDiagnostics.emitError( + finalPromptError ?? "codex app-server attempt interrupted", + { + failureKind: modelCallFailureKind, + }, + ); + } else if (finalPromptError) { + codexModelCallDiagnostics.emitError(finalPromptError); + } else { + codexModelCallDiagnostics.emitCompleted(result); + } + const assistantTranscriptOwned = await codexTranscriptMirrorRuntime.mirrorBestEffort({ + params, + agentId: sessionAgentId, + notifyUserMessagePersisted, + result, + sessionKey: contextSessionKey, + cwd: effectiveCwd, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + }); + if (activeContextEngine) { + const contextEnginePluginId = resolveContextEngineOwnerPluginId(activeContextEngine); + const isHeartbeat = + params.bootstrapContextRunKind === "heartbeat" || + params.bootstrapContextRunKind === "commitment-only"; + const finalMessages = + (await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? + historyState.messages.concat(result.messagesSnapshot); + await finalizeHarnessContextEngineTurn({ + contextEngine: activeContextEngine, + promptError: Boolean(finalPromptError), + aborted: finalAborted, + yieldAborted: Boolean(result.yieldDetected), + sessionIdUsed: activeSessionId, + sessionKey: contextSessionKey, + sessionFile: activeSessionFile, + sessionTarget: params.sessionTarget, + messagesSnapshot: finalMessages, + prePromptMessageCount: promptState.prePromptMessageCount, + tokenBudget: effectiveContextTokenBudget, + runtimeContext: buildHarnessContextEngineRuntimeContextFromUsage({ + attempt: buildActiveRunAttemptParams(), + workspaceDir: effectiveWorkspace, + cwd: effectiveCwd, + agentDir, + activeAgentId: sessionAgentId, + contextEnginePluginId, + tokenBudget: effectiveContextTokenBudget, + lastCallUsage: result.attemptUsage, + promptCache: result.promptCache, + }), + contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + providerId: usesSupervisionConnection + ? (resourceState.thread.modelProvider ?? effectiveRuntimeProviderId) + : params.provider, + requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, + modelId: usesSupervisionConnection + ? (resourceState.thread.model ?? effectiveRuntimeModelId) + : params.modelId, + fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, + degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, + runMaintenance: runHarnessContextEngineMaintenance, + config: params.config, + warn: (message) => embeddedAgentLog.warn(message), + isHeartbeat, + }); + } + runAgentHarnessLlmOutputHook({ + event: { + runId: params.runId, + sessionId: params.sessionId, + provider: usesSupervisionConnection + ? (resourceState.thread.modelProvider ?? effectiveRuntimeProviderId) + : params.provider, + model: usesSupervisionConnection + ? (resourceState.thread.model ?? effectiveRuntimeModelId) + : params.modelId, + ...hookContextWindowFields, + resolvedRef: usesSupervisionConnection + ? `${resourceState.thread.modelProvider ?? effectiveRuntimeProviderId}/${resourceState.thread.model ?? effectiveRuntimeModelId}` + : (params.runtimePlan?.observability.resolvedRef ?? `${params.provider}/${params.modelId}`), + ...(!usesSupervisionConnection && params.runtimePlan?.observability.harnessId + ? { harnessId: params.runtimePlan.observability.harnessId } + : {}), + assistantTexts: result.assistantTexts, + ...(result.lastAssistant ? { lastAssistant: result.lastAssistant } : {}), + ...(result.attemptUsage ? { usage: result.attemptUsage } : {}), + }, + ctx: hookContext, + hookRunner, + }); + await runCodexAgentEndHook(params, { + event: { + messages: result.messagesSnapshot, + success: !finalAborted && !finalPromptError, + ...(finalPromptError ? { error: formatErrorMessage(finalPromptError) } : {}), + durationMs: Date.now() - attemptStartedAt, + }, + ctx: hookContext, + hookRunner, + }); + state.shouldDelayNativeHookRelayUnregister = + completedTurnStatus === "completed" && + !effectiveTimedOut && + !runAbortController.signal.aborted && + !finalAborted && + !finalPromptError; + if (state.shouldDelayNativeHookRelayUnregister) { + try { + await markCodexAppServerBindingCoveredThroughTurn({ + bindingStore, + identity: bindingIdentity, + threadId: resourceState.thread.threadId, + }); + } catch (error) { + if (resourceState.thread.connectionScope === "supervision") { + throw error; + } + const cleared = await bindingStore.mutate(bindingIdentity, { + kind: "clear", + threadId: resourceState.thread.threadId, + }); + if (!cleared) { + throw error; + } + embeddedAgentLog.warn( + "codex app-server binding coverage update failed after completed turn; cleared stale binding", + { threadId: resourceState.thread.threadId, turnId: activeTurnId, error }, + ); + } + } + recordCodexTrajectoryCompletion(trajectoryRecorder, { + attempt: params, + result, + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + timedOut: effectiveTimedOut, + yieldDetected: toolState.yieldDetected, + }); + trajectoryRecorder?.recordEvent("session.ended", { + status: finalPromptError + ? "error" + : finalAborted || effectiveTimedOut + ? "interrupted" + : "success", + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + timedOut: effectiveTimedOut, + yieldDetected: toolState.yieldDetected, + promptError: normalizeCodexTrajectoryError(finalPromptError), + }); + markTrajectoryEndRecorded(); + const terminalAssistantText = collectTerminalAssistantText(result); + if ( + terminalAssistantText && + (!streamState.eventEmitted || streamState.needsTerminalSnapshot) && + !finalAborted && + !finalPromptError + ) { + void emitCodexAppServerEvent(params, { + stream: "assistant", + data: { text: terminalAssistantText }, + }); + } + emitLifecycleTerminal( + finalPromptError + ? { + phase: "error", + error: formatErrorMessage(finalPromptError), + ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), + } + : { + phase: "end", + ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), + }, + ); + return { + ...result, + timedOut: effectiveTimedOut, + aborted: finalAborted, + promptError: finalPromptError, + promptErrorSource: finalPromptErrorSource, + ...(codexAppServerFailure ? { codexAppServerFailure } : {}), + ...(promptTimeoutOutcome ? { promptTimeoutOutcome } : {}), + ...(assistantTranscriptOwned ? { assistantTranscriptOwned: true } : {}), + ...(resourceState.runtimeArtifact ? { runtimeArtifact: resourceState.runtimeArtifact } : {}), + ...(!finalAborted && !effectiveTimedOut && !finalPromptError && preparedAuthBinding + ? { authBindingFingerprint: preparedAuthBinding.fingerprint } + : {}), + systemPromptReport, + }; +} diff --git a/extensions/codex/src/app-server/run-attempt-lifecycle-controller.ts b/extensions/codex/src/app-server/run-attempt-lifecycle-controller.ts new file mode 100644 index 000000000000..5be651f85e44 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-lifecycle-controller.ts @@ -0,0 +1,262 @@ +import { + embeddedAgentLog, + FAST_MODE_AUTO_PROGRESS_KIND, + formatErrorMessage, + formatFastModeAutoProgressText, + resolveAgentRunAbortLifecycleFields, + resolveFastModeForElapsed, + type EmbeddedRunAttemptParams, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + interruptCodexTurnBestEffort, +} from "./attempt-client-cleanup.js"; +import { reportCodexExecutionNotification } from "./attempt-notification-state.js"; +import { + resolveTerminalDynamicToolBatchAction, + shouldReleaseTurnAfterTerminalDynamicTool, +} from "./dynamic-tool-execution.js"; +import type { + CodexDynamicToolCallParams, + CodexDynamicToolCallResponse, + CodexServerNotification, +} from "./protocol.js"; +import { emitCodexAppServerEvent } from "./run-attempt-lifecycle.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; + +export function createCodexAttemptLifecycleController( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, +) { + const { prompt, state: resourceState, trajectoryRecorder } = resources; + const { connection } = prompt.context.runtime; + const { + params, + attemptStartedAt, + runAbortController, + fastModeAutoStartedAtMs, + fastModeAutoProgressState, + } = connection; + const { state, activeTurnItemIds, pendingOpenClawDynamicToolCompletionIds, turnWatches } = + turnRuntime; + const releaseTurnAfterTerminalDynamicTool = (value: { + call: CodexDynamicToolCallParams; + response: CodexDynamicToolCallResponse; + durationMs: number; + }) => { + if ( + !shouldReleaseTurnAfterTerminalDynamicTool({ + completed: state.completed, + aborted: runAbortController.signal.aborted, + responseSuccess: value.response.success, + currentTurnHadNonTerminalDynamicToolResult: + state.currentTurnHadNonTerminalDynamicToolResult, + activeAppServerTurnRequests: state.activeAppServerTurnRequests, + activeTurnItemIdsCount: activeTurnItemIds.size, + pendingOpenClawDynamicToolCompletionIdsCount: pendingOpenClawDynamicToolCompletionIds.size, + }) + ) { + return; + } + state.pendingTerminalDynamicToolRelease = undefined; + trajectoryRecorder?.recordEvent("turn.dynamic_tool_terminal_release", { + threadId: value.call.threadId, + turnId: value.call.turnId, + toolCallId: value.call.callId, + name: value.call.tool, + durationMs: value.durationMs, + }); + embeddedAgentLog.info("codex app-server turn released after terminal dynamic tool result", { + threadId: value.call.threadId, + turnId: value.call.turnId, + toolCallId: value.call.callId, + tool: value.call.tool, + durationMs: value.durationMs, + }); + interruptCodexTurnBestEffort(resourceState.client, { + threadId: value.call.threadId, + turnId: value.call.turnId, + timeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + }); + state.completed = true; + turnWatches.clearCompletionIdleTimer(); + turnWatches.clearAssistantCompletionIdleTimer(); + turnWatches.clearTerminalIdleTimer(); + state.resolveCompletion?.(); + }; + const scheduleTerminalDynamicToolReleaseCheck = () => { + if ( + state.terminalDynamicToolReleaseCheckScheduled || + (!state.pendingTerminalDynamicToolRelease && + !state.currentTurnHadNonTerminalDynamicToolResult) + ) { + return; + } + // The JSON-RPC response must flush before the terminal tool interrupts its turn. + state.terminalDynamicToolReleaseCheckScheduled = true; + const immediate = setImmediate(() => { + state.terminalDynamicToolReleaseCheckScheduled = false; + const action = resolveTerminalDynamicToolBatchAction({ + activeAppServerTurnRequests: state.activeAppServerTurnRequests, + activeTurnItemIdsCount: activeTurnItemIds.size, + pendingOpenClawDynamicToolCompletionIdsCount: pendingOpenClawDynamicToolCompletionIds.size, + currentTurnHadNonTerminalDynamicToolResult: + state.currentTurnHadNonTerminalDynamicToolResult, + hasPendingTerminalDynamicToolRelease: state.pendingTerminalDynamicToolRelease !== undefined, + }); + if (action === "release-pending-terminal" && state.pendingTerminalDynamicToolRelease) { + releaseTurnAfterTerminalDynamicTool(state.pendingTerminalDynamicToolRelease); + } else if (action === "clear-nonterminal-batch") { + state.pendingTerminalDynamicToolRelease = undefined; + state.currentTurnHadNonTerminalDynamicToolResult = false; + } + }); + immediate.unref?.(); + }; + const scheduleTurnReleaseAfterTerminalDynamicTool = (value: { + call: CodexDynamicToolCallParams; + response: CodexDynamicToolCallResponse; + durationMs: number; + }) => { + state.pendingTerminalDynamicToolRelease = value; + scheduleTerminalDynamicToolReleaseCheck(); + }; + const emitLifecycleStart = () => { + void emitCodexAppServerEvent(params, { + stream: "lifecycle", + data: { phase: "start", startedAt: attemptStartedAt }, + }); + state.lifecycleStarted = true; + }; + const emitLifecycleTerminal = (data: Record & { phase: "end" | "error" }) => { + if (!state.lifecycleStarted || state.lifecycleTerminalEmitted) { + return; + } + void emitCodexAppServerEvent(params, { + stream: "lifecycle", + data: { + startedAt: attemptStartedAt, + endedAt: Date.now(), + ...data, + ...((params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd) + ? { phase: "finishing" } + : {}), + }, + }); + state.lifecycleTerminalEmitted = true; + }; + const buildLifecycleTerminalMeta = (input: { aborted: boolean; timedOut: boolean }) => { + const abortFields = input.aborted + ? resolveAgentRunAbortLifecycleFields(runAbortController.signal) + : undefined; + if (input.timedOut || abortFields?.stopReason === "timeout") { + return { + aborted: true, + status: "timed_out", + stopReason: "timeout", + timeoutPhase: "provider", + providerStarted: true, + } as const; + } + return input.aborted + ? ({ aborted: true, status: "cancelled", stopReason: "stop" } as const) + : undefined; + }; + const executionPhaseKeys = new Set(); + const emitExecutionPhaseOnce = ( + key: string, + info: Parameters>[0], + ) => { + if (executionPhaseKeys.has(key)) { + return; + } + executionPhaseKeys.add(key); + params.onExecutionPhase?.({ + provider: params.provider, + model: params.modelId, + backend: "codex-app-server", + ...info, + }); + }; + const reportExecutionNotification = (notification: CodexServerNotification) => { + reportCodexExecutionNotification({ notification, emitExecutionPhaseOnce }); + }; + const emitFastModeAutoProgress = async (payload: { + enabled: boolean; + elapsedSeconds: number; + fastAutoOnSeconds?: number; + }) => { + const summary = formatFastModeAutoProgressText(payload); + await emitCodexAppServerEvent(params, { + stream: "item", + data: { kind: "status", title: "Fast", phase: "update", summary }, + }); + try { + await params.onToolResult?.({ + text: summary, + channelData: { openclawProgressKind: FAST_MODE_AUTO_PROGRESS_KIND }, + }); + } catch (error) { + embeddedAgentLog.debug("codex app-server fast mode auto progress delivery failed", { error }); + } + }; + const maybeAnnounceFastModeAutoOff = async () => { + if ( + params.fastModeAuto !== true || + fastModeAutoStartedAtMs === undefined || + fastModeAutoProgressState.offAnnounced + ) { + return; + } + const next = resolveFastModeForElapsed({ + mode: "auto", + startedAtMs: fastModeAutoStartedAtMs, + fastAutoOnSeconds: params.fastModeAutoOnSeconds, + }); + if (next.enabled) { + return; + } + fastModeAutoProgressState.offAnnounced = true; + await emitFastModeAutoProgress(next); + }; + const maybeEmitFastModeAutoReset = async () => { + if ( + params.fastModeAuto !== true || + !fastModeAutoProgressState.offAnnounced || + fastModeAutoProgressState.resetAnnounced + ) { + return; + } + fastModeAutoProgressState.resetAnnounced = true; + await emitFastModeAutoProgress({ + enabled: true, + elapsedSeconds: 0, + fastAutoOnSeconds: params.fastModeAutoOnSeconds, + }); + }; + const maybeEmitFastModeAutoResetBestEffort = async () => { + try { + await maybeEmitFastModeAutoReset(); + } catch (error) { + embeddedAgentLog.warn( + `codex app-server fast mode auto reset progress failed: ${formatErrorMessage(error)}`, + ); + } + }; + return { + scheduleTerminalDynamicToolReleaseCheck, + scheduleTurnReleaseAfterTerminalDynamicTool, + emitLifecycleStart, + emitLifecycleTerminal, + buildLifecycleTerminalMeta, + emitExecutionPhaseOnce, + reportExecutionNotification, + maybeAnnounceFastModeAutoOff, + maybeEmitFastModeAutoResetBestEffort, + }; +} + +export type CodexAttemptLifecycleController = ReturnType< + typeof createCodexAttemptLifecycleController +>; diff --git a/extensions/codex/src/app-server/run-attempt-notification-controller.ts b/extensions/codex/src/app-server/run-attempt-notification-controller.ts new file mode 100644 index 000000000000..382196b96e26 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-notification-controller.ts @@ -0,0 +1,266 @@ +import { embeddedAgentLog } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + applyCodexTurnNotificationState, + isTerminalCodexTurnNotificationForTurn, +} from "./attempt-notification-state.js"; +import { + describeNotificationActivity, + isAssistantCompletionReleaseNotification, + isRawFunctionToolOutputCompletionNotification, + readCodexNotificationItem, + readRawResponseToolCallId, +} from "./attempt-notifications.js"; +import type { CodexServerNotification } from "./protocol.js"; +import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { + readCodexFinalizationHookNotification, + waitForCodexNotificationDispatchTurn, +} from "./run-attempt-state.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { CODEX_APP_SERVER_NATIVE_TURN_WAIT_TIMEOUT_MS } from "./turn-router.js"; +import type { CodexThreadRouteScope } from "./turn-router.js"; + +export function createCodexAttemptNotificationController( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + lifecycle: CodexAttemptLifecycleController, +) { + const { prompt, state: resourceState, projectorRef, registerNativeSubagentMonitor } = resources; + const { context, turnState } = prompt; + const { attemptTools, runtime } = context; + const { connection } = runtime; + const { appServer, runAbortController } = connection; + const { allocateCodexToolOutcomeOrdinal } = attemptTools; + const { + state, + turnIdRef, + userInputBridgeRef, + steeringQueueRef, + turnWatches, + activeTurnItemIds, + activeCompletionBlockerItemIds, + activeFinalizationHookRunIds, + finalizationHookBatchStatuses, + pendingOpenClawDynamicToolCompletionIds, + postToolRawAssistantCompletionIdleTimeoutMs, + } = turnRuntime; + const { + scheduleTerminalDynamicToolReleaseCheck, + reportExecutionNotification, + maybeAnnounceFastModeAutoOff, + } = lifecycle; + const isTerminalTurnNotificationForTurn = ( + notification: CodexServerNotification, + notificationTurnId: string, + ) => + isTerminalCodexTurnNotificationForTurn({ + notification, + threadId: resourceState.thread.threadId, + turnId: notificationTurnId, + currentPromptTexts: [turnState.codexTurnPromptText], + }); + const handleNotification = async (notification: CodexServerNotification) => { + const projector = projectorRef.current; + const turnId = turnIdRef.current; + const steeringQueue = steeringQueueRef.current; + userInputBridgeRef.current?.handleNotification(notification); + if (!projector || !turnId) { + if (notification.method === "error") { + state.latestStartupErrorNotification = notification; + } + return; + } + const notificationState = applyCodexTurnNotificationState({ + notification, + threadId: resourceState.thread.threadId, + turnId, + currentPromptTexts: [turnState.codexTurnPromptText], + turnWatches, + activeTurnItemIds, + activeCompletionBlockerItemIds, + activeAppServerTurnRequests: state.activeAppServerTurnRequests, + pendingOpenClawDynamicToolCompletionIds, + turnCrossedToolHandoff: state.turnCrossedToolHandoff, + postToolRawAssistantCompletionIdleTimeoutMs, + onScheduleTerminalDynamicToolReleaseCheck: scheduleTerminalDynamicToolReleaseCheck, + onReportExecutionNotification: reportExecutionNotification, + }); + state.turnCrossedToolHandoff = notificationState.turnCrossedToolHandoff; + const hookNotification = readCodexFinalizationHookNotification( + notification, + resourceState.thread.threadId, + turnId, + ); + if (hookNotification?.phase === "started") { + if (activeFinalizationHookRunIds.size === 0) { + finalizationHookBatchStatuses.clear(); + } + activeFinalizationHookRunIds.add(hookNotification.runId); + turnWatches.disarmAssistantCompletionIdleWatch(); + } + if (notificationState.isTurnTerminal) { + state.terminalTurnNotificationQueued = true; + } + try { + await waitForCodexNotificationDispatchTurn(); + await projector.handleNotification(notification); + const canRelease = + isAssistantCompletionReleaseNotification(notification, state.turnCrossedToolHandoff) || + (notificationState.isCurrentTurnNotification && + state.turnCrossedToolHandoff && + notification.method === "rawResponseItem/completed" && + projector.canReleaseLatestTerminalAssistantAfterToolHandoff()); + if (notificationState.isCurrentTurnNotification && canRelease) { + const itemId = projector.getLatestTerminalAssistantCandidate()?.itemId; + if ( + state.rejectedFinalizationHookAssistant && + itemId && + itemId !== state.rejectedFinalizationHookAssistant.itemId + ) { + state.rejectedFinalizationHookAssistant = undefined; + } else if (state.rejectedFinalizationHookAssistant) { + turnWatches.disarmAssistantCompletionIdleWatch(); + } else if ( + activeFinalizationHookRunIds.size === 0 && + !state.terminalTurnNotificationQueued && + state.activeAppServerTurnRequests === 0 && + activeTurnItemIds.size === 0 && + activeCompletionBlockerItemIds.size === 0 && + pendingOpenClawDynamicToolCompletionIds.size === 0 && + projector.hasLatestTerminalAssistantCandidateText() + ) { + turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification)); + } + } + if ( + notificationState.isCurrentTurnNotification && + activeTurnItemIds.size === 0 && + isRawFunctionToolOutputCompletionNotification(notification) + ) { + await maybeAnnounceFastModeAutoOff(); + } + } catch (error) { + embeddedAgentLog.debug("codex app-server projector notification threw", { + method: notification.method, + error, + }); + } finally { + if (hookNotification?.phase === "completed") { + state.unsettledFinalizationHookCount = Math.max( + 0, + state.unsettledFinalizationHookCount - 1, + ); + activeFinalizationHookRunIds.delete(hookNotification.runId); + finalizationHookBatchStatuses.set(hookNotification.runId, hookNotification.status); + if (activeFinalizationHookRunIds.size === 0) { + const statuses = new Set(finalizationHookBatchStatuses.values()); + if (statuses.has("blocked") && !statuses.has("stopped")) { + const itemId = projector.getLatestTerminalAssistantCandidate()?.itemId; + state.rejectedFinalizationHookAssistant = itemId ? { itemId } : {}; + turnWatches.disarmAssistantCompletionIdleWatch(); + } else { + state.rejectedFinalizationHookAssistant = undefined; + } + } + if ( + activeFinalizationHookRunIds.size === 0 && + state.rejectedFinalizationHookAssistant === undefined && + !state.terminalTurnNotificationQueued && + state.activeAppServerTurnRequests === 0 && + activeTurnItemIds.size === 0 && + activeCompletionBlockerItemIds.size === 0 && + pendingOpenClawDynamicToolCompletionIds.size === 0 && + projector.hasLatestTerminalAssistantCandidateText() + ) { + turnWatches.armAssistantCompletionIdleWatch({ + lastNotificationMethod: notification.method, + hookRunId: hookNotification.runId, + hookStatus: hookNotification.status, + }); + } + } + if (notificationState.isTurnTerminal) { + if (notificationState.isTurnAbortMarker) { + projector.markAborted(); + } + if (!state.timedOut && !runAbortController.signal.aborted) { + await steeringQueue?.flushPending(); + } + state.completed = true; + turnWatches.clearCompletionIdleTimer(); + turnWatches.clearAssistantCompletionIdleTimer(); + turnWatches.clearTerminalIdleTimer(); + state.resolveCompletion?.(); + } + } + }; + const waitForActiveNativeTurnCompletion = async () => { + const route = resourceState.turnRoute; + if (!route) { + return false; + } + return await route.waitForTurnCompletion({ + timeoutMs: Math.min(appServer.requestTimeoutMs, CODEX_APP_SERVER_NATIVE_TURN_WAIT_TIMEOUT_MS), + signal: runAbortController.signal, + }); + }; + const noteNotificationReceived = ( + notification: CodexServerNotification, + scope: CodexThreadRouteScope, + receivedAtMs: number, + ) => { + const projector = projectorRef.current; + const turnId = turnIdRef.current; + if (!projector || !turnId) { + return; + } + if (isTerminalTurnNotificationForTurn(notification, turnId)) { + state.terminalTurnNotificationQueued = true; + } + if (scope.turnId === turnId) { + const modelToolCallId = readRawResponseToolCallId(notification); + if (modelToolCallId) { + allocateCodexToolOutcomeOrdinal?.(modelToolCallId); + } + const nativeItem = readCodexNotificationItem(notification.params); + if (nativeItem?.type === "webSearch") { + projector.recordNativeToolOutcome(nativeItem); + } + } + const hookNotification = readCodexFinalizationHookNotification( + notification, + resourceState.thread.threadId, + turnId, + ); + if (hookNotification?.phase === "started") { + state.unsettledFinalizationHookCount += 1; + turnWatches.disarmAssistantCompletionIdleWatch(); + } + turnWatches.noteNotificationReceived(notification.method, { receivedAtMs }); + }; + const enqueueNotification = async ( + notification: CodexServerNotification, + scope: CodexThreadRouteScope, + ) => { + embeddedAgentLog.trace("codex app-server raw notification received", { + method: notification.method, + ...scope, + }); + await handleNotification(notification); + }; + const drainNotificationQueue = async () => { + await resourceState.turnRoute?.drain(); + }; + registerNativeSubagentMonitor(resourceState.thread.threadId); + return { + waitForActiveNativeTurnCompletion, + noteNotificationReceived, + enqueueNotification, + drainNotificationQueue, + }; +} + +export type CodexAttemptNotificationController = ReturnType< + typeof createCodexAttemptNotificationController +>; diff --git a/extensions/codex/src/app-server/run-attempt-prompt.ts b/extensions/codex/src/app-server/run-attempt-prompt.ts new file mode 100644 index 000000000000..9981d9662e00 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-prompt.ts @@ -0,0 +1,480 @@ +import { + assembleHarnessContextEngine, + CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + embeddedAgentLog, + formatErrorMessage, + resolveAgentHarnessBeforePromptBuildResult, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + buildCodexSystemPromptReport, + prependCodexOpenClawPromptContext, + readContextEngineThreadBootstrapProjection, + resolveCodexDeliveryHintPreservedInputRange, + resolveContextEngineBootstrapProjectionDecision, +} from "./attempt-context.js"; +import { + fitCodexProjectedContextForTurnStart, + projectContextEngineAssemblyForCodex, + type CodexProjectedContextRange, +} from "./context-engine-projection.js"; +import { flattenCodexDynamicToolFunctions } from "./protocol.js"; +import type { CodexAttemptContext } from "./run-attempt-context.js"; +import { estimateCodexAppServerProjectedTurnTokens } from "./run-attempt-lifecycle.js"; +import { + isNonEmptyString, + joinPresentSections, + prependCurrentInboundContext, +} from "./run-attempt-state.js"; +import { rotateOversizedCodexAppServerStartupBinding } from "./startup-binding.js"; +import { + buildContextEngineBinding, + buildTurnCollaborationMode, + codexDynamicToolsFingerprint, + codexLegacyDynamicToolsFingerprint, +} from "./thread-lifecycle.js"; + +export async function prepareCodexAttemptPrompt(context: CodexAttemptContext) { + const { + runtime, + attemptTools, + historyState, + hookContext, + workspaceBootstrapContext, + baseDeveloperInstructions, + openClawPromptContext, + skillsCollaborationInstructions, + promptState, + codexContextProjectionMaxChars, + } = context; + const { + connection, + buildActiveRunAttemptParams, + effectiveContextTokenBudget, + effectiveRuntimeModelId, + effectiveRuntimeProviderId, + } = runtime; + const { + params, + activeContextEngine, + usesSupervisionConnection, + mutable, + isInactiveThreadBootstrapBinding, + bindingStore, + bindingIdentity, + agentDir, + appServer, + contextSessionKey, + effectiveWorkspace, + } = connection; + const { toolBridge } = attemptTools; + const applyFreshThreadContinuityProjection = () => { + const projection = projectContextEngineAssemblyForCodex({ + assembledMessages: historyState.messages, + originalHistoryMessages: historyState.messages, + prompt: params.prompt, + maxRenderedContextChars: codexContextProjectionMaxChars, + }); + promptState.promptText = projection.promptText; + promptState.promptContextRange = projection.promptContextRange; + promptState.prePromptMessageCount = projection.prePromptMessageCount; + }; + const applyActiveContextEngineProjection = async ( + decisionStartupBinding: typeof mutable.startupBinding, + ) => { + if (!activeContextEngine) { + return; + } + const assembled = await assembleHarnessContextEngine({ + contextEngine: activeContextEngine, + sessionId: runtime.activeSessionId, + sessionKey: contextSessionKey, + messages: historyState.messages, + tokenBudget: effectiveContextTokenBudget, + availableTools: new Set( + flattenCodexDynamicToolFunctions(toolBridge.availableSpecs) + .map((tool) => tool.name) + .filter(isNonEmptyString), + ), + citationsMode: params.config?.memory?.citations, + modelId: effectiveRuntimeModelId, + contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + providerId: effectiveRuntimeProviderId, + requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, + fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, + degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, + prompt: params.prompt, + }); + if (!assembled) { + throw new Error("context engine assemble returned no result"); + } + promptState.contextEngineProjection = readContextEngineThreadBootstrapProjection( + assembled.contextProjection, + ); + const projection = projectContextEngineAssemblyForCodex({ + assembledMessages: assembled.messages, + originalHistoryMessages: historyState.messages, + prompt: params.prompt, + systemPromptAddition: assembled.systemPromptAddition, + maxRenderedContextChars: codexContextProjectionMaxChars, + toolPayloadMode: promptState.contextEngineProjection ? "preserve" : "elide", + }); + const projectionDecision = promptState.contextEngineProjection + ? resolveContextEngineBootstrapProjectionDecision({ + startupBinding: decisionStartupBinding, + expectedBinding: buildContextEngineBinding( + buildActiveRunAttemptParams(), + promptState.contextEngineProjection, + ), + projection: promptState.contextEngineProjection, + dynamicToolsFingerprint: codexDynamicToolsFingerprint(toolBridge.specs), + legacyDynamicToolsFingerprint: codexLegacyDynamicToolsFingerprint(toolBridge.specs), + }) + : { project: true, reason: "per-turn-projection" }; + const decisionBinding = decisionStartupBinding; + embeddedAgentLog.info("codex app-server context-engine projection decision", { + sessionId: params.sessionId, + sessionKey: contextSessionKey, + engineId: activeContextEngine.info.id, + mode: + promptState.contextEngineProjection?.mode ?? + assembled.contextProjection?.mode ?? + "per_turn", + epoch: promptState.contextEngineProjection?.epoch, + fingerprint: promptState.contextEngineProjection?.fingerprint, + previousThreadId: decisionBinding?.threadId, + previousEpoch: decisionBinding?.contextEngine?.projection?.epoch, + previousFingerprint: decisionBinding?.contextEngine?.projection?.fingerprint, + projected: projectionDecision.project, + reason: projectionDecision.reason, + assembledMessages: assembled.messages.length, + originalHistoryMessages: historyState.messages.length, + projectedPromptChars: projection.promptText.length, + developerInstructionAdditionChars: projection.developerInstructionAddition?.length ?? 0, + }); + promptState.promptText = projectionDecision.project ? projection.promptText : params.prompt; + promptState.promptContextRange = projectionDecision.project + ? projection.promptContextRange + : undefined; + promptState.developerInstructions = joinPresentSections( + baseDeveloperInstructions, + projection.developerInstructionAddition, + ); + promptState.prePromptMessageCount = projection.prePromptMessageCount; + }; + if (activeContextEngine) { + try { + await applyActiveContextEngineProjection( + runtime.nativeToolSurfaceEnabled ? mutable.startupBinding : undefined, + ); + } catch (assembleErr) { + embeddedAgentLog.warn("context engine assemble failed; using Codex baseline prompt", { + error: formatErrorMessage(assembleErr), + }); + } + } + const codexModelInputHistoryMessages: typeof historyState.messages = []; + const buildPromptFromCurrentInputs = () => + resolveAgentHarnessBeforePromptBuildResult({ + prompt: prependCurrentInboundContext(promptState.promptText, params.currentInboundContext), + developerInstructions: promptState.developerInstructions, + messages: codexModelInputHistoryMessages, + ctx: hookContext, + bootstrapContextRunKind: params.bootstrapContextRunKind, + ...("beforeAgentStartResult" in params + ? { beforeAgentStartResult: params.beforeAgentStartResult } + : {}), + }); + const resolveShiftedPromptInputRange = ( + prompt: string, + promptInputRange: { start: number; end: number } | undefined, + turnPromptText: string, + ): CodexProjectedContextRange | undefined => { + if ( + !promptInputRange || + promptInputRange.start < 0 || + promptInputRange.end < promptInputRange.start || + promptInputRange.end > prompt.length || + !turnPromptText.endsWith(prompt) + ) { + return undefined; + } + const turnPromptOffset = turnPromptText.length - prompt.length; + return { + start: turnPromptOffset + promptInputRange.start, + end: turnPromptOffset + promptInputRange.end, + }; + }; + const resolveShiftedPromptContextRange = ( + prompt: string, + promptInputRange: { start: number; end: number } | undefined, + turnPromptText: string, + ) => { + const promptTextInputOffset = promptInputRange + ? promptInputRange.end - promptState.promptText.length + : undefined; + if ( + !promptState.promptContextRange || + !promptInputRange || + promptTextInputOffset === undefined || + promptInputRange.start < 0 || + promptInputRange.end < promptInputRange.start || + promptInputRange.end > prompt.length || + promptTextInputOffset < promptInputRange.start || + prompt.slice(promptTextInputOffset, promptInputRange.end) !== promptState.promptText || + !turnPromptText.endsWith(prompt) + ) { + return undefined; + } + const promptTextOffset = prompt.endsWith(promptState.promptText) + ? prompt.length - promptState.promptText.length + : promptTextInputOffset; + if (promptTextOffset < 0) { + return undefined; + } + const turnPromptOffset = turnPromptText.length - prompt.length + promptTextOffset; + const contextRange = { + start: turnPromptOffset + promptState.promptContextRange.start, + end: turnPromptOffset + promptState.promptContextRange.end, + }; + return { + contextRange, + requestRange: { + start: contextRange.end, + end: turnPromptOffset + promptState.promptText.length, + }, + }; + }; + const decorateCodexTurnPromptText = (promptBuildResult: { + prompt: string; + promptInputRange?: { start: number; end: number }; + }) => { + const turnPromptText = prependCodexOpenClawPromptContext( + promptBuildResult.prompt, + openClawPromptContext, + { + preservePromptWithoutContext: + params.bootstrapContextMode === "lightweight" && + params.bootstrapContextRunKind === "cron", + }, + ); + const projectedRanges = resolveShiftedPromptContextRange( + promptBuildResult.prompt, + promptBuildResult.promptInputRange, + turnPromptText, + ); + const preservedRange = + resolveShiftedPromptInputRange( + promptBuildResult.prompt, + promptBuildResult.promptInputRange, + turnPromptText, + ) ?? + resolveCodexDeliveryHintPreservedInputRange({ + prompt: promptBuildResult.prompt, + promptInputRange: promptBuildResult.promptInputRange, + decoratedPrompt: turnPromptText, + }); + return fitCodexProjectedContextForTurnStart({ + promptText: turnPromptText, + contextRange: projectedRanges?.contextRange, + requestRange: projectedRanges?.requestRange, + preservedRange, + }); + }; + const firstPromptBuild = await buildPromptFromCurrentInputs(); + const turnState = { + promptBuild: firstPromptBuild, + codexTurnPromptText: decorateCodexTurnPromptText(firstPromptBuild), + }; + const buildRenderedCodexDeveloperInstructions = () => + joinPresentSections( + turnState.promptBuild.developerInstructions, + buildTurnCollaborationMode(params, { + turnScopedDeveloperInstructions: workspaceBootstrapContext.turnScopedDeveloperInstructions, + skillsCollaborationInstructions, + memoryCollaborationInstructions: workspaceBootstrapContext.memoryCollaborationInstructions, + heartbeatCollaborationInstructions: + workspaceBootstrapContext.heartbeatCollaborationInstructions, + }).settings.developer_instructions ?? undefined, + ); + const rebuildCodexPromptBuildFromCurrentProjection = async () => { + turnState.promptBuild = await buildPromptFromCurrentInputs(); + turnState.codexTurnPromptText = decorateCodexTurnPromptText(turnState.promptBuild); + }; + const rebuildCodexTurnPromptTextFromCurrentProjection = async () => { + const nextPromptBuild = await buildPromptFromCurrentInputs(); + turnState.promptBuild = { + ...turnState.promptBuild, + prompt: nextPromptBuild.prompt, + promptInputRange: nextPromptBuild.promptInputRange, + }; + turnState.codexTurnPromptText = decorateCodexTurnPromptText(nextPromptBuild); + }; + const selectNewerVisibleHistoryAfterBinding = ( + binding: NonNullable, + ) => { + const cutoff = Date.parse(binding.historyCoveredThrough ?? ""); + return historyState.messages.filter((message) => { + if (message.role !== "user" && message.role !== "assistant") { + return false; + } + const record = message as unknown as Record; + const meta = record["__openclaw"]; + const mirrorIdentity = + meta && typeof meta === "object" && !Array.isArray(meta) + ? (meta as Record).mirrorIdentity + : undefined; + const mirrorOrigin = + meta && typeof meta === "object" && !Array.isArray(meta) + ? (meta as Record).mirrorOrigin + : undefined; + const timestamp = + typeof message.timestamp === "number" + ? message.timestamp + : typeof message.timestamp === "string" + ? Date.parse(message.timestamp) + : Number.NaN; + return ( + !( + typeof record.idempotencyKey === "string" && + record.idempotencyKey.startsWith("codex-app-server:") + ) && + mirrorOrigin !== "codex-app-server" && + !(typeof mirrorIdentity === "string" && mirrorIdentity.startsWith("codex-app-server:")) && + Number.isFinite(timestamp) && + timestamp > (Number.isFinite(cutoff) ? cutoff : 0) + ); + }); + }; + const applyResumeStaleBindingContinuityProjection = ( + binding: NonNullable, + ) => { + const newerVisibleMessages = selectNewerVisibleHistoryAfterBinding(binding); + if (newerVisibleMessages.length === 0) { + return false; + } + const projection = projectContextEngineAssemblyForCodex({ + assembledMessages: newerVisibleMessages, + originalHistoryMessages: historyState.messages, + prompt: params.prompt, + maxRenderedContextChars: codexContextProjectionMaxChars, + }); + promptState.promptText = projection.promptText; + promptState.promptContextRange = projection.promptContextRange; + promptState.prePromptMessageCount = projection.prePromptMessageCount; + return true; + }; + const precomputeNoContextEngineStaleBindingProjection = () => { + promptState.precomputedStaleBindingContinuityProjectionApplied = false; + promptState.staleBindingContinuityForcedFreshStart = false; + const binding = mutable.startupBinding; + if (activeContextEngine || !binding?.threadId || binding.pendingSupervisionBranch) { + return false; + } + if (isInactiveThreadBootstrapBinding(binding)) { + promptState.inactiveThreadBootstrapBindingForcedFreshStart = true; + return false; + } + const projected = applyResumeStaleBindingContinuityProjection(binding); + promptState.precomputedStaleBindingContinuityProjectionApplied = projected; + return projected; + }; + const applyNoContextEngineContinuityProjection = ( + action: "started" | "resumed" | "forked", + binding?: NonNullable, + ) => { + if (activeContextEngine || !historyState.messages.some((message) => message.role === "user")) { + return false; + } + if (action === "resumed" && promptState.precomputedStaleBindingContinuityProjectionApplied) { + return true; + } + if (action === "started" && promptState.staleBindingContinuityForcedFreshStart) { + return true; + } + if (action === "started" && promptState.inactiveThreadBootstrapBindingForcedFreshStart) { + return false; + } + if (action === "resumed" && binding) { + return applyResumeStaleBindingContinuityProjection(binding); + } + if (action === "started") { + applyFreshThreadContinuityProjection(); + return true; + } + return false; + }; + if (precomputeNoContextEngineStaleBindingProjection()) { + await rebuildCodexPromptBuildFromCurrentProjection(); + } + const rotateStartupBindingForProjectedTurn = async () => { + const binding = mutable.startupBinding; + if (!binding?.threadId) { + return; + } + const previousThreadId = binding.threadId; + const hadInactiveThreadBootstrapBinding = isInactiveThreadBootstrapBinding(binding); + mutable.startupBinding = await rotateOversizedCodexAppServerStartupBinding({ + binding, + bindingStore, + identity: bindingIdentity, + sessionFile: params.sessionFile, + agentDir, + codexHome: appServer.start.env?.CODEX_HOME, + config: params.config, + contextEngineActive: Boolean(activeContextEngine), + projectedTurnTokens: estimateCodexAppServerProjectedTurnTokens({ + prompt: turnState.codexTurnPromptText, + developerInstructions: buildRenderedCodexDeveloperInstructions(), + }), + }); + if (mutable.startupBinding?.threadId) { + return; + } + promptState.inactiveThreadBootstrapBindingForcedFreshStart = hadInactiveThreadBootstrapBinding; + promptState.staleBindingContinuityForcedFreshStart = + promptState.precomputedStaleBindingContinuityProjectionApplied && + !promptState.inactiveThreadBootstrapBindingForcedFreshStart; + if (promptState.staleBindingContinuityForcedFreshStart) { + applyFreshThreadContinuityProjection(); + } + if (activeContextEngine) { + promptState.contextEngineProjection = undefined; + try { + await applyActiveContextEngineProjection(undefined); + } catch (assembleErr) { + embeddedAgentLog.warn("context engine assemble failed; using Codex baseline prompt", { + error: formatErrorMessage(assembleErr), + }); + } + } + await rebuildCodexPromptBuildFromCurrentProjection(); + embeddedAgentLog.info("codex app-server rebuilt turn prompt after native thread rotation", { + sessionId: params.sessionId, + sessionKey: contextSessionKey, + previousThreadId, + promptChars: turnState.codexTurnPromptText.length, + developerInstructionChars: buildRenderedCodexDeveloperInstructions()?.length ?? 0, + }); + }; + await rotateStartupBindingForProjectedTurn(); + const systemPromptReport = buildCodexSystemPromptReport({ + attempt: params, + sessionKey: contextSessionKey, + workspaceDir: effectiveWorkspace, + developerInstructions: buildRenderedCodexDeveloperInstructions(), + workspaceBootstrapContext, + skillsPrompt: skillsCollaborationInstructions ? (params.skillsSnapshot?.prompt ?? "") : "", + tools: toolBridge.availableSpecs, + }); + return { + context, + codexModelInputHistoryMessages, + turnState, + buildRenderedCodexDeveloperInstructions, + rebuildCodexTurnPromptTextFromCurrentProjection, + applyNoContextEngineContinuityProjection, + systemPromptReport, + }; +} + +export type CodexAttemptPrompt = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-resources.ts b/extensions/codex/src/app-server/run-attempt-resources.ts new file mode 100644 index 000000000000..861eecd7b385 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-resources.ts @@ -0,0 +1,251 @@ +import { + embeddedAgentLog, + type AgentHarnessRuntimeArtifactBinding, + type NativeHookRelayRegistrationHandle, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { resolveCodexStartupTimeoutMs } from "./attempt-timeouts.js"; +import type { CodexAppServerClient } from "./client.js"; +import { resolveCodexToolAbortTerminalReason } from "./dynamic-tool-execution.js"; +import { CodexAppServerEventProjector } from "./event-projector.js"; +import { + buildCodexNativeHookRelayDisabledConfig, + buildCodexNativeHookRelayConfig, + CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS, + createCodexNativeHookRelay, + emitCodexNativePreToolUseFailureDiagnostic, + type CodexNativePreToolUseFailure, +} from "./native-hook-relay.js"; +import { codexNativeSubagentMonitorRuntime } from "./native-subagent-monitor.js"; +import type { CodexSandboxPolicy, CodexTurnEnvironmentParams } from "./protocol.js"; +import type { CodexAttemptPrompt } from "./run-attempt-prompt.js"; +import { releaseCodexSandboxExecServerEnvironment } from "./sandbox-exec-server.js"; +import type { CodexAppServerThreadBinding } from "./session-binding.js"; +import { + retainSharedCodexAppServerClientIfCurrent, + retireSharedCodexAppServerClientIfCurrent, +} from "./shared-client.js"; +import type { CodexAppServerThreadLifecycleBinding } from "./thread-lifecycle.js"; +import { createCodexTrajectoryRecorder, type CodexHostTrajectoryRecorder } from "./trajectory.js"; +import type { CodexAppServerTurnRouter, CodexThreadRouteReservation } from "./turn-router.js"; + +export function prepareCodexAttemptResources(prompt: CodexAttemptPrompt) { + const { context, turnState, buildRenderedCodexDeveloperInstructions } = prompt; + const { runtime, attemptTools } = context; + const { connection, hookChannelId } = runtime; + const { + params, + effectiveCwd, + sessionAgentId, + sandboxSessionKey, + runAbortController, + sandbox, + options, + nativeHookRelayEvents, + } = connection; + const { toolBridge } = attemptTools; + const hostTrajectoryRecorder = ( + params as typeof params & { trajectoryRecorder?: CodexHostTrajectoryRecorder | null } + ).trajectoryRecorder; + const trajectoryRecorder = createCodexTrajectoryRecorder({ + attempt: params, + cwd: effectiveCwd, + developerInstructions: buildRenderedCodexDeveloperInstructions(), + prompt: turnState.codexTurnPromptText, + trajectoryRecorder: hostTrajectoryRecorder, + trajectorySessionFile: params.trajectorySessionFile, + tools: toolBridge.availableSpecs, + warn: (message, fields) => embeddedAgentLog.warn(message, fields), + }); + const state = { + client: undefined as unknown as CodexAppServerClient, + thread: undefined as unknown as CodexAppServerThreadLifecycleBinding, + runtimeArtifact: undefined as AgentHarnessRuntimeArtifactBinding | undefined, + turnRouter: undefined as unknown as CodexAppServerTurnRouter, + turnRoute: undefined as CodexThreadRouteReservation | undefined, + routeActivated: false, + detachRouteAbort: (() => undefined) as () => void, + trajectoryEndRecorded: false, + nativeHookRelay: undefined as NativeHookRelayRegistrationHandle | undefined, + nativeSubagentMonitor: undefined as + | ReturnType + | undefined, + nativePreToolUseFailureFallbackActive: false, + nativePreToolUseFailureFallbackTerminalReason: undefined as + | CodexNativePreToolUseFailure["disposition"] + | undefined, + releaseSharedClientLease: undefined as (() => void) | undefined, + sharedCodexClientRetiredForOneShotCleanup: false, + sandboxExecEnvironmentAcquired: false, + codexEnvironmentSelection: undefined as CodexTurnEnvironmentParams[] | undefined, + codexExecutionCwd: effectiveCwd, + codexSandboxPolicy: undefined as CodexSandboxPolicy | undefined, + restartContextEngineCodexThread: undefined as + | (() => Promise) + | undefined, + }; + const pendingNativePreToolUseFailures: CodexNativePreToolUseFailure[] = []; + const projectorRef: { current?: CodexAppServerEventProjector } = {}; + const emitNativePreToolUseFailure = (failure: CodexNativePreToolUseFailure) => { + emitCodexNativePreToolUseFailureDiagnostic({ + agentId: sessionAgentId, + sessionId: params.sessionId, + sessionKey: sandboxSessionKey, + runId: params.runId, + signal: runAbortController.signal, + failure, + ...(state.nativePreToolUseFailureFallbackActive + ? { + terminalReason: + state.nativePreToolUseFailureFallbackTerminalReason ?? failure.disposition, + } + : {}), + }); + }; + const flushPendingNativePreToolUseFailures = () => { + for (const failure of pendingNativePreToolUseFailures.splice(0)) { + emitNativePreToolUseFailure(failure); + } + }; + const activateNativePreToolUseFailureFallback = () => { + if (!state.nativePreToolUseFailureFallbackActive) { + state.nativePreToolUseFailureFallbackTerminalReason = runAbortController.signal.aborted + ? resolveCodexToolAbortTerminalReason(runAbortController.signal) + : undefined; + state.nativePreToolUseFailureFallbackActive = true; + } + flushPendingNativePreToolUseFailures(); + }; + const releaseSharedClientLeaseOnce = () => { + const release = state.releaseSharedClientLease; + if (!release) { + return; + } + state.releaseSharedClientLease = undefined; + release(); + }; + const retireSharedCodexClientForOneShotCleanup = async () => { + if ( + params.cleanupBundleMcpOnRunEnd !== true || + state.sharedCodexClientRetiredForOneShotCleanup + ) { + return; + } + state.sharedCodexClientRetiredForOneShotCleanup = true; + const retired = retireSharedCodexAppServerClientIfCurrent(state.client); + embeddedAgentLog.info("codex app-server one-shot cleanup retired shared client", { + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + activeLeases: retired?.activeLeases ?? null, + closed: retired?.closed ?? false, + matchedSharedClient: Boolean(retired), + }); + if (retired?.closed) { + await state.client.closeAndWait({ exitTimeoutMs: 2_000, forceKillDelayMs: 250 }); + } + }; + const releaseSharedClientLeaseAndRetireOneShotClient = async () => { + releaseSharedClientLeaseOnce(); + await retireSharedCodexClientForOneShotCleanup(); + }; + const releaseSandboxExecEnvironment = async () => { + if (state.sandboxExecEnvironmentAcquired) { + state.sandboxExecEnvironmentAcquired = false; + await releaseCodexSandboxExecServerEnvironment(sandbox); + } + }; + const unregisterNativeSubagentMonitor = () => { + state.nativeSubagentMonitor?.unregister(); + state.nativeSubagentMonitor = undefined; + }; + const registerNativeSubagentMonitor = (parentThreadId: string) => { + unregisterNativeSubagentMonitor(); + state.nativeSubagentMonitor = codexNativeSubagentMonitorRuntime.register({ + client: state.client, + parentThreadId, + requesterSessionKey: params.sessionKey, + taskRuntimeScope: params.agentHarnessTaskRuntimeScope, + agentId: sessionAgentId, + retainClient: () => retainSharedCodexAppServerClientIfCurrent(state.client), + }); + }; + const releaseCurrentRoute = () => { + state.detachRouteAbort(); + state.detachRouteAbort = () => undefined; + state.turnRoute?.release(); + state.turnRoute = undefined; + state.routeActivated = false; + unregisterNativeSubagentMonitor(); + }; + const startupTimeoutMs = resolveCodexStartupTimeoutMs({ + timeoutMs: params.timeoutMs, + timeoutFloorMs: options.startupTimeoutFloorMs, + }); + const buildNativeHookRelayFinalConfigPatch = ( + decision: { action: "resume"; binding: CodexAppServerThreadBinding } | { action: "start" }, + ) => { + state.nativeHookRelay?.unregister(); + state.nativeHookRelay = createCodexNativeHookRelay({ + options: options.nativeHookRelay, + generation: + decision.action === "resume" ? decision.binding.nativeHookRelayGeneration : undefined, + generationMismatchGraceMs: + decision.action === "resume" && !decision.binding.nativeHookRelayGeneration + ? CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS + : undefined, + events: nativeHookRelayEvents, + agentId: sessionAgentId, + sessionId: params.sessionId, + sessionKey: sandboxSessionKey, + config: params.config, + runId: params.runId, + channelId: hookChannelId, + attemptTimeoutMs: params.timeoutMs, + startupTimeoutMs, + turnStartTimeoutMs: params.timeoutMs, + signal: runAbortController.signal, + onPreToolUseFailure: (failure) => { + const projector = projectorRef.current; + if (projector) { + projector.recordNativeToolPreToolUseFailure(failure); + } else if (state.nativePreToolUseFailureFallbackActive) { + emitNativePreToolUseFailure(failure); + } else { + pendingNativePreToolUseFailures.push(failure); + } + }, + }); + return { + configPatch: state.nativeHookRelay + ? buildCodexNativeHookRelayConfig({ + relay: state.nativeHookRelay, + events: nativeHookRelayEvents, + hookTimeoutSec: options.nativeHookRelay?.hookTimeoutSec, + }) + : options.nativeHookRelay?.enabled === false + ? buildCodexNativeHookRelayDisabledConfig() + : undefined, + nativeHookRelayGeneration: state.nativeHookRelay?.generation, + }; + }; + return { + prompt, + trajectoryRecorder, + state, + projectorRef, + pendingNativePreToolUseFailures, + markTrajectoryEndRecorded: () => { + state.trajectoryEndRecorded = true; + }, + activateNativePreToolUseFailureFallback, + releaseSharedClientLeaseOnce, + releaseSharedClientLeaseAndRetireOneShotClient, + releaseSandboxExecEnvironment, + registerNativeSubagentMonitor, + releaseCurrentRoute, + startupTimeoutMs, + buildNativeHookRelayFinalConfigPatch, + }; +} + +export type CodexAttemptResources = ReturnType; diff --git a/extensions/codex/src/app-server/run-attempt-route.ts b/extensions/codex/src/app-server/run-attempt-route.ts new file mode 100644 index 000000000000..ce91ad188ba5 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-route.ts @@ -0,0 +1,104 @@ +import { embeddedAgentLog, formatErrorMessage } from "openclaw/plugin-sdk/agent-harness-runtime"; +import type { CodexAttemptNotificationController } from "./run-attempt-notification-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import type { createCodexAttemptServerRequestController } from "./run-attempt-server-requests.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import type { CodexThreadRouteReservation } from "./turn-router.js"; + +export async function prepareCodexAttemptRoute( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + notifications: CodexAttemptNotificationController, + handleServerRequest: ReturnType< + typeof createCodexAttemptServerRequestController + >["handleServerRequest"], +) { + const { + prompt, + state: resourceState, + trajectoryRecorder, + releaseCurrentRoute, + registerNativeSubagentMonitor, + activateNativePreToolUseFailureFallback, + releaseSandboxExecEnvironment, + releaseSharedClientLeaseOnce, + } = resources; + const { connection } = prompt.context.runtime; + const { params, runAbortController, abortFromUpstream } = connection; + const { state, turnIdRef, turnWatches } = turnRuntime; + const { noteNotificationReceived, enqueueNotification } = notifications; + const attachRouteAbort = (route: CodexThreadRouteReservation) => { + const onAbort = () => { + if ( + state.completed || + state.terminalTurnNotificationQueued || + runAbortController.signal.aborted + ) { + return; + } + const reasonText = formatErrorMessage(route.signal.reason); + const closedClient = reasonText.includes("turn router closed"); + state.clientClosedPromptError = closedClient + ? "codex app-server client closed before turn completed" + : `codex app-server turn route closed before turn completed: ${reasonText}`; + state.clientClosedAbort = closedClient; + const activeTurnId = turnIdRef.current; + if (activeTurnId) { + trajectoryRecorder?.recordEvent("turn.client_closed", { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + }); + } + embeddedAgentLog.warn(state.clientClosedPromptError, { + threadId: resourceState.thread.threadId, + turnId: activeTurnId, + }); + runAbortController.abort(closedClient ? "client_closed" : "turn_route_closed"); + state.completed = true; + turnWatches.clearAllTimers(); + state.resolveCompletion?.(); + }; + route.signal.addEventListener("abort", onAbort, { once: true }); + if (route.signal.aborted) { + onAbort(); + } + return () => route.signal.removeEventListener("abort", onAbort); + }; + const ensureCurrentThreadRoute = async () => { + if (resourceState.turnRoute?.threadId !== resourceState.thread.threadId) { + releaseCurrentRoute(); + resourceState.turnRoute = resourceState.turnRouter.reserveThread({ + threadId: resourceState.thread.threadId, + releaseOn: runAbortController.signal, + }); + } + if (!resourceState.turnRoute) { + throw new Error("codex app-server turn route was not reserved"); + } + if (!resourceState.routeActivated) { + if (!resourceState.nativeSubagentMonitor) { + registerNativeSubagentMonitor(resourceState.thread.threadId); + } + resourceState.detachRouteAbort = attachRouteAbort(resourceState.turnRoute); + await resourceState.turnRoute.activate({ + onNotificationReceived: noteNotificationReceived, + onNotification: enqueueNotification, + onRequest: handleServerRequest, + }); + resourceState.routeActivated = true; + } + return resourceState.turnRoute; + }; + try { + await ensureCurrentThreadRoute(); + } catch (error) { + activateNativePreToolUseFailureFallback(); + releaseCurrentRoute(); + resourceState.nativeHookRelay?.unregister(); + await releaseSandboxExecEnvironment(); + releaseSharedClientLeaseOnce(); + params.abortSignal?.removeEventListener("abort", abortFromUpstream); + throw error; + } + return { ensureCurrentThreadRoute }; +} diff --git a/extensions/codex/src/app-server/run-attempt-runtime.ts b/extensions/codex/src/app-server/run-attempt-runtime.ts new file mode 100644 index 000000000000..a96d9e261470 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-runtime.ts @@ -0,0 +1,211 @@ +import { + assertContextEngineHostSupport, + CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + embeddedAgentLog, + loadCodexBundleMcpThreadConfig, + supportsModelTools, + type EmbeddedRunAttemptParams, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { prepareCodexAppServerAuthBinding } from "./auth-binding.js"; +import { + resolveCodexAppServerAuthAccountCacheKey, + resolveCodexAppServerFallbackApiKeyCacheKey, + resolveCodexAppServerPreparedApiKeyCacheKey, +} from "./auth-bridge.js"; +import { isCodexSandboxExecServerEnabled } from "./config.js"; +import { + resolveCodexAppServerHookChannelId, + shouldEnableCodexAppServerNativeToolSurface, +} from "./dynamic-tool-build.js"; +import { resolveCodexProviderWebSearchSupport } from "./provider-capabilities.js"; +import type { CodexAttemptConnection } from "./run-attempt-connection.js"; +import { resolveCodexAppServerThreadModelSelection } from "./thread-lifecycle.js"; +import { resolveCodexWebSearchPlan } from "./web-search.js"; + +export async function prepareCodexAttemptRuntime(connection: CodexAttemptConnection) { + const { + params, + pluginConfig, + usesSupervisionConnection, + appServer, + startupAuthProfileId, + startupPreparedAuth, + startupClientAuthProfileId, + agentDir, + preDynamicStartupStages, + effectiveWorkspace, + contextSessionKey, + sandboxSessionKey, + sessionAgentId, + sandbox, + attemptClientFactory, + runAbortController, + activeContextEngine, + mutable, + } = connection; + const preparedAuthBinding = + !usesSupervisionConnection && appServer.start.homeScope !== "user" && startupAuthProfileId + ? await prepareCodexAppServerAuthBinding({ + authProfileId: startupAuthProfileId, + authProfileStore: params.authProfileStore, + agentDir, + config: params.config, + }) + : undefined; + const attemptAuthProfileStore = preparedAuthBinding?.authProfileStore ?? params.authProfileStore; + const effectiveContextWindowInfo = usesSupervisionConnection + ? undefined + : params.contextWindowInfo; + const effectiveContextTokenBudget = usesSupervisionConnection + ? undefined + : params.contextTokenBudget; + const effectiveRuntimeProviderId = usesSupervisionConnection + ? (mutable.startupBinding?.modelProvider ?? "codex") + : params.provider; + const effectiveRuntimeModelId = usesSupervisionConnection + ? (mutable.startupBinding?.model ?? "codex-native") + : params.modelId; + const { + authProfileId: _outerAuthProfileId, + contextWindowInfo: _outerContextWindowInfo, + contextTokenBudget: _outerContextTokenBudget, + model: _outerModel, + modelId: _outerModelId, + provider: _outerProvider, + runtimePlan: _outerRuntimePlan, + requestedModelId: _outerRequestedModelId, + fallbackReason: _outerFallbackReason, + degradedReason: _outerDegradedReason, + thinkLevel: _outerThinkLevel, + fastMode: _outerFastMode, + ...paramsWithoutOuterNativeOwnership + } = params; + const supervisedRuntimeModel = { + id: effectiveRuntimeModelId, + name: effectiveRuntimeModelId, + provider: effectiveRuntimeProviderId, + api: "openai-chatgpt-responses", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: undefined, + maxTokens: undefined, + } as unknown as EmbeddedRunAttemptParams["model"]; + const runtimeParams: EmbeddedRunAttemptParams = usesSupervisionConnection + ? { + ...paramsWithoutOuterNativeOwnership, + provider: "codex", + modelId: effectiveRuntimeModelId, + model: supervisedRuntimeModel, + thinkLevel: _outerThinkLevel, + sessionKey: contextSessionKey, + } + : { + ...params, + authProfileStore: attemptAuthProfileStore, + sessionKey: contextSessionKey, + ...(startupAuthProfileId ? { authProfileId: startupAuthProfileId } : {}), + }; + const activeSessionId = params.sessionId; + const activeSessionFile = params.sessionFile; + const buildActiveRunAttemptParams = (): EmbeddedRunAttemptParams => ({ + ...runtimeParams, + sessionId: activeSessionId, + sessionFile: activeSessionFile, + }); + const startupAuthAccountCacheKey = usesSupervisionConnection + ? undefined + : startupPreparedAuth?.kind === "api-key" + ? resolveCodexAppServerPreparedApiKeyCacheKey(startupPreparedAuth.apiKey) + : startupPreparedAuth?.kind === "profile" + ? startupPreparedAuth.snapshot?.secretFreeCacheKey + : await resolveCodexAppServerAuthAccountCacheKey({ + authProfileId: startupAuthProfileId, + authProfileStore: attemptAuthProfileStore, + agentDir, + config: params.config, + }); + const startupEnvApiKeyCacheKey = usesSupervisionConnection + ? undefined + : startupPreparedAuth || startupAuthProfileId + ? undefined + : resolveCodexAppServerFallbackApiKeyCacheKey({ startOptions: appServer.start }); + preDynamicStartupStages.mark("auth-cache"); + const bundleMcpThreadConfig = await loadCodexBundleMcpThreadConfig({ + workspaceDir: effectiveWorkspace, + cfg: params.config, + toolsEnabled: usesSupervisionConnection || supportsModelTools(params.model), + disableTools: params.disableTools, + toolsAllow: params.toolsAllow, + }); + preDynamicStartupStages.mark("bundle-mcp"); + const sandboxExecServerEnabled = isCodexSandboxExecServerEnabled(pluginConfig); + const nativeToolSurfaceEnabled = shouldEnableCodexAppServerNativeToolSurface( + runtimeParams, + sandbox, + { agentId: sessionAgentId, runtimeSessionKey: sandboxSessionKey, sandboxExecServerEnabled }, + ); + preDynamicStartupStages.mark("native-tool-surface"); + const nativeProviderWebSearchSupport = + resolveCodexWebSearchPlan({ + config: params.config, + disableTools: params.disableTools, + nativeToolSurfaceEnabled, + }).kind === "native-hosted" + ? await resolveCodexProviderWebSearchSupport({ + clientFactory: attemptClientFactory, + appServer, + authProfileId: startupClientAuthProfileId, + preparedAuth: startupPreparedAuth, + agentDir, + config: params.config, + modelProviderOverride: usesSupervisionConnection + ? mutable.startupBinding?.modelProvider + : resolveCodexAppServerThreadModelSelection({ + provider: params.provider, + model: params.modelId, + binding: mutable.startupBinding, + authProfileId: startupAuthProfileId, + authProfileStore: attemptAuthProfileStore, + agentDir, + config: params.config, + }).modelProvider, + signal: runAbortController.signal, + }) + : "unsupported"; + preDynamicStartupStages.mark("provider-capabilities"); + for (const diagnostic of bundleMcpThreadConfig.diagnostics) { + embeddedAgentLog.warn(`bundle-mcp: ${diagnostic.pluginId}: ${diagnostic.message}`); + } + if (activeContextEngine) { + assertContextEngineHostSupport({ + contextEngine: activeContextEngine, + operation: "agent-run", + host: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, + }); + } + const hookChannelId = resolveCodexAppServerHookChannelId(params, sandboxSessionKey); + preDynamicStartupStages.mark("context-engine-support"); + return { + connection, + preparedAuthBinding, + runtimeParams, + activeSessionId, + activeSessionFile, + buildActiveRunAttemptParams, + attemptAuthProfileStore, + effectiveContextWindowInfo, + effectiveContextTokenBudget, + effectiveRuntimeProviderId, + effectiveRuntimeModelId, + startupAuthAccountCacheKey, + startupEnvApiKeyCacheKey, + bundleMcpThreadConfig, + sandboxExecServerEnabled, + nativeToolSurfaceEnabled, + nativeProviderWebSearchSupport, + hookChannelId, + }; +} + +export type CodexAttemptRuntime = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-server-requests.ts b/extensions/codex/src/app-server/run-attempt-server-requests.ts new file mode 100644 index 000000000000..40f052a6ed91 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-server-requests.ts @@ -0,0 +1,360 @@ +import { onInternalDiagnosticEvent } from "openclaw/plugin-sdk/diagnostic-runtime"; +import { isCodexAppServerApprovalRequest } from "./client.js"; +import { shouldAutoApproveCodexAppServerApprovals } from "./config.js"; +import { + emitDynamicToolErrorDiagnostic, + emitDynamicToolStartedDiagnostic, + emitDynamicToolTerminalDiagnostic, +} from "./dynamic-tool-diagnostics.js"; +import { + handleDynamicToolCallWithTimeout, + hasPendingDynamicToolTerminalDiagnostic, + isDynamicToolTerminalDiagnosticEvent, + isMatchingDynamicToolTerminalDiagnostic, + resolveDynamicToolCallTimeoutMs, + shouldBlockTerminalReleaseForNonTerminalDynamicToolResult, + toCodexDynamicToolProgressResponse, + toCodexDynamicToolProtocolResponse, +} from "./dynamic-tool-execution.js"; +import { handleCodexAppServerElicitationRequest } from "./elicitation-bridge.js"; +import { shouldEmitTranscriptToolProgress } from "./event-projector.js"; +import { readCodexDynamicToolCallParams } from "./protocol-validators.js"; +import type { JsonValue } from "./protocol.js"; +import type { CodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import { emitCodexAppServerEvent } from "./run-attempt-lifecycle.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { handleApprovalRequest, toTranscriptToolResult } from "./run-attempt-tools.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { + inferCodexDynamicToolMeta, + resolveCodexToolProgressDetailMode, + sanitizeCodexToolArguments, +} from "./tool-progress-normalization.js"; +import type { CodexAppServerServerRequest, CodexThreadRouteScope } from "./turn-router.js"; + +export function createCodexAttemptServerRequestController( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + lifecycle: CodexAttemptLifecycleController, +) { + const { prompt, state: resourceState, projectorRef, trajectoryRecorder } = resources; + const { context } = prompt; + const { runtime, attemptTools } = context; + const { connection } = runtime; + const { params, computerUseConfig, runAbortController, appServer, sessionAgentId } = connection; + const { + toolBridge, + toolOutcomeOrdinals, + suppressedDynamicToolOutcomeOrdinals, + allocateCodexToolOutcomeOrdinal, + } = attemptTools; + const { + state, + turnIdRef, + userInputBridgeRef, + openClawDynamicToolExecutions, + pendingOpenClawDynamicToolCompletionIds, + postToolRawAssistantCompletionIdleTimeoutMs, + turnWatches, + } = turnRuntime; + const { + emitExecutionPhaseOnce, + scheduleTurnReleaseAfterTerminalDynamicTool, + scheduleTerminalDynamicToolReleaseCheck, + } = lifecycle; + const handleServerRequest = async ( + request: CodexAppServerServerRequest, + scope: CodexThreadRouteScope, + ) => { + const turnId = turnIdRef.current; + const projector = projectorRef.current; + let armCompletionWatchOnResponse = false; + let requestCountsAsTurnActivity = false; + const markCurrentTurnRequestProgress = () => { + state.activeAppServerTurnRequests += 1; + turnWatches.clearCompletionIdleTimer(); + turnWatches.disarmAssistantCompletionIdleWatch(); + requestCountsAsTurnActivity = true; + turnWatches.touchActivity(`request:${request.method}:start`, { attemptProgress: true }); + }; + try { + if (!turnId) { + return undefined; + } + if (request.method === "mcpServer/elicitation/request") { + if (!scope.turnId || scope.turnId === turnId) { + armCompletionWatchOnResponse = true; + markCurrentTurnRequestProgress(); + } + return await handleCodexAppServerElicitationRequest({ + requestParams: request.params, + paramsForRun: params, + threadId: resourceState.thread.threadId, + turnId, + pluginAppPolicyContext: resourceState.thread.pluginAppPolicyContext, + ...(computerUseConfig.enabled + ? { computerUseMcpServerName: computerUseConfig.mcpServerName } + : {}), + signal: runAbortController.signal, + }); + } + if (request.method === "item/tool/requestUserInput") { + if (scope.turnId === turnId) { + armCompletionWatchOnResponse = true; + markCurrentTurnRequestProgress(); + } + return userInputBridgeRef.current?.handleRequest({ + id: request.id, + params: request.params, + }); + } + if (request.method !== "item/tool/call") { + if (isCodexAppServerApprovalRequest(request.method)) { + if (scope.turnId === turnId) { + armCompletionWatchOnResponse = true; + markCurrentTurnRequestProgress(); + } + return handleApprovalRequest({ + method: request.method, + params: request.params, + paramsForRun: params, + threadId: resourceState.thread.threadId, + turnId, + nativeHookRelay: resourceState.nativeHookRelay, + autoApprove: shouldAutoApproveCodexAppServerApprovals(appServer), + signal: runAbortController.signal, + onNativeToolFailureDisposition: (itemId, disposition) => + projector?.recordNativeToolApprovalFailure(itemId, disposition), + }); + } + return undefined; + } + const call = readCodexDynamicToolCallParams(request.params); + if (!call || call.threadId !== resourceState.thread.threadId || call.turnId !== turnId) { + return undefined; + } + const replayedExecution = openClawDynamicToolExecutions.get(call); + if (replayedExecution) { + armCompletionWatchOnResponse = true; + markCurrentTurnRequestProgress(); + state.turnCrossedToolHandoff = true; + return toCodexDynamicToolProtocolResponse(await replayedExecution) as JsonValue; + } + const toolCallOrdinal = allocateCodexToolOutcomeOrdinal?.(call.callId); + armCompletionWatchOnResponse = true; + markCurrentTurnRequestProgress(); + state.turnCrossedToolHandoff = true; + pendingOpenClawDynamicToolCompletionIds.add(call.callId); + trajectoryRecorder?.recordEvent("tool.call", { + threadId: call.threadId, + turnId: call.turnId, + toolCallId: call.callId, + name: call.tool, + arguments: call.arguments, + }); + projector?.recordDynamicToolCall({ + callId: call.callId, + tool: call.tool, + arguments: call.arguments, + }); + emitExecutionPhaseOnce(`tool:${call.callId}`, { + phase: "tool_execution_started", + tool: call.tool, + toolCallId: call.callId, + }); + emitDynamicToolStartedDiagnostic({ + call, + agentId: sessionAgentId, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + }); + const toolMeta = inferCodexDynamicToolMeta( + call, + resolveCodexToolProgressDetailMode(params.toolProgressDetail), + ); + const toolArgs = sanitizeCodexToolArguments(call.arguments); + const shouldEmitDynamicToolProgress = shouldEmitTranscriptToolProgress(call.tool, toolArgs); + if (shouldEmitDynamicToolProgress) { + void emitCodexAppServerEvent(params, { + stream: "tool", + data: { + phase: "start", + name: call.tool, + toolCallId: call.callId, + ...(toolMeta ? { meta: toolMeta } : {}), + ...(toolArgs ? { args: toolArgs } : {}), + }, + }); + } + const dynamicToolTimeoutMs = resolveDynamicToolCallTimeoutMs({ call, config: params.config }); + const toolStartedAt = Date.now(); + let terminalDiagnosticObserved = false; + const unsubscribeToolDiagnosticObserver = onInternalDiagnosticEvent((event) => { + if ( + isDynamicToolTerminalDiagnosticEvent(event) && + isMatchingDynamicToolTerminalDiagnostic({ + event, + call, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + }) + ) { + terminalDiagnosticObserved = true; + } + }); + try { + const { execution } = openClawDynamicToolExecutions.claim(call, () => + handleDynamicToolCallWithTimeout({ + call, + toolBridge, + signal: runAbortController.signal, + timeoutMs: dynamicToolTimeoutMs, + toolCallOrdinal, + onAgentToolResult: params.onAgentToolResult, + onFallbackSelected: () => { + if (toolCallOrdinal !== undefined) { + suppressedDynamicToolOutcomeOrdinals.add(toolCallOrdinal); + } + }, + onTimeout: () => { + trajectoryRecorder?.recordEvent("tool.timeout", { + threadId: call.threadId, + turnId: call.turnId, + toolCallId: call.callId, + name: call.tool, + timeoutMs: dynamicToolTimeoutMs, + }); + }, + }), + ); + const response = await execution; + const protocolResponse = toCodexDynamicToolProtocolResponse(response); + if (!protocolResponse.success && toolCallOrdinal !== undefined) { + suppressedDynamicToolOutcomeOrdinals.add(toolCallOrdinal); + params.onToolOutcome?.({ + toolName: call.tool, + argsHash: "", + resultHash: "", + toolCallOrdinal, + terminalPresentation: undefined, + presentationOnly: true, + }); + } + const toolDurationMs = Math.max(0, Date.now() - toolStartedAt); + trajectoryRecorder?.recordEvent("tool.result", { + threadId: call.threadId, + turnId: call.turnId, + toolCallId: call.callId, + name: call.tool, + success: protocolResponse.success, + contentItems: protocolResponse.contentItems, + }); + projector?.recordDynamicToolResult({ + callId: call.callId, + tool: call.tool, + asyncStarted: response.asyncStarted === true, + success: protocolResponse.success, + terminalType: + response.diagnosticTerminalType ?? (protocolResponse.success ? "completed" : "error"), + sideEffectEvidence: response.sideEffectEvidence === true, + contentItems: protocolResponse.contentItems, + }); + if (shouldEmitDynamicToolProgress) { + const progressResponse = toCodexDynamicToolProgressResponse(response, protocolResponse); + void emitCodexAppServerEvent(params, { + stream: "tool", + data: { + phase: "result", + name: call.tool, + toolCallId: call.callId, + ...(toolMeta ? { meta: toolMeta } : {}), + isError: !protocolResponse.success, + result: toTranscriptToolResult(progressResponse), + }, + }); + } + if ( + !terminalDiagnosticObserved && + !hasPendingDynamicToolTerminalDiagnostic({ + call, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + }) + ) { + emitDynamicToolTerminalDiagnostic({ + response, + call, + agentId: sessionAgentId, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + durationMs: toolDurationMs, + }); + } + pendingOpenClawDynamicToolCompletionIds.delete(call.callId); + if (response.terminate === true) { + scheduleTurnReleaseAfterTerminalDynamicTool({ + call, + response, + durationMs: toolDurationMs, + }); + } else if (!shouldBlockTerminalReleaseForNonTerminalDynamicToolResult(response)) { + scheduleTerminalDynamicToolReleaseCheck(); + } else { + state.currentTurnHadNonTerminalDynamicToolResult = true; + state.pendingTerminalDynamicToolRelease = undefined; + } + return protocolResponse as JsonValue; + } catch (error) { + pendingOpenClawDynamicToolCompletionIds.delete(call.callId); + if ( + !terminalDiagnosticObserved && + !hasPendingDynamicToolTerminalDiagnostic({ + call, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + }) + ) { + emitDynamicToolErrorDiagnostic({ + call, + agentId: sessionAgentId, + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + durationMs: Math.max(0, Date.now() - toolStartedAt), + }); + } + throw error; + } finally { + toolOutcomeOrdinals.delete(call.callId); + unsubscribeToolDiagnosticObserver(); + } + } finally { + if (requestCountsAsTurnActivity) { + state.activeAppServerTurnRequests = Math.max(0, state.activeAppServerTurnRequests - 1); + const postToolContinuationTimeoutMs = + request.method === "item/tool/call" && state.turnCrossedToolHandoff + ? postToolRawAssistantCompletionIdleTimeoutMs + : undefined; + turnWatches.touchActivity(`request:${request.method}:response`, { + arm: armCompletionWatchOnResponse, + attemptProgress: true, + ...(postToolContinuationTimeoutMs !== undefined + ? { attemptTimeoutMs: postToolContinuationTimeoutMs } + : {}), + }); + if (armCompletionWatchOnResponse && postToolContinuationTimeoutMs !== undefined) { + turnWatches.armCompletionIdleWatch({ timeoutMs: postToolContinuationTimeoutMs }); + } + scheduleTerminalDynamicToolReleaseCheck(); + } else { + turnWatches.scheduleProgressWatches(); + } + } + }; + return { handleServerRequest }; +} diff --git a/extensions/codex/src/app-server/run-attempt-start.ts b/extensions/codex/src/app-server/run-attempt-start.ts new file mode 100644 index 000000000000..fe4432c867b9 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-start.ts @@ -0,0 +1,186 @@ +import { embeddedAgentLog } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { resolveCodexAppServerForModelProvider } from "./app-server-policy.js"; +import { startCodexAttemptThread } from "./attempt-startup.js"; +import { flattenCodexDynamicToolFunctions } from "./protocol.js"; +import { + emitCodexAppServerEvent, + withCodexAppServerFastModeServiceTier, +} from "./run-attempt-lifecycle.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { recordCodexTrajectoryContext } from "./trajectory.js"; + +export async function startCodexAttemptRuntime(resources: CodexAttemptResources) { + const { + prompt, + state, + trajectoryRecorder, + activateNativePreToolUseFailureFallback, + releaseSandboxExecEnvironment, + releaseCurrentRoute, + startupTimeoutMs, + buildNativeHookRelayFinalConfigPatch, + } = resources; + const { + context, + turnState, + buildRenderedCodexDeveloperInstructions, + rebuildCodexTurnPromptTextFromCurrentProjection, + applyNoContextEngineContinuityProjection, + } = prompt; + const { runtime, attemptTools, promptState } = context; + const { + connection, + runtimeParams, + preparedAuthBinding, + buildActiveRunAttemptParams, + startupAuthAccountCacheKey, + startupEnvApiKeyCacheKey, + bundleMcpThreadConfig, + nativeToolSurfaceEnabled, + nativeProviderWebSearchSupport, + sandboxExecServerEnabled, + } = runtime; + const { toolBridge, toolState } = attemptTools; + const { + params, + attemptClientFactory, + bindingStore, + appServer, + pluginConfig, + computerUseConfig, + startupClientAuthProfileId, + runtimeArtifactRequest, + startupPreparedAuth, + agentDir, + sessionAgentId, + effectiveWorkspace, + effectiveCwd, + sandbox, + runAbortController, + usesSupervisionConnection, + resolveReviewerPolicyContext, + resolveRuntimeOptionsForCurrentBinding, + startupAuthProfileId, + abortFromUpstream, + } = connection; + let pluginAppServer = withCodexAppServerFastModeServiceTier(appServer, runtimeParams); + try { + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { phase: "startup" }, + }); + const startupResult = await startCodexAttemptThread({ + attemptClientFactory, + bindingStore, + appServer: pluginAppServer, + pluginConfig, + computerUseConfig, + startupAuthProfileId: startupClientAuthProfileId, + startupAuthBindingFingerprint: preparedAuthBinding?.fingerprint, + ...(runtimeArtifactRequest ? { runtimeArtifactRequest } : {}), + startupPreparedAuth, + startupAuthAccountCacheKey, + startupEnvApiKeyCacheKey, + agentDir, + config: params.config, + buildAttemptParams: buildActiveRunAttemptParams, + sessionAgentId, + effectiveWorkspace, + effectiveCwd, + dynamicTools: toolBridge.specs, + persistentWebSearchAllowed: toolState.persistentWebSearchAllowed, + webSearchAllowed: toolState.webSearchAllowed, + developerInstructions: turnState.promptBuild.developerInstructions, + buildFinalConfigPatch: buildNativeHookRelayFinalConfigPatch, + bundleMcpThreadConfig, + nativeToolSurfaceEnabled, + nativeProviderWebSearchSupport, + sandboxExecServerEnabled, + sandbox, + contextEngineProjection: promptState.contextEngineProjection, + startupTimeoutMs, + signal: runAbortController.signal, + onStartupTimeout: () => runAbortController.abort("codex_startup_timeout"), + spawnedBy: params.spawnedBy, + }); + state.client = startupResult.client; + state.thread = startupResult.thread; + state.runtimeArtifact = startupResult.runtimeArtifact; + state.turnRouter = startupResult.turnRouter; + state.turnRoute = startupResult.turnRoute; + pluginAppServer = startupResult.pluginAppServer; + if ( + usesSupervisionConnection && + (state.thread.connectionScope !== "supervision" || + state.thread.supervisionSourceThreadId !== + connection.mutable.startupBinding?.supervisionSourceThreadId) + ) { + throw new Error("Codex supervised thread lost its private connection ownership"); + } + if (state.thread.lifecycle.action === "started" || state.thread.lifecycle.action === "forked") { + const activePolicy = resolveReviewerPolicyContext(state.thread); + const activeConfig = resolveRuntimeOptionsForCurrentBinding({ + modelProvider: activePolicy.modelProvider, + model: activePolicy.model, + }); + const activeAppServer = resolveCodexAppServerForModelProvider({ + appServer: activeConfig, + provider: activePolicy.modelProvider, + model: activePolicy.model, + config: params.config, + env: process.env, + agentDir, + }); + const previousReviewer = pluginAppServer.approvalsReviewer; + pluginAppServer = { + ...pluginAppServer, + approvalsReviewer: activeAppServer.approvalsReviewer, + }; + if (pluginAppServer.approvalsReviewer !== previousReviewer) { + embeddedAgentLog.info( + "codex app-server approval reviewer updated from active thread model provider", + { + from: previousReviewer, + to: pluginAppServer.approvalsReviewer, + modelProvider: activePolicy.modelProvider, + }, + ); + } + } + state.sandboxExecEnvironmentAcquired = Boolean(startupResult.sandboxEnvironment); + state.codexEnvironmentSelection = startupResult.environmentSelection; + state.codexExecutionCwd = startupResult.executionCwd; + state.codexSandboxPolicy = startupResult.sandboxPolicy; + state.releaseSharedClientLease = startupResult.releaseSharedClientLease; + state.restartContextEngineCodexThread = startupResult.restartContextEngineCodexThread; + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { phase: "thread_ready", threadId: state.thread.threadId }, + }); + } catch (error) { + activateNativePreToolUseFailureFallback(); + releaseCurrentRoute(); + state.nativeHookRelay?.unregister(); + await releaseSandboxExecEnvironment(); + params.abortSignal?.removeEventListener("abort", abortFromUpstream); + throw error; + } + if (applyNoContextEngineContinuityProjection(state.thread.lifecycle.action, state.thread)) { + await rebuildCodexTurnPromptTextFromCurrentProjection(); + } + trajectoryRecorder?.recordEvent("session.started", { + sessionFile: params.sessionFile, + threadId: state.thread.threadId, + authProfileId: startupAuthProfileId, + workspaceDir: effectiveWorkspace, + toolCount: flattenCodexDynamicToolFunctions(toolBridge.specs).length, + }); + recordCodexTrajectoryContext(trajectoryRecorder, { + attempt: params, + cwd: effectiveCwd, + developerInstructions: buildRenderedCodexDeveloperInstructions(), + prompt: turnState.codexTurnPromptText, + tools: toolBridge.availableSpecs, + }); + connection.mutable.pluginAppServer = pluginAppServer; +} diff --git a/extensions/codex/src/app-server/run-attempt-tool-setup.ts b/extensions/codex/src/app-server/run-attempt-tool-setup.ts new file mode 100644 index 000000000000..9b08df1cf1ee --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-tool-setup.ts @@ -0,0 +1,187 @@ +import { + embeddedAgentLog, + isHostScopedAgentToolActive, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + buildDynamicTools, + formatCodexDynamicToolBuildStageSummary, + resolveCodexMessageToolProvider, + shouldWarnCodexDynamicToolBuildStageSummary, +} from "./dynamic-tool-build.js"; +import { resolveCodexDynamicToolsLoadingForRuntime } from "./dynamic-tool-profile.js"; +import { createCodexDynamicToolBridge } from "./dynamic-tools.js"; +import { emitCodexAppServerEvent } from "./run-attempt-lifecycle.js"; +import type { CodexAttemptRuntime } from "./run-attempt-runtime.js"; +import { resolveCodexDynamicToolDirectNames } from "./run-attempt-tools.js"; + +export async function prepareCodexAttemptTools(runtime: CodexAttemptRuntime) { + const { + connection, + bundleMcpThreadConfig, + runtimeParams, + effectiveRuntimeModelId, + nativeToolSurfaceEnabled, + nativeProviderWebSearchSupport, + hookChannelId, + } = runtime; + const { + params, + preDynamicStartupStages, + mutable, + startupAuthProfileId, + resolvedWorkspace, + effectiveWorkspace, + effectiveCwd, + sandboxSessionKey, + sandbox, + runAbortController, + sessionAgentId, + pluginConfig, + profilerEnabled, + } = connection; + const preDynamicSummary = preDynamicStartupStages.snapshot(); + if (shouldWarnCodexDynamicToolBuildStageSummary(preDynamicSummary)) { + embeddedAgentLog.warn( + `codex app-server pre-dynamic startup timings runId=${params.runId} sessionId=${params.sessionId} totalMs=${preDynamicSummary.totalMs} stages=${formatCodexDynamicToolBuildStageSummary(preDynamicSummary)}`, + { + runId: params.runId, + sessionId: params.sessionId, + totalMs: preDynamicSummary.totalMs, + stages: preDynamicSummary.stages, + hasStartupBinding: Boolean(mutable.startupBinding?.threadId), + startupAuthProfileId: startupAuthProfileId ?? null, + bundleMcpDiagnosticCount: bundleMcpThreadConfig.diagnostics.length, + nativeToolSurfaceEnabled, + }, + ); + } + const toolState = { + yieldDetected: false, + persistentWebSearchAllowed: undefined as boolean | undefined, + webSearchAllowed: false, + }; + const toolOutcomeOrdinals = new Map(); + const suppressedDynamicToolOutcomeOrdinals = new Set(); + const onCodexToolOutcome = params.onToolOutcome + ? (observation: Parameters>[0]) => { + if ( + observation.toolCallOrdinal !== undefined && + suppressedDynamicToolOutcomeOrdinals.has(observation.toolCallOrdinal) + ) { + return; + } + params.onToolOutcome?.(observation); + } + : undefined; + const baseAllocateToolOutcomeOrdinal = params.allocateToolOutcomeOrdinal; + const allocateCodexToolOutcomeOrdinal = baseAllocateToolOutcomeOrdinal + ? (toolCallId?: string): number => { + const reservedOrdinal = toolCallId ? toolOutcomeOrdinals.get(toolCallId) : undefined; + if (reservedOrdinal !== undefined) { + return reservedOrdinal; + } + const ordinal = baseAllocateToolOutcomeOrdinal(toolCallId); + if (toolCallId) { + toolOutcomeOrdinals.set(toolCallId, ordinal); + } + return ordinal; + } + : undefined; + const dynamicToolParams = + allocateCodexToolOutcomeOrdinal || onCodexToolOutcome + ? { + ...runtimeParams, + ...(allocateCodexToolOutcomeOrdinal + ? { allocateToolOutcomeOrdinal: allocateCodexToolOutcomeOrdinal } + : {}), + ...(onCodexToolOutcome ? { onToolOutcome: onCodexToolOutcome } : {}), + } + : runtimeParams; + const computerContextEpoch: { + value: number; + frameToolCallId?: string; + frameImageIdentity?: string; + } = { value: 0 }; + const commonToolParams = { + params: dynamicToolParams, + resolvedWorkspace, + effectiveWorkspace, + effectiveCwd, + sandboxSessionKey, + sandbox, + nativeToolSurfaceEnabled, + nativeProviderWebSearchSupport, + runAbortController, + sessionAgentId, + pluginConfig, + profilerEnabled, + onYieldDetected: () => { + toolState.yieldDetected = true; + }, + onCodexAppServerEvent: (event: Parameters[1]) => { + void emitCodexAppServerEvent(params, event); + }, + computerContextEpoch, + }; + const tools = await buildDynamicTools({ + ...commonToolParams, + onPersistentWebSearchPolicyResolved: (allowed) => { + toolState.persistentWebSearchAllowed = allowed; + }, + onWebSearchPolicyResolved: (allowed) => { + toolState.webSearchAllowed = allowed; + }, + }); + const registeredTools = await buildDynamicTools({ + ...commonToolParams, + forceHeartbeatTool: true, + ignoreDisableMessageTool: true, + ignoreRuntimePlan: true, + }); + const toolBridge = createCodexDynamicToolBridge({ + tools, + registeredTools, + signal: runAbortController.signal, + computerContextEpoch, + loading: resolveCodexDynamicToolsLoadingForRuntime(pluginConfig, effectiveRuntimeModelId, { + connectionClass: connection.appServer.connectionClass, + }), + directToolNames: resolveCodexDynamicToolDirectNames( + params, + isHostScopedAgentToolActive("crestodian"), + ), + hookContext: { + agentId: sessionAgentId, + config: params.config, + workspaceDir: effectiveWorkspace, + sessionId: params.sessionId, + sessionKey: sandboxSessionKey, + runId: params.runId, + channelId: hookChannelId, + currentChannelProvider: resolveCodexMessageToolProvider(params), + currentChannelId: params.currentChannelId, + currentMessagingTarget: params.currentMessagingTarget, + currentMessageId: params.currentMessageId, + currentThreadId: params.currentThreadTs, + replyToMode: params.replyToMode, + hasRepliedRef: params.hasRepliedRef, + sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, + onToolOutcome: onCodexToolOutcome, + allocateToolOutcomeOrdinal: allocateCodexToolOutcomeOrdinal, + }, + }); + return { + tools, + registeredTools, + dynamicToolParams, + computerContextEpoch, + toolBridge, + toolState, + toolOutcomeOrdinals, + suppressedDynamicToolOutcomeOrdinals, + onCodexToolOutcome, + allocateCodexToolOutcomeOrdinal, + }; +} + +export type CodexAttemptTools = Awaited>; diff --git a/extensions/codex/src/app-server/run-attempt-turn-request.ts b/extensions/codex/src/app-server/run-attempt-turn-request.ts new file mode 100644 index 000000000000..ac6631f4c48d --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-turn-request.ts @@ -0,0 +1,198 @@ +import { embeddedAgentLog, formatErrorMessage } from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + interruptCodexTurnBestEffort, +} from "./attempt-client-cleanup.js"; +import { + createCodexModelCallDiagnosticEmitter, + utf8JsonByteLength, +} from "./attempt-diagnostics.js"; +import { assertCodexTurnStartResponse } from "./protocol-validators.js"; +import type { CodexTurnStartResponse } from "./protocol.js"; +import { readCodexRateLimitsRevision } from "./rate-limit-cache.js"; +import { + emitCodexAppServerEvent, + withCodexAppServerFastModeServiceTier, +} from "./run-attempt-lifecycle.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { buildTurnStartParams } from "./thread-lifecycle.js"; +import { buildCodexUserPromptMessage } from "./transcript-mirror.js"; + +export async function prepareCodexAttemptTurnRequest( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + ensureCurrentThreadRoute: () => Promise, + waitForActiveNativeTurnCompletion: () => Promise, +) { + const { prompt, state: resourceState, releaseCurrentRoute } = resources; + const { context, turnState, buildRenderedCodexDeveloperInstructions } = prompt; + const { runtime, attemptTools, hookContextWindowFields, workspaceBootstrapContext } = context; + const { connection, runtimeParams, effectiveRuntimeProviderId, effectiveRuntimeModelId } = + runtime; + const { tools } = attemptTools; + const { + params, + usesSupervisionConnection, + codexModelCallId, + codexModelCallTrace, + codexModelContentCapture, + appServer, + runAbortController, + } = connection; + const { state } = turnRuntime; + const buildCodexModelInputMessages = () => [ + ...prompt.codexModelInputHistoryMessages, + buildCodexUserPromptMessage({ ...runtimeParams, prompt: turnState.codexTurnPromptText }), + ]; + const codexModelCallDiagnostics = createCodexModelCallDiagnosticEmitter({ + baseFields: { + runId: params.runId, + callId: codexModelCallId, + ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), + sessionId: params.sessionId, + provider: usesSupervisionConnection + ? (resourceState.thread.modelProvider ?? effectiveRuntimeProviderId) + : params.provider, + model: usesSupervisionConnection + ? (resourceState.thread.model ?? effectiveRuntimeModelId) + : params.modelId, + api: usesSupervisionConnection ? runtimeParams.model.api : params.model.api, + transport: appServer.start.transport, + ...hookContextWindowFields, + trace: codexModelCallTrace, + }, + capture: codexModelContentCapture, + tools, + buildInputMessages: buildCodexModelInputMessages, + buildSystemPrompt: buildRenderedCodexDeveloperInstructions, + onErrorDiagnostic: (error) => { + embeddedAgentLog.debug("codex app-server model call diagnostic ended with error", { + error: formatErrorMessage(error), + }); + }, + }); + const throwIfTurnStartAcceptedAfterAbort = () => { + if (!runAbortController.signal.aborted) { + return; + } + const reason = runAbortController.signal.reason; + if (reason instanceof Error) { + throw reason; + } + const error = new Error( + typeof reason === "string" && reason.length > 0 + ? reason + : "codex app-server turn start aborted before acceptance", + ); + error.name = "AbortError"; + throw error; + }; + const startCodexTurn = async (): Promise => { + const activeTurnRoute = (await ensureCurrentThreadRoute()) as { + armTurn(): void; + cancelTurn(): Promise; + }; + const turnAppServer = withCodexAppServerFastModeServiceTier( + connection.mutable.pluginAppServer, + runtimeParams, + ); + connection.mutable.pluginAppServer = turnAppServer; + const turnStartParams = buildTurnStartParams(runtimeParams, { + threadId: resourceState.thread.threadId, + cwd: resourceState.codexExecutionCwd, + appServer: turnAppServer, + promptText: turnState.codexTurnPromptText, + sandboxPolicy: resourceState.codexSandboxPolicy, + environmentSelection: resourceState.codexEnvironmentSelection, + ...(usesSupervisionConnection + ? {} + : { model: resourceState.thread.model, modelProvider: resourceState.thread.modelProvider }), + turnScopedDeveloperInstructions: workspaceBootstrapContext.turnScopedDeveloperInstructions, + skillsCollaborationInstructions: context.skillsCollaborationInstructions, + memoryCollaborationInstructions: workspaceBootstrapContext.memoryCollaborationInstructions, + heartbeatCollaborationInstructions: + workspaceBootstrapContext.heartbeatCollaborationInstructions, + preserveNativeTurnSettings: usesSupervisionConnection, + }); + codexModelCallDiagnostics.setRequestPayloadBytes(utf8JsonByteLength(turnStartParams)); + state.latestStartupErrorNotification = undefined; + state.rateLimitsRevisionBeforeLastTurnStart = readCodexRateLimitsRevision(resourceState.client); + activeTurnRoute.armTurn(); + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { + phase: "turn_starting", + threadId: resourceState.thread.threadId, + model: turnStartParams.model, + effort: turnStartParams.effort, + collaborationEffort: turnStartParams.collaborationMode?.settings.reasoning_effort, + }, + }); + let acceptedTurnId: string | undefined; + try { + const startedTurn = assertCodexTurnStartResponse( + await resourceState.client.request("turn/start", turnStartParams, { + timeoutMs: params.timeoutMs, + signal: runAbortController.signal, + }), + ); + acceptedTurnId = startedTurn.turn.id; + throwIfTurnStartAcceptedAfterAbort(); + return startedTurn; + } catch (error) { + if (acceptedTurnId) { + interruptCodexTurnBestEffort(resourceState.client, { + threadId: resourceState.thread.threadId, + turnId: acceptedTurnId, + timeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + }); + releaseCurrentRoute(); + } else { + await activeTurnRoute.cancelTurn(); + } + throw error; + } + }; + if ( + resourceState.thread.lifecycle.action === "resumed" && + (resourceState.thread.lifecycle.activeTurnIds?.length ?? 0) > 0 + ) { + embeddedAgentLog.info( + "codex app-server resumed thread has active native turn; waiting before turn/start", + { threadId: resourceState.thread.threadId }, + ); + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { + phase: "turn_start_waiting_for_native_turn", + threadId: resourceState.thread.threadId, + }, + }); + const nativeTurnCompleted = await waitForActiveNativeTurnCompletion(); + if (nativeTurnCompleted) { + await resourceState.turnRoute?.drain(); + } else if (!runAbortController.signal.aborted) { + embeddedAgentLog.warn( + "codex app-server active native turn did not complete before turn/start wait timed out", + { threadId: resourceState.thread.threadId }, + ); + } + } + const buildLlmInputEvent = () => ({ + runId: params.runId, + sessionId: params.sessionId, + provider: usesSupervisionConnection + ? (resourceState.thread.modelProvider ?? effectiveRuntimeProviderId) + : params.provider, + model: usesSupervisionConnection + ? (resourceState.thread.model ?? effectiveRuntimeModelId) + : params.modelId, + systemPrompt: buildRenderedCodexDeveloperInstructions(), + prompt: turnState.codexTurnPromptText, + historyMessages: prompt.codexModelInputHistoryMessages, + imagesCount: params.images?.length ?? 0, + tools, + }); + return { codexModelCallDiagnostics, startCodexTurn, buildLlmInputEvent }; +} diff --git a/extensions/codex/src/app-server/run-attempt-turn-start.ts b/extensions/codex/src/app-server/run-attempt-turn-start.ts new file mode 100644 index 000000000000..168d5f172ecd --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-turn-start.ts @@ -0,0 +1,298 @@ +import { + embeddedAgentLog, + formatErrorMessage, + runAgentCleanupStep, + runAgentHarnessLlmInputHook, + runAgentHarnessLlmOutputHook, + type EmbeddedRunAttemptResult, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { + CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, + unsubscribeCodexThreadBestEffort, +} from "./attempt-client-cleanup.js"; +import { classifyCodexModelCallFailureKind } from "./attempt-diagnostics.js"; +import { + buildCodexTurnStartFailureResult, + isInvalidCodexImagePayloadError, +} from "./attempt-results.js"; +import { isCodexContextRestartSelectionChangedError } from "./attempt-startup.js"; +import type { CodexTurnStartResponse } from "./protocol.js"; +import { emitCodexAppServerEvent, runCodexAgentEndHook } from "./run-attempt-lifecycle.js"; +import type { CodexAttemptNotificationController } from "./run-attempt-notification-controller.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { + isCodexActiveCompactTurnError, + clearCodexBindingAfterInvalidImagePayload, + shouldUseFreshCodexThreadAfterContextEngineOverflow, +} from "./run-attempt-state.js"; +import type { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.js"; +import type { CodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import { buildCodexUserPromptMessage } from "./transcript-mirror.js"; +import { + formatCodexTurnStartUsageLimitError, + markCodexAuthProfileBlockedFromRateLimits, +} from "./usage-limit-error.js"; + +export async function startCodexAttemptTurn( + resources: CodexAttemptResources, + turnRuntime: CodexAttemptTurnState, + notifications: CodexAttemptNotificationController, + requestRuntime: Awaited>, +): Promise<{ result: EmbeddedRunAttemptResult } | { turn: CodexTurnStartResponse }> { + const { + prompt, + state: resourceState, + trajectoryRecorder, + markTrajectoryEndRecorded, + activateNativePreToolUseFailureFallback, + releaseCurrentRoute, + releaseSandboxExecEnvironment, + releaseSharedClientLeaseAndRetireOneShotClient, + } = resources; + const { context, turnState, systemPromptReport } = prompt; + const { runtime, historyState, hookContext, hookContextWindowFields, hookRunner } = context; + const { connection, runtimeParams, effectiveRuntimeProviderId, effectiveRuntimeModelId } = + runtime; + const { + params, + usesSupervisionConnection, + runAbortController, + activeContextEngine, + bindingStore, + bindingIdentity, + appServer, + attemptStartedAt, + startupAuthProfileId, + abortFromUpstream, + } = connection; + const { state, turnIdRef } = turnRuntime; + const { waitForActiveNativeTurnCompletion } = notifications; + const { codexModelCallDiagnostics, startCodexTurn, buildLlmInputEvent } = requestRuntime; + let turn: CodexTurnStartResponse | undefined; + try { + codexModelCallDiagnostics.emitStarted(); + runAgentHarnessLlmInputHook({ event: buildLlmInputEvent(), ctx: hookContext, hookRunner }); + turn = await startCodexTurn(); + } catch (error) { + let turnStartError = error; + if (isCodexActiveCompactTurnError(turnStartError)) { + embeddedAgentLog.info( + "codex app-server turn/start blocked by active compact turn; waiting to retry", + { threadId: resourceState.thread.threadId }, + ); + const compactTurnCompleted = await waitForActiveNativeTurnCompletion(); + if (compactTurnCompleted && !runAbortController.signal.aborted) { + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { + phase: "turn_start_retry_after_compact", + threadId: resourceState.thread.threadId, + }, + }); + try { + turn = await startCodexTurn(); + } catch (retryError) { + turnStartError = retryError; + } + } + } + if ( + turn === undefined && + resourceState.thread.connectionScope !== "supervision" && + shouldUseFreshCodexThreadAfterContextEngineOverflow({ + error: turnStartError, + contextEngineActive: Boolean(activeContextEngine), + thread: resourceState.thread, + }) && + resourceState.restartContextEngineCodexThread + ) { + embeddedAgentLog.warn( + "codex app-server context-engine turn overflowed on resume; retrying with fresh thread", + { threadId: resourceState.thread.threadId, error: formatErrorMessage(turnStartError) }, + ); + try { + const clearedBinding = await bindingStore.mutate(bindingIdentity, { + kind: "clear", + threadId: resourceState.thread.threadId, + }); + if (!clearedBinding) { + embeddedAgentLog.warn( + "codex app-server preserved newer context-engine binding after resume overflow; skipping fresh retry", + { threadId: resourceState.thread.threadId, error: formatErrorMessage(turnStartError) }, + ); + } else { + resourceState.thread = await resourceState.restartContextEngineCodexThread(); + const retryBinding = await bindingStore.read(bindingIdentity); + if ( + retryBinding && + retryBinding.threadId === resourceState.thread.threadId && + retryBinding.contextEngine?.projection + ) { + await bindingStore.mutate(bindingIdentity, { + kind: "patch", + threadId: retryBinding.threadId, + patch: { + contextEngine: { ...retryBinding.contextEngine, projection: undefined }, + }, + }); + embeddedAgentLog.info( + "codex app-server cleared stale context-engine projection after overflow retry", + { + threadId: resourceState.thread.threadId, + previousEpoch: retryBinding.contextEngine.projection.epoch, + }, + ); + } + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { phase: "thread_ready_retry", threadId: resourceState.thread.threadId }, + }); + try { + turn = await startCodexTurn(); + } catch (retryError) { + turnStartError = retryError; + } + } + } catch (retrySetupError) { + turnStartError = retrySetupError; + } + } + if (turn === undefined) { + const usageLimitError = await formatCodexTurnStartUsageLimitError({ + client: resourceState.client, + error: turnStartError, + errorNotification: state.latestStartupErrorNotification, + rateLimitsRevisionBeforeTurnStart: state.rateLimitsRevisionBeforeLastTurnStart, + timeoutMs: appServer.requestTimeoutMs, + signal: runAbortController.signal, + }); + const message = usageLimitError?.message ?? formatErrorMessage(turnStartError); + if (isInvalidCodexImagePayloadError(message)) { + await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, { + phase: "turn_start", + threadId: resourceState.thread.threadId, + error: message, + }); + } + void emitCodexAppServerEvent(params, { + stream: "codex_app_server.lifecycle", + data: { phase: "turn_start_failed", error: message }, + }); + trajectoryRecorder?.recordEvent("session.ended", { + status: "error", + threadId: resourceState.thread.threadId, + timedOut: state.timedOut, + aborted: runAbortController.signal.aborted, + promptError: message, + }); + markTrajectoryEndRecorded(); + runAgentHarnessLlmOutputHook({ + event: { + runId: params.runId, + sessionId: params.sessionId, + provider: usesSupervisionConnection + ? (resourceState.thread.modelProvider ?? effectiveRuntimeProviderId) + : params.provider, + model: usesSupervisionConnection + ? (resourceState.thread.model ?? effectiveRuntimeModelId) + : params.modelId, + ...hookContextWindowFields, + resolvedRef: usesSupervisionConnection + ? `${resourceState.thread.modelProvider ?? effectiveRuntimeProviderId}/${resourceState.thread.model ?? effectiveRuntimeModelId}` + : (params.runtimePlan?.observability.resolvedRef ?? + `${params.provider}/${params.modelId}`), + ...(!usesSupervisionConnection && params.runtimePlan?.observability.harnessId + ? { harnessId: params.runtimePlan.observability.harnessId } + : {}), + assistantTexts: [], + }, + ctx: hookContext, + hookRunner, + }); + const failureKind = classifyCodexModelCallFailureKind({ + error: turnStartError, + timedOut: state.timedOut, + turnCompletionIdleTimedOut: state.turnCompletionIdleTimedOut, + runAborted: runAbortController.signal.aborted, + abortReason: runAbortController.signal.reason, + clientClosedAbort: state.clientClosedAbort, + formatError: formatErrorMessage, + }); + codexModelCallDiagnostics.emitError(message, failureKind ? { failureKind } : {}); + const messagesSnapshot = [ + ...historyState.messages, + buildCodexUserPromptMessage({ ...runtimeParams, prompt: turnState.codexTurnPromptText }), + ]; + await runCodexAgentEndHook(params, { + event: { + messages: messagesSnapshot, + success: false, + error: message, + durationMs: Date.now() - attemptStartedAt, + }, + ctx: hookContext, + hookRunner, + }); + if (!state.timedOut) { + await unsubscribeCodexThreadBestEffort(resourceState.client, { + threadId: resourceState.thread.threadId, + timeoutMs: CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, + }); + } + releaseCurrentRoute(); + activateNativePreToolUseFailureFallback(); + resourceState.nativeHookRelay?.unregister(); + await releaseSandboxExecEnvironment(); + await runAgentCleanupStep({ + runId: params.runId, + sessionId: params.sessionId, + step: "codex-trajectory-flush-startup-failure", + log: embeddedAgentLog, + cleanup: async () => trajectoryRecorder?.flush(), + }); + params.abortSignal?.removeEventListener("abort", abortFromUpstream); + await releaseSharedClientLeaseAndRetireOneShotClient(); + if (usageLimitError) { + await markCodexAuthProfileBlockedFromRateLimits({ + params, + authProfileId: startupAuthProfileId, + rateLimits: usageLimitError.rateLimitsForProfile, + }); + return { + result: buildCodexTurnStartFailureResult({ + params, + message: usageLimitError.message, + messagesSnapshot, + systemPromptReport, + }), + }; + } + if (isCodexContextRestartSelectionChangedError(turnStartError)) { + return { + result: { + ...buildCodexTurnStartFailureResult({ + params, + message, + messagesSnapshot, + systemPromptReport, + }), + codexAppServerFailure: { + kind: "client_closed_before_turn_completed" as const, + transport: appServer.start.transport, + threadId: resourceState.thread.threadId, + replaySafe: true, + }, + }, + }; + } + throw turnStartError; + } + } + if (!turn) { + activateNativePreToolUseFailureFallback(); + await releaseSharedClientLeaseAndRetireOneShotClient(); + throw new Error("codex app-server turn/start failed without an error"); + } + turnIdRef.current = turn.turn.id; + return { turn }; +} diff --git a/extensions/codex/src/app-server/run-attempt-turn-state.ts b/extensions/codex/src/app-server/run-attempt-turn-state.ts new file mode 100644 index 000000000000..c29550dd8a7c --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-turn-state.ts @@ -0,0 +1,207 @@ +import { emitTrustedDiagnosticEvent } from "openclaw/plugin-sdk/diagnostic-runtime"; +import { + CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + interruptCodexTurnBestEffort, +} from "./attempt-client-cleanup.js"; +import { createCodexSteeringQueue } from "./attempt-steering.js"; +import { + resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs, + resolveCodexTurnAssistantCompletionIdleTimeoutMs, + resolveCodexTurnCompletionIdleTimeoutMs, + resolveCodexTurnTerminalIdleTimeoutMs, +} from "./attempt-timeouts.js"; +import { + createCodexAttemptTurnWatchController, + type CodexAttemptTurnWatchTimeoutKind, +} from "./attempt-turn-watches.js"; +import { + resolveCodexNativeHookRelayTtlMs, + CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS, +} from "./native-hook-relay.js"; +import type { + CodexServerNotification, + CodexDynamicToolCallParams, + CodexDynamicToolCallResponse, +} from "./protocol.js"; +import type { CodexAttemptResources } from "./run-attempt-resources.js"; +import { createCodexDynamicToolExecutionRegistry } from "./run-attempt-tools.js"; +import { createCodexUserInputBridge } from "./user-input-bridge.js"; + +const CODEX_NATIVE_HOOK_RELAY_RENEW_INTERVAL_MS = 60_000; + +export function createCodexAttemptTurnState(resources: CodexAttemptResources) { + const { + prompt, + state: resourceState, + projectorRef, + trajectoryRecorder, + startupTimeoutMs, + } = resources; + const { context } = prompt; + const { connection } = context.runtime; + const { params, options, appServer, runAbortController } = connection; + const state = { + latestStartupErrorNotification: undefined as CodexServerNotification | undefined, + rateLimitsRevisionBeforeLastTurnStart: undefined as number | undefined, + completed: false, + terminalTurnNotificationQueued: false, + timedOut: false, + turnCompletionIdleTimedOut: false, + turnWatchTimeoutKind: undefined as CodexAttemptTurnWatchTimeoutKind | undefined, + turnWatchTimeoutIdleMs: undefined as number | undefined, + turnWatchTimeoutMs: undefined as number | undefined, + turnWatchTimeoutLastActivityReason: undefined as string | undefined, + turnWatchTimeoutDetails: undefined as Record | undefined, + turnCompletionIdleTimeoutMessage: undefined as string | undefined, + clientClosedPromptError: undefined as string | undefined, + clientClosedAbort: false, + shouldDelayNativeHookRelayUnregister: false, + lifecycleStarted: false, + lifecycleTerminalEmitted: false, + resolveCompletion: undefined as (() => void) | undefined, + nativeHookRelayLastRenewedAt: 0, + activeAppServerTurnRequests: 0, + unsettledFinalizationHookCount: 0, + rejectedFinalizationHookAssistant: undefined as { itemId?: string } | undefined, + turnCrossedToolHandoff: false, + pendingTerminalDynamicToolRelease: undefined as + | { + call: CodexDynamicToolCallParams; + response: CodexDynamicToolCallResponse; + durationMs: number; + } + | undefined, + terminalDynamicToolReleaseCheckScheduled: false, + currentTurnHadNonTerminalDynamicToolResult: false, + }; + const completion = new Promise((resolve) => { + state.resolveCompletion = resolve; + }); + const turnCompletionIdleTimeoutMs = resolveCodexTurnCompletionIdleTimeoutMs( + options.turnCompletionIdleTimeoutMs ?? appServer.turnCompletionIdleTimeoutMs, + ); + const turnAssistantCompletionIdleTimeoutMs = resolveCodexTurnAssistantCompletionIdleTimeoutMs( + options.turnAssistantCompletionIdleTimeoutMs, + ); + const postToolRawAssistantCompletionIdleTimeoutMs = + resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs( + options.postToolRawAssistantCompletionIdleTimeoutMs ?? + appServer.postToolRawAssistantCompletionIdleTimeoutMs, + turnAssistantCompletionIdleTimeoutMs, + ); + const turnTerminalIdleTimeoutMs = resolveCodexTurnTerminalIdleTimeoutMs( + options.turnTerminalIdleTimeoutMs, + params.runTimeoutOverrideMs, + ); + const turnAttemptIdleTimeoutMs = Math.max(100, Math.floor(params.timeoutMs)); + const pendingOpenClawDynamicToolCompletionIds = new Set(); + // One execution promise per call id prevents duplicate delivery from + // repeating non-idempotent computer input while the attempt remains active. + const openClawDynamicToolExecutions = createCodexDynamicToolExecutionRegistry(); + const activeTurnItemIds = new Set(); + const activeCompletionBlockerItemIds = new Set(); + const activeFinalizationHookRunIds = new Set(); + const finalizationHookBatchStatuses = new Map(); + const turnIdRef: { current?: string } = {}; + const userInputBridgeRef: { current?: ReturnType } = {}; + const steeringQueueRef: { current?: ReturnType } = {}; + const renewNativeHookRelayForTurnProgress = () => { + if (!resourceState.nativeHookRelay || options.nativeHookRelay?.ttlMs !== undefined) { + return; + } + const now = Date.now(); + const renewsRecently = + now - state.nativeHookRelayLastRenewedAt < CODEX_NATIVE_HOOK_RELAY_RENEW_INTERVAL_MS; + const expiresSoon = + now >= resourceState.nativeHookRelay.expiresAtMs - CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS; + if (renewsRecently && !expiresSoon) { + return; + } + state.nativeHookRelayLastRenewedAt = now; + resourceState.nativeHookRelay.renew( + resolveCodexNativeHookRelayTtlMs({ + explicitTtlMs: undefined, + attemptTimeoutMs: turnAttemptIdleTimeoutMs, + startupTimeoutMs, + turnStartTimeoutMs: params.timeoutMs, + }), + ); + }; + const turnWatches = createCodexAttemptTurnWatchController({ + threadId: resourceState.thread.threadId, + signal: runAbortController.signal, + getTurnId: () => turnIdRef.current, + isCompleted: () => state.completed, + isTerminalTurnNotificationQueued: () => state.terminalTurnNotificationQueued, + getActiveAppServerTurnRequests: () => state.activeAppServerTurnRequests, + getActiveTurnItemCount: () => activeTurnItemIds.size, + getActiveCompletionBlockerItemCount: () => activeCompletionBlockerItemIds.size, + getActiveFinalizationHookCount: () => state.unsettledFinalizationHookCount, + canReleaseAssistantCompletionIdle: () => + projectorRef.current?.hasLatestTerminalAssistantCandidateText() === true, + turnCompletionIdleTimeoutMs, + turnAssistantCompletionIdleTimeoutMs, + turnAttemptIdleTimeoutMs, + turnTerminalIdleTimeoutMs, + interruptTimeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, + onInterruptTurn: (input) => interruptCodexTurnBestEffort(resourceState.client, input), + onTimeout: (timeout) => { + state.timedOut = true; + state.turnCompletionIdleTimedOut = true; + state.turnWatchTimeoutKind = timeout.kind; + state.turnWatchTimeoutIdleMs = timeout.idleMs; + state.turnWatchTimeoutMs = timeout.timeoutMs; + state.turnWatchTimeoutLastActivityReason = timeout.lastActivityReason; + state.turnWatchTimeoutDetails = timeout.details; + state.turnCompletionIdleTimeoutMessage = + "codex app-server turn idle timed out waiting for turn/completed"; + }, + onMarkTimedOut: () => projectorRef.current?.markTimedOut(), + onAbort: (reason) => runAbortController.abort(reason), + onCompleted: () => { + state.completed = true; + }, + onResolveCompletion: () => state.resolveCompletion?.(), + onRecordEvent: (name, fields) => trajectoryRecorder?.recordEvent(name, fields), + onAttemptProgress: (reason) => { + renewNativeHookRelayForTurnProgress(); + params.onRunProgress?.({ + reason, + provider: params.provider, + model: params.modelId, + backend: "codex-app-server", + }); + }, + onProgressDiagnostic: (reason) => { + emitTrustedDiagnosticEvent({ + type: "run.progress", + runId: params.runId, + sessionId: params.sessionId, + sessionKey: params.sessionKey, + reason: `codex_app_server:${reason}`, + }); + }, + }); + return { + state, + completion, + turnCompletionIdleTimeoutMs, + turnAssistantCompletionIdleTimeoutMs, + postToolRawAssistantCompletionIdleTimeoutMs, + turnTerminalIdleTimeoutMs, + turnAttemptIdleTimeoutMs, + pendingOpenClawDynamicToolCompletionIds, + openClawDynamicToolExecutions, + activeTurnItemIds, + activeCompletionBlockerItemIds, + activeFinalizationHookRunIds, + finalizationHookBatchStatuses, + turnIdRef, + userInputBridgeRef, + steeringQueueRef, + renewNativeHookRelayForTurnProgress, + turnWatches, + }; +} + +export type CodexAttemptTurnState = ReturnType; diff --git a/extensions/codex/src/app-server/run-attempt-types.ts b/extensions/codex/src/app-server/run-attempt-types.ts new file mode 100644 index 000000000000..652c11c19538 --- /dev/null +++ b/extensions/codex/src/app-server/run-attempt-types.ts @@ -0,0 +1,29 @@ +import type { + EmbeddedRunAttemptParams, + NativeHookRelayEvent, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import type { CodexAppServerBindingStore } from "./session-binding.js"; +import type { CodexAppServerClientFactory } from "./shared-client.js"; + +export type CodexRunAttemptOptions = { + bindingStore: CodexAppServerBindingStore; + pluginConfig?: unknown; + startupTimeoutFloorMs?: number; + nativeHookRelay?: { + enabled?: boolean; + events?: readonly NativeHookRelayEvent[]; + ttlMs?: number; + gatewayTimeoutMs?: number; + hookTimeoutSec?: number; + }; + turnCompletionIdleTimeoutMs?: number; + turnAssistantCompletionIdleTimeoutMs?: number; + postToolRawAssistantCompletionIdleTimeoutMs?: number; + turnTerminalIdleTimeoutMs?: number; + clientFactory?: CodexAppServerClientFactory; +}; + +export type CodexRunAttemptInput = { + params: EmbeddedRunAttemptParams; + options: CodexRunAttemptOptions; +}; diff --git a/extensions/codex/src/app-server/run-attempt.ts b/extensions/codex/src/app-server/run-attempt.ts index 698d3e0263b5..37eabb9efcee 100644 --- a/extensions/codex/src/app-server/run-attempt.ts +++ b/extensions/codex/src/app-server/run-attempt.ts @@ -1,3731 +1,81 @@ // Codex plugin module implements run attempt behavior. -import { - assembleHarnessContextEngine, - assertContextEngineHostSupport, - bootstrapHarnessContextEngine, - buildHarnessContextEngineRuntimeContext, - buildHarnessContextEngineRuntimeContextFromUsage, - CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, - clearActiveEmbeddedRun, - embeddedAgentLog, - finalizeHarnessContextEngineTurn, - FAST_MODE_AUTO_PROGRESS_KIND, - formatFastModeAutoProgressText, - formatErrorMessage, - getAgentHarnessHookRunner, - getBeforeToolCallPolicyDiagnosticState, - isHostScopedAgentToolActive, - isActiveHarnessContextEngine, - loadCodexBundleMcpThreadConfig, - resolveAgentHarnessBeforePromptBuildResult, - resolveAgentRunAbortLifecycleFields, - resolveContextEngineOwnerPluginId, - resolveSandboxContext, - resolveSessionAgentIds, - resolveUserPath, - runAgentHarnessLlmInputHook, - runAgentHarnessLlmOutputHook, - runHarnessContextEngineMaintenance, - resolveFastModeForElapsed, - setActiveEmbeddedRun, - supportsModelTools, - runAgentCleanupStep, - type AgentHarnessRuntimeArtifactBinding, - type FastModeAutoProgressState, - type EmbeddedRunAttemptParams, - type EmbeddedRunAttemptResult, - type NativeHookRelayEvent, - type NativeHookRelayRegistrationHandle, +import type { + EmbeddedRunAttemptParams, + EmbeddedRunAttemptResult, } from "openclaw/plugin-sdk/agent-harness-runtime"; -import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime"; -import { - createDiagnosticTraceContextFromActiveScope, - emitTrustedDiagnosticEvent, - freezeDiagnosticTraceContext, - onInternalDiagnosticEvent, - resolveDiagnosticModelContentCapturePolicy, -} from "openclaw/plugin-sdk/diagnostic-runtime"; -import { loadExecApprovals } from "openclaw/plugin-sdk/exec-approvals-runtime"; -import { - resolveCodexAppServerForModelProvider, - resolveCodexAppServerForOpenClawToolPolicy, -} from "./app-server-policy.js"; -import { - CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, - CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, - interruptCodexTurnBestEffort, - retireCodexAppServerClientAfterTimedOutTurn, - unsubscribeCodexThreadBestEffort, -} from "./attempt-client-cleanup.js"; -import { - buildCodexOpenClawPromptContext, - buildCodexSystemPromptReport, - buildCodexWorkspaceBootstrapContext, - getCodexWorkspaceMemoryToolNames, - prependCodexOpenClawPromptContext, - readContextEngineThreadBootstrapProjection, - readMirroredSessionHistoryMessages, - renderCodexSkillsCollaborationInstructions, - resolveCodexDeliveryHintPreservedInputRange, - resolveContextEngineBootstrapProjectionDecision, -} from "./attempt-context.js"; -import { - classifyCodexModelCallFailureKind, - createCodexModelCallDiagnosticEmitter, - utf8JsonByteLength, -} from "./attempt-diagnostics.js"; -import { - applyCodexTurnNotificationState, - isTerminalCodexTurnNotificationForTurn, - reportCodexExecutionNotification, -} from "./attempt-notification-state.js"; -import { - describeNotificationActivity, - isAssistantCompletionReleaseNotification, - isRawFunctionToolOutputCompletionNotification, - isTerminalTurnStatus, - readCodexNotificationItem, - readRawResponseToolCallId, -} from "./attempt-notifications.js"; -import { - buildCodexAppServerPromptTimeoutOutcome, - buildCodexTurnStartFailureResult, - collectTerminalAssistantText, - isInvalidCodexImagePayloadError, - resolveCodexAppServerReplayBlockedReason, -} from "./attempt-results.js"; -import { - isCodexContextRestartSelectionChangedError, - startCodexAttemptThread, -} from "./attempt-startup.js"; -import { createCodexSteeringQueue, type CodexSteeringQueueOptions } from "./attempt-steering.js"; -import { - resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs, - resolveCodexStartupTimeoutMs, - resolveCodexTurnAssistantCompletionIdleTimeoutMs, - resolveCodexTurnCompletionIdleTimeoutMs, - resolveCodexTurnTerminalIdleTimeoutMs, -} from "./attempt-timeouts.js"; -import { - createCodexAttemptTurnWatchController, - type CodexAttemptTurnWatchTimeoutKind, -} from "./attempt-turn-watches.js"; -import { prepareCodexAppServerAuthBinding } from "./auth-binding.js"; -import { - resolveCodexAppServerAuthAccountCacheKey, - resolveCodexAppServerFallbackApiKeyCacheKey, - resolveCodexAppServerAuthProfileId, - resolveCodexAppServerAuthProfileIdForAgent, - resolveCodexAppServerPreparedAuthHandoff, - resolveCodexAppServerPreparedApiKeyCacheKey, -} from "./auth-bridge.js"; -import { resolveCodexBindingAppServerConnection } from "./binding-connection.js"; -import { isCodexAppServerApprovalRequest, type CodexAppServerClient } from "./client.js"; -import { - isCodexAppServerApprovalPolicyAllowedByRequirements, - isCodexSandboxExecServerEnabled, - readCodexPluginConfig, - resolveCodexComputerUseConfig, - resolveCodexModelBackedReviewerPolicyContext, - resolveOpenClawExecPolicyForCodexAppServer, - shouldAutoApproveCodexAppServerApprovals, - type CodexAppServerRuntimeOptions, -} from "./config.js"; -import { - type CodexProjectedContextRange, - fitCodexProjectedContextForTurnStart, - projectContextEngineAssemblyForCodex, - resolveCodexContextEngineProjectionMaxChars, - resolveCodexContextEngineProjectionReserveTokens, -} from "./context-engine-projection.js"; -import { - buildDynamicTools, - createCodexDynamicToolBuildStageTracker, - formatCodexDynamicToolBuildStageSummary, - resolveCodexAppServerHookChannelId, - resolveCodexMessageToolProvider, - shouldEnableCodexAppServerNativeToolSurface, - shouldWarnCodexDynamicToolBuildStageSummary, -} from "./dynamic-tool-build.js"; -import { - emitDynamicToolErrorDiagnostic, - emitDynamicToolStartedDiagnostic, - emitDynamicToolTerminalDiagnostic, -} from "./dynamic-tool-diagnostics.js"; -import { - handleDynamicToolCallWithTimeout, - hasPendingDynamicToolTerminalDiagnostic, - isDynamicToolTerminalDiagnosticEvent, - isMatchingDynamicToolTerminalDiagnostic, - resolveCodexToolAbortTerminalReason, - resolveDynamicToolCallTimeoutMs, - resolveTerminalDynamicToolBatchAction, - shouldBlockTerminalReleaseForNonTerminalDynamicToolResult, - shouldReleaseTurnAfterTerminalDynamicTool, - toCodexDynamicToolProgressResponse, - toCodexDynamicToolProtocolResponse, -} from "./dynamic-tool-execution.js"; -import { resolveCodexDynamicToolsLoadingForRuntime } from "./dynamic-tool-profile.js"; -import { createCodexDynamicToolBridge } from "./dynamic-tools.js"; -import { handleCodexAppServerElicitationRequest } from "./elicitation-bridge.js"; -import { - CodexAppServerEventProjector, - shouldEmitTranscriptToolProgress, -} from "./event-projector.js"; -import { - buildCodexNativeHookRelayDisabledConfig, - buildCodexNativeHookRelayConfig, - CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS, - createCodexNativeHookRelay, - emitCodexNativePreToolUseFailureDiagnostic, - resolveCodexNativeHookRelayEvents, - resolveCodexNativeHookRelayTtlMs, - scheduleCodexNativeHookRelayUnregister, - type CodexNativePreToolUseFailure, -} from "./native-hook-relay.js"; -import { codexNativeSubagentMonitorRuntime } from "./native-subagent-monitor.js"; -import { isCodexAppServerProfilerEnabled } from "./profiler-flag.js"; -import { - assertCodexTurnStartResponse, - readCodexDynamicToolCallParams, -} from "./protocol-validators.js"; -import { - flattenCodexDynamicToolFunctions, - type CodexSandboxPolicy, - type CodexTurnEnvironmentParams, - type CodexServerNotification, - type CodexDynamicToolCallParams, - type CodexDynamicToolCallResponse, - type CodexTurnStartResponse, - type JsonObject, - type JsonValue, -} from "./protocol.js"; -import { resolveCodexProviderWebSearchSupport } from "./provider-capabilities.js"; -import { readCodexRateLimitsRevision, readRecentCodexRateLimits } from "./rate-limit-cache.js"; -import { - emitCodexAppServerEvent, - ensureCodexWorkspaceDirOnce, - estimateCodexAppServerProjectedTurnTokens, - runCodexAgentEndHook, - shouldKeepCodexSharedAbortOpen, - withCodexAppServerFastModeServiceTier, -} from "./run-attempt-lifecycle.js"; -import { - buildCodexAppServerTimeoutDiagnostics, - clearCodexBindingAfterInvalidImagePayload, - isCodexActiveCompactTurnError, - isNonEmptyString, - joinPresentSections, - markCodexAppServerBindingCoveredThroughTurn, - prependCurrentInboundContext, - readCodexFinalizationHookNotification, - shouldUseFreshCodexThreadAfterContextEngineOverflow, - waitForCodexNotificationDispatchTurn, -} from "./run-attempt-state.js"; -import { - createCodexDynamicToolExecutionRegistry, - handleApprovalRequest, - resolveCodexDynamicToolDirectNames, - toTranscriptToolResult, -} from "./run-attempt-tools.js"; -import { releaseCodexSandboxExecServerEnvironment } from "./sandbox-exec-server.js"; -import { - reclaimCurrentCodexSessionGeneration, - sessionBindingIdentity, - type CodexAppServerBindingIdentity, - type CodexAppServerBindingStore, - type CodexAppServerThreadBinding, -} from "./session-binding.js"; -import { - getLeasedSharedCodexAppServerClient, - retainSharedCodexAppServerClientIfCurrent, - retireSharedCodexAppServerClientIfCurrent, - type CodexAppServerClientFactory, -} from "./shared-client.js"; -import { rotateOversizedCodexAppServerStartupBinding } from "./startup-binding.js"; -import { - buildDeveloperInstructions, - buildContextEngineBinding, - buildTurnCollaborationMode, - buildTurnStartParams, - codexDynamicToolsFingerprint, - codexLegacyDynamicToolsFingerprint, - resolveCodexAppServerThreadModelSelection, - type CodexAppServerThreadLifecycleBinding, - type CodexContextEngineThreadBootstrapProjection, -} from "./thread-lifecycle.js"; -import { - inferCodexDynamicToolMeta, - resolveCodexToolProgressDetailMode, - sanitizeCodexToolArguments, -} from "./tool-progress-normalization.js"; -import { - createCodexTrajectoryRecorder, - type CodexHostTrajectoryRecorder, - normalizeCodexTrajectoryError, - recordCodexTrajectoryCompletion, - recordCodexTrajectoryContext, -} from "./trajectory.js"; -import { - buildCodexUserPromptMessage, - codexTranscriptMirrorRuntime, - createCodexAppServerUserMessagePersistenceNotifier, - mirrorPromptAtTurnStartBestEffort, -} from "./transcript-mirror.js"; -import { - CODEX_APP_SERVER_NATIVE_TURN_WAIT_TIMEOUT_MS, - type CodexAppServerServerRequest, - type CodexAppServerTurnRouter, - type CodexThreadRouteReservation, - type CodexThreadRouteScope, -} from "./turn-router.js"; -import { - formatCodexTurnStartUsageLimitError, - markCodexAuthProfileBlockedFromRateLimits, - refreshCodexUsageLimitPromptError, -} from "./usage-limit-error.js"; -import { createCodexUserInputBridge } from "./user-input-bridge.js"; -import { resolveCodexWebSearchPlan } from "./web-search.js"; - -const CODEX_NATIVE_HOOK_RELAY_RENEW_INTERVAL_MS = 60_000; +import { activateCodexAttemptTurn } from "./run-attempt-active-turn.js"; +import { cleanupCodexAttempt } from "./run-attempt-cleanup.js"; +import { prepareCodexAttemptConnection } from "./run-attempt-connection.js"; +import { prepareCodexAttemptContext } from "./run-attempt-context.js"; +import { finalizeCodexAttempt } from "./run-attempt-finalize.js"; +import { createCodexAttemptLifecycleController } from "./run-attempt-lifecycle-controller.js"; +import { createCodexAttemptNotificationController } from "./run-attempt-notification-controller.js"; +import { prepareCodexAttemptPrompt } from "./run-attempt-prompt.js"; +import { prepareCodexAttemptResources } from "./run-attempt-resources.js"; +import { prepareCodexAttemptRoute } from "./run-attempt-route.js"; +import { prepareCodexAttemptRuntime } from "./run-attempt-runtime.js"; +import { createCodexAttemptServerRequestController } from "./run-attempt-server-requests.js"; +import { startCodexAttemptRuntime } from "./run-attempt-start.js"; +import { prepareCodexAttemptTools } from "./run-attempt-tool-setup.js"; +import { prepareCodexAttemptTurnRequest } from "./run-attempt-turn-request.js"; +import { startCodexAttemptTurn } from "./run-attempt-turn-start.js"; +import { createCodexAttemptTurnState } from "./run-attempt-turn-state.js"; +import type { CodexRunAttemptOptions } from "./run-attempt-types.js"; export async function runCodexAppServerAttempt( params: EmbeddedRunAttemptParams, - options: { - bindingStore: CodexAppServerBindingStore; - pluginConfig?: unknown; - startupTimeoutFloorMs?: number; - nativeHookRelay?: { - enabled?: boolean; - events?: readonly NativeHookRelayEvent[]; - ttlMs?: number; - gatewayTimeoutMs?: number; - hookTimeoutSec?: number; - }; - turnCompletionIdleTimeoutMs?: number; - turnAssistantCompletionIdleTimeoutMs?: number; - postToolRawAssistantCompletionIdleTimeoutMs?: number; - turnTerminalIdleTimeoutMs?: number; - clientFactory?: CodexAppServerClientFactory; - }, + options: CodexRunAttemptOptions, ): Promise { - const attemptStartedAt = Date.now(); - const profilerEnabled = isCodexAppServerProfilerEnabled(params.config); - const codexModelCallTrace = freezeDiagnosticTraceContext( - createDiagnosticTraceContextFromActiveScope(), + const connection = await prepareCodexAttemptConnection({ params, options }); + const runtime = await prepareCodexAttemptRuntime(connection); + const attemptTools = await prepareCodexAttemptTools(runtime); + const attemptContext = await prepareCodexAttemptContext(runtime, attemptTools); + const attemptPrompt = await prepareCodexAttemptPrompt(attemptContext); + const resources = prepareCodexAttemptResources(attemptPrompt); + await startCodexAttemptRuntime(resources); + + const turnRuntime = createCodexAttemptTurnState(resources); + const lifecycle = createCodexAttemptLifecycleController(resources, turnRuntime); + const notifications = createCodexAttemptNotificationController(resources, turnRuntime, lifecycle); + const serverRequests = createCodexAttemptServerRequestController( + resources, + turnRuntime, + lifecycle, ); - const codexModelContentCapture = resolveDiagnosticModelContentCapturePolicy(params.config); - const codexModelCallId = `${params.runId}:codex-model:1`; - const fastModeAutoStartedAtMs = - typeof params.fastModeStartedAtMs === "number" && Number.isFinite(params.fastModeStartedAtMs) - ? params.fastModeStartedAtMs - : undefined; - const fastModeAutoProgressState: FastModeAutoProgressState = params.fastModeAutoProgressState ?? { - offAnnounced: false, - resetAnnounced: false, - }; - // Startup phase timings are profiler-gated because this function runs before - // every Codex turn; normal production should not do timing bookkeeping here. - const preDynamicStartupStages = createCodexDynamicToolBuildStageTracker({ - enabled: profilerEnabled, - }); - const attemptClientFactory = options.clientFactory ?? getLeasedSharedCodexAppServerClient; - const runtimeArtifactRequest = - params.captureRuntimeArtifact || params.expectedRuntimeArtifact - ? params.expectedRuntimeArtifact - ? { expected: params.expectedRuntimeArtifact } - : {} - : undefined; - const pluginConfig = readCodexPluginConfig(options.pluginConfig); - const computerUseConfig = resolveCodexComputerUseConfig({ pluginConfig }); - const { sessionAgentId } = resolveSessionAgentIds({ - sessionKey: params.sessionKey, - config: params.config, - agentId: params.agentId, - }); - const beforeToolCallPolicy = getBeforeToolCallPolicyDiagnosticState(); - preDynamicStartupStages.mark("config"); - const resolvedWorkspace = resolveUserPath(params.workspaceDir); - await ensureCodexWorkspaceDirOnce(resolvedWorkspace); - preDynamicStartupStages.mark("workspace"); - const sandboxSessionKey = - params.sandboxSessionKey?.trim() || params.sessionKey?.trim() || params.sessionId; - const contextSessionKey = params.sessionKey?.trim() || sandboxSessionKey; - const sandbox = await resolveSandboxContext({ - config: params.config, - sessionKey: sandboxSessionKey, - workspaceDir: resolvedWorkspace, - }); - preDynamicStartupStages.mark("sandbox"); - const execPolicy = resolveOpenClawExecPolicyForCodexAppServer({ - execOverrides: params.execOverrides, - approvals: loadExecApprovals(), - config: params.config, - agentId: sessionAgentId, - }); - const agentDir = params.agentDir ?? resolveAgentDir(params.config ?? {}, sessionAgentId); - const bindingIdentity: CodexAppServerBindingIdentity = sessionBindingIdentity({ - sessionId: params.sessionId, - sessionKey: params.sessionKey, - agentId: params.agentId, - config: params.config, - }); - const bindingStore = options.bindingStore; - preDynamicStartupStages.mark("session-agent"); - let activeContextEngine = isActiveHarnessContextEngine(params.contextEngine) - ? params.contextEngine - : undefined; - const isInactiveThreadBootstrapBinding = (binding: CodexAppServerThreadBinding | undefined) => - !activeContextEngine && binding?.contextEngine?.projection?.mode === "thread_bootstrap"; - let startupBinding = await bindingStore.read(bindingIdentity); - if (!startupBinding && bindingIdentity.kind === "session" && bindingIdentity.sessionKey) { - const reclaimed = await reclaimCurrentCodexSessionGeneration({ - bindingStore, - identity: bindingIdentity, - config: params.config, - }); - if (!reclaimed) { - throw new Error( - `Codex session generation is no longer current: ${bindingIdentity.sessionId}`, - ); - } - startupBinding = await bindingStore.read(bindingIdentity); - } - preDynamicStartupStages.mark("read-binding"); - // Only the private binding store may authorize the native user-home runtime. - // Public session metadata and preserveNativeModel are intentionally insufficient. - const usesSupervisionConnection = startupBinding?.connectionScope === "supervision"; - if (usesSupervisionConnection) { - activeContextEngine = undefined; - } - if (usesSupervisionConnection && pluginConfig.supervision?.enabled !== true) { - throw new Error( - "Codex supervision is disabled; refusing to open a native user-home supervised session", - ); - } - const resolveRuntimeOptionsForBinding = (paramsLocal: { - modelProvider?: string; - model?: string; - }) => - resolveCodexBindingAppServerConnection({ - binding: startupBinding, - pluginConfig, - execPolicy, - modelProvider: paramsLocal.modelProvider, - model: paramsLocal.model, - config: params.config, - agentDir, - openClawSandboxActive: sandbox?.enabled === true, - }).appServer; - const initialStartupBindingHadInactiveThreadBootstrap = - isInactiveThreadBootstrapBinding(startupBinding); - const preparedAuthRoute = usesSupervisionConnection - ? undefined - : params.runtimePlan?.auth.modelRoute; - const startupAuthProfileCandidate = usesSupervisionConnection - ? undefined - : preparedAuthRoute - ? params.runtimePlan?.auth.forwardedAuthProfileId - : (params.runtimePlan?.auth.forwardedAuthProfileId ?? - params.authProfileId ?? - startupBinding?.authProfileId); - const resolvedStartupAuthProfileId = usesSupervisionConnection - ? undefined - : preparedAuthRoute - ? startupAuthProfileCandidate - : params.authProfileStore - ? resolveCodexAppServerAuthProfileId({ - authProfileId: startupAuthProfileCandidate, - store: params.authProfileStore, - config: params.config, - }) - : resolveCodexAppServerAuthProfileIdForAgent({ - authProfileId: startupAuthProfileCandidate, - agentDir, - config: params.config, - }); - const authHandoff = usesSupervisionConnection - ? { authProfileId: undefined, nativeAuthProfile: true, preparedAuth: undefined } - : await resolveCodexAppServerPreparedAuthHandoff({ - authRequirement: preparedAuthRoute?.authRequirement, - resolvedApiKey: params.resolvedApiKey, - authProfileId: resolvedStartupAuthProfileId, - authProfileStore: params.authProfileStore, - agentDir, - config: params.config, - subscriptionProfileRequiredError: - "Prepared Codex subscription route requires a forwarded OpenAI OAuth or token profile.", - subscriptionProfileUnusableError: "Prepared Codex subscription auth profile is unusable.", - }); - const { - authProfileId: startupAuthProfileId, - nativeAuthProfile, - preparedAuth: startupPreparedAuth, - } = authHandoff; - const startupClientAuthProfileId = - usesSupervisionConnection || startupPreparedAuth?.kind === "api-key" - ? null - : startupAuthProfileId; - const resolveReviewerPolicyContext = (binding: CodexAppServerThreadBinding | undefined) => { - const nativeModelOwned = binding?.preserveNativeModel === true; - // A supervised Codex branch owns its model. The outer OpenClaw default may - // be Anthropic (or anything else) and must not select this thread's reviewer. - return resolveCodexModelBackedReviewerPolicyContext({ - provider: nativeModelOwned ? "codex" : params.provider, - model: nativeModelOwned ? binding.model : params.modelId, - bindingModelProvider: binding?.modelProvider, - bindingModel: binding?.model, - nativeAuthProfile, - }); - }; - let reviewerPolicyContext = resolveReviewerPolicyContext(startupBinding); - preDynamicStartupStages.mark("auth-profile"); - let configuredAppServer = resolveRuntimeOptionsForBinding({ - modelProvider: reviewerPolicyContext.modelProvider, - model: reviewerPolicyContext.model, - }); - const effectiveWorkspace = sandbox?.enabled - ? sandbox.workspaceAccess === "rw" - ? resolvedWorkspace - : sandbox.workspaceDir - : resolvedWorkspace; - const requestedCwd = params.cwd ? resolveUserPath(params.cwd) : undefined; - if (sandbox?.enabled && requestedCwd && requestedCwd !== resolvedWorkspace) { - throw new Error( - "cwd override is not supported for sandboxed Codex app-server runs; omit cwd or use the agent workspace as cwd", - ); - } - const effectiveCwd = sandbox?.enabled ? effectiveWorkspace : (requestedCwd ?? effectiveWorkspace); - await ensureCodexWorkspaceDirOnce(effectiveWorkspace); - preDynamicStartupStages.mark("effective-workspace"); - let policyAppServer = resolveCodexAppServerForOpenClawToolPolicy({ - appServer: configuredAppServer, - pluginConfig, - env: process.env, - shouldPromote: - beforeToolCallPolicy.hasBeforeToolCallHook || - beforeToolCallPolicy.trustedToolPolicies.length > 0, - execPolicy, - canUseUntrustedApprovalPolicy: - configuredAppServer.start.transport !== "stdio" || - isCodexAppServerApprovalPolicyAllowedByRequirements("untrusted"), - }); - let appServer = resolveCodexAppServerForModelProvider({ - appServer: policyAppServer, - provider: reviewerPolicyContext.modelProvider, - model: reviewerPolicyContext.model, - config: params.config, - env: process.env, - agentDir, - }); - if (configuredAppServer.approvalPolicy === "never" && appServer.approvalPolicy === "untrusted") { - embeddedAgentLog.info("codex app-server approval policy promoted for OpenClaw tool policy", { - from: "never", - to: "untrusted", - beforeToolCallHook: beforeToolCallPolicy.hasBeforeToolCallHook, - trustedToolPolicies: beforeToolCallPolicy.trustedToolPolicies, - }); - } - preDynamicStartupStages.mark("app-server-policy"); - let pluginAppServer: CodexAppServerRuntimeOptions = appServer; - let nativeHookRelayEvents = resolveCodexNativeHookRelayEvents({ - configuredEvents: options.nativeHookRelay?.events, - appServer, - }); - preDynamicStartupStages.mark("native-hook-relay"); - - const runAbortController = new AbortController(); - // AbortController preserves its first reason, so retain explicit cancellation - // that can arrive after the timeout abort while cleanup is still draining. - let explicitCancellationObserved = false; - let explicitCancellationReason: unknown; - let terminalOutcomeFrozen = false; - let sharedAbortAllowedAfterTerminalOutcome = false; - let attemptAbortNotified = false; - const notifyAttemptAbort = () => { - if (attemptAbortNotified) { - return; - } - attemptAbortNotified = true; - params.onAttemptAbort?.(); - }; - const abortExplicitly = (reason: unknown) => { - if (terminalOutcomeFrozen) { - if (sharedAbortAllowedAfterTerminalOutcome) { - notifyAttemptAbort(); - } - return; - } - notifyAttemptAbort(); - explicitCancellationObserved = true; - explicitCancellationReason ??= reason; - runAbortController.abort(reason); - }; - const abortFromUpstream = () => { - abortExplicitly(params.abortSignal?.reason ?? "upstream_abort"); - }; - if (params.abortSignal?.aborted) { - abortFromUpstream(); - } else { - params.abortSignal?.addEventListener("abort", abortFromUpstream, { once: true }); - } - - startupBinding = await rotateOversizedCodexAppServerStartupBinding({ - binding: startupBinding, - bindingStore, - identity: bindingIdentity, - sessionFile: params.sessionFile, - agentDir, - codexHome: appServer.start.env?.CODEX_HOME, - config: params.config, - contextEngineActive: Boolean(activeContextEngine), - }); - const initialInactiveThreadBootstrapBindingForcedFreshStart = - initialStartupBindingHadInactiveThreadBootstrap && !startupBinding?.threadId; - preDynamicStartupStages.mark("rotate-binding"); - reviewerPolicyContext = resolveReviewerPolicyContext(startupBinding); - configuredAppServer = resolveRuntimeOptionsForBinding({ - modelProvider: reviewerPolicyContext.modelProvider, - model: reviewerPolicyContext.model, - }); - policyAppServer = resolveCodexAppServerForOpenClawToolPolicy({ - appServer: configuredAppServer, - pluginConfig, - env: process.env, - shouldPromote: - beforeToolCallPolicy.hasBeforeToolCallHook || - beforeToolCallPolicy.trustedToolPolicies.length > 0, - execPolicy, - canUseUntrustedApprovalPolicy: - configuredAppServer.start.transport !== "stdio" || - isCodexAppServerApprovalPolicyAllowedByRequirements("untrusted"), - }); - appServer = resolveCodexAppServerForModelProvider({ - appServer: policyAppServer, - provider: reviewerPolicyContext.modelProvider, - model: reviewerPolicyContext.model, - config: params.config, - env: process.env, - agentDir, - }); - pluginAppServer = appServer; - nativeHookRelayEvents = resolveCodexNativeHookRelayEvents({ - configuredEvents: options.nativeHookRelay?.events, - appServer, - }); - const preparedAuthBinding = - !usesSupervisionConnection && appServer.start.homeScope !== "user" && startupAuthProfileId - ? await prepareCodexAppServerAuthBinding({ - authProfileId: startupAuthProfileId, - authProfileStore: params.authProfileStore, - agentDir, - config: params.config, - }) - : undefined; - const attemptAuthProfileStore = preparedAuthBinding?.authProfileStore ?? params.authProfileStore; - const effectiveContextWindowInfo = usesSupervisionConnection - ? undefined - : params.contextWindowInfo; - const effectiveContextTokenBudget = usesSupervisionConnection - ? undefined - : params.contextTokenBudget; - const effectiveRuntimeProviderId = usesSupervisionConnection - ? (startupBinding?.modelProvider ?? "codex") - : params.provider; - // Pending branches learn the authoritative model only inside App Server. - // This placeholder prevents outer-model metadata from shaping pre-start context policy. - const effectiveRuntimeModelId = usesSupervisionConnection - ? (startupBinding?.model ?? "codex-native") - : params.modelId; - const { - authProfileId: _outerAuthProfileId, - contextWindowInfo: _outerContextWindowInfo, - contextTokenBudget: _outerContextTokenBudget, - model: _outerModel, - modelId: _outerModelId, - provider: _outerProvider, - runtimePlan: _outerRuntimePlan, - requestedModelId: _outerRequestedModelId, - fallbackReason: _outerFallbackReason, - degradedReason: _outerDegradedReason, - thinkLevel: _outerThinkLevel, - fastMode: _outerFastMode, - ...paramsWithoutOuterNativeOwnership - } = params; - const supervisedRuntimeModel = { - id: effectiveRuntimeModelId, - name: effectiveRuntimeModelId, - provider: effectiveRuntimeProviderId, - api: "openai-chatgpt-responses", - reasoning: true, - input: ["text", "image"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: undefined, - maxTokens: undefined, - } as unknown as EmbeddedRunAttemptParams["model"]; - const runtimeParams: EmbeddedRunAttemptParams = usesSupervisionConnection - ? { - ...paramsWithoutOuterNativeOwnership, - provider: "codex", - modelId: effectiveRuntimeModelId, - model: supervisedRuntimeModel, - thinkLevel: _outerThinkLevel, - sessionKey: contextSessionKey, - } - : { - ...params, - authProfileStore: attemptAuthProfileStore, - sessionKey: contextSessionKey, - ...(startupAuthProfileId ? { authProfileId: startupAuthProfileId } : {}), - }; - const activeSessionId = params.sessionId; - const activeSessionFile = params.sessionFile; - const buildActiveRunAttemptParams = (): EmbeddedRunAttemptParams => ({ - ...runtimeParams, - sessionId: activeSessionId, - sessionFile: activeSessionFile, - }); - const startupAuthAccountCacheKey = usesSupervisionConnection - ? undefined - : startupPreparedAuth?.kind === "api-key" - ? resolveCodexAppServerPreparedApiKeyCacheKey(startupPreparedAuth.apiKey) - : startupPreparedAuth?.kind === "profile" - ? startupPreparedAuth.snapshot?.secretFreeCacheKey - : await resolveCodexAppServerAuthAccountCacheKey({ - authProfileId: startupAuthProfileId, - authProfileStore: attemptAuthProfileStore, - agentDir, - config: params.config, - }); - const startupEnvApiKeyCacheKey = usesSupervisionConnection - ? undefined - : startupPreparedAuth || startupAuthProfileId - ? undefined - : resolveCodexAppServerFallbackApiKeyCacheKey({ - startOptions: appServer.start, - }); - preDynamicStartupStages.mark("auth-cache"); - const bundleMcpThreadConfig = await loadCodexBundleMcpThreadConfig({ - workspaceDir: effectiveWorkspace, - cfg: params.config, - toolsEnabled: usesSupervisionConnection || supportsModelTools(params.model), - disableTools: params.disableTools, - toolsAllow: params.toolsAllow, - }); - preDynamicStartupStages.mark("bundle-mcp"); - const sandboxExecServerEnabled = isCodexSandboxExecServerEnabled(pluginConfig); - const nativeToolSurfaceEnabled = shouldEnableCodexAppServerNativeToolSurface( - runtimeParams, - sandbox, - { - agentId: sessionAgentId, - runtimeSessionKey: sandboxSessionKey, - sandboxExecServerEnabled, - }, + const { ensureCurrentThreadRoute } = await prepareCodexAttemptRoute( + resources, + turnRuntime, + notifications, + serverRequests.handleServerRequest, ); - preDynamicStartupStages.mark("native-tool-surface"); - const nativeProviderWebSearchSupport = - resolveCodexWebSearchPlan({ - config: params.config, - disableTools: params.disableTools, - nativeToolSurfaceEnabled, - }).kind === "native-hosted" - ? await resolveCodexProviderWebSearchSupport({ - clientFactory: attemptClientFactory, - appServer, - authProfileId: startupClientAuthProfileId, - preparedAuth: startupPreparedAuth, - agentDir, - config: params.config, - modelProviderOverride: usesSupervisionConnection - ? startupBinding?.modelProvider - : resolveCodexAppServerThreadModelSelection({ - provider: params.provider, - model: params.modelId, - binding: startupBinding, - authProfileId: startupAuthProfileId, - authProfileStore: attemptAuthProfileStore, - agentDir, - config: params.config, - }).modelProvider, - signal: runAbortController.signal, - }) - : "unsupported"; - preDynamicStartupStages.mark("provider-capabilities"); - for (const diagnostic of bundleMcpThreadConfig.diagnostics) { - embeddedAgentLog.warn(`bundle-mcp: ${diagnostic.pluginId}: ${diagnostic.message}`); - } - if (activeContextEngine) { - assertContextEngineHostSupport({ - contextEngine: activeContextEngine, - operation: "agent-run", - host: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, - }); - } - const hookChannelId = resolveCodexAppServerHookChannelId(params, sandboxSessionKey); - preDynamicStartupStages.mark("context-engine-support"); - const preDynamicSummary = preDynamicStartupStages.snapshot(); - if (shouldWarnCodexDynamicToolBuildStageSummary(preDynamicSummary)) { - embeddedAgentLog.warn( - `codex app-server pre-dynamic startup timings runId=${params.runId} sessionId=${params.sessionId} totalMs=${preDynamicSummary.totalMs} stages=${formatCodexDynamicToolBuildStageSummary(preDynamicSummary)}`, - { - runId: params.runId, - sessionId: params.sessionId, - totalMs: preDynamicSummary.totalMs, - stages: preDynamicSummary.stages, - hasStartupBinding: Boolean(startupBinding?.threadId), - startupAuthProfileId: startupAuthProfileId ?? null, - bundleMcpDiagnosticCount: bundleMcpThreadConfig.diagnostics.length, - nativeToolSurfaceEnabled, - }, - ); - } - let yieldDetected = false; - const toolOutcomeOrdinals = new Map(); - const suppressedDynamicToolOutcomeOrdinals = new Set(); - const onCodexToolOutcome = params.onToolOutcome - ? (observation: Parameters>[0]) => { - if ( - observation.toolCallOrdinal !== undefined && - suppressedDynamicToolOutcomeOrdinals.has(observation.toolCallOrdinal) - ) { - return; - } - params.onToolOutcome?.(observation); - } - : undefined; - const baseAllocateToolOutcomeOrdinal = params.allocateToolOutcomeOrdinal; - const allocateCodexToolOutcomeOrdinal = baseAllocateToolOutcomeOrdinal - ? (toolCallId?: string): number => { - const reservedOrdinal = toolCallId ? toolOutcomeOrdinals.get(toolCallId) : undefined; - if (reservedOrdinal !== undefined) { - return reservedOrdinal; - } - const ordinal = baseAllocateToolOutcomeOrdinal(toolCallId); - if (toolCallId) { - toolOutcomeOrdinals.set(toolCallId, ordinal); - } - return ordinal; - } - : undefined; - const dynamicToolParams = - allocateCodexToolOutcomeOrdinal || onCodexToolOutcome - ? { - ...runtimeParams, - ...(allocateCodexToolOutcomeOrdinal - ? { allocateToolOutcomeOrdinal: allocateCodexToolOutcomeOrdinal } - : {}), - ...(onCodexToolOutcome ? { onToolOutcome: onCodexToolOutcome } : {}), - } - : runtimeParams; - let persistentWebSearchAllowed: boolean | undefined; - let webSearchAllowed = false; - // Codex can compact a thread while keeping the same dynamic-tool bridge. - // Bind screenshot coordinates to the context generation the pixels reached. - const computerContextEpoch: { - value: number; - frameToolCallId?: string; - frameImageIdentity?: string; - } = { value: 0 }; - const tools = await buildDynamicTools({ - params: dynamicToolParams, - resolvedWorkspace, - effectiveWorkspace, - effectiveCwd, - sandboxSessionKey, - sandbox, - nativeToolSurfaceEnabled, - nativeProviderWebSearchSupport, - runAbortController, - sessionAgentId, - pluginConfig, - profilerEnabled, - onYieldDetected: () => { - yieldDetected = true; - }, - onCodexAppServerEvent: (event) => { - void emitCodexAppServerEvent(params, event); - }, - onPersistentWebSearchPolicyResolved: (allowed) => { - persistentWebSearchAllowed = allowed; - }, - onWebSearchPolicyResolved: (allowed) => { - webSearchAllowed = allowed; - }, - computerContextEpoch, - }); - const registeredTools = await buildDynamicTools({ - params: dynamicToolParams, - resolvedWorkspace, - effectiveWorkspace, - effectiveCwd, - sandboxSessionKey, - sandbox, - nativeToolSurfaceEnabled, - nativeProviderWebSearchSupport, - runAbortController, - sessionAgentId, - pluginConfig, - profilerEnabled, - forceHeartbeatTool: true, - ignoreDisableMessageTool: true, - ignoreRuntimePlan: true, - onYieldDetected: () => { - yieldDetected = true; - }, - onCodexAppServerEvent: (event) => { - void emitCodexAppServerEvent(params, event); - }, - computerContextEpoch, - }); - const toolBridge = createCodexDynamicToolBridge({ - tools, - registeredTools, - signal: runAbortController.signal, - computerContextEpoch, - loading: resolveCodexDynamicToolsLoadingForRuntime(pluginConfig, effectiveRuntimeModelId, { - connectionClass: appServer.connectionClass, - }), - directToolNames: resolveCodexDynamicToolDirectNames( - params, - isHostScopedAgentToolActive("crestodian"), - ), - hookContext: { - agentId: sessionAgentId, - config: params.config, - workspaceDir: effectiveWorkspace, - sessionId: params.sessionId, - sessionKey: sandboxSessionKey, - runId: params.runId, - channelId: hookChannelId, - currentChannelProvider: resolveCodexMessageToolProvider(params), - currentChannelId: params.currentChannelId, - currentMessagingTarget: params.currentMessagingTarget, - currentMessageId: params.currentMessageId, - currentThreadId: params.currentThreadTs, - replyToMode: params.replyToMode, - hasRepliedRef: params.hasRepliedRef, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - onToolOutcome: onCodexToolOutcome, - allocateToolOutcomeOrdinal: allocateCodexToolOutcomeOrdinal, - }, - }); - const activeTranscriptTarget = { - agentId: sessionAgentId, - sessionFile: activeSessionFile, - sessionId: activeSessionId, - sessionKey: contextSessionKey, - }; - let historyMessages = - !activeContextEngine && initialStartupBindingHadInactiveThreadBootstrap - ? [] - : ((await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? []); - const hadSessionTranscriptState = historyMessages.length > 0; - const hookContextWindowFields = { - ...(effectiveContextWindowInfo?.tokens - ? { contextTokenBudget: effectiveContextWindowInfo.tokens } - : effectiveContextTokenBudget - ? { contextTokenBudget: effectiveContextTokenBudget } - : {}), - ...(effectiveContextWindowInfo?.source - ? { contextWindowSource: effectiveContextWindowInfo.source } - : {}), - ...(effectiveContextWindowInfo?.referenceTokens - ? { contextWindowReferenceTokens: effectiveContextWindowInfo.referenceTokens } - : {}), - }; - const hookContext = { - runId: params.runId, - agentId: sessionAgentId, - sessionKey: sandboxSessionKey, - sessionId: params.sessionId, - workspaceDir: params.workspaceDir, - messageProvider: params.messageProvider ?? undefined, - trigger: params.trigger, - channelId: hookChannelId, - ...hookContextWindowFields, - }; - const hookRunner = getAgentHarnessHookRunner(); - const activeContextEnginePluginId = activeContextEngine - ? resolveContextEngineOwnerPluginId(activeContextEngine) - : undefined; - const buildActiveContextEngineRuntimeContext = () => - buildHarnessContextEngineRuntimeContext({ - attempt: buildActiveRunAttemptParams(), - workspaceDir: effectiveWorkspace, - cwd: effectiveCwd, - agentDir, - activeAgentId: sessionAgentId, - contextEnginePluginId: activeContextEnginePluginId, - tokenBudget: effectiveContextTokenBudget, - }); - if (activeContextEngine) { - await bootstrapHarnessContextEngine({ - hadSessionFile: hadSessionTranscriptState, - contextEngine: activeContextEngine, - sessionId: activeSessionId, - sessionKey: contextSessionKey, - sessionFile: activeSessionFile, - sessionTarget: params.sessionTarget, - runtimeContext: buildActiveContextEngineRuntimeContext(), - contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, - providerId: effectiveRuntimeProviderId, - requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, - modelId: effectiveRuntimeModelId, - fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, - degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, - runMaintenance: runHarnessContextEngineMaintenance, - config: params.config, - warn: (message) => embeddedAgentLog.warn(message), - }); - historyMessages = - (await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? historyMessages; - } - const memoryToolNames = getCodexWorkspaceMemoryToolNames(toolBridge.availableSpecs); - const workspaceBootstrapContext = await buildCodexWorkspaceBootstrapContext({ - params: runtimeParams, - resolvedWorkspace, - effectiveWorkspace, - sessionKey: contextSessionKey, - sessionAgentId, - memoryToolNames, - }); - const baseDeveloperInstructions = joinPresentSections( - buildDeveloperInstructions(runtimeParams, { - dynamicTools: toolBridge.availableSpecs, - }), - workspaceBootstrapContext.developerInstructions, + const turnRequest = await prepareCodexAttemptTurnRequest( + resources, + turnRuntime, + ensureCurrentThreadRoute, + notifications.waitForActiveNativeTurnCompletion, ); - const openClawPromptContext = buildCodexOpenClawPromptContext({ - params: runtimeParams, - workspacePromptContext: workspaceBootstrapContext.promptContext, - }); - const skillsCollaborationInstructions = renderCodexSkillsCollaborationInstructions({ - attempt: runtimeParams, - skillsPrompt: params.skillsSnapshot?.prompt, - }); - let promptText = params.prompt; - let promptContextRange: CodexProjectedContextRange | undefined; - let developerInstructions = baseDeveloperInstructions; - let prePromptMessageCount = historyMessages.length; - const codexContextProjectionMaxChars = resolveCodexContextEngineProjectionMaxChars({ - contextTokenBudget: effectiveContextTokenBudget, - reserveTokens: resolveCodexContextEngineProjectionReserveTokens({ - config: params.config, - }), - }); - let contextEngineProjection: CodexContextEngineThreadBootstrapProjection | undefined; - let precomputedStaleBindingContinuityProjectionApplied = false; - let staleBindingContinuityForcedFreshStart = false; - let inactiveThreadBootstrapBindingForcedFreshStart = - initialInactiveThreadBootstrapBindingForcedFreshStart; - const applyFreshThreadContinuityProjection = () => { - const projection = projectContextEngineAssemblyForCodex({ - assembledMessages: historyMessages, - originalHistoryMessages: historyMessages, - prompt: params.prompt, - maxRenderedContextChars: codexContextProjectionMaxChars, - }); - promptText = projection.promptText; - promptContextRange = projection.promptContextRange; - prePromptMessageCount = projection.prePromptMessageCount; - }; - const applyActiveContextEngineProjection = async ( - decisionStartupBinding: CodexAppServerThreadBinding | undefined, - ) => { - if (!activeContextEngine) { - return; - } - const assembled = await assembleHarnessContextEngine({ - contextEngine: activeContextEngine, - sessionId: activeSessionId, - sessionKey: contextSessionKey, - messages: historyMessages, - tokenBudget: effectiveContextTokenBudget, - availableTools: new Set( - flattenCodexDynamicToolFunctions(toolBridge.availableSpecs) - .map((tool) => tool.name) - .filter(isNonEmptyString), - ), - citationsMode: params.config?.memory?.citations, - modelId: effectiveRuntimeModelId, - contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, - providerId: effectiveRuntimeProviderId, - requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, - fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, - degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, - prompt: params.prompt, - }); - if (!assembled) { - throw new Error("context engine assemble returned no result"); - } - contextEngineProjection = readContextEngineThreadBootstrapProjection( - assembled.contextProjection, - ); - const projection = projectContextEngineAssemblyForCodex({ - assembledMessages: assembled.messages, - originalHistoryMessages: historyMessages, - prompt: params.prompt, - systemPromptAddition: assembled.systemPromptAddition, - maxRenderedContextChars: codexContextProjectionMaxChars, - toolPayloadMode: contextEngineProjection ? "preserve" : "elide", - }); - const projectionDecision = contextEngineProjection - ? resolveContextEngineBootstrapProjectionDecision({ - startupBinding: decisionStartupBinding, - expectedBinding: buildContextEngineBinding( - buildActiveRunAttemptParams(), - contextEngineProjection, - ), - projection: contextEngineProjection, - dynamicToolsFingerprint: codexDynamicToolsFingerprint(toolBridge.specs), - legacyDynamicToolsFingerprint: codexLegacyDynamicToolsFingerprint(toolBridge.specs), - }) - : { project: true, reason: "per-turn-projection" }; - embeddedAgentLog.info("codex app-server context-engine projection decision", { - sessionId: params.sessionId, - sessionKey: contextSessionKey, - engineId: activeContextEngine.info.id, - mode: contextEngineProjection?.mode ?? assembled.contextProjection?.mode ?? "per_turn", - epoch: contextEngineProjection?.epoch, - fingerprint: contextEngineProjection?.fingerprint, - previousThreadId: decisionStartupBinding?.threadId, - previousEpoch: decisionStartupBinding?.contextEngine?.projection?.epoch, - previousFingerprint: decisionStartupBinding?.contextEngine?.projection?.fingerprint, - projected: projectionDecision.project, - reason: projectionDecision.reason, - assembledMessages: assembled.messages.length, - originalHistoryMessages: historyMessages.length, - projectedPromptChars: projection.promptText.length, - developerInstructionAdditionChars: projection.developerInstructionAddition?.length ?? 0, - }); - promptText = projectionDecision.project ? projection.promptText : params.prompt; - promptContextRange = projectionDecision.project ? projection.promptContextRange : undefined; - developerInstructions = joinPresentSections( - baseDeveloperInstructions, - projection.developerInstructionAddition, - ); - prePromptMessageCount = projection.prePromptMessageCount; - }; - if (activeContextEngine) { - try { - await applyActiveContextEngineProjection( - !nativeToolSurfaceEnabled ? undefined : startupBinding, - ); - } catch (assembleErr) { - embeddedAgentLog.warn("context engine assemble failed; using Codex baseline prompt", { - error: formatErrorMessage(assembleErr), - }); - } + const turnStart = await startCodexAttemptTurn(resources, turnRuntime, notifications, turnRequest); + if ("result" in turnStart) { + return turnStart.result; } - // Codex app-server threads own conversation continuity. The mirrored - // OpenClaw transcript is persistence/search state. Context-engine output is - // rendered into the prompt/developer instructions, not parallel history. - const codexModelInputHistoryMessages: typeof historyMessages = []; - const buildPromptFromCurrentInputs = () => - resolveAgentHarnessBeforePromptBuildResult({ - prompt: prependCurrentInboundContext(promptText, params.currentInboundContext), - developerInstructions, - messages: codexModelInputHistoryMessages, - ctx: hookContext, - bootstrapContextRunKind: params.bootstrapContextRunKind, - ...("beforeAgentStartResult" in params - ? { beforeAgentStartResult: params.beforeAgentStartResult } - : {}), - }); - const resolveShiftedPromptInputRange = ( - prompt: string, - promptInputRange: { start: number; end: number } | undefined, - turnPromptText: string, - ): CodexProjectedContextRange | undefined => { - if ( - !promptInputRange || - promptInputRange.start < 0 || - promptInputRange.end < promptInputRange.start || - promptInputRange.end > prompt.length || - !turnPromptText.endsWith(prompt) - ) { - return undefined; - } - const turnPromptOffset = turnPromptText.length - prompt.length; - return { - start: turnPromptOffset + promptInputRange.start, - end: turnPromptOffset + promptInputRange.end, - }; - }; - const resolveShiftedPromptContextRange = ( - prompt: string, - promptInputRange: { start: number; end: number } | undefined, - turnPromptText: string, - ): - | { - contextRange: CodexProjectedContextRange; - requestRange: CodexProjectedContextRange; - } - | undefined => { - // promptInputRange ends before hook appendContext. Measure from the - // immutable projected prompt instead of the hook-expanded prompt so that - // the suffix remains available for bounded fitting as newer context. - const promptTextInputOffset = promptInputRange - ? promptInputRange.end - promptText.length - : undefined; - if ( - !promptContextRange || - !promptInputRange || - promptTextInputOffset === undefined || - promptInputRange.start < 0 || - promptInputRange.end < promptInputRange.start || - promptInputRange.end > prompt.length || - promptTextInputOffset < promptInputRange.start || - prompt.slice(promptTextInputOffset, promptInputRange.end) !== promptText || - !turnPromptText.endsWith(prompt) - ) { - return undefined; - } - // A hook can append the full projected prompt as newer transient context. - // Fit that suffix so truncation retains its latest context rather than the - // earlier input span. The exact input range still covers prepend-only hooks. - const promptTextOffset = prompt.endsWith(promptText) - ? prompt.length - promptText.length - : promptTextInputOffset; - if (promptTextOffset < 0) { - return undefined; - } - const turnPromptOffset = turnPromptText.length - prompt.length + promptTextOffset; - const contextRange = { - start: turnPromptOffset + promptContextRange.start, - end: turnPromptOffset + promptContextRange.end, - }; - return { - contextRange, - requestRange: { - start: contextRange.end, - end: turnPromptOffset + promptText.length, - }, - }; - }; - let promptBuild = await buildPromptFromCurrentInputs(); - const decorateCodexTurnPromptText = (promptBuildResult: { - prompt: string; - promptInputRange?: { start: number; end: number }; - }) => { - const turnPromptText = prependCodexOpenClawPromptContext( - promptBuildResult.prompt, - openClawPromptContext, - { - preservePromptWithoutContext: - params.bootstrapContextMode === "lightweight" && - params.bootstrapContextRunKind === "cron", - }, - ); - const projectedRanges = resolveShiftedPromptContextRange( - promptBuildResult.prompt, - promptBuildResult.promptInputRange, - turnPromptText, - ); - const preservedRange = - resolveShiftedPromptInputRange( - promptBuildResult.prompt, - promptBuildResult.promptInputRange, - turnPromptText, - ) ?? - resolveCodexDeliveryHintPreservedInputRange({ - prompt: promptBuildResult.prompt, - promptInputRange: promptBuildResult.promptInputRange, - decoratedPrompt: turnPromptText, - }); - return fitCodexProjectedContextForTurnStart({ - promptText: turnPromptText, - contextRange: projectedRanges?.contextRange, - requestRange: projectedRanges?.requestRange, - preservedRange, - }); - }; - let codexTurnPromptText = decorateCodexTurnPromptText(promptBuild); - const buildCodexTurnCollaborationDeveloperInstructions = () => - buildTurnCollaborationMode(params, { - turnScopedDeveloperInstructions: workspaceBootstrapContext.turnScopedDeveloperInstructions, - skillsCollaborationInstructions, - memoryCollaborationInstructions: workspaceBootstrapContext.memoryCollaborationInstructions, - heartbeatCollaborationInstructions: - workspaceBootstrapContext.heartbeatCollaborationInstructions, - }).settings.developer_instructions ?? undefined; - const buildRenderedCodexDeveloperInstructions = () => - joinPresentSections( - promptBuild.developerInstructions, - buildCodexTurnCollaborationDeveloperInstructions(), - ); - const rebuildCodexPromptBuildFromCurrentProjection = async () => { - promptBuild = await buildPromptFromCurrentInputs(); - codexTurnPromptText = decorateCodexTurnPromptText(promptBuild); - }; - const rebuildCodexTurnPromptTextFromCurrentProjection = async () => { - const nextPromptBuild = await buildPromptFromCurrentInputs(); - // Native Codex thread instructions are fixed once thread/start or - // thread/resume completes; recovery continuity after that is turn input. - promptBuild = { - ...promptBuild, - prompt: nextPromptBuild.prompt, - promptInputRange: nextPromptBuild.promptInputRange, - }; - codexTurnPromptText = decorateCodexTurnPromptText(nextPromptBuild); - }; - const selectNewerVisibleHistoryAfterBinding = (binding: CodexAppServerThreadBinding) => { - const historyCoveredThrough = Date.parse(binding.historyCoveredThrough ?? ""); - const cutoff = Number.isFinite(historyCoveredThrough) ? historyCoveredThrough : 0; - return historyMessages.filter((message) => { - if (message.role !== "user" && message.role !== "assistant") { - return false; - } - const record = message as unknown as Record; - const idempotencyKey = record.idempotencyKey; - if (typeof idempotencyKey === "string" && idempotencyKey.startsWith("codex-app-server:")) { - return false; - } - const meta = record["__openclaw"]; - const mirrorIdentity = - meta && typeof meta === "object" && !Array.isArray(meta) - ? (meta as Record).mirrorIdentity - : undefined; - const mirrorOrigin = - meta && typeof meta === "object" && !Array.isArray(meta) - ? (meta as Record).mirrorOrigin - : undefined; - if (mirrorOrigin === "codex-app-server") { - return false; - } - if (typeof mirrorIdentity === "string" && mirrorIdentity.startsWith("codex-app-server:")) { - return false; - } - const timestamp = - typeof message.timestamp === "number" - ? message.timestamp - : typeof message.timestamp === "string" - ? Date.parse(message.timestamp) - : Number.NaN; - return Number.isFinite(timestamp) && timestamp > cutoff; - }); - }; - const applyResumeStaleBindingContinuityProjection = (binding: CodexAppServerThreadBinding) => { - const newerVisibleMessages = selectNewerVisibleHistoryAfterBinding(binding); - if (newerVisibleMessages.length === 0) { - return false; - } - const projection = projectContextEngineAssemblyForCodex({ - assembledMessages: newerVisibleMessages, - originalHistoryMessages: historyMessages, - prompt: params.prompt, - maxRenderedContextChars: codexContextProjectionMaxChars, - }); - promptText = projection.promptText; - promptContextRange = projection.promptContextRange; - prePromptMessageCount = projection.prePromptMessageCount; - return true; - }; - const precomputeNoContextEngineStaleBindingProjection = ( - binding: CodexAppServerThreadBinding | undefined, - ) => { - precomputedStaleBindingContinuityProjectionApplied = false; - staleBindingContinuityForcedFreshStart = false; - if (activeContextEngine || !binding?.threadId || binding.pendingSupervisionBranch) { - return false; - } - if (isInactiveThreadBootstrapBinding(binding)) { - inactiveThreadBootstrapBindingForcedFreshStart = true; - return false; - } - const projected = applyResumeStaleBindingContinuityProjection(binding); - precomputedStaleBindingContinuityProjectionApplied = projected; - return projected; - }; - const applyNoContextEngineContinuityProjection = ( - action: "started" | "resumed" | "forked", - binding?: CodexAppServerThreadBinding, - ) => { - if (activeContextEngine || !historyMessages.some((message) => message.role === "user")) { - return false; - } - if (action === "resumed" && precomputedStaleBindingContinuityProjectionApplied) { - return true; - } - if (action === "started" && staleBindingContinuityForcedFreshStart) { - return true; - } - if (action === "started" && inactiveThreadBootstrapBindingForcedFreshStart) { - // A retired thread-bootstrap context engine already forced Codex onto a - // clean native thread; without that engine active, mirrored history would - // re-inject stale bootstrap context as a new user turn. - return false; - } - if (action === "resumed" && binding) { - return applyResumeStaleBindingContinuityProjection(binding); - } - if (action === "started") { - applyFreshThreadContinuityProjection(); - return true; - } - return false; - }; - if (precomputeNoContextEngineStaleBindingProjection(startupBinding)) { - await rebuildCodexPromptBuildFromCurrentProjection(); - } - const rotateStartupBindingForProjectedTurn = async () => { - if (!startupBinding?.threadId) { - return; - } - const previousThreadId = startupBinding.threadId; - const hadInactiveThreadBootstrapBinding = isInactiveThreadBootstrapBinding(startupBinding); - const projectedTurnTokens = estimateCodexAppServerProjectedTurnTokens({ - prompt: codexTurnPromptText, - developerInstructions: buildRenderedCodexDeveloperInstructions(), - }); - startupBinding = await rotateOversizedCodexAppServerStartupBinding({ - binding: startupBinding, - bindingStore, - identity: bindingIdentity, - sessionFile: params.sessionFile, - agentDir, - codexHome: appServer.start.env?.CODEX_HOME, - config: params.config, - contextEngineActive: Boolean(activeContextEngine), - projectedTurnTokens, - }); - if (startupBinding?.threadId) { - return; - } - inactiveThreadBootstrapBindingForcedFreshStart = hadInactiveThreadBootstrapBinding; - staleBindingContinuityForcedFreshStart = - precomputedStaleBindingContinuityProjectionApplied && - !inactiveThreadBootstrapBindingForcedFreshStart; - if (staleBindingContinuityForcedFreshStart) { - // Once the native thread id is discarded, Codex no longer owns the - // pre-binding history; rebuild from the mirrored transcript. - applyFreshThreadContinuityProjection(); - } - if (activeContextEngine) { - contextEngineProjection = undefined; - try { - await applyActiveContextEngineProjection(undefined); - } catch (assembleErr) { - embeddedAgentLog.warn("context engine assemble failed; using Codex baseline prompt", { - error: formatErrorMessage(assembleErr), - }); - } - } - await rebuildCodexPromptBuildFromCurrentProjection(); - embeddedAgentLog.info("codex app-server rebuilt turn prompt after native thread rotation", { - sessionId: params.sessionId, - sessionKey: contextSessionKey, - previousThreadId, - promptChars: codexTurnPromptText.length, - developerInstructionChars: buildRenderedCodexDeveloperInstructions()?.length ?? 0, - }); - }; - await rotateStartupBindingForProjectedTurn(); - const systemPromptReport = buildCodexSystemPromptReport({ - attempt: params, - sessionKey: contextSessionKey, - workspaceDir: effectiveWorkspace, - developerInstructions: buildRenderedCodexDeveloperInstructions(), - workspaceBootstrapContext, - skillsPrompt: skillsCollaborationInstructions ? (params.skillsSnapshot?.prompt ?? "") : "", - tools: toolBridge.availableSpecs, - }); - const hostTrajectoryRecorder = ( - params as EmbeddedRunAttemptParams & { - trajectoryRecorder?: CodexHostTrajectoryRecorder | null; - } - ).trajectoryRecorder; - const trajectoryRecorder = createCodexTrajectoryRecorder({ - attempt: params, - cwd: effectiveCwd, - developerInstructions: buildRenderedCodexDeveloperInstructions(), - prompt: codexTurnPromptText, - trajectoryRecorder: hostTrajectoryRecorder, - trajectorySessionFile: params.trajectorySessionFile, - tools: toolBridge.availableSpecs, - warn: (message, fields) => embeddedAgentLog.warn(message, fields), - }); - let client: CodexAppServerClient; - let thread: CodexAppServerThreadLifecycleBinding; - let runtimeArtifact: AgentHarnessRuntimeArtifactBinding | undefined; - let turnRouter: CodexAppServerTurnRouter; - let turnRoute: CodexThreadRouteReservation | undefined; - let routeActivated = false; - let detachRouteAbort: () => void = () => undefined; - let trajectoryEndRecorded = false; - const markTrajectoryEndRecorded = () => { - trajectoryEndRecorded = true; - }; - let nativeHookRelay: NativeHookRelayRegistrationHandle | undefined; - let nativeSubagentMonitor: - | ReturnType - | undefined; - const pendingNativePreToolUseFailures: CodexNativePreToolUseFailure[] = []; - const projectorRef: { current?: CodexAppServerEventProjector } = {}; - let nativePreToolUseFailureFallbackActive = false; - let nativePreToolUseFailureFallbackTerminalReason: - | CodexNativePreToolUseFailure["disposition"] - | undefined; - const emitNativePreToolUseFailure = (failure: CodexNativePreToolUseFailure) => { - emitCodexNativePreToolUseFailureDiagnostic({ - agentId: sessionAgentId, - sessionId: params.sessionId, - sessionKey: sandboxSessionKey, - runId: params.runId, - signal: runAbortController.signal, - failure, - ...(nativePreToolUseFailureFallbackActive - ? { - terminalReason: nativePreToolUseFailureFallbackTerminalReason ?? failure.disposition, - } - : {}), - }); - }; - const flushPendingNativePreToolUseFailures = () => { - for (const failure of pendingNativePreToolUseFailures.splice(0)) { - emitNativePreToolUseFailure(failure); - } - }; - const activateNativePreToolUseFailureFallback = () => { - if (!nativePreToolUseFailureFallbackActive) { - nativePreToolUseFailureFallbackTerminalReason = runAbortController.signal.aborted - ? resolveCodexToolAbortTerminalReason(runAbortController.signal) - : undefined; - nativePreToolUseFailureFallbackActive = true; - } - flushPendingNativePreToolUseFailures(); - }; - let releaseSharedClientLease: (() => void) | undefined; - let sharedCodexClientRetiredForOneShotCleanup = false; - const releaseSharedClientLeaseOnce = () => { - const release = releaseSharedClientLease; - if (!release) { - return; - } - releaseSharedClientLease = undefined; - release(); - }; - const retireSharedCodexClientForOneShotCleanup = async () => { - if (params.cleanupBundleMcpOnRunEnd !== true) { - return; - } - if (sharedCodexClientRetiredForOneShotCleanup) { - return; - } - sharedCodexClientRetiredForOneShotCleanup = true; - const retired = retireSharedCodexAppServerClientIfCurrent(client); - embeddedAgentLog.info("codex app-server one-shot cleanup retired shared client", { - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - activeLeases: retired?.activeLeases ?? null, - closed: retired?.closed ?? false, - matchedSharedClient: Boolean(retired), - }); - if (retired?.closed) { - await client.closeAndWait({ exitTimeoutMs: 2_000, forceKillDelayMs: 250 }); - } - }; - const releaseSharedClientLeaseAndRetireOneShotClient = async () => { - releaseSharedClientLeaseOnce(); - await retireSharedCodexClientForOneShotCleanup(); - }; - let sandboxExecEnvironmentAcquired = false; - const releaseSandboxExecEnvironment = async () => { - if (sandboxExecEnvironmentAcquired) { - sandboxExecEnvironmentAcquired = false; - await releaseCodexSandboxExecServerEnvironment(sandbox); - } - }; - const unregisterNativeSubagentMonitor = () => { - nativeSubagentMonitor?.unregister(); - nativeSubagentMonitor = undefined; - }; - const registerNativeSubagentMonitor = (parentThreadId: string) => { - unregisterNativeSubagentMonitor(); - nativeSubagentMonitor = codexNativeSubagentMonitorRuntime.register({ - client, - parentThreadId, - requesterSessionKey: params.sessionKey, - taskRuntimeScope: params.agentHarnessTaskRuntimeScope, - agentId: sessionAgentId, - retainClient: () => retainSharedCodexAppServerClientIfCurrent(client), - }); - }; - const releaseCurrentRoute = () => { - detachRouteAbort(); - detachRouteAbort = () => undefined; - turnRoute?.release(); - turnRoute = undefined; - routeActivated = false; - unregisterNativeSubagentMonitor(); - }; - let codexEnvironmentSelection: CodexTurnEnvironmentParams[] | undefined; - let codexExecutionCwd = effectiveCwd; - let codexSandboxPolicy: CodexSandboxPolicy | undefined; - let restartContextEngineCodexThread: - | (() => Promise) - | undefined; - const startupTimeoutMs = resolveCodexStartupTimeoutMs({ - timeoutMs: params.timeoutMs, - timeoutFloorMs: options.startupTimeoutFloorMs, - }); - const buildNativeHookRelayFinalConfigPatch = ( - decision: { action: "resume"; binding: CodexAppServerThreadBinding } | { action: "start" }, - ) => { - nativeHookRelay?.unregister(); - nativeHookRelay = createCodexNativeHookRelay({ - options: options.nativeHookRelay, - generation: - decision.action === "resume" ? decision.binding.nativeHookRelayGeneration : undefined, - generationMismatchGraceMs: - decision.action === "resume" && !decision.binding.nativeHookRelayGeneration - ? CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS - : undefined, - events: nativeHookRelayEvents, - agentId: sessionAgentId, - sessionId: params.sessionId, - sessionKey: sandboxSessionKey, - config: params.config, - runId: params.runId, - channelId: hookChannelId, - attemptTimeoutMs: params.timeoutMs, - startupTimeoutMs, - turnStartTimeoutMs: params.timeoutMs, - signal: runAbortController.signal, - onPreToolUseFailure: (failure) => { - const projector = projectorRef.current; - if (projector) { - projector.recordNativeToolPreToolUseFailure(failure); - } else if (nativePreToolUseFailureFallbackActive) { - emitNativePreToolUseFailure(failure); - } else { - pendingNativePreToolUseFailures.push(failure); - } - }, - }); - return { - configPatch: nativeHookRelay - ? buildCodexNativeHookRelayConfig({ - relay: nativeHookRelay, - events: nativeHookRelayEvents, - hookTimeoutSec: options.nativeHookRelay?.hookTimeoutSec, - }) - : options.nativeHookRelay?.enabled === false - ? buildCodexNativeHookRelayDisabledConfig() - : undefined, - nativeHookRelayGeneration: nativeHookRelay?.generation, - }; - }; - try { - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { phase: "startup" }, - }); - const attemptAppServer = withCodexAppServerFastModeServiceTier(appServer, runtimeParams); - pluginAppServer = attemptAppServer; - const startupResult = await startCodexAttemptThread({ - attemptClientFactory, - bindingStore, - appServer: attemptAppServer, - pluginConfig, - computerUseConfig, - startupAuthProfileId: startupClientAuthProfileId, - startupAuthBindingFingerprint: preparedAuthBinding?.fingerprint, - ...(runtimeArtifactRequest ? { runtimeArtifactRequest } : {}), - startupPreparedAuth, - startupAuthAccountCacheKey, - startupEnvApiKeyCacheKey, - agentDir, - config: params.config, - buildAttemptParams: buildActiveRunAttemptParams, - sessionAgentId, - effectiveWorkspace, - effectiveCwd, - dynamicTools: toolBridge.specs, - persistentWebSearchAllowed, - webSearchAllowed, - developerInstructions: promptBuild.developerInstructions, - buildFinalConfigPatch: buildNativeHookRelayFinalConfigPatch, - bundleMcpThreadConfig, - nativeToolSurfaceEnabled, - nativeProviderWebSearchSupport, - sandboxExecServerEnabled, - sandbox, - contextEngineProjection, - startupTimeoutMs, - signal: runAbortController.signal, - onStartupTimeout: () => { - runAbortController.abort("codex_startup_timeout"); - }, - spawnedBy: params.spawnedBy, - }); - client = startupResult.client; - thread = startupResult.thread; - runtimeArtifact = startupResult.runtimeArtifact; - turnRouter = startupResult.turnRouter; - turnRoute = startupResult.turnRoute; - pluginAppServer = startupResult.pluginAppServer; - if ( - usesSupervisionConnection && - (thread.connectionScope !== "supervision" || - thread.supervisionSourceThreadId !== startupBinding?.supervisionSourceThreadId) - ) { - throw new Error("Codex supervised thread lost its private connection ownership"); - } - if (thread.lifecycle.action === "started" || thread.lifecycle.action === "forked") { - const activeThreadReviewerPolicyContext = resolveReviewerPolicyContext(thread); - const activeThreadConfiguredAppServer = resolveRuntimeOptionsForBinding({ - modelProvider: activeThreadReviewerPolicyContext.modelProvider, - model: activeThreadReviewerPolicyContext.model, - }); - const activeThreadAppServer = resolveCodexAppServerForModelProvider({ - appServer: activeThreadConfiguredAppServer, - provider: activeThreadReviewerPolicyContext.modelProvider, - model: activeThreadReviewerPolicyContext.model, - config: params.config, - env: process.env, - agentDir, - }); - const previousApprovalsReviewer = pluginAppServer.approvalsReviewer; - pluginAppServer = { - ...pluginAppServer, - approvalsReviewer: activeThreadAppServer.approvalsReviewer, - }; - if (pluginAppServer.approvalsReviewer !== previousApprovalsReviewer) { - embeddedAgentLog.info( - "codex app-server approval reviewer updated from active thread model provider", - { - from: previousApprovalsReviewer, - to: pluginAppServer.approvalsReviewer, - modelProvider: activeThreadReviewerPolicyContext.modelProvider, - }, - ); - } - } - sandboxExecEnvironmentAcquired = Boolean(startupResult.sandboxEnvironment); - codexEnvironmentSelection = startupResult.environmentSelection; - codexExecutionCwd = startupResult.executionCwd; - codexSandboxPolicy = startupResult.sandboxPolicy; - releaseSharedClientLease = startupResult.releaseSharedClientLease; - restartContextEngineCodexThread = startupResult.restartContextEngineCodexThread; - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { phase: "thread_ready", threadId: thread.threadId }, - }); - } catch (error) { - activateNativePreToolUseFailureFallback(); - releaseCurrentRoute(); - nativeHookRelay?.unregister(); - await releaseSandboxExecEnvironment(); - params.abortSignal?.removeEventListener("abort", abortFromUpstream); - throw error; - } - if (applyNoContextEngineContinuityProjection(thread.lifecycle.action, thread)) { - await rebuildCodexTurnPromptTextFromCurrentProjection(); - } - trajectoryRecorder?.recordEvent("session.started", { - sessionFile: params.sessionFile, - threadId: thread.threadId, - authProfileId: startupAuthProfileId, - workspaceDir: effectiveWorkspace, - toolCount: flattenCodexDynamicToolFunctions(toolBridge.specs).length, - }); - recordCodexTrajectoryContext(trajectoryRecorder, { - attempt: params, - cwd: effectiveCwd, - developerInstructions: buildRenderedCodexDeveloperInstructions(), - prompt: codexTurnPromptText, - tools: toolBridge.availableSpecs, - }); - let latestStartupErrorNotification: CodexServerNotification | undefined; - let rateLimitsRevisionBeforeLastTurnStart: number | undefined; - let completed = false; - let terminalTurnNotificationQueued = false; - let timedOut = false; - let turnCompletionIdleTimedOut = false; - let turnWatchTimeoutKind: CodexAttemptTurnWatchTimeoutKind | undefined; - let turnWatchTimeoutIdleMs: number | undefined; - let turnWatchTimeoutMs: number | undefined; - let turnWatchTimeoutLastActivityReason: string | undefined; - let turnWatchTimeoutDetails: Record | undefined; - let turnCompletionIdleTimeoutMessage: string | undefined; - let clientClosedPromptError: string | undefined; - let clientClosedAbort = false; - let shouldDelayNativeHookRelayUnregister = false; - let lifecycleStarted = false; - let lifecycleTerminalEmitted = false; - let resolveCompletion: (() => void) | undefined; - const completion = new Promise((resolve) => { - resolveCompletion = resolve; - }); - const turnCompletionIdleTimeoutMs = resolveCodexTurnCompletionIdleTimeoutMs( - options.turnCompletionIdleTimeoutMs ?? appServer.turnCompletionIdleTimeoutMs, + const activeTurn = await activateCodexAttemptTurn( + resources, + turnRuntime, + lifecycle, + notifications, + turnStart.turn, ); - const turnAssistantCompletionIdleTimeoutMs = resolveCodexTurnAssistantCompletionIdleTimeoutMs( - options.turnAssistantCompletionIdleTimeoutMs, - ); - const postToolRawAssistantCompletionIdleTimeoutMs = - resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs( - options.postToolRawAssistantCompletionIdleTimeoutMs ?? - appServer.postToolRawAssistantCompletionIdleTimeoutMs, - turnAssistantCompletionIdleTimeoutMs, - ); - const turnTerminalIdleTimeoutMs = resolveCodexTurnTerminalIdleTimeoutMs( - options.turnTerminalIdleTimeoutMs, - params.runTimeoutOverrideMs, - ); - const turnAttemptIdleTimeoutMs = Math.max(100, Math.floor(params.timeoutMs)); - let nativeHookRelayLastRenewedAt = 0; - let activeAppServerTurnRequests = 0; - const pendingOpenClawDynamicToolCompletionIds = new Set(); - // Codex can redeliver one pending server request while this attempt remains - // active. Keep one execution promise so duplicate delivery never repeats a - // non-idempotent action such as computer input. - const openClawDynamicToolExecutions = createCodexDynamicToolExecutionRegistry(); - const activeTurnItemIds = new Set(); - const activeCompletionBlockerItemIds = new Set(); - const activeFinalizationHookRunIds = new Set(); - const finalizationHookBatchStatuses = new Map(); - let unsettledFinalizationHookCount = 0; - let rejectedFinalizationHookAssistant: { itemId?: string } | undefined; - let turnCrossedToolHandoff = false; - let pendingTerminalDynamicToolRelease: - | { - call: CodexDynamicToolCallParams; - response: CodexDynamicToolCallResponse; - durationMs: number; - } - | undefined; - let terminalDynamicToolReleaseCheckScheduled = false; - let currentTurnHadNonTerminalDynamicToolResult = false; - const turnIdRef: { current?: string } = {}; - const userInputBridgeRef: { - current?: ReturnType; - } = {}; - const steeringQueueRef: { - current?: ReturnType; - } = {}; - - const renewNativeHookRelayForTurnProgress = () => { - if (!nativeHookRelay || options.nativeHookRelay?.ttlMs !== undefined) { - return; - } - const now = Date.now(); - const renewsRecently = - now - nativeHookRelayLastRenewedAt < CODEX_NATIVE_HOOK_RELAY_RENEW_INTERVAL_MS; - const expiresSoon = now >= nativeHookRelay.expiresAtMs - CODEX_NATIVE_HOOK_RELAY_TTL_GRACE_MS; - if (renewsRecently && !expiresSoon) { - return; - } - nativeHookRelayLastRenewedAt = now; - nativeHookRelay.renew( - resolveCodexNativeHookRelayTtlMs({ - explicitTtlMs: undefined, - attemptTimeoutMs: turnAttemptIdleTimeoutMs, - startupTimeoutMs, - turnStartTimeoutMs: params.timeoutMs, - }), - ); - }; - - const turnWatches = createCodexAttemptTurnWatchController({ - threadId: thread.threadId, - signal: runAbortController.signal, - getTurnId: () => turnIdRef.current, - isCompleted: () => completed, - isTerminalTurnNotificationQueued: () => terminalTurnNotificationQueued, - getActiveAppServerTurnRequests: () => activeAppServerTurnRequests, - getActiveTurnItemCount: () => activeTurnItemIds.size, - getActiveCompletionBlockerItemCount: () => activeCompletionBlockerItemIds.size, - getActiveFinalizationHookCount: () => unsettledFinalizationHookCount, - canReleaseAssistantCompletionIdle: () => - projectorRef.current?.hasLatestTerminalAssistantCandidateText() === true, - turnCompletionIdleTimeoutMs, - turnAssistantCompletionIdleTimeoutMs, - turnAttemptIdleTimeoutMs, - turnTerminalIdleTimeoutMs, - interruptTimeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, - onInterruptTurn: (input) => interruptCodexTurnBestEffort(client, input), - onTimeout: (timeout) => { - timedOut = true; - turnCompletionIdleTimedOut = true; - turnWatchTimeoutKind = timeout.kind; - turnWatchTimeoutIdleMs = timeout.idleMs; - turnWatchTimeoutMs = timeout.timeoutMs; - turnWatchTimeoutLastActivityReason = timeout.lastActivityReason; - turnWatchTimeoutDetails = timeout.details; - turnCompletionIdleTimeoutMessage = - "codex app-server turn idle timed out waiting for turn/completed"; - }, - onMarkTimedOut: () => projectorRef.current?.markTimedOut(), - onAbort: (reason) => runAbortController.abort(reason), - onCompleted: () => { - completed = true; - }, - onResolveCompletion: () => resolveCompletion?.(), - onRecordEvent: (name, fields) => trajectoryRecorder?.recordEvent(name, fields), - onAttemptProgress: (reason) => { - renewNativeHookRelayForTurnProgress(); - params.onRunProgress?.({ - reason, - provider: params.provider, - model: params.modelId, - backend: "codex-app-server", - }); - }, - onProgressDiagnostic: (reason) => { - emitTrustedDiagnosticEvent({ - type: "run.progress", - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - reason: `codex_app_server:${reason}`, - }); - }, - }); - - const releaseTurnAfterTerminalDynamicTool = (paramsValue: { - call: CodexDynamicToolCallParams; - response: CodexDynamicToolCallResponse; - durationMs: number; - }) => { - if ( - !shouldReleaseTurnAfterTerminalDynamicTool({ - completed, - aborted: runAbortController.signal.aborted, - responseSuccess: paramsValue.response.success, - currentTurnHadNonTerminalDynamicToolResult, - activeAppServerTurnRequests, - activeTurnItemIdsCount: activeTurnItemIds.size, - pendingOpenClawDynamicToolCompletionIdsCount: pendingOpenClawDynamicToolCompletionIds.size, - }) - ) { - return; - } - pendingTerminalDynamicToolRelease = undefined; - trajectoryRecorder?.recordEvent("turn.dynamic_tool_terminal_release", { - threadId: paramsValue.call.threadId, - turnId: paramsValue.call.turnId, - toolCallId: paramsValue.call.callId, - name: paramsValue.call.tool, - durationMs: paramsValue.durationMs, - }); - embeddedAgentLog.info("codex app-server turn released after terminal dynamic tool result", { - threadId: paramsValue.call.threadId, - turnId: paramsValue.call.turnId, - toolCallId: paramsValue.call.callId, - tool: paramsValue.call.tool, - durationMs: paramsValue.durationMs, - }); - interruptCodexTurnBestEffort(client, { - threadId: paramsValue.call.threadId, - turnId: paramsValue.call.turnId, - timeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, - }); - completed = true; - turnWatches.clearCompletionIdleTimer(); - turnWatches.clearAssistantCompletionIdleTimer(); - turnWatches.clearTerminalIdleTimer(); - resolveCompletion?.(); - }; - - const scheduleTerminalDynamicToolReleaseCheck = () => { - if ( - terminalDynamicToolReleaseCheckScheduled || - (!pendingTerminalDynamicToolRelease && !currentTurnHadNonTerminalDynamicToolResult) - ) { - return; - } - // Let the JSON-RPC tool-call response flush before interrupting the turn. - terminalDynamicToolReleaseCheckScheduled = true; - const immediate = setImmediate(() => { - terminalDynamicToolReleaseCheckScheduled = false; - const action = resolveTerminalDynamicToolBatchAction({ - activeAppServerTurnRequests, - activeTurnItemIdsCount: activeTurnItemIds.size, - pendingOpenClawDynamicToolCompletionIdsCount: pendingOpenClawDynamicToolCompletionIds.size, - currentTurnHadNonTerminalDynamicToolResult, - hasPendingTerminalDynamicToolRelease: pendingTerminalDynamicToolRelease !== undefined, - }); - if (action === "release-pending-terminal" && pendingTerminalDynamicToolRelease) { - releaseTurnAfterTerminalDynamicTool(pendingTerminalDynamicToolRelease); - } else if (action === "clear-nonterminal-batch") { - pendingTerminalDynamicToolRelease = undefined; - currentTurnHadNonTerminalDynamicToolResult = false; - } - }); - immediate.unref?.(); - }; - - const scheduleTurnReleaseAfterTerminalDynamicTool = (paramsLocal: { - call: CodexDynamicToolCallParams; - response: CodexDynamicToolCallResponse; - durationMs: number; - }) => { - pendingTerminalDynamicToolRelease = paramsLocal; - scheduleTerminalDynamicToolReleaseCheck(); - }; - - const emitLifecycleStart = () => { - void emitCodexAppServerEvent(params, { - stream: "lifecycle", - data: { phase: "start", startedAt: attemptStartedAt }, - }); - lifecycleStarted = true; - }; - - const emitLifecycleTerminal = (data: Record & { phase: "end" | "error" }) => { - if (!lifecycleStarted || lifecycleTerminalEmitted) { - return; - } - void emitCodexAppServerEvent(params, { - stream: "lifecycle", - data: { - startedAt: attemptStartedAt, - endedAt: Date.now(), - ...data, - ...((params.deferTerminalLifecycle ?? params.deferTerminalLifecycleEnd) - ? { phase: "finishing" } - : {}), - }, - }); - lifecycleTerminalEmitted = true; - }; - const buildLifecycleTerminalMeta = (input: { aborted: boolean; timedOut: boolean }) => { - const abortFields = input.aborted - ? resolveAgentRunAbortLifecycleFields(runAbortController.signal) - : undefined; - if (input.timedOut || abortFields?.stopReason === "timeout") { - return { - aborted: true, - status: "timed_out", - stopReason: "timeout", - timeoutPhase: "provider", - providerStarted: true, - } as const; - } - return input.aborted - ? ({ aborted: true, status: "cancelled", stopReason: "stop" } as const) - : undefined; - }; - - const executionPhaseKeys = new Set(); - const emitExecutionPhaseOnce = ( - key: string, - info: Parameters>[0], - ) => { - if (executionPhaseKeys.has(key)) { - return; - } - executionPhaseKeys.add(key); - params.onExecutionPhase?.({ - provider: params.provider, - model: params.modelId, - backend: "codex-app-server", - ...info, - }); - }; - const reportExecutionNotification = (notification: CodexServerNotification) => { - reportCodexExecutionNotification({ - notification, - emitExecutionPhaseOnce, - }); - }; - const emitFastModeAutoProgress = async (payload: { - enabled: boolean; - elapsedSeconds: number; - fastAutoOnSeconds?: number; - }): Promise => { - const summary = formatFastModeAutoProgressText(payload); - await emitCodexAppServerEvent(params, { - stream: "item", - data: { - kind: "status", - title: "Fast", - phase: "update", - summary, - }, - }); - try { - await params.onToolResult?.({ - text: summary, - channelData: { openclawProgressKind: FAST_MODE_AUTO_PROGRESS_KIND }, - }); - } catch (error) { - embeddedAgentLog.debug("codex app-server fast mode auto progress delivery failed", { - error, - }); - } - }; - const maybeAnnounceFastModeAutoOff = async (): Promise => { - if ( - params.fastModeAuto !== true || - fastModeAutoStartedAtMs === undefined || - fastModeAutoProgressState.offAnnounced - ) { - return; - } - const next = resolveFastModeForElapsed({ - mode: "auto", - startedAtMs: fastModeAutoStartedAtMs, - fastAutoOnSeconds: params.fastModeAutoOnSeconds, - }); - if (next.enabled) { - return; - } - fastModeAutoProgressState.offAnnounced = true; - await emitFastModeAutoProgress(next); - }; - const maybeEmitFastModeAutoReset = async (): Promise => { - if ( - params.fastModeAuto !== true || - !fastModeAutoProgressState.offAnnounced || - fastModeAutoProgressState.resetAnnounced - ) { - return; - } - fastModeAutoProgressState.resetAnnounced = true; - await emitFastModeAutoProgress({ - enabled: true, - elapsedSeconds: 0, - fastAutoOnSeconds: params.fastModeAutoOnSeconds, - }); - }; - const maybeEmitFastModeAutoResetBestEffort = async (): Promise => { - try { - await maybeEmitFastModeAutoReset(); - } catch (error) { - embeddedAgentLog.warn( - `codex app-server fast mode auto reset progress failed: ${formatErrorMessage(error)}`, - ); - } - }; - - const isTerminalTurnNotificationForTurn = ( - notification: CodexServerNotification, - notificationTurnId: string, - ): boolean => - isTerminalCodexTurnNotificationForTurn({ - notification, - threadId: thread.threadId, - turnId: notificationTurnId, - currentPromptTexts: [codexTurnPromptText], - }); - - const handleNotification = async (notification: CodexServerNotification) => { - const projector = projectorRef.current; - const turnId = turnIdRef.current; - const userInputBridge = userInputBridgeRef.current; - const steeringQueue = steeringQueueRef.current; - userInputBridge?.handleNotification(notification); - if (!projector || !turnId) { - // Pre-turn traffic on an open route is a resumed native turn. Keep the - // last error so a failed turn/start can still explain usage limits. - if (notification.method === "error") { - latestStartupErrorNotification = notification; - } - return; - } - const notificationState = applyCodexTurnNotificationState({ - notification, - threadId: thread.threadId, - turnId, - currentPromptTexts: [codexTurnPromptText], - turnWatches, - activeTurnItemIds, - activeCompletionBlockerItemIds, - activeAppServerTurnRequests, - pendingOpenClawDynamicToolCompletionIds, - turnCrossedToolHandoff, - postToolRawAssistantCompletionIdleTimeoutMs, - onScheduleTerminalDynamicToolReleaseCheck: scheduleTerminalDynamicToolReleaseCheck, - onReportExecutionNotification: reportExecutionNotification, - }); - turnCrossedToolHandoff = notificationState.turnCrossedToolHandoff; - const finalizationHookNotification = readCodexFinalizationHookNotification( - notification, - thread.threadId, - turnId, - ); - if (finalizationHookNotification?.phase === "started") { - // Codex emits every start in one Stop/SubagentStop batch before it runs - // any handler, then emits completions. An empty active set starts a batch. - if (activeFinalizationHookRunIds.size === 0) { - finalizationHookBatchStatuses.clear(); - } - activeFinalizationHookRunIds.add(finalizationHookNotification.runId); - // The receive-time disarm may precede an earlier queued assistant - // completion. Repeat it in notification order after that completion. - turnWatches.disarmAssistantCompletionIdleWatch(); - } - // Determine terminal-turn status before invoking the projector so a throw - // inside projector.handleNotification still releases the session lane. - // See openclaw/openclaw#67996. - if (notificationState.isTurnTerminal) { - terminalTurnNotificationQueued = true; - } - try { - await waitForCodexNotificationDispatchTurn(); - await projector.handleNotification(notification); - const projectedAssistantCompletionCanRelease = - isAssistantCompletionReleaseNotification(notification, turnCrossedToolHandoff) || - (notificationState.isCurrentTurnNotification && - turnCrossedToolHandoff && - notification.method === "rawResponseItem/completed" && - projector.canReleaseLatestTerminalAssistantAfterToolHandoff()); - if (notificationState.isCurrentTurnNotification && projectedAssistantCompletionCanRelease) { - const completedAssistantItemId = projector.getLatestTerminalAssistantCandidate()?.itemId; - if ( - rejectedFinalizationHookAssistant !== undefined && - completedAssistantItemId !== undefined && - completedAssistantItemId !== rejectedFinalizationHookAssistant.itemId - ) { - rejectedFinalizationHookAssistant = undefined; - } else if (rejectedFinalizationHookAssistant !== undefined) { - // A delayed raw echo still belongs to the rejected assistant. - turnWatches.disarmAssistantCompletionIdleWatch(); - } else { - const canArmProjectedAssistantCompletion = - activeFinalizationHookRunIds.size === 0 && - !terminalTurnNotificationQueued && - activeAppServerTurnRequests === 0 && - activeTurnItemIds.size === 0 && - activeCompletionBlockerItemIds.size === 0 && - pendingOpenClawDynamicToolCompletionIds.size === 0 && - projector.hasLatestTerminalAssistantCandidateText(); - if (canArmProjectedAssistantCompletion) { - // Receive-time arming can expire while an earlier queued projection - // is still running. Restart the idle window from committed output. - turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification)); - } - } - } - if ( - notificationState.isCurrentTurnNotification && - activeTurnItemIds.size === 0 && - isRawFunctionToolOutputCompletionNotification(notification) - ) { - await maybeAnnounceFastModeAutoOff(); - } - } catch (error) { - embeddedAgentLog.debug("codex app-server projector notification threw", { - method: notification.method, - error, - }); - } finally { - if (finalizationHookNotification?.phase === "completed") { - unsettledFinalizationHookCount = Math.max(0, unsettledFinalizationHookCount - 1); - activeFinalizationHookRunIds.delete(finalizationHookNotification.runId); - finalizationHookBatchStatuses.set( - finalizationHookNotification.runId, - finalizationHookNotification.status, - ); - if (activeFinalizationHookRunIds.size === 0) { - const statuses = new Set(finalizationHookBatchStatuses.values()); - const aggregateBlocked = statuses.has("blocked") && !statuses.has("stopped"); - if (aggregateBlocked) { - const itemId = projector.getLatestTerminalAssistantCandidate()?.itemId; - rejectedFinalizationHookAssistant = itemId ? { itemId } : {}; - turnWatches.disarmAssistantCompletionIdleWatch(); - } else { - rejectedFinalizationHookAssistant = undefined; - } - } - const canRearmAssistantCompletionWatch = - activeFinalizationHookRunIds.size === 0 && - rejectedFinalizationHookAssistant === undefined && - !terminalTurnNotificationQueued && - activeAppServerTurnRequests === 0 && - activeTurnItemIds.size === 0 && - activeCompletionBlockerItemIds.size === 0 && - pendingOpenClawDynamicToolCompletionIds.size === 0 && - projector.hasLatestTerminalAssistantCandidateText(); - if (canRearmAssistantCompletionWatch) { - turnWatches.armAssistantCompletionIdleWatch({ - lastNotificationMethod: notification.method, - hookRunId: finalizationHookNotification.runId, - hookStatus: finalizationHookNotification.status, - }); - } - } - if (notificationState.isTurnTerminal) { - if (notificationState.isTurnAbortMarker) { - projector.markAborted(); - } - if (!timedOut && !runAbortController.signal.aborted) { - await steeringQueue?.flushPending(); - } - completed = true; - turnWatches.clearCompletionIdleTimer(); - turnWatches.clearAssistantCompletionIdleTimer(); - turnWatches.clearTerminalIdleTimer(); - resolveCompletion?.(); - } - } - }; - const waitForActiveNativeTurnCompletion = async (): Promise => { - const route = turnRoute; - if (!route) { - return false; - } - return await route.waitForTurnCompletion({ - timeoutMs: Math.min(appServer.requestTimeoutMs, CODEX_APP_SERVER_NATIVE_TURN_WAIT_TIMEOUT_MS), - signal: runAbortController.signal, - }); - }; - const noteNotificationReceived = ( - notification: CodexServerNotification, - scope: CodexThreadRouteScope, - receivedAtMs: number, - ) => { - const projector = projectorRef.current; - const turnId = turnIdRef.current; - if (!projector || !turnId) { - return; - } - if (isTerminalTurnNotificationForTurn(notification, turnId)) { - terminalTurnNotificationQueued = true; - } - if (scope.turnId === turnId) { - const modelToolCallId = readRawResponseToolCallId(notification); - if (modelToolCallId) { - // Raw response items arrive in model order before Codex schedules tool - // futures, so later lifecycle races reuse this authoritative position. - allocateCodexToolOutcomeOrdinal?.(modelToolCallId); - } - const nativeItem = readCodexNotificationItem(notification.params); - if (nativeItem?.type === "webSearch") { - // Upstream omits the raw web-search id. Its lifecycle still follows the - // model stream, so reserve synchronously before queued projection. - projector.recordNativeToolOutcome(nativeItem); - } - } - const finalizationHookNotification = readCodexFinalizationHookNotification( - notification, - thread.threadId, - turnId, - ); - if (finalizationHookNotification?.phase === "started") { - unsettledFinalizationHookCount += 1; - // Codex runs finalization hooks after completing the assistant item. - // Suspend recovery until those hooks accept or replace that answer. - turnWatches.disarmAssistantCompletionIdleWatch(); - } - // Touch idle-watch timestamps at receive time, not just after queued - // projection. A queued terminal event should suppress short false-idle - // guards, while the full attempt watchdog still releases a wedged queue. - turnWatches.noteNotificationReceived(notification.method, { receivedAtMs }); - }; - const enqueueNotification = async ( - notification: CodexServerNotification, - scope: CodexThreadRouteScope, - ): Promise => { - embeddedAgentLog.trace("codex app-server raw notification received", { - method: notification.method, - ...scope, - }); - await handleNotification(notification); - }; - const drainNotificationQueue = async (): Promise => { - await turnRoute?.drain(); - }; - - registerNativeSubagentMonitor(thread.threadId); - const handleServerRequest = async ( - request: CodexAppServerServerRequest, - scope: CodexThreadRouteScope, - ) => { - const turnId = turnIdRef.current; - const userInputBridge = userInputBridgeRef.current; - const projector = projectorRef.current; - let armCompletionWatchOnResponse = false; - let requestCountsAsTurnActivity = false; - const markCurrentTurnRequestProgress = () => { - activeAppServerTurnRequests += 1; - turnWatches.clearCompletionIdleTimer(); - turnWatches.disarmAssistantCompletionIdleWatch(); - requestCountsAsTurnActivity = true; - turnWatches.touchActivity(`request:${request.method}:start`, { - attemptProgress: true, - }); - }; - try { - if (!turnId) { - return undefined; - } - if (request.method === "mcpServer/elicitation/request") { - if (!scope.turnId || scope.turnId === turnId) { - armCompletionWatchOnResponse = true; - markCurrentTurnRequestProgress(); - } - return await handleCodexAppServerElicitationRequest({ - requestParams: request.params, - paramsForRun: params, - threadId: thread.threadId, - turnId, - pluginAppPolicyContext: thread.pluginAppPolicyContext, - ...(computerUseConfig.enabled - ? { computerUseMcpServerName: computerUseConfig.mcpServerName } - : {}), - signal: runAbortController.signal, - }); - } - if (request.method === "item/tool/requestUserInput") { - if (scope.turnId === turnId) { - armCompletionWatchOnResponse = true; - markCurrentTurnRequestProgress(); - } - return userInputBridge?.handleRequest({ - id: request.id, - params: request.params, - }); - } - if (request.method !== "item/tool/call") { - if (isCodexAppServerApprovalRequest(request.method)) { - if (scope.turnId === turnId) { - armCompletionWatchOnResponse = true; - markCurrentTurnRequestProgress(); - } - return handleApprovalRequest({ - method: request.method, - params: request.params, - paramsForRun: params, - threadId: thread.threadId, - turnId, - nativeHookRelay, - autoApprove: shouldAutoApproveCodexAppServerApprovals(appServer), - signal: runAbortController.signal, - onNativeToolFailureDisposition: (itemId, disposition) => - projector?.recordNativeToolApprovalFailure(itemId, disposition), - }); - } - return undefined; - } - const call = readCodexDynamicToolCallParams(request.params); - if (!call || call.threadId !== thread.threadId || call.turnId !== turnId) { - return undefined; - } - const replayedExecution = openClawDynamicToolExecutions.get(call); - if (replayedExecution) { - armCompletionWatchOnResponse = true; - markCurrentTurnRequestProgress(); - turnCrossedToolHandoff = true; - return toCodexDynamicToolProtocolResponse(await replayedExecution) as JsonValue; - } - const toolCallOrdinal = allocateCodexToolOutcomeOrdinal?.(call.callId); - armCompletionWatchOnResponse = true; - markCurrentTurnRequestProgress(); - turnCrossedToolHandoff = true; - pendingOpenClawDynamicToolCompletionIds.add(call.callId); - trajectoryRecorder?.recordEvent("tool.call", { - threadId: call.threadId, - turnId: call.turnId, - toolCallId: call.callId, - name: call.tool, - arguments: call.arguments, - }); - projector?.recordDynamicToolCall({ - callId: call.callId, - tool: call.tool, - arguments: call.arguments, - }); - emitExecutionPhaseOnce(`tool:${call.callId}`, { - phase: "tool_execution_started", - tool: call.tool, - toolCallId: call.callId, - }); - emitDynamicToolStartedDiagnostic({ - call, - agentId: sessionAgentId, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - }); - const toolProgressDetailMode = resolveCodexToolProgressDetailMode(params.toolProgressDetail); - const toolMeta = inferCodexDynamicToolMeta(call, toolProgressDetailMode); - const toolArgs = sanitizeCodexToolArguments(call.arguments); - const shouldEmitDynamicToolProgress = shouldEmitTranscriptToolProgress(call.tool, toolArgs); - if (shouldEmitDynamicToolProgress) { - void emitCodexAppServerEvent(params, { - stream: "tool", - data: { - phase: "start", - name: call.tool, - toolCallId: call.callId, - ...(toolMeta ? { meta: toolMeta } : {}), - ...(toolArgs ? { args: toolArgs } : {}), - }, - }); - } - const dynamicToolTimeoutMs = resolveDynamicToolCallTimeoutMs({ - call, - config: params.config, - }); - const toolStartedAt = Date.now(); - let terminalDiagnosticObserved = false; - const unsubscribeToolDiagnosticObserver = onInternalDiagnosticEvent((event) => { - if (isDynamicToolTerminalDiagnosticEvent(event)) { - if ( - isMatchingDynamicToolTerminalDiagnostic({ - event, - call, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - }) - ) { - terminalDiagnosticObserved = true; - } - } - }); - try { - const { execution } = openClawDynamicToolExecutions.claim(call, () => - handleDynamicToolCallWithTimeout({ - call, - toolBridge, - signal: runAbortController.signal, - timeoutMs: dynamicToolTimeoutMs, - toolCallOrdinal, - onAgentToolResult: params.onAgentToolResult, - onFallbackSelected: () => { - if (toolCallOrdinal !== undefined) { - suppressedDynamicToolOutcomeOrdinals.add(toolCallOrdinal); - } - }, - onTimeout: () => { - trajectoryRecorder?.recordEvent("tool.timeout", { - threadId: call.threadId, - turnId: call.turnId, - toolCallId: call.callId, - name: call.tool, - timeoutMs: dynamicToolTimeoutMs, - }); - }, - }), - ); - const response = await execution; - const protocolResponse = toCodexDynamicToolProtocolResponse(response); - if (!protocolResponse.success && toolCallOrdinal !== undefined) { - // The underlying tool may ignore cancellation and finish after the - // timeout response. Its late presentation must not replace this failure. - suppressedDynamicToolOutcomeOrdinals.add(toolCallOrdinal); - params.onToolOutcome?.({ - toolName: call.tool, - argsHash: "", - resultHash: "", - toolCallOrdinal, - terminalPresentation: undefined, - presentationOnly: true, - }); - } - const toolDurationMs = Math.max(0, Date.now() - toolStartedAt); - trajectoryRecorder?.recordEvent("tool.result", { - threadId: call.threadId, - turnId: call.turnId, - toolCallId: call.callId, - name: call.tool, - success: protocolResponse.success, - contentItems: protocolResponse.contentItems, - }); - projector?.recordDynamicToolResult({ - callId: call.callId, - tool: call.tool, - asyncStarted: response.asyncStarted === true, - success: protocolResponse.success, - terminalType: - response.diagnosticTerminalType ?? (protocolResponse.success ? "completed" : "error"), - sideEffectEvidence: response.sideEffectEvidence === true, - contentItems: protocolResponse.contentItems, - }); - if (shouldEmitDynamicToolProgress) { - const progressResponse = toCodexDynamicToolProgressResponse(response, protocolResponse); - void emitCodexAppServerEvent(params, { - stream: "tool", - data: { - phase: "result", - name: call.tool, - toolCallId: call.callId, - ...(toolMeta ? { meta: toolMeta } : {}), - isError: !protocolResponse.success, - result: toTranscriptToolResult(progressResponse), - }, - }); - } - if ( - !terminalDiagnosticObserved && - !hasPendingDynamicToolTerminalDiagnostic({ - call, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - }) - ) { - emitDynamicToolTerminalDiagnostic({ - response, - call, - agentId: sessionAgentId, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - durationMs: toolDurationMs, - }); - } - pendingOpenClawDynamicToolCompletionIds.delete(call.callId); - if (response.terminate === true) { - scheduleTurnReleaseAfterTerminalDynamicTool({ - call, - response, - durationMs: toolDurationMs, - }); - } else if (!shouldBlockTerminalReleaseForNonTerminalDynamicToolResult(response)) { - scheduleTerminalDynamicToolReleaseCheck(); - } else { - currentTurnHadNonTerminalDynamicToolResult = true; - pendingTerminalDynamicToolRelease = undefined; - } - return protocolResponse as JsonValue; - } catch (error) { - pendingOpenClawDynamicToolCompletionIds.delete(call.callId); - if ( - !terminalDiagnosticObserved && - !hasPendingDynamicToolTerminalDiagnostic({ - call, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - }) - ) { - emitDynamicToolErrorDiagnostic({ - call, - agentId: sessionAgentId, - runId: params.runId, - sessionId: params.sessionId, - sessionKey: params.sessionKey, - durationMs: Math.max(0, Date.now() - toolStartedAt), - }); - } - throw error; - } finally { - toolOutcomeOrdinals.delete(call.callId); - unsubscribeToolDiagnosticObserver(); - } - } finally { - if (requestCountsAsTurnActivity) { - activeAppServerTurnRequests = Math.max(0, activeAppServerTurnRequests - 1); - const postToolContinuationTimeoutMs = - request.method === "item/tool/call" && turnCrossedToolHandoff - ? postToolRawAssistantCompletionIdleTimeoutMs - : undefined; - turnWatches.touchActivity(`request:${request.method}:response`, { - arm: armCompletionWatchOnResponse, - attemptProgress: true, - ...(postToolContinuationTimeoutMs !== undefined - ? { attemptTimeoutMs: postToolContinuationTimeoutMs } - : {}), - }); - if (armCompletionWatchOnResponse && postToolContinuationTimeoutMs !== undefined) { - turnWatches.armCompletionIdleWatch({ timeoutMs: postToolContinuationTimeoutMs }); - } - scheduleTerminalDynamicToolReleaseCheck(); - } else { - turnWatches.scheduleProgressWatches(); - } - } - }; - - const attachRouteAbort = (route: CodexThreadRouteReservation) => { - const onAbort = () => { - if (completed || terminalTurnNotificationQueued || runAbortController.signal.aborted) { - return; - } - const reasonText = formatErrorMessage(route.signal.reason); - const closedClient = reasonText.includes("turn router closed"); - clientClosedPromptError = closedClient - ? "codex app-server client closed before turn completed" - : `codex app-server turn route closed before turn completed: ${reasonText}`; - clientClosedAbort = closedClient; - const activeTurnId = turnIdRef.current; - if (activeTurnId) { - trajectoryRecorder?.recordEvent("turn.client_closed", { - threadId: thread.threadId, - turnId: activeTurnId, - }); - } - embeddedAgentLog.warn(clientClosedPromptError, { - threadId: thread.threadId, - turnId: activeTurnId, - }); - runAbortController.abort(closedClient ? "client_closed" : "turn_route_closed"); - completed = true; - turnWatches.clearAllTimers(); - resolveCompletion?.(); - }; - route.signal.addEventListener("abort", onAbort, { once: true }); - if (route.signal.aborted) { - onAbort(); - } - return () => route.signal.removeEventListener("abort", onAbort); - }; - const ensureCurrentThreadRoute = async (): Promise => { - if (turnRoute?.threadId !== thread.threadId) { - releaseCurrentRoute(); - turnRoute = turnRouter.reserveThread({ - threadId: thread.threadId, - releaseOn: runAbortController.signal, - }); - } - if (!turnRoute) { - throw new Error("codex app-server turn route was not reserved"); - } - if (!routeActivated) { - if (!nativeSubagentMonitor) { - registerNativeSubagentMonitor(thread.threadId); - } - detachRouteAbort = attachRouteAbort(turnRoute); - await turnRoute.activate({ - onNotificationReceived: noteNotificationReceived, - onNotification: enqueueNotification, - onRequest: handleServerRequest, - }); - routeActivated = true; - } - return turnRoute; - }; - try { - await ensureCurrentThreadRoute(); - } catch (error) { - activateNativePreToolUseFailureFallback(); - releaseCurrentRoute(); - nativeHookRelay?.unregister(); - await releaseSandboxExecEnvironment(); - releaseSharedClientLeaseOnce(); - params.abortSignal?.removeEventListener("abort", abortFromUpstream); - throw error; - } - - const buildLlmInputEvent = () => ({ - runId: params.runId, - sessionId: params.sessionId, - provider: usesSupervisionConnection - ? (thread.modelProvider ?? effectiveRuntimeProviderId) - : params.provider, - model: usesSupervisionConnection ? (thread.model ?? effectiveRuntimeModelId) : params.modelId, - systemPrompt: buildRenderedCodexDeveloperInstructions(), - prompt: codexTurnPromptText, - historyMessages: codexModelInputHistoryMessages, - imagesCount: params.images?.length ?? 0, - tools, - }); - const buildCodexModelInputMessages = () => [ - ...codexModelInputHistoryMessages, - buildCodexUserPromptMessage({ ...runtimeParams, prompt: codexTurnPromptText }), - ]; - const codexModelCallBaseFields = { - runId: params.runId, - callId: codexModelCallId, - ...(params.sessionKey ? { sessionKey: params.sessionKey } : {}), - sessionId: params.sessionId, - provider: usesSupervisionConnection - ? (thread.modelProvider ?? effectiveRuntimeProviderId) - : params.provider, - model: usesSupervisionConnection ? (thread.model ?? effectiveRuntimeModelId) : params.modelId, - api: usesSupervisionConnection ? runtimeParams.model.api : params.model.api, - transport: appServer.start.transport, - ...hookContextWindowFields, - trace: codexModelCallTrace, - }; - const codexModelCallDiagnostics = createCodexModelCallDiagnosticEmitter({ - baseFields: codexModelCallBaseFields, - capture: codexModelContentCapture, - tools, - buildInputMessages: buildCodexModelInputMessages, - buildSystemPrompt: buildRenderedCodexDeveloperInstructions, - onErrorDiagnostic: (error) => { - embeddedAgentLog.debug("codex app-server model call diagnostic ended with error", { - error: formatErrorMessage(error), - }); - }, - }); - - let turn: CodexTurnStartResponse | undefined; - const throwIfTurnStartAcceptedAfterAbort = () => { - if (!runAbortController.signal.aborted) { - return; - } - const reason = runAbortController.signal.reason; - if (reason instanceof Error) { - throw reason; - } - const error = new Error( - typeof reason === "string" && reason.length > 0 - ? reason - : "codex app-server turn start aborted before acceptance", - ); - error.name = "AbortError"; - throw error; - }; - const startCodexTurn = async (): Promise => { - const activeTurnRoute = await ensureCurrentThreadRoute(); - const turnAppServer = withCodexAppServerFastModeServiceTier(pluginAppServer, runtimeParams); - pluginAppServer = turnAppServer; - const turnStartParams = buildTurnStartParams(runtimeParams, { - threadId: thread.threadId, - cwd: codexExecutionCwd, - appServer: turnAppServer, - promptText: codexTurnPromptText, - sandboxPolicy: codexSandboxPolicy, - environmentSelection: codexEnvironmentSelection, - ...(usesSupervisionConnection - ? {} - : { model: thread.model, modelProvider: thread.modelProvider }), - turnScopedDeveloperInstructions: workspaceBootstrapContext.turnScopedDeveloperInstructions, - skillsCollaborationInstructions, - memoryCollaborationInstructions: workspaceBootstrapContext.memoryCollaborationInstructions, - heartbeatCollaborationInstructions: - workspaceBootstrapContext.heartbeatCollaborationInstructions, - preserveNativeTurnSettings: usesSupervisionConnection, - }); - codexModelCallDiagnostics.setRequestPayloadBytes(utf8JsonByteLength(turnStartParams)); - // Keep turn/start diagnostics scoped to this attempt: resumed native work - // can emit unrelated errors, and only a primary rate-limit update observed - // after this point may be trusted for the attempt's auth profile. - latestStartupErrorNotification = undefined; - rateLimitsRevisionBeforeLastTurnStart = readCodexRateLimitsRevision(client); - activeTurnRoute.armTurn(); - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { - phase: "turn_starting", - threadId: thread.threadId, - model: turnStartParams.model, - effort: turnStartParams.effort, - collaborationEffort: turnStartParams.collaborationMode?.settings.reasoning_effort, - }, - }); - let acceptedTurnId: string | undefined; - try { - const startedTurn = assertCodexTurnStartResponse( - await client.request("turn/start", turnStartParams, { - timeoutMs: params.timeoutMs, - signal: runAbortController.signal, - }), - ); - acceptedTurnId = startedTurn.turn.id; - throwIfTurnStartAcceptedAfterAbort(); - return startedTurn; - } catch (error) { - if (acceptedTurnId) { - // The turn was accepted but this run is failing. Interrupt it before - // dropping the route, so an accepted turn can never keep feeding a - // released route or wedge the shared client mid-flight. - interruptCodexTurnBestEffort(client, { - threadId: thread.threadId, - turnId: acceptedTurnId, - timeoutMs: CODEX_APP_SERVER_INTERRUPT_TIMEOUT_MS, - }); - releaseCurrentRoute(); - } else { - await activeTurnRoute.cancelTurn(); - } - throw error; - } - }; - const resumedWithActiveNativeTurn = - thread.lifecycle.action === "resumed" && (thread.lifecycle.activeTurnIds?.length ?? 0) > 0; - if (resumedWithActiveNativeTurn) { - // A resumed Codex thread can already be running a native compact/review turn. - // Starting an OpenClaw turn before that native turn completes can wedge the - // accepted turn behind a completion event we intentionally ignore. - embeddedAgentLog.info( - "codex app-server resumed thread has active native turn; waiting before turn/start", - { threadId: thread.threadId }, - ); - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { - phase: "turn_start_waiting_for_native_turn", - threadId: thread.threadId, - }, - }); - const nativeTurnCompleted = await waitForActiveNativeTurnCompletion(); - if (nativeTurnCompleted) { - await turnRoute?.drain(); - } - if (!nativeTurnCompleted && !runAbortController.signal.aborted) { - embeddedAgentLog.warn( - "codex app-server active native turn did not complete before turn/start wait timed out", - { threadId: thread.threadId }, - ); - } - } - try { - codexModelCallDiagnostics.emitStarted(); - runAgentHarnessLlmInputHook({ - event: buildLlmInputEvent(), - ctx: hookContext, - hookRunner, - }); - turn = await startCodexTurn(); - } catch (error) { - let turnStartError = error; - if (isCodexActiveCompactTurnError(turnStartError)) { - // Codex native compaction returns before its compact turn finishes. If - // the next OpenClaw turn collides with that compact turn, wait for the - // terminal notification and retry once instead of surfacing drift. - embeddedAgentLog.info( - "codex app-server turn/start blocked by active compact turn; waiting to retry", - { threadId: thread.threadId }, - ); - const compactTurnCompleted = await waitForActiveNativeTurnCompletion(); - if (compactTurnCompleted && !runAbortController.signal.aborted) { - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { phase: "turn_start_retry_after_compact", threadId: thread.threadId }, - }); - try { - turn = await startCodexTurn(); - } catch (retryError) { - turnStartError = retryError; - } - } - } - if ( - turn === undefined && - thread.connectionScope !== "supervision" && - shouldUseFreshCodexThreadAfterContextEngineOverflow({ - error: turnStartError, - contextEngineActive: Boolean(activeContextEngine), - thread, - }) && - restartContextEngineCodexThread - ) { - // Do not try to pre-compact or summarize through OpenClaw here. Codex owns - // automatic compaction; OpenClaw may only discard a stale projection thread - // and let Codex start cleanly. - embeddedAgentLog.warn( - "codex app-server context-engine turn overflowed on resume; retrying with fresh thread", - { - threadId: thread.threadId, - error: formatErrorMessage(turnStartError), - }, - ); - try { - const clearedBinding = await bindingStore.mutate(bindingIdentity, { - kind: "clear", - threadId: thread.threadId, - }); - if (!clearedBinding) { - embeddedAgentLog.warn( - "codex app-server preserved newer context-engine binding after resume overflow; skipping fresh retry", - { - threadId: thread.threadId, - error: formatErrorMessage(turnStartError), - }, - ); - } else { - thread = await restartContextEngineCodexThread(); - // The fresh retry thread was not bootstrapped with the - // context-engine projection. Clear the stale projection from - // the saved binding so the next run will re-project instead - // of assuming the old epoch is still in the thread. - { - const retryBinding = await bindingStore.read(bindingIdentity); - if ( - retryBinding && - retryBinding.threadId === thread.threadId && - retryBinding.contextEngine?.projection - ) { - await bindingStore.mutate(bindingIdentity, { - kind: "patch", - threadId: retryBinding.threadId, - patch: { - contextEngine: retryBinding.contextEngine - ? { ...retryBinding.contextEngine, projection: undefined } - : undefined, - }, - }); - embeddedAgentLog.info( - "codex app-server cleared stale context-engine projection after overflow retry", - { - threadId: thread.threadId, - previousEpoch: retryBinding.contextEngine.projection.epoch, - }, - ); - } - } - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { phase: "thread_ready_retry", threadId: thread.threadId }, - }); - try { - turn = await startCodexTurn(); - } catch (retryError) { - turnStartError = retryError; - } - } - } catch (retrySetupError) { - turnStartError = retrySetupError; - } - } - if (turn === undefined) { - const usageLimitError = await formatCodexTurnStartUsageLimitError({ - client, - error: turnStartError, - errorNotification: latestStartupErrorNotification, - rateLimitsRevisionBeforeTurnStart: rateLimitsRevisionBeforeLastTurnStart, - timeoutMs: appServer.requestTimeoutMs, - signal: runAbortController.signal, - }); - const turnStartErrorMessage = usageLimitError?.message ?? formatErrorMessage(turnStartError); - if (isInvalidCodexImagePayloadError(turnStartErrorMessage)) { - await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, { - phase: "turn_start", - threadId: thread.threadId, - error: turnStartErrorMessage, - }); - } - void emitCodexAppServerEvent(params, { - stream: "codex_app_server.lifecycle", - data: { phase: "turn_start_failed", error: turnStartErrorMessage }, - }); - trajectoryRecorder?.recordEvent("session.ended", { - status: "error", - threadId: thread.threadId, - timedOut, - aborted: runAbortController.signal.aborted, - promptError: turnStartErrorMessage, - }); - markTrajectoryEndRecorded(); - runAgentHarnessLlmOutputHook({ - event: { - runId: params.runId, - sessionId: params.sessionId, - provider: usesSupervisionConnection - ? (thread.modelProvider ?? effectiveRuntimeProviderId) - : params.provider, - model: usesSupervisionConnection - ? (thread.model ?? effectiveRuntimeModelId) - : params.modelId, - ...hookContextWindowFields, - resolvedRef: usesSupervisionConnection - ? `${thread.modelProvider ?? effectiveRuntimeProviderId}/${thread.model ?? effectiveRuntimeModelId}` - : (params.runtimePlan?.observability.resolvedRef ?? - `${params.provider}/${params.modelId}`), - ...(!usesSupervisionConnection && params.runtimePlan?.observability.harnessId - ? { harnessId: params.runtimePlan.observability.harnessId } - : {}), - assistantTexts: [], - }, - ctx: hookContext, - hookRunner, - }); - const turnStartFailureKind = classifyCodexModelCallFailureKind({ - error: turnStartError, - timedOut, - turnCompletionIdleTimedOut, - runAborted: runAbortController.signal.aborted, - abortReason: runAbortController.signal.reason, - clientClosedAbort, - formatError: formatErrorMessage, - }); - codexModelCallDiagnostics.emitError( - turnStartErrorMessage, - turnStartFailureKind ? { failureKind: turnStartFailureKind } : {}, - ); - const turnStartFailureMessages = [ - ...historyMessages, - buildCodexUserPromptMessage({ ...runtimeParams, prompt: codexTurnPromptText }), - ]; - await runCodexAgentEndHook(params, { - event: { - messages: turnStartFailureMessages, - success: false, - error: turnStartErrorMessage, - durationMs: Date.now() - attemptStartedAt, - }, - ctx: hookContext, - hookRunner, - }); - if (!timedOut) { - await unsubscribeCodexThreadBestEffort(client, { - threadId: thread.threadId, - timeoutMs: CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, - }); - } - releaseCurrentRoute(); - activateNativePreToolUseFailureFallback(); - nativeHookRelay?.unregister(); - await releaseSandboxExecEnvironment(); - await runAgentCleanupStep({ - runId: params.runId, - sessionId: params.sessionId, - step: "codex-trajectory-flush-startup-failure", - log: embeddedAgentLog, - cleanup: async () => { - await trajectoryRecorder?.flush(); - }, - }); - params.abortSignal?.removeEventListener("abort", abortFromUpstream); - await releaseSharedClientLeaseAndRetireOneShotClient(); - if (usageLimitError) { - await markCodexAuthProfileBlockedFromRateLimits({ - params, - authProfileId: startupAuthProfileId, - rateLimits: usageLimitError.rateLimitsForProfile, - }); - return { - ...buildCodexTurnStartFailureResult({ - params, - message: usageLimitError.message, - messagesSnapshot: turnStartFailureMessages, - systemPromptReport, - }), - }; - } - if (isCodexContextRestartSelectionChangedError(turnStartError)) { - return { - ...buildCodexTurnStartFailureResult({ - params, - message: turnStartErrorMessage, - messagesSnapshot: turnStartFailureMessages, - systemPromptReport, - }), - codexAppServerFailure: { - kind: "client_closed_before_turn_completed", - transport: appServer.start.transport, - threadId: thread.threadId, - replaySafe: true, - }, - }; - } - throw turnStartError; - } - } - if (!turn) { - activateNativePreToolUseFailureFallback(); - await releaseSharedClientLeaseAndRetireOneShotClient(); - throw new Error("codex app-server turn/start failed without an error"); - } - turnIdRef.current = turn.turn.id; - const activeTurnId = turn.turn.id; - let assistantStreamEventEmitted = false; - let assistantStreamNeedsTerminalSnapshot = false; - emitExecutionPhaseOnce("turn_accepted", { phase: "turn_accepted" }); - userInputBridgeRef.current = createCodexUserInputBridge({ - paramsForRun: params, - threadId: thread.threadId, - turnId: activeTurnId, - signal: runAbortController.signal, - }); - trajectoryRecorder?.recordEvent("prompt.submitted", { - threadId: thread.threadId, - turnId: activeTurnId, - prompt: codexTurnPromptText, - imagesCount: params.images?.length ?? 0, - }); - projectorRef.current = new CodexAppServerEventProjector( - { - ...dynamicToolParams, - onAgentEvent: (event) => { - if (event.stream === "assistant" && typeof event.data.delta === "string") { - assistantStreamEventEmitted = true; - assistantStreamNeedsTerminalSnapshot ||= event.data.replaceable === true; - } - return dynamicToolParams.onAgentEvent?.(event); - }, - }, - thread.threadId, - activeTurnId, - { - nativePostToolUseRelayEnabled: - nativeHookRelay?.allowedEvents.includes("post_tool_use") === true && - nativeHookRelay.shouldRelayEvent("post_tool_use"), - readRecentRateLimits: () => readRecentCodexRateLimits(client), - runAbortSignal: runAbortController.signal, - trajectoryRecorder, - onNativeToolResultRecorded: maybeAnnounceFastModeAutoOff, - onContextCompacted: () => { - computerContextEpoch.value += 1; - delete computerContextEpoch.frameToolCallId; - delete computerContextEpoch.frameImageIdentity; - }, - }, - ); - if (isTerminalTurnStatus(turn.turn.status)) { - terminalTurnNotificationQueued = true; - } - emitLifecycleStart(); - const activeProjector = projectorRef.current; - if (!activeProjector) { - throw new Error("codex app-server projector was not initialized"); - } - turnWatches.armTerminalIdleWatch(); - turnWatches.touchActivity("turn:start", { arm: true }); - turnWatches.armAttemptIdleWatch(); - turnWatches.touchActivity("turn:start", { attemptProgress: true }); - for (const failure of pendingNativePreToolUseFailures.splice(0)) { - activeProjector.recordNativeToolPreToolUseFailure(failure); - } - // Codex can emit notifications and requests before turn/start returns. The - // route buffered them while armed; publish the full turn context first, then - // release them in wire order. - if (turnRoute) { - try { - await turnRoute.bindTurn(activeTurnId); - } catch (error) { - if (!terminalTurnNotificationQueued) { - throw error; - } - await turnRoute.drain(); - if (!completed) { - turnWatches.clearAllTimers(); - throw error; - } - } - } - if (!completed && isTerminalTurnStatus(turn.turn.status)) { - await enqueueNotification( - { - method: "turn/completed", - params: { - threadId: thread.threadId, - turnId: activeTurnId, - turn: turn.turn as unknown as JsonObject, - }, - }, - { threadId: thread.threadId, turnId: activeTurnId }, - ); - } - - const activeSteeringQueue = createCodexSteeringQueue({ - client, - threadId: thread.threadId, - turnId: activeTurnId, - answerPendingUserInput: (text) => - userInputBridgeRef.current?.handleQueuedMessage(text) ?? false, - signal: runAbortController.signal, - }); - steeringQueueRef.current = activeSteeringQueue; - const handle = { - kind: "embedded" as const, - runId: params.runId, - queueMessage: async (text: string, optionsLocal?: CodexSteeringQueueOptions) => - activeSteeringQueue.queue(text, optionsLocal), - isStreaming: () => !completed && !runAbortController.signal.aborted, - isStopped: () => completed || timedOut || runAbortController.signal.aborted, - isAbortable: () => !terminalOutcomeFrozen || sharedAbortAllowedAfterTerminalOutcome, - isCompacting: () => projectorRef.current?.isCompacting() ?? false, - sourceReplyDeliveryMode: params.sourceReplyDeliveryMode, - cancel: () => abortExplicitly("cancelled"), - abort: () => abortExplicitly("aborted"), - }; - params.replyOperation?.attachBackend(handle); - setActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile); - const freezeRunTerminalOutcome = () => { - if (terminalOutcomeFrozen) { - return; - } - terminalOutcomeFrozen = true; - params.abortSignal?.removeEventListener("abort", abortFromUpstream); - }; - const notifyUserMessagePersisted = createCodexAppServerUserMessagePersistenceNotifier(params); - void mirrorPromptAtTurnStartBestEffort({ - params, - agentId: sessionAgentId, - notifyUserMessagePersisted, - sessionKey: sandboxSessionKey, - cwd: effectiveCwd, - threadId: thread.threadId, - turnId: activeTurnId, - }); - - const abortListener = () => { - const shouldRetireClient = timedOut; - if (shouldRetireClient) { - void (async () => { - // Supervised sessions stay native even after a suspect turn. Clearing - // their private scope would make the next attempt silently agent-home. - if (thread.connectionScope !== "supervision") { - await bindingStore.mutate(bindingIdentity, { - kind: "clear", - threadId: thread.threadId, - }); - } - await retireCodexAppServerClientAfterTimedOutTurn(client, { - threadId: thread.threadId, - turnId: activeTurnId, - reason: String(runAbortController.signal.reason ?? "timeout"), - suspectPhysicalClient: turnWatchTimeoutKind === "terminal", - }); - })().finally(() => { - resolveCompletion?.(); - }); - return; - } - interruptCodexTurnBestEffort(client, { - threadId: thread.threadId, - turnId: activeTurnId, - }); - resolveCompletion?.(); - }; - runAbortController.signal.addEventListener("abort", abortListener, { once: true }); - if (runAbortController.signal.aborted) { - abortListener(); - } try { - await completion; - // Timeout completion can win while a received notification is still being - // projected, for example while persisting raw image-generation media. Wait - // for already-queued projection work so the final result includes artifacts - // from the notification that triggered the idle watchdog. - await drainNotificationQueue(); - const hasQuiescentCompletedAssistant = - activeProjector.hasCompletedTerminalAssistantText() && - activeAppServerTurnRequests === 0 && - activeTurnItemIds.size === 0 && - activeCompletionBlockerItemIds.size === 0 && - pendingOpenClawDynamicToolCompletionIds.size === 0 && - activeFinalizationHookRunIds.size === 0 && - unsettledFinalizationHookCount === 0 && - rejectedFinalizationHookAssistant === undefined; - const hasRecoverableCompletedAssistant = - !turnWatches.isCompletionIdleWatchPinnedByTerminalError() && - turnWatches.isAssistantCompletionIdleWatchArmed() && - hasQuiescentCompletedAssistant; - const recoveredTurnWatchTimeout = - turnCompletionIdleTimedOut && - !explicitCancellationObserved && - !terminalTurnNotificationQueued && - hasRecoverableCompletedAssistant && - activeProjector.recoverCompletedTerminalAssistantAfterTurnWatchTimeout(); - if (recoveredTurnWatchTimeout) { - embeddedAgentLog.warn( - "codex app-server recovered completed assistant output after missing turn completion", - { - threadId: thread.threadId, - turnId: activeTurnId, - timeoutKind: turnWatchTimeoutKind, - idleMs: turnWatchTimeoutIdleMs, - timeoutMs: turnWatchTimeoutMs, - }, - ); - trajectoryRecorder?.recordEvent("turn.watch_timeout_recovered", { - threadId: thread.threadId, - turnId: activeTurnId, - timeoutKind: turnWatchTimeoutKind, - idleMs: turnWatchTimeoutIdleMs, - timeoutMs: turnWatchTimeoutMs, - }); - } - const result = activeProjector.buildResult(toolBridge.telemetry, { yieldDetected }); - const effectiveTimedOut = timedOut && !recoveredTurnWatchTimeout; - const effectiveTurnCompletionIdleTimedOut = - turnCompletionIdleTimedOut && !recoveredTurnWatchTimeout; - const isFinalAborted = () => - result.aborted || - explicitCancellationObserved || - (runAbortController.signal.aborted && !clientClosedAbort && !recoveredTurnWatchTimeout); - const clientClosedPromptErrorForFinal = - clientClosedPromptError && hasRecoverableCompletedAssistant - ? undefined - : clientClosedPromptError; - let finalPromptError = - clientClosedPromptErrorForFinal ?? - (effectiveTurnCompletionIdleTimedOut - ? turnCompletionIdleTimeoutMessage - : effectiveTimedOut - ? "codex app-server attempt timed out" - : result.promptError); - const finalPromptErrorMessage = - typeof finalPromptError === "string" - ? finalPromptError - : finalPromptError - ? formatErrorMessage(finalPromptError) - : undefined; - if (isInvalidCodexImagePayloadError(finalPromptErrorMessage)) { - await clearCodexBindingAfterInvalidImagePayload(bindingStore, bindingIdentity, { - phase: "turn_completed", - threadId: thread.threadId, - turnId: activeTurnId, - error: finalPromptErrorMessage, - }); - } - if ( - thread.connectionScope !== "supervision" && - shouldUseFreshCodexThreadAfterContextEngineOverflow({ - error: finalPromptError, - contextEngineActive: Boolean(activeContextEngine), - thread, - }) - ) { - embeddedAgentLog.warn( - "codex app-server context-engine turn overflowed after resume; clearing thread binding for recovery", - { - threadId: thread.threadId, - turnId: activeTurnId, - error: finalPromptErrorMessage, - }, - ); - await bindingStore.mutate(bindingIdentity, { kind: "clear", threadId: thread.threadId }); - } - const refreshedUsageLimitPromptError = await refreshCodexUsageLimitPromptError({ - client, - message: finalPromptErrorMessage, - timeoutMs: appServer.requestTimeoutMs, - signal: runAbortController.signal, - }); - if (refreshedUsageLimitPromptError) { - finalPromptError = refreshedUsageLimitPromptError; - } - const finalPromptErrorSource = - effectiveTimedOut || clientClosedPromptErrorForFinal ? "prompt" : result.promptErrorSource; - const codexAppServerFailureKind = clientClosedPromptErrorForFinal - ? "client_closed_before_turn_completed" - : effectiveTurnCompletionIdleTimedOut - ? "turn_completion_idle_timeout" - : undefined; - const codexAppServerReplayBlockedReason = codexAppServerFailureKind - ? resolveCodexAppServerReplayBlockedReason(result) - : undefined; - const promptTimeoutOutcome = buildCodexAppServerPromptTimeoutOutcome({ - result, - turnCompletionIdleTimedOut: effectiveTurnCompletionIdleTimedOut, - turnWatchTimeoutKind, - }); - const codexAppServerFailureDiagnostics = - codexAppServerFailureKind === "turn_completion_idle_timeout" && - turnWatchTimeoutKind === "completion" - ? buildCodexAppServerTimeoutDiagnostics({ - idleMs: turnWatchTimeoutIdleMs, - timeoutMs: turnWatchTimeoutMs, - lastActivityReason: turnWatchTimeoutLastActivityReason, - details: turnWatchTimeoutDetails, - }) - : undefined; - const codexAppServerFailure = codexAppServerFailureKind - ? ({ - kind: codexAppServerFailureKind, - ...(codexAppServerFailureKind === "turn_completion_idle_timeout" && turnWatchTimeoutKind - ? { turnWatchTimeoutKind } - : {}), - transport: appServer.start.transport, - threadId: thread.threadId, - turnId: activeTurnId, - replaySafe: codexAppServerReplayBlockedReason === undefined, - ...(codexAppServerReplayBlockedReason - ? { replayBlockedReason: codexAppServerReplayBlockedReason } - : {}), - ...(codexAppServerFailureDiagnostics - ? { diagnostics: codexAppServerFailureDiagnostics } - : {}), - } satisfies NonNullable) - : undefined; - const finalAborted = isFinalAborted(); - const completedTurnStatus = activeProjector.getCompletedTurnStatus(); - const completedWithoutTerminalNotification = - completed && - !terminalTurnNotificationQueued && - !timedOut && - clientClosedPromptErrorForFinal === undefined; - const attemptSucceeded = - !finalAborted && - !effectiveTimedOut && - (finalPromptError === null || finalPromptError === undefined) && - result.agentHarnessResultClassification === undefined && - (completedTurnStatus === "completed" || - recoveredTurnWatchTimeout || - completedWithoutTerminalNotification); - sharedAbortAllowedAfterTerminalOutcome = shouldKeepCodexSharedAbortOpen({ - trigger: params.trigger, - result, - attemptSucceeded, - explicitCancellationObserved, - }); - // Terminal diagnostics, transcript mirroring, hooks, and lifecycle events - // must all observe one immutable attempt outcome. Failed attempts still - // allow the shared reply operation to cancel retries or model fallback. - freezeRunTerminalOutcome(); - const modelCallFailureKind = - classifyCodexModelCallFailureKind({ - error: finalPromptError, - timedOut: effectiveTimedOut, - turnCompletionIdleTimedOut: effectiveTurnCompletionIdleTimedOut, - runAborted: finalAborted, - abortReason: explicitCancellationReason ?? runAbortController.signal.reason, - clientClosedAbort, - formatError: formatErrorMessage, - }) ?? (finalAborted ? "aborted" : undefined); - if (modelCallFailureKind) { - codexModelCallDiagnostics.emitError( - finalPromptError ?? "codex app-server attempt interrupted", - { - failureKind: modelCallFailureKind, - }, - ); - } else if (finalPromptError) { - codexModelCallDiagnostics.emitError(finalPromptError); - } else { - codexModelCallDiagnostics.emitCompleted(result); - } - const assistantTranscriptOwned = await codexTranscriptMirrorRuntime.mirrorBestEffort({ - params, - agentId: sessionAgentId, - notifyUserMessagePersisted, - result, - sessionKey: contextSessionKey, - cwd: effectiveCwd, - threadId: thread.threadId, - turnId: activeTurnId, - }); - if (activeContextEngine) { - const activeContextEnginePluginIdLocal = - resolveContextEngineOwnerPluginId(activeContextEngine); - // Gateway command runs use a generic `user` trigger, so the bootstrap run - // kind is the canonical heartbeat lifecycle signal at this boundary. - const isHeartbeatLifecycleRun = - params.bootstrapContextRunKind === "heartbeat" || - params.bootstrapContextRunKind === "commitment-only"; - const finalMessages = - (await readMirroredSessionHistoryMessages(activeTranscriptTarget)) ?? - historyMessages.concat(result.messagesSnapshot); - await finalizeHarnessContextEngineTurn({ - contextEngine: activeContextEngine, - promptError: Boolean(finalPromptError), - aborted: finalAborted, - yieldAborted: Boolean(result.yieldDetected), - sessionIdUsed: activeSessionId, - sessionKey: contextSessionKey, - sessionFile: activeSessionFile, - sessionTarget: params.sessionTarget, - messagesSnapshot: finalMessages, - prePromptMessageCount, - tokenBudget: effectiveContextTokenBudget, - runtimeContext: buildHarnessContextEngineRuntimeContextFromUsage({ - attempt: buildActiveRunAttemptParams(), - workspaceDir: effectiveWorkspace, - cwd: effectiveCwd, - agentDir, - activeAgentId: sessionAgentId, - contextEnginePluginId: activeContextEnginePluginIdLocal, - tokenBudget: effectiveContextTokenBudget, - lastCallUsage: result.attemptUsage, - promptCache: result.promptCache, - }), - contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, - providerId: usesSupervisionConnection - ? (thread.modelProvider ?? effectiveRuntimeProviderId) - : params.provider, - requestedModelId: usesSupervisionConnection ? undefined : params.requestedModelId, - modelId: usesSupervisionConnection - ? (thread.model ?? effectiveRuntimeModelId) - : params.modelId, - fallbackReason: usesSupervisionConnection ? undefined : params.fallbackReason, - degradedReason: usesSupervisionConnection ? undefined : params.degradedReason, - runMaintenance: runHarnessContextEngineMaintenance, - config: params.config, - warn: (message) => embeddedAgentLog.warn(message), - isHeartbeat: isHeartbeatLifecycleRun, - }); - } - runAgentHarnessLlmOutputHook({ - event: { - runId: params.runId, - sessionId: params.sessionId, - provider: usesSupervisionConnection - ? (thread.modelProvider ?? effectiveRuntimeProviderId) - : params.provider, - model: usesSupervisionConnection - ? (thread.model ?? effectiveRuntimeModelId) - : params.modelId, - ...hookContextWindowFields, - resolvedRef: usesSupervisionConnection - ? `${thread.modelProvider ?? effectiveRuntimeProviderId}/${thread.model ?? effectiveRuntimeModelId}` - : (params.runtimePlan?.observability.resolvedRef ?? - `${params.provider}/${params.modelId}`), - ...(!usesSupervisionConnection && params.runtimePlan?.observability.harnessId - ? { harnessId: params.runtimePlan.observability.harnessId } - : {}), - assistantTexts: result.assistantTexts, - ...(result.lastAssistant ? { lastAssistant: result.lastAssistant } : {}), - ...(result.attemptUsage ? { usage: result.attemptUsage } : {}), - }, - ctx: hookContext, - hookRunner, - }); - await runCodexAgentEndHook(params, { - event: { - messages: result.messagesSnapshot, - success: !finalAborted && !finalPromptError, - ...(finalPromptError ? { error: formatErrorMessage(finalPromptError) } : {}), - durationMs: Date.now() - attemptStartedAt, - }, - ctx: hookContext, - hookRunner, - }); - shouldDelayNativeHookRelayUnregister = - completedTurnStatus === "completed" && - !effectiveTimedOut && - !runAbortController.signal.aborted && - !finalAborted && - !finalPromptError; - if (shouldDelayNativeHookRelayUnregister) { - try { - await markCodexAppServerBindingCoveredThroughTurn({ - bindingStore, - identity: bindingIdentity, - threadId: thread.threadId, - }); - } catch (error) { - if (thread.connectionScope === "supervision") { - throw error; - } - const clearedStaleBinding = await bindingStore.mutate(bindingIdentity, { - kind: "clear", - threadId: thread.threadId, - }); - if (!clearedStaleBinding) { - throw error; - } - embeddedAgentLog.warn( - "codex app-server binding coverage update failed after completed turn; cleared stale binding", - { - threadId: thread.threadId, - turnId: activeTurnId, - error, - }, - ); - } - } - recordCodexTrajectoryCompletion(trajectoryRecorder, { - attempt: params, - result, - threadId: thread.threadId, - turnId: activeTurnId, - timedOut: effectiveTimedOut, - yieldDetected, - }); - trajectoryRecorder?.recordEvent("session.ended", { - status: finalPromptError - ? "error" - : finalAborted || effectiveTimedOut - ? "interrupted" - : "success", - threadId: thread.threadId, - turnId: activeTurnId, - timedOut: effectiveTimedOut, - yieldDetected, - promptError: normalizeCodexTrajectoryError(finalPromptError), - }); - markTrajectoryEndRecorded(); - const terminalAssistantText = collectTerminalAssistantText(result); - if ( - terminalAssistantText && - (!assistantStreamEventEmitted || assistantStreamNeedsTerminalSnapshot) && - !finalAborted && - !finalPromptError - ) { - void emitCodexAppServerEvent(params, { - stream: "assistant", - data: { text: terminalAssistantText }, - }); - } - if (finalPromptError) { - emitLifecycleTerminal({ - phase: "error", - error: formatErrorMessage(finalPromptError), - ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), - }); - } else { - emitLifecycleTerminal({ - phase: "end", - ...buildLifecycleTerminalMeta({ aborted: finalAborted, timedOut: effectiveTimedOut }), - }); - } - return { - ...result, - timedOut: effectiveTimedOut, - aborted: finalAborted, - promptError: finalPromptError, - promptErrorSource: finalPromptErrorSource, - ...(codexAppServerFailure ? { codexAppServerFailure } : {}), - ...(promptTimeoutOutcome ? { promptTimeoutOutcome } : {}), - ...(assistantTranscriptOwned ? { assistantTranscriptOwned: true } : {}), - ...(runtimeArtifact ? { runtimeArtifact } : {}), - ...(!finalAborted && !effectiveTimedOut && !finalPromptError && preparedAuthBinding - ? { authBindingFingerprint: preparedAuthBinding.fingerprint } - : {}), - systemPromptReport, - }; + return await finalizeCodexAttempt( + resources, + turnRuntime, + lifecycle, + notifications, + turnRequest, + activeTurn, + ); } finally { - if (params.isFinalFallbackAttempt !== false) { - await maybeEmitFastModeAutoResetBestEffort(); - } - codexModelCallDiagnostics.emitError( - "codex app-server run completed without model-call terminal event", - ); - emitLifecycleTerminal({ - phase: "error", - error: "codex app-server run completed without lifecycle terminal event", - ...buildLifecycleTerminalMeta({ - aborted: runAbortController.signal.aborted && !clientClosedAbort, - timedOut, - }), - }); - if (trajectoryRecorder && !trajectoryEndRecorded) { - trajectoryRecorder.recordEvent("session.ended", { - status: - timedOut || (runAbortController.signal.aborted && !clientClosedAbort) - ? "interrupted" - : "cleanup", - threadId: thread.threadId, - turnId: activeTurnId, - timedOut, - aborted: runAbortController.signal.aborted && !clientClosedAbort, - }); - } - await runAgentCleanupStep({ - runId: params.runId, - sessionId: params.sessionId, - step: "codex-trajectory-flush", - log: embeddedAgentLog, - cleanup: async () => { - await trajectoryRecorder?.flush(); - }, - }); - if (!timedOut && !runAbortController.signal.aborted) { - await steeringQueueRef.current?.flushPending(); - } - if (!timedOut) { - await unsubscribeCodexThreadBestEffort(client, { - threadId: thread.threadId, - timeoutMs: CODEX_APP_SERVER_UNSUBSCRIBE_TIMEOUT_MS, - }); - } - userInputBridgeRef.current?.cancelPending(); - turnWatches.clearAllTimers(); - releaseCurrentRoute(); - await releaseSharedClientLeaseAndRetireOneShotClient(); - if (nativeHookRelay) { - if (shouldDelayNativeHookRelayUnregister) { - // Codex hook subprocesses can outlive a completed app-server turn by a - // few seconds. Keep the relay available briefly so late - // nativeHook.invoke RPCs can still reach before_tool_call enforcement. - scheduleCodexNativeHookRelayUnregister({ - relay: nativeHookRelay, - hookTimeoutSec: options.nativeHookRelay?.hookTimeoutSec, - }); - } else { - nativeHookRelay.unregister(); - } - } - await releaseSandboxExecEnvironment(); - runAbortController.signal.removeEventListener("abort", abortListener); - steeringQueueRef.current?.cancel(); - freezeRunTerminalOutcome(); - params.replyOperation?.detachBackend(handle); - clearActiveEmbeddedRun(params.sessionId, handle, params.sessionKey, params.sessionFile); + await cleanupCodexAttempt(resources, turnRuntime, lifecycle, turnRequest, activeTurn); } }