openclaw/src/agents/command/run-embedded-attempt.ts

import { sanitizeForLog } from "../../../packages/terminal-core/src/ansi.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import { emitAgentEvent } from "../../infra/agent-events.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import {
  MODEL_SELECTION_LOCKED_MESSAGE,
  ModelSelectionLockedError,
  isModelSelectionLocked,
} from "../../sessions/model-overrides.js";
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
import {
  getGeneratedMediaTaskIdsForSessionKey,
  hasNewGeneratedMediaTaskForSessionKey,
} from "../../tasks/task-status-access.js";
import { createTrajectoryRuntimeRecorder } from "../../trajectory/runtime.js";
import { resolveMessageChannel } from "../../utils/message-channel.js";
import {
  clearAutoFallbackPrimaryProbeSelection,
  entryMatchesAutoFallbackPrimaryProbe,
  markAutoFallbackPrimaryProbe,
  resolveEffectiveModelFallbacks,
} from "../agent-scope.js";
import {
  runEmbeddedAgentEntry,
  type EmbeddedAgentRunEntryTerminal,
} from "../embedded-agent-runner/run-entry.js";
import { resolveFastModeState } from "../fast-mode.js";
import { runAgentHarnessBeforeMessageWriteHook } from "../harness/hook-helpers.js";
import { prepareInternalSessionEffectsSession } from "../internal-session-effects.js";
import { LiveSessionModelSwitchError } from "../live-model-switch.js";
import { modelKey, resolveThinkingDefault } from "../model-selection.js";
import type { AgentRunSessionTarget } from "../run-session-target.js";
import {
  isAgentRunDirectAbortReason,
  isAgentRunRestartAbortReason,
  resolveAgentRunErrorLifecycleFields,
} from "../run-termination.js";
import { resolveSessionRuntimeOverrideForProvider } from "../session-runtime-compat.js";
import {
  resolveCandidateThinkingLevel,
  resolveEffectiveAgentRuntime,
} from "../thinking-runtime.js";
import {
  createAgentAttemptLifecycleCallbacks,
  type AgentAttemptLifecycleState,
} from "./attempt-callbacks.js";
import { createAgentCommandLifecycle } from "./lifecycle.js";
import { normalizeAgentCommandModelRef } from "./model-ref.js";
import type { EmbeddedModelSelection } from "./model-selection.js";
import type { PreparedAgentCommandExecution } from "./prepare.js";
import { loadAttemptExecutionRuntime, type AgentAttemptResult } from "./runtime-loaders.js";
import { persistSessionEntry, resolveInternalSessionEffectsSource } from "./session-helpers.js";
import type { EmbeddedSessionState } from "./session-preparation.js";
import type { AgentCommandOpts } from "./types.js";

const log = createSubsystemLogger("agents/agent-command");
const MAX_LIVE_SWITCH_RETRIES = 5;

export async function runEmbeddedAgentAttempt(params: {
  prepared: PreparedAgentCommandExecution;
  opts: AgentCommandOpts;
  sessionEntry?: SessionEntry;
  lifecycleGeneration: string;
  onLifecycleGenerationChanged: (lifecycleGeneration: string) => void;
  suppressVisibleSessionEffects: boolean;
  preserveUserFacingSessionModelState: boolean;
  modelSelection: EmbeddedModelSelection;
  embeddedSessionState: EmbeddedSessionState;
  trackInternalModelRunTarget: (target: AgentRunSessionTarget | undefined) => void;
}) {
  const {
    cfg,
    body,
    transcriptBody,
    sessionId,
    sessionKey,
    sessionStore,
    storePath,
    sessionAgentId,
    workspaceDir,
    cwd,
    agentDir,
    runId,
    pluginsEnabled,
    manifestMetadataSnapshot,
    modelManifestContext,
    normalizedSpawned,
    isNewSession,
    timeoutMs,
    runTimeoutOverrideMs,
  } = params.prepared;
  const { runContext, skillsSnapshot, resolvedVerboseLevel } = params.embeddedSessionState;
  const {
    defaultProvider,
    defaultModel,
    configuredDefaultAuthProfileId,
    visibilityPolicy,
    hasExplicitRunOverride,
    storedProviderOverride,
    hasStoredAutoFallbackProvenance,
    autoFallbackPrimaryProbe,
    thinkingCatalog,
    immutableThinkLevel,
    sessionFile,
  } = params.modelSelection;
  let {
    provider,
    model,
    providerForAuthProfileValidation,
    sessionEntryForAttempt,
    storedModelOverride,
    storedModelOverrideSource,
    effectiveTurnThinkLevel,
  } = params.modelSelection;
  let sessionEntry = params.sessionEntry;
  let lifecycleGeneration = params.lifecycleGeneration;

  const sessionEffectsSource = resolveInternalSessionEffectsSource({
    agentId: sessionAgentId,
    sessionId,
    sessionKey,
    storePath,
  });
  const internalSessionTarget = params.suppressVisibleSessionEffects
    ? await prepareInternalSessionEffectsSession({
        agentId: sessionAgentId,
        cwd: cwd ?? workspaceDir,
        runId,
        source: sessionEffectsSource,
        storePath,
      })
    : undefined;
  params.trackInternalModelRunTarget(internalSessionTarget);
  const attemptSessionTarget =
    internalSessionTarget ??
    (sessionKey && storePath
      ? {
          agentId: sessionAgentId,
          sessionId,
          sessionKey,
          storePath,
        }
      : undefined);
  const attemptSessionFile = internalSessionTarget?.sessionFile ?? sessionFile;

  const startedAt = Date.now();
  const attemptLifecycleState: AgentAttemptLifecycleState = {
    currentTurnUserMessagePersisted: false,
    lifecycleFinishing: false,
    lifecycleEnded: false,
  };
  const attemptLifecycleCallbacks = createAgentAttemptLifecycleCallbacks(attemptLifecycleState);
  const transcriptMedia = params.opts.transcriptMedia ?? [];
  const hasTranscriptMedia = transcriptMedia.length > 0;
  const suppressUserTurnPersistence =
    params.opts.suppressPromptPersistence === true ||
    (params.opts.transcriptMessage === "" && !hasTranscriptMedia);
  const recorderTranscriptText = transcriptBody || undefined;
  const userTurnTranscriptRecorder = createUserTurnTranscriptRecorder({
    ...(!suppressUserTurnPersistence && (recorderTranscriptText || hasTranscriptMedia)
      ? {
          input: {
            text: recorderTranscriptText,
            ...(hasTranscriptMedia ? { media: transcriptMedia } : {}),
          },
        }
      : {}),
    target: {
      sessionId: internalSessionTarget?.sessionId ?? sessionId,
      agentId: internalSessionTarget?.agentId ?? sessionAgentId,
      sessionKey: internalSessionTarget?.sessionKey ?? sessionKey ?? sessionId,
      sessionEntry: internalSessionTarget?.sessionEntry ?? sessionEntry,
      sessionStore: params.suppressVisibleSessionEffects ? undefined : sessionStore,
      storePath: internalSessionTarget?.storePath ?? storePath,
      cwd: cwd ?? workspaceDir,
      config: cfg,
    },
    beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook,
    errorContext: "agent command user turn transcript",
  });
  if (suppressUserTurnPersistence) {
    userTurnTranscriptRecorder.markBlocked();
  }
  const lifecycle = createAgentCommandLifecycle({
    runId,
    lifecycleGeneration: () => lifecycleGeneration,
    startedAt,
    abortSignal: params.opts.abortSignal,
    state: attemptLifecycleState,
  });
  const attemptExecutionRuntime = await loadAttemptExecutionRuntime();
  const messageChannel = resolveMessageChannel(
    runContext.messageChannel,
    params.opts.replyChannel ?? params.opts.channel,
  );

  let result: AgentAttemptResult;
  let fallbackProvider = provider;
  let fallbackModel = model;
  let fallbackExhausted = false;
  let terminal: EmbeddedAgentRunEntryTerminal;
  let liveSwitchRetries = 0;
  let autoFallbackPrimaryProbeInterruptedByLiveSwitch = false;
  const fastModeStartedAtMs = Date.now();
  const fallbackTrajectoryRecorder = createTrajectoryRuntimeRecorder({
    cfg,
    runId,
    sessionId,
    sessionKey,
    sessionFile: attemptSessionFile,
    provider,
    modelId: model,
    workspaceDir,
  });
  let liveSwitchMediaTaskIds: ReadonlySet<string> = new Set();
  for (;;) {
    try {
      liveSwitchMediaTaskIds = sessionKey
        ? getGeneratedMediaTaskIdsForSessionKey(sessionKey)
        : new Set<string>();
      const spawnedBy = normalizedSpawned.spawnedBy ?? sessionEntry?.spawnedBy;
      const effectiveFallbacksOverride = isModelSelectionLocked(sessionEntry)
        ? []
        : (params.opts.modelFallbacksOverride ??
          resolveEffectiveModelFallbacks({
            cfg,
            agentId: sessionAgentId,
            sessionKey,
            hasSessionModelOverride:
              hasExplicitRunOverride || Boolean(storedProviderOverride || storedModelOverride),
            modelOverrideSource: hasExplicitRunOverride ? "user" : storedModelOverrideSource,
            hasAutoFallbackProvenance: hasExplicitRunOverride
              ? false
              : hasStoredAutoFallbackProvenance,
          }));

      const fallbackRuntimeState: { originRuntime?: "cli" | "embedded" } = {};
      attemptLifecycleState.currentTurnUserMessagePersisted = false;
      let attemptMediaTaskIds = liveSwitchMediaTaskIds;
      const currentAttemptCommittedCronMedia = () =>
        Boolean(
          sessionKey && hasNewGeneratedMediaTaskForSessionKey(sessionKey, attemptMediaTaskIds),
        );
      const fallbackResult = await runEmbeddedAgentEntry<AgentAttemptResult>({
        selection: {
          cfg,
          provider,
          model,
          requestedRouteResolution: params.modelSelection.requestedRouteResolution,
          agentDir,
          fallbacksOverride: effectiveFallbacksOverride,
          ...modelManifestContext,
        },
        identity: {
          runId,
          agentId: sessionAgentId,
          sessionId,
          sessionKey: sessionKey ?? sessionId,
        },
        harness: {
          workspaceDir,
          sessionKey,
          preparation: { kind: "direct" },
          resolveRuntimeOverride: (candidateProvider) =>
            resolveSessionRuntimeOverrideForProvider({
              provider: candidateProvider,
              entry: sessionEntryForAttempt,
              cfg,
            }),
        },
        behavior: {
          kind: "command-rpc",
          hasCommittedSideEffect: currentAttemptCommittedCronMedia,
        },
        sessionOverride: {
          kind: "reconcile-completed",
          reconcile: async ({ provider: winnerProvider, model: winnerModel }) => {
            if (
              !autoFallbackPrimaryProbe ||
              autoFallbackPrimaryProbeInterruptedByLiveSwitch ||
              !sessionEntry ||
              !sessionStore ||
              !sessionKey ||
              isModelSelectionLocked(sessionEntry) ||
              params.suppressVisibleSessionEffects ||
              params.preserveUserFacingSessionModelState ||
              !entryMatchesAutoFallbackPrimaryProbe(sessionEntry, autoFallbackPrimaryProbe) ||
              winnerProvider !== autoFallbackPrimaryProbe.provider ||
              winnerModel !== autoFallbackPrimaryProbe.model
            ) {
              return;
            }
            const nextSessionEntry = { ...sessionEntry };
            clearAutoFallbackPrimaryProbeSelection(nextSessionEntry);
            sessionEntry = await persistSessionEntry({
              sessionStore,
              sessionKey,
              storePath,
              initialEntry: sessionEntry,
              entry: nextSessionEntry,
              shouldPersist: (current) =>
                Boolean(
                  current &&
                  entryMatchesAutoFallbackPrimaryProbe(current, autoFallbackPrimaryProbe),
                ),
            });
          },
        },
        abortSignal: params.opts.abortSignal,
        onFallbackStep: (step) => {
          fallbackTrajectoryRecorder?.recordEvent("model.fallback_step", step);
        },
        runCandidate: async (providerOverride, modelOverride, runOptions) => {
          attemptMediaTaskIds = sessionKey
            ? getGeneratedMediaTaskIdsForSessionKey(sessionKey)
            : new Set<string>();
          attemptLifecycleState.lifecycleError = undefined;
          attemptLifecycleState.lifecycleFinishing = false;
          attemptLifecycleState.lifecycleEnded = false;
          const isAutoFallbackPrimaryProbeCandidate =
            autoFallbackPrimaryProbe &&
            providerOverride === autoFallbackPrimaryProbe.provider &&
            modelOverride === autoFallbackPrimaryProbe.model;
          const attemptSessionEntry =
            autoFallbackPrimaryProbe &&
            providerOverride === autoFallbackPrimaryProbe.fallbackProvider &&
            !isAutoFallbackPrimaryProbeCandidate
              ? sessionEntry
              : sessionEntryForAttempt;
          if (isAutoFallbackPrimaryProbeCandidate) {
            markAutoFallbackPrimaryProbe({ probe: autoFallbackPrimaryProbe, sessionKey });
          }
          await params.opts.onActiveModelSelected?.({
            provider: providerOverride,
            model: modelOverride,
          });
          const fastModeState = resolveFastModeState({
            cfg,
            provider: providerOverride,
            model: modelOverride,
            agentId: sessionAgentId,
            sessionEntry,
          });
          const fastMode = params.opts.fastMode ?? fastModeState.mode;
          const configuredAuthProfileId =
            providerOverride === defaultProvider && modelOverride === defaultModel
              ? configuredDefaultAuthProfileId
              : undefined;
          const agentHarnessRuntimeOverride = resolveSessionRuntimeOverrideForProvider({
            provider: providerOverride,
            entry: attemptSessionEntry,
            cfg,
          });
          const candidateRuntime = resolveEffectiveAgentRuntime({
            cfg,
            provider: providerOverride,
            modelId: modelOverride,
            agentId: sessionAgentId,
            sessionKey,
            sessionEntry: attemptSessionEntry,
          });
          const candidateRequestedThinkLevel =
            immutableThinkLevel ??
            resolveThinkingDefault({
              cfg,
              provider: providerOverride,
              model: modelOverride,
              catalog: thinkingCatalog,
              agentRuntime: candidateRuntime,
            });
          const candidateThinkLevel =
            resolveCandidateThinkingLevel({
              cfg,
              provider: providerOverride,
              modelId: modelOverride,
              level: candidateRequestedThinkLevel,
              catalog: thinkingCatalog,
              agentId: sessionAgentId,
              sessionKey,
              sessionEntry: attemptSessionEntry,
              agentRuntime: candidateRuntime,
            }) ?? candidateRequestedThinkLevel;
          effectiveTurnThinkLevel = candidateThinkLevel;
          return attemptExecutionRuntime.runAgentAttempt({
            providerOverride,
            modelOverride,
            configuredAuthProfileId,
            modelFallbacksOverride: effectiveFallbacksOverride,
            originalProvider: provider,
            cfg,
            sessionEntry: attemptSessionEntry,
            agentHarnessRuntimeOverride,
            sessionId,
            sessionKey,
            ...(attemptSessionTarget ? { sessionTarget: attemptSessionTarget } : {}),
            sessionAgentId,
            sessionFile: attemptSessionFile,
            workspaceDir,
            cwd,
            body,
            transcriptBody,
            isFallbackRetry: runOptions.isFallbackRetry,
            resolvedThinkLevel: candidateThinkLevel,
            fastMode,
            fastModeStartedAtMs,
            fastModeAutoOnSeconds:
              fastMode === "auto"
                ? (params.opts.fastModeAutoOnSeconds ?? fastModeState.fastAutoOnSeconds)
                : fastModeState.fastAutoOnSeconds,
            isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
            timeoutMs,
            runTimeoutOverrideMs,
            runId,
            lifecycleGeneration,
            opts: params.opts,
            runContext,
            spawnedBy,
            messageChannel,
            skillsSnapshot,
            resolvedVerboseLevel,
            agentDir,
            authProfileProvider: providerForAuthProfileValidation,
            sessionStore: params.suppressVisibleSessionEffects ? undefined : sessionStore,
            storePath: params.suppressVisibleSessionEffects ? undefined : storePath,
            pluginsEnabled,
            ...(manifestMetadataSnapshot ? { metadataSnapshot: manifestMetadataSnapshot } : {}),
            allowTransientCooldownProbe: runOptions?.allowTransientCooldownProbe,
            sessionHasHistory:
              !isNewSession ||
              (await attemptExecutionRuntime.sessionFileHasContent(attemptSessionFile)),
            fallbackRuntimeState,
            suppressPromptPersistenceOnRetry:
              suppressUserTurnPersistence ||
              userTurnTranscriptRecorder.hasPersisted() ||
              userTurnTranscriptRecorder.isBlocked() ||
              (runOptions.isFallbackRetry && attemptLifecycleState.currentTurnUserMessagePersisted),
            userTurnTranscriptRecorder,
            onUserMessagePersisted: attemptLifecycleCallbacks.onUserMessagePersisted,
            onLifecycleGenerationChanged: (nextLifecycleGeneration) => {
              lifecycleGeneration = nextLifecycleGeneration;
              // Outer cleanup owns the run context, so publish before the attempt can reject.
              params.onLifecycleGenerationChanged(nextLifecycleGeneration);
            },
            onAgentEvent: attemptLifecycleCallbacks.onAgentEvent,
            deferTerminalLifecycle: true,
          });
        },
      });
      result = fallbackResult.result;
      terminal = fallbackResult.terminal;
      if (isAgentRunRestartAbortReason(params.opts.abortSignal?.reason)) {
        throw params.opts.abortSignal?.reason;
      }
      fallbackProvider = fallbackResult.provider;
      fallbackModel = fallbackResult.model;
      fallbackExhausted = fallbackResult.outcome === "exhausted";
      await fallbackResult.settleSessionOverride();
      if (fallbackResult.attempts.length > 0 && result.meta.agentMeta) {
        result = {
          ...result,
          meta: {
            ...result.meta,
            agentMeta: {
              ...result.meta.agentMeta,
              fallbackAttempts: fallbackResult.attempts,
            },
          },
        };
      }
      if (!fallbackExhausted) {
        lifecycle.emitFinishing(terminal);
      }
      break;
    } catch (err) {
      if (err instanceof LiveSessionModelSwitchError) {
        if (isModelSelectionLocked(sessionEntry)) {
          if (!attemptLifecycleState.lifecycleEnded) {
            emitAgentEvent({
              runId,
              lifecycleGeneration,
              stream: "lifecycle",
              data: {
                phase: "error",
                startedAt,
                endedAt: Date.now(),
                error: MODEL_SELECTION_LOCKED_MESSAGE,
              },
            });
          }
          await fallbackTrajectoryRecorder?.flush();
          throw new ModelSelectionLockedError();
        }
        if (
          sessionKey &&
          hasNewGeneratedMediaTaskForSessionKey(sessionKey, liveSwitchMediaTaskIds)
        ) {
          throw err;
        }
        liveSwitchRetries += 1;
        if (liveSwitchRetries > MAX_LIVE_SWITCH_RETRIES) {
          log.error(
            `Live session model switch in subagent run ${runId}: exceeded maximum retries (${MAX_LIVE_SWITCH_RETRIES})`,
          );
          if (!attemptLifecycleState.lifecycleEnded) {
            emitAgentEvent({
              runId,
              lifecycleGeneration,
              stream: "lifecycle",
              data: {
                phase: "error",
                startedAt,
                endedAt: Date.now(),
                error: "Agent run failed",
              },
            });
          }
          await fallbackTrajectoryRecorder?.flush();
          throw new Error(
            `Exceeded maximum live model switch retries (${MAX_LIVE_SWITCH_RETRIES})`,
            {
              cause: err,
            },
          );
        }
        const switchRef = normalizeAgentCommandModelRef(
          cfg,
          err.provider,
          err.model,
          modelManifestContext,
        );
        if (!visibilityPolicy.allowsKey(modelKey(switchRef.provider, switchRef.model))) {
          log.info(
            `Live session model switch in subagent run ${runId}: ` +
              `rejected ${sanitizeForLog(err.provider)}/${sanitizeForLog(err.model)} (not in allowlist)`,
          );
          if (!attemptLifecycleState.lifecycleEnded) {
            emitAgentEvent({
              runId,
              lifecycleGeneration,
              stream: "lifecycle",
              data: {
                phase: "error",
                startedAt,
                endedAt: Date.now(),
                error: "Agent run failed",
              },
            });
          }
          await fallbackTrajectoryRecorder?.flush();
          throw new Error(
            `Live model switch rejected: ${sanitizeForLog(err.provider)}/${sanitizeForLog(err.model)} is not in the agent allowlist`,
            { cause: err },
          );
        }
        const previousProvider = provider;
        const previousModel = model;
        if (autoFallbackPrimaryProbe) {
          autoFallbackPrimaryProbeInterruptedByLiveSwitch = true;
        }
        provider = err.provider;
        model = err.model;
        fallbackProvider = err.provider;
        fallbackModel = err.model;
        providerForAuthProfileValidation = err.provider;
        if (sessionEntry) {
          sessionEntry = { ...sessionEntry };
          if (err.agentRuntimeOverride) {
            sessionEntry.agentRuntimeOverride = err.agentRuntimeOverride;
          } else {
            delete sessionEntry.agentRuntimeOverride;
          }
          sessionEntry.authProfileOverride = err.authProfileId;
          sessionEntry.authProfileOverrideSource = err.authProfileId
            ? err.authProfileIdSource
            : undefined;
          sessionEntry.authProfileOverrideCompactionCount = undefined;
          sessionEntryForAttempt = sessionEntry;
        }
        if (
          storedModelOverride ||
          err.model !== previousModel ||
          err.provider !== previousProvider
        ) {
          storedModelOverride = err.model;
          storedModelOverrideSource = "user";
        }
        attemptLifecycleState.lifecycleEnded = false;
        log.info(
          `Live session model switch in subagent run ${runId}: switching to ${sanitizeForLog(err.provider)}/${sanitizeForLog(err.model)}`,
        );
        continue;
      }
      if (!attemptLifecycleState.lifecycleEnded) {
        const errorLifecycleFields = isAgentRunDirectAbortReason(err)
          ? { aborted: true as const, stopReason: "aborted" as const }
          : resolveAgentRunErrorLifecycleFields(err, params.opts.abortSignal);
        emitAgentEvent({
          runId,
          lifecycleGeneration,
          stream: "lifecycle",
          data: {
            phase: "error",
            startedAt,
            endedAt: Date.now(),
            error: err instanceof Error ? err.message : "Agent run failed",
            ...errorLifecycleFields,
          },
        });
      }
      await fallbackTrajectoryRecorder?.flush();
      throw err;
    }
  }

  return {
    result,
    fallbackProvider,
    fallbackModel,
    fallbackExhausted,
    provider,
    model,
    sessionEntry,
    lifecycleGeneration,
    effectiveTurnThinkLevel,
    internalSessionTarget,
    attemptExecutionRuntime,
    messageChannel,
    suppressUserTurnPersistence,
    userTurnTranscriptRecorder,
    fallbackTrajectoryRecorder,
    lifecycle,
    terminal,
  };
}

export type EmbeddedAgentAttempt = Awaited<ReturnType<typeof runEmbeddedAgentAttempt>>;