diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.test.ts b/src/agents/embedded-agent-runner/run/assistant-failure.test.ts new file mode 100644 index 000000000000..edfb4a80a784 --- /dev/null +++ b/src/agents/embedded-agent-runner/run/assistant-failure.test.ts @@ -0,0 +1,202 @@ +import { describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../../../config/types.openclaw.js"; +import { FailoverError } from "../../failover-error.js"; +import { runWithModelFallback } from "../../model-fallback-runner.js"; +import { + buildEmbeddedRunnerAssistant, + makeEmbeddedRunnerAttempt, +} from "../../test-helpers/embedded-agent-runner-e2e-fixtures.js"; +import { handleEmbeddedAssistantFailure } from "./assistant-failure.js"; +import { resolveEmbeddedRunAttemptTerminalState } from "./terminal-outcome.js"; + +const CREDENTIAL_FILE_ENOENT_MESSAGE = + "ENOENT: no such file or directory, open '/home/operator/.claude/.credentials.json'"; + +type AssistantFailureInput = Parameters[0]; + +function makeExhaustedCredentialFailureInput(options?: { replaySafe?: boolean }) { + const replaySafe = options?.replaySafe !== false; + const assistant = buildEmbeddedRunnerAssistant({ + provider: "anthropic", + model: "mock-1", + stopReason: "error", + errorMessage: CREDENTIAL_FILE_ENOENT_MESSAGE, + }); + const attempt = makeEmbeddedRunnerAttempt({ + lastAssistant: assistant, + currentAttemptAssistant: assistant, + toolMetas: replaySafe ? [] : [{ toolName: "write", replaySafe: false }], + }); + const advanceAttemptAuthProfile = vi.fn(async () => true); + const maybeMarkAuthProfileFailure = vi.fn(async () => {}); + const traceAttempts: AssistantFailureInput["traceAttempts"] = []; + const input: AssistantFailureInput = { + runParams: { + sessionId: "session:credential-enoent", + runId: "run:credential-enoent", + config: undefined, + } as AssistantFailureInput["runParams"], + attempt, + attemptAssistant: assistant, + currentAttemptAssistant: assistant, + terminalState: resolveEmbeddedRunAttemptTerminalState({ + attempt, + assistant, + }), + activeErrorContext: { provider: "anthropic", model: "mock-1" }, + provider: "anthropic", + modelId: "mock-1", + model: "mock-1", + thinkLevel: "off", + getThinkLevel: () => "off", + attemptedThinking: new Set(["off"]), + fallbackConfigured: true, + pluginHarnessOwnsTransport: false, + canRestartForLiveSwitch: false, + authProfileId: "anthropic:p1", + authProfileStore: { + version: 1, + profiles: { + "anthropic:p1": { + type: "api_key", + provider: "anthropic", + key: "test-key", + }, + "anthropic:p2": { + type: "api_key", + provider: "anthropic", + key: "test-key-2", + }, + }, + usageStats: { + "anthropic:p1": { lastUsed: 1 }, + "anthropic:p2": { lastUsed: 2 }, + }, + }, + runtimeAuthRetry: false, + maybeRefreshRuntimeAuthForAuthError: vi.fn(async () => false), + resolveAuthProfileFailureReason: () => null, + emptyErrorRetries: 3, + overloadProfileRotations: 0, + overloadProfileRotationLimit: 1, + rateLimitProfileRotations: 0, + rateLimitProfileRotationLimit: 1, + sameModelIdleTimeoutRetries: 0, + previousRetryFailoverReason: null, + maybeMarkAuthProfileFailure, + maybeEscalateRateLimitProfileFallback: vi.fn(), + maybeRetrySameModelRateLimit: vi.fn(async () => false), + maybeBackoffBeforeOverloadFailover: vi.fn(async () => {}), + advanceAttemptAuthProfile, + traceAttempts, + suspendForFailure: vi.fn(), + suspensionSessionId: "session:credential-enoent", + agentDir: "/tmp/openclaw-assistant-failure-test", + isProbeSession: false, + }; + return { + advanceAttemptAuthProfile, + input, + maybeMarkAuthProfileFailure, + traceAttempts, + }; +} + +describe("handleEmbeddedAssistantFailure", () => { + it("falls back after exhausted replay-safe credential-file retries without touching auth state", async () => { + const fixture = makeExhaustedCredentialFailureInput(); + + await expect(handleEmbeddedAssistantFailure(fixture.input)).rejects.toMatchObject({ + reason: "unknown", + provider: "anthropic", + model: "mock-1", + rawError: CREDENTIAL_FILE_ENOENT_MESSAGE, + }); + + expect(fixture.advanceAttemptAuthProfile).not.toHaveBeenCalled(); + expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled(); + expect(fixture.input.authProfileStore.usageStats).toEqual({ + "anthropic:p1": { lastUsed: 1 }, + "anthropic:p2": { lastUsed: 2 }, + }); + expect(fixture.traceAttempts).toEqual([ + { + provider: "anthropic", + model: "mock-1", + result: "fallback_model", + reason: "unknown", + stage: "assistant", + }, + ]); + }); + + it("does not fallback credential-file ENOENT after replay-unsafe tool activity", async () => { + const fixture = makeExhaustedCredentialFailureInput({ replaySafe: false }); + + const outcome = await handleEmbeddedAssistantFailure(fixture.input); + + expect(outcome.action).toBe("proceed"); + expect(fixture.advanceAttemptAuthProfile).not.toHaveBeenCalled(); + expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled(); + expect(fixture.traceAttempts).toEqual([]); + }); + + it("does not cache an exact credential-file failure from a fallback candidate", async () => { + const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS; + process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000"; + try { + const config = { + agents: { + defaults: { + model: { + primary: "openai/mock-0", + fallbacks: ["anthropic/mock-1", "groq/mock-2"], + }, + }, + }, + } satisfies OpenClawConfig; + const calls: string[] = []; + const run = async (provider: string, model: string) => { + calls.push(`${provider}/${model}`); + if (provider === "openai") { + throw new FailoverError("primary rate limited", { + provider, + model, + reason: "rate_limit", + }); + } + if (provider === "anthropic") { + await handleEmbeddedAssistantFailure(makeExhaustedCredentialFailureInput().input); + } + return "ok"; + }; + + for (let turn = 0; turn < 2; turn += 1) { + const result = await runWithModelFallback({ + cfg: config, + provider: "openai", + model: "mock-0", + sessionId: "session:credential-enoent-no-skip", + skipAuthProfileRuntime: true, + run, + }); + expect(result.result).toBe("ok"); + } + + expect(calls).toEqual([ + "openai/mock-0", + "anthropic/mock-1", + "groq/mock-2", + "openai/mock-0", + "anthropic/mock-1", + "groq/mock-2", + ]); + } finally { + if (previous === undefined) { + delete process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS; + } else { + process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = previous; + } + } + }); +}); diff --git a/src/agents/embedded-agent-runner/run/assistant-failure.ts b/src/agents/embedded-agent-runner/run/assistant-failure.ts index 1d8d3ff170fd..38ade3e55778 100644 --- a/src/agents/embedded-agent-runner/run/assistant-failure.ts +++ b/src/agents/embedded-agent-runner/run/assistant-failure.ts @@ -148,7 +148,7 @@ export async function handleEmbeddedAssistantFailure(input: { assistantFailoverReason === "unclassified" || assistantFailoverReason === "unknown" || assistantFailoverReason === "server_error"; - if ( + const replaySafeSilentErrorFailure = !authFailure && !rateLimitFailure && !billingFailure && @@ -160,9 +160,8 @@ export async function handleEmbeddedAssistantFailure(input: { shouldRetrySilentErrorAssistantTurn({ attempt: input.attempt, assistant: input.attemptAssistant, - }) && - input.emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES - ) { + }); + if (replaySafeSilentErrorFailure && input.emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES) { const emptyErrorRetries = input.emptyErrorRetries + 1; log.warn( `[empty-error-retry] stopReason=error non-visible-output; resubmitting ` + @@ -179,12 +178,24 @@ export async function handleEmbeddedAssistantFailure(input: { }); } + // The bounded same-model retry already proved this attempt had no visible output + // or replay-unsafe effects. Once those retries are exhausted, skip profile + // rotation and let the configured model fallback recover the invisible failure. + const exhaustedUnclassifiedSilentError = + input.fallbackConfigured && + assistantFailoverReason === null && + replaySafeSilentErrorFailure && + input.emptyErrorRetries >= MAX_EMPTY_ERROR_RETRIES; + const effectiveFailoverReason = exhaustedUnclassifiedSilentError + ? ("unknown" as const) + : assistantFailoverReason; + const failedProfileId = input.authProfileId; const logFailoverDecision = createFailoverDecisionLogger({ stage: "assistant", runId: input.runParams.runId, rawError: input.attemptAssistant?.errorMessage?.trim(), - failoverReason: assistantFailoverReason, + failoverReason: effectiveFailoverReason, profileFailureReason: assistantProfileFailureReason, provider: input.activeErrorContext.provider, model: input.activeErrorContext.model, @@ -229,17 +240,19 @@ export async function handleEmbeddedAssistantFailure(input: { ); } - const initialDecision = resolveRunFailoverDecision({ - stage: "assistant", - allowFormatRetry: cloudCodeAssistFormatError, - terminal: input.attempt.terminal, - signalOwnedInterruption, - fallbackConfigured: input.fallbackConfigured, - failoverFailure, - failoverReason: assistantFailoverReason, - harnessOwnsTransport: input.pluginHarnessOwnsTransport, - profileRotated: false, - }); + const initialDecision = exhaustedUnclassifiedSilentError + ? ({ action: "fallback_model", reason: "unknown" } as const) + : resolveRunFailoverDecision({ + stage: "assistant", + allowFormatRetry: cloudCodeAssistFormatError, + terminal: input.attempt.terminal, + signalOwnedInterruption, + fallbackConfigured: input.fallbackConfigured, + failoverFailure, + failoverReason: assistantFailoverReason, + harnessOwnsTransport: input.pluginHarnessOwnsTransport, + profileRotated: false, + }); const outcome = await handleAssistantFailover({ initialDecision, terminal: input.attempt.terminal, @@ -288,14 +301,14 @@ export async function handleEmbeddedAssistantFailure(input: { const retryTraceResult = outcome.retryKind === "same_model_rate_limit" ? "same_model_rate_limit" - : outcome.retryKind === "same_model_idle_timeout" || assistantFailoverReason === "timeout" + : outcome.retryKind === "same_model_idle_timeout" || effectiveFailoverReason === "timeout" ? "timeout" : "rotate_profile"; input.traceAttempts.push({ provider: input.activeErrorContext.provider, model: input.activeErrorContext.model, result: retryTraceResult, - ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), + ...(effectiveFailoverReason ? { reason: effectiveFailoverReason } : {}), stage: "assistant", }); return buildOutcome(input, { @@ -316,12 +329,12 @@ export async function handleEmbeddedAssistantFailure(input: { provider: input.activeErrorContext.provider, model: input.activeErrorContext.model, result: - assistantFailoverReason === "timeout" + effectiveFailoverReason === "timeout" ? "timeout" : initialDecision.action === "fallback_model" ? "fallback_model" : "error", - ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}), + ...(effectiveFailoverReason ? { reason: effectiveFailoverReason } : {}), stage: "assistant", ...(typeof outcome.error.status === "number" ? { status: outcome.error.status } : {}), });