fix(agents): fallback after replay-safe silent errors (#117356)

Co-authored-by: wenri <13506193475@163.com>

src/agents/embedded-agent-runner/run/assistant-failure.test.ts

@@ -0,0 +1,202 @@
+import { describe, expect, it, vi } from "vitest";
+import type { OpenClawConfig } from "../../../config/types.openclaw.js";
+import { FailoverError } from "../../failover-error.js";
+import { runWithModelFallback } from "../../model-fallback-runner.js";
+import {
+  buildEmbeddedRunnerAssistant,
+  makeEmbeddedRunnerAttempt,
+} from "../../test-helpers/embedded-agent-runner-e2e-fixtures.js";
+import { handleEmbeddedAssistantFailure } from "./assistant-failure.js";
+import { resolveEmbeddedRunAttemptTerminalState } from "./terminal-outcome.js";
+
+const CREDENTIAL_FILE_ENOENT_MESSAGE =
+  "ENOENT: no such file or directory, open '/home/operator/.claude/.credentials.json'";
+
+type AssistantFailureInput = Parameters<typeof handleEmbeddedAssistantFailure>[0];
+
+function makeExhaustedCredentialFailureInput(options?: { replaySafe?: boolean }) {
+  const replaySafe = options?.replaySafe !== false;
+  const assistant = buildEmbeddedRunnerAssistant({
+    provider: "anthropic",
+    model: "mock-1",
+    stopReason: "error",
+    errorMessage: CREDENTIAL_FILE_ENOENT_MESSAGE,
+  });
+  const attempt = makeEmbeddedRunnerAttempt({
+    lastAssistant: assistant,
+    currentAttemptAssistant: assistant,
+    toolMetas: replaySafe ? [] : [{ toolName: "write", replaySafe: false }],
+  });
+  const advanceAttemptAuthProfile = vi.fn(async () => true);
+  const maybeMarkAuthProfileFailure = vi.fn(async () => {});
+  const traceAttempts: AssistantFailureInput["traceAttempts"] = [];
+  const input: AssistantFailureInput = {
+    runParams: {
+      sessionId: "session:credential-enoent",
+      runId: "run:credential-enoent",
+      config: undefined,
+    } as AssistantFailureInput["runParams"],
+    attempt,
+    attemptAssistant: assistant,
+    currentAttemptAssistant: assistant,
+    terminalState: resolveEmbeddedRunAttemptTerminalState({
+      attempt,
+      assistant,
+    }),
+    activeErrorContext: { provider: "anthropic", model: "mock-1" },
+    provider: "anthropic",
+    modelId: "mock-1",
+    model: "mock-1",
+    thinkLevel: "off",
+    getThinkLevel: () => "off",
+    attemptedThinking: new Set(["off"]),
+    fallbackConfigured: true,
+    pluginHarnessOwnsTransport: false,
+    canRestartForLiveSwitch: false,
+    authProfileId: "anthropic:p1",
+    authProfileStore: {
+      version: 1,
+      profiles: {
+        "anthropic:p1": {
+          type: "api_key",
+          provider: "anthropic",
+          key: "test-key",
+        },
+        "anthropic:p2": {
+          type: "api_key",
+          provider: "anthropic",
+          key: "test-key-2",
+        },
+      },
+      usageStats: {
+        "anthropic:p1": { lastUsed: 1 },
+        "anthropic:p2": { lastUsed: 2 },
+      },
+    },
+    runtimeAuthRetry: false,
+    maybeRefreshRuntimeAuthForAuthError: vi.fn(async () => false),
+    resolveAuthProfileFailureReason: () => null,
+    emptyErrorRetries: 3,
+    overloadProfileRotations: 0,
+    overloadProfileRotationLimit: 1,
+    rateLimitProfileRotations: 0,
+    rateLimitProfileRotationLimit: 1,
+    sameModelIdleTimeoutRetries: 0,
+    previousRetryFailoverReason: null,
+    maybeMarkAuthProfileFailure,
+    maybeEscalateRateLimitProfileFallback: vi.fn(),
+    maybeRetrySameModelRateLimit: vi.fn(async () => false),
+    maybeBackoffBeforeOverloadFailover: vi.fn(async () => {}),
+    advanceAttemptAuthProfile,
+    traceAttempts,
+    suspendForFailure: vi.fn(),
+    suspensionSessionId: "session:credential-enoent",
+    agentDir: "/tmp/openclaw-assistant-failure-test",
+    isProbeSession: false,
+  };
+  return {
+    advanceAttemptAuthProfile,
+    input,
+    maybeMarkAuthProfileFailure,
+    traceAttempts,
+  };
+}
+
+describe("handleEmbeddedAssistantFailure", () => {
+  it("falls back after exhausted replay-safe credential-file retries without touching auth state", async () => {
+    const fixture = makeExhaustedCredentialFailureInput();
+
+    await expect(handleEmbeddedAssistantFailure(fixture.input)).rejects.toMatchObject({
+      reason: "unknown",
+      provider: "anthropic",
+      model: "mock-1",
+      rawError: CREDENTIAL_FILE_ENOENT_MESSAGE,
+    });
+
+    expect(fixture.advanceAttemptAuthProfile).not.toHaveBeenCalled();
+    expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled();
+    expect(fixture.input.authProfileStore.usageStats).toEqual({
+      "anthropic:p1": { lastUsed: 1 },
+      "anthropic:p2": { lastUsed: 2 },
+    });
+    expect(fixture.traceAttempts).toEqual([
+      {
+        provider: "anthropic",
+        model: "mock-1",
+        result: "fallback_model",
+        reason: "unknown",
+        stage: "assistant",
+      },
+    ]);
+  });
+
+  it("does not fallback credential-file ENOENT after replay-unsafe tool activity", async () => {
+    const fixture = makeExhaustedCredentialFailureInput({ replaySafe: false });
+
+    const outcome = await handleEmbeddedAssistantFailure(fixture.input);
+
+    expect(outcome.action).toBe("proceed");
+    expect(fixture.advanceAttemptAuthProfile).not.toHaveBeenCalled();
+    expect(fixture.maybeMarkAuthProfileFailure).not.toHaveBeenCalled();
+    expect(fixture.traceAttempts).toEqual([]);
+  });
+
+  it("does not cache an exact credential-file failure from a fallback candidate", async () => {
+    const previous = process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS;
+    process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = "60000";
+    try {
+      const config = {
+        agents: {
+          defaults: {
+            model: {
+              primary: "openai/mock-0",
+              fallbacks: ["anthropic/mock-1", "groq/mock-2"],
+            },
+          },
+        },
+      } satisfies OpenClawConfig;
+      const calls: string[] = [];
+      const run = async (provider: string, model: string) => {
+        calls.push(`${provider}/${model}`);
+        if (provider === "openai") {
+          throw new FailoverError("primary rate limited", {
+            provider,
+            model,
+            reason: "rate_limit",
+          });
+        }
+        if (provider === "anthropic") {
+          await handleEmbeddedAssistantFailure(makeExhaustedCredentialFailureInput().input);
+        }
+        return "ok";
+      };
+
+      for (let turn = 0; turn < 2; turn += 1) {
+        const result = await runWithModelFallback({
+          cfg: config,
+          provider: "openai",
+          model: "mock-0",
+          sessionId: "session:credential-enoent-no-skip",
+          skipAuthProfileRuntime: true,
+          run,
+        });
+        expect(result.result).toBe("ok");
+      }
+
+      expect(calls).toEqual([
+        "openai/mock-0",
+        "anthropic/mock-1",
+        "groq/mock-2",
+        "openai/mock-0",
+        "anthropic/mock-1",
+        "groq/mock-2",
+      ]);
+    } finally {
+      if (previous === undefined) {
+        delete process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS;
+      } else {
+        process.env.OPENCLAW_FALLBACK_SKIP_TTL_MS = previous;
+      }
+    }
+  });
+});

src/agents/embedded-agent-runner/run/assistant-failure.ts

@@ -148,7 +148,7 @@ export async function handleEmbeddedAssistantFailure(input: {
     assistantFailoverReason === "unclassified" ||
     assistantFailoverReason === "unknown" ||
     assistantFailoverReason === "server_error";
-  if (
+  const replaySafeSilentErrorFailure =
     !authFailure &&
     !rateLimitFailure &&
     !billingFailure &&
@@ -160,9 +160,8 @@ export async function handleEmbeddedAssistantFailure(input: {
     shouldRetrySilentErrorAssistantTurn({
       attempt: input.attempt,
       assistant: input.attemptAssistant,
-    }) &&
-    input.emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES
-  ) {
+    });
+  if (replaySafeSilentErrorFailure && input.emptyErrorRetries < MAX_EMPTY_ERROR_RETRIES) {
     const emptyErrorRetries = input.emptyErrorRetries + 1;
     log.warn(
       `[empty-error-retry] stopReason=error non-visible-output; resubmitting ` +
@@ -179,12 +178,24 @@ export async function handleEmbeddedAssistantFailure(input: {
     });
   }
 
+  // The bounded same-model retry already proved this attempt had no visible output
+  // or replay-unsafe effects. Once those retries are exhausted, skip profile
+  // rotation and let the configured model fallback recover the invisible failure.
+  const exhaustedUnclassifiedSilentError =
+    input.fallbackConfigured &&
+    assistantFailoverReason === null &&
+    replaySafeSilentErrorFailure &&
+    input.emptyErrorRetries >= MAX_EMPTY_ERROR_RETRIES;
+  const effectiveFailoverReason = exhaustedUnclassifiedSilentError
+    ? ("unknown" as const)
+    : assistantFailoverReason;
+
   const failedProfileId = input.authProfileId;
   const logFailoverDecision = createFailoverDecisionLogger({
     stage: "assistant",
     runId: input.runParams.runId,
     rawError: input.attemptAssistant?.errorMessage?.trim(),
-    failoverReason: assistantFailoverReason,
+    failoverReason: effectiveFailoverReason,
     profileFailureReason: assistantProfileFailureReason,
     provider: input.activeErrorContext.provider,
     model: input.activeErrorContext.model,
@@ -229,17 +240,19 @@ export async function handleEmbeddedAssistantFailure(input: {
     );
   }
 
-  const initialDecision = resolveRunFailoverDecision({
-    stage: "assistant",
-    allowFormatRetry: cloudCodeAssistFormatError,
-    terminal: input.attempt.terminal,
-    signalOwnedInterruption,
-    fallbackConfigured: input.fallbackConfigured,
-    failoverFailure,
-    failoverReason: assistantFailoverReason,
-    harnessOwnsTransport: input.pluginHarnessOwnsTransport,
-    profileRotated: false,
-  });
+  const initialDecision = exhaustedUnclassifiedSilentError
+    ? ({ action: "fallback_model", reason: "unknown" } as const)
+    : resolveRunFailoverDecision({
+        stage: "assistant",
+        allowFormatRetry: cloudCodeAssistFormatError,
+        terminal: input.attempt.terminal,
+        signalOwnedInterruption,
+        fallbackConfigured: input.fallbackConfigured,
+        failoverFailure,
+        failoverReason: assistantFailoverReason,
+        harnessOwnsTransport: input.pluginHarnessOwnsTransport,
+        profileRotated: false,
+      });
   const outcome = await handleAssistantFailover({
     initialDecision,
     terminal: input.attempt.terminal,
@@ -288,14 +301,14 @@ export async function handleEmbeddedAssistantFailure(input: {
     const retryTraceResult =
       outcome.retryKind === "same_model_rate_limit"
         ? "same_model_rate_limit"
-        : outcome.retryKind === "same_model_idle_timeout" || assistantFailoverReason === "timeout"
+        : outcome.retryKind === "same_model_idle_timeout" || effectiveFailoverReason === "timeout"
           ? "timeout"
           : "rotate_profile";
     input.traceAttempts.push({
       provider: input.activeErrorContext.provider,
       model: input.activeErrorContext.model,
       result: retryTraceResult,
-      ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}),
+      ...(effectiveFailoverReason ? { reason: effectiveFailoverReason } : {}),
       stage: "assistant",
     });
     return buildOutcome(input, {
@@ -316,12 +329,12 @@ export async function handleEmbeddedAssistantFailure(input: {
       provider: input.activeErrorContext.provider,
       model: input.activeErrorContext.model,
       result:
-        assistantFailoverReason === "timeout"
+        effectiveFailoverReason === "timeout"
           ? "timeout"
           : initialDecision.action === "fallback_model"
             ? "fallback_model"
             : "error",
-      ...(assistantFailoverReason ? { reason: assistantFailoverReason } : {}),
+      ...(effectiveFailoverReason ? { reason: effectiveFailoverReason } : {}),
       stage: "assistant",
       ...(typeof outcome.error.status === "number" ? { status: outcome.error.status } : {}),
     });