fix(agents): recover genericized Anthropic thinking errors (#92916)

Recover invalid Anthropic thinking replays when provider details survive genericization in SDK, failover, cause-chain, or terminal stream error fields.

The recovery matcher now uses cycle-safe named error carriers, avoids scanning assistant content and tool arguments, and retains one retry per provider call. Focused regressions cover each carrier, cyclic causes, terminal errors, and false-positive payload text.

Addresses the recovery path in #92201. The separate root cause that creates or persists invalid signatures remains open for investigation.

Co-authored-by: wlzeng0668001202 <ceng.wenlong@xydigit.com>

src/agents/embedded-agent-runner/thinking.test.ts

@@ -491,6 +491,8 @@ describe("wrapAnthropicStreamWithRecovery", () => {
   const anthropicThinkingError = new Error(
     "thinking or redacted_thinking blocks in the latest assistant message cannot be modified",
   );
+  const genericizedProviderError =
+    "LLM request failed: provider rejected the request schema or tool payload.";
   const terminalThinkingSignatureError =
     "ValidationException: invalid signature on thinking block in message history";
 
@@ -757,6 +759,57 @@ describe("wrapAnthropicStreamWithRecovery", () => {
     expect(callCount).toBe(2);
   });
 
+  it.each([
+    {
+      name: "failover rawError",
+      createError: () =>
+        Object.assign(new Error(genericizedProviderError), {
+          rawError: terminalThinkingSignatureError,
+        }),
+    },
+    {
+      name: "Anthropic SDK error body",
+      createError: () =>
+        Object.assign(new Error(genericizedProviderError), {
+          error: { error: { message: terminalThinkingSignatureError } },
+        }),
+    },
+    {
+      name: "direct errorMessage",
+      createError: () =>
+        Object.assign(new Error(genericizedProviderError), {
+          errorMessage: terminalThinkingSignatureError,
+        }),
+    },
+    {
+      name: "cyclic cause graph",
+      createError: () => {
+        const root = new Error(genericizedProviderError) as Error & { cause?: unknown };
+        const nested = { cause: root, message: terminalThinkingSignatureError };
+        root.cause = nested;
+        return root;
+      },
+    },
+  ])(
+    "retries genericized request errors carrying provider detail in $name",
+    async ({ createError }) => {
+      const providerError = createError();
+      let callCount = 0;
+      const wrapped = wrapAnthropicStreamWithRecovery(
+        (() => {
+          callCount += 1;
+          return Promise.reject(providerError);
+        }) as Parameters<typeof wrapAnthropicStreamWithRecovery>[0],
+        { id: "test-session" },
+      );
+
+      await expect(wrapped({} as never, { messages: [] } as never, {} as never)).rejects.toBe(
+        providerError,
+      );
+      expect(callCount).toBe(2);
+    },
+  );
+
   it("retries pre-content terminal stream-error events with omitted-reasoning text", async () => {
     let callCount = 0;
     const contexts: Array<{ messages?: AgentMessage[] }> = [];
@@ -818,7 +871,11 @@ describe("wrapAnthropicStreamWithRecovery", () => {
 
   it("does not retry non-thinking terminal stream-error events", async () => {
     let callCount = 0;
-    const errorMessage = createTestStreamErrorMessage("rate limit exceeded");
+    const errorMessage = createTestAssistantMessage({
+      content: [{ type: "text", text: terminalThinkingSignatureError }],
+      stopReason: "error",
+      errorMessage: "rate limit exceeded",
+    });
     const wrapped = wrapAnthropicStreamWithRecovery(
       (() => {
         callCount += 1;

src/agents/embedded-agent-runner/thinking.ts

@@ -1,7 +1,7 @@
 /**
  * Sanitizes reasoning/thinking blocks for replay and recovery.
  */
-import { formatErrorMessage } from "../../infra/errors.js";
+import { collectErrorGraphCandidates, formatErrorMessage } from "../../infra/errors.js";
 import type { AssistantMessageEvent } from "../../llm/types.js";
 import { createAssistantMessageEventStream } from "../../llm/utils/event-stream.js";
 import type { AgentMessage, StreamFn } from "../runtime/index.js";
@@ -571,7 +571,24 @@ function shouldRecoverAnthropicThinkingError(
   error: unknown,
   sessionMeta: RecoverySessionMeta,
 ): boolean {
-  return shouldRecoverAnthropicThinkingErrorMessage(formatErrorMessage(error), sessionMeta);
+  // Provider detail survives genericization in different carriers across the
+  // Anthropic SDK, failover wrapping, and terminal stream messages.
+  const candidates = collectErrorGraphCandidates(error, (current) => [
+    current.cause,
+    current.error,
+    current.rawError,
+    current.errorMessage,
+    current.message,
+  ]);
+  for (const candidate of candidates) {
+    if (
+      typeof candidate === "string" &&
+      shouldRecoverAnthropicThinkingErrorMessage(candidate, sessionMeta)
+    ) {
+      return true;
+    }
+  }
+  return false;
 }
 
 function shouldRecoverAnthropicThinkingErrorMessage(
@@ -598,13 +615,6 @@ function isAssistantMessageErrorEvent(
   );
 }
 
-function getAssistantMessageErrorText(
-  event: Extract<AssistantMessageEvent, { type: "error" }>,
-): string {
-  const errorMessage = (event.error as { errorMessage?: unknown }).errorMessage;
-  return typeof errorMessage === "string" ? errorMessage : "";
-}
-
 async function notifyRecoveredAnthropicThinking(
   sessionMeta: RecoverySessionMeta,
   recovery: AnthropicThinkingRecovery,
@@ -682,12 +692,7 @@ async function pumpStreamWithRecovery(
     const resolved = stream instanceof Promise ? await stream : stream;
     for await (const chunk of resolved as AsyncIterable<unknown>) {
       if (isAssistantMessageErrorEvent(chunk)) {
-        if (
-          shouldRecoverAnthropicThinkingErrorMessage(
-            getAssistantMessageErrorText(chunk),
-            sessionMeta,
-          )
-        ) {
+        if (shouldRecoverAnthropicThinkingError(chunk.error, sessionMeta)) {
           if (yieldedOutput) {
             log.warn(
               `[session-recovery] Anthropic thinking error occurred after streaming began; skipping retry to avoid duplicate chunks: sessionId=${sessionMeta.id}`,