fix: cron stream stalls fail over before job timeout (#96096)

* fix(agents): cap cron stream idle stalls

* fix(agents): preserve cron hostname timeout

* fix: bound cron idle timeout local exceptions

* fix: bound cron idle timeout local exceptions

---------

Co-authored-by: Radek Sienkiewicz <mail@velvetshark.com>
(cherry picked from commit 7fefc5ff58)

docs/concepts/agent-loop.md

@@ -167,7 +167,7 @@ surfaces, while Codex native hooks remain a separate lower-level Codex mechanism
 - Agent runtime: `agents.defaults.timeoutSeconds` default 172800s (48 hours); enforced in `runEmbeddedAgent` abort timer.
 - Cron runtime: isolated agent-turn `timeoutSeconds` is owned by cron. The scheduler starts that timer when execution begins, aborts the underlying run at the configured deadline, then runs bounded cleanup before recording the timeout so a stale child session cannot keep the lane stuck.
 - Session liveness diagnostics: with diagnostics enabled, `diagnostics.stuckSessionWarnMs` classifies long `processing` sessions that have no observed reply, tool, status, block, or ACP progress. Active embedded runs, model calls, and tool calls report as `session.long_running`; owned silent model calls also stay `session.long_running` until `diagnostics.stuckSessionAbortMs` so slow or non-streaming providers are not reported as stalled too early. Active work with no recent progress reports as `session.stalled`; owned model calls switch to `session.stalled` at or after the abort threshold, and ownerless stale model/tool activity is not hidden as long-running. `session.stuck` is reserved for recoverable stale session bookkeeping, including idle queued sessions with stale ownerless model/tool activity. Stale session bookkeeping releases the affected session lane immediately after recovery gates pass; stalled embedded runs are abort-drained only after `diagnostics.stuckSessionAbortMs` (default: at least 5 minutes and 3x the warning threshold) so queued work can resume without cutting off merely slow runs. Recovery emits structured requested/completed outcomes, and diagnostic state is marked idle only if the same processing generation is still current. Repeated `session.stuck` diagnostics back off while the session remains unchanged.
-- Model idle timeout: OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers.<id>.timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but it is still bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout because those control the whole agent run. Otherwise OpenClaw uses `agents.defaults.timeoutSeconds` when configured, capped at 120s by default. Cron-triggered cloud model runs with no explicit model or agent timeout use the same default idle watchdog; cron-triggered local or self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured, so slow local providers should set `models.providers.<id>.timeoutSeconds`.
+- Model idle timeout: OpenClaw aborts a model request when no response chunks arrive before the idle window. `models.providers.<id>.timeoutSeconds` extends this idle watchdog for slow local/self-hosted providers, but it is still bounded by any lower `agents.defaults.timeoutSeconds` or run-specific timeout because those control the whole agent run. Otherwise OpenClaw uses `agents.defaults.timeoutSeconds` when configured, capped at 120s by default. Cron-triggered cloud model runs with no explicit model or agent timeout use the same default idle watchdog; with an explicit cron run timeout, cloud model stream stalls are capped at 60s so configured model fallbacks can run before the outer cron deadline. Cron-triggered local or self-hosted model runs disable the implicit watchdog unless an explicit timeout is configured, and explicit cron run timeouts remain the idle window for local/self-hosted providers, so slow local providers should set `models.providers.<id>.timeoutSeconds`.
 - Provider HTTP request timeout: `models.providers.<id>.timeoutSeconds` applies to that provider's model HTTP fetches, including connect, headers, body, SDK request timeout, total guarded-fetch abort handling, and model stream idle watchdog. Use this for slow local/self-hosted providers such as Ollama before raising the whole agent runtime timeout, and keep the agent/runtime timeout at least as high when the model request needs to run longer.
 
 ## Where things can end early

src/agents/embedded-agent-runner/run/attempt.ts

@@ -3129,7 +3129,11 @@ export async function runEmbeddedAttempt(
         trigger: params.trigger,
         runTimeoutMs: resolvedRunTimeoutMs,
         modelRequestTimeoutMs: (params.model as { requestTimeoutMs?: number }).requestTimeoutMs,
-        model: params.model as { baseUrl?: string },
+        model: {
+          baseUrl: params.model.baseUrl,
+          id: params.modelId,
+          provider: params.provider,
+        },
       });
       if (idleTimeoutMs > 0) {
         activeSession.agent.streamFn = streamWithIdleTimeout(

src/agents/embedded-agent-runner/run/llm-idle-timeout.test.ts

@@ -12,6 +12,7 @@ import type { StreamFn } from "../../runtime/index.js";
 import { resolveLlmIdleTimeoutMs, streamWithIdleTimeout } from "./llm-idle-timeout.js";
 
 const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000;
+const CRON_LLM_IDLE_TIMEOUT_MS = 60_000;
 
 describe("resolveLlmIdleTimeoutMs", () => {
   it("returns default when config is undefined", () => {
@@ -41,8 +42,153 @@ describe("resolveLlmIdleTimeoutMs", () => {
     expect(resolveLlmIdleTimeoutMs({ runTimeoutMs: 30_000 })).toBe(30_000);
   });
 
-  it("honors explicit cron run timeouts as the idle watchdog ceiling", () => {
-    expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe(600_000);
+  it("caps explicit cron run timeouts so stream stalls can reach model fallbacks", () => {
+    expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 600_000 })).toBe(
+      CRON_LLM_IDLE_TIMEOUT_MS,
+    );
+  });
+
+  it("uses shorter explicit cron run timeouts as the idle watchdog ceiling", () => {
+    expect(resolveLlmIdleTimeoutMs({ trigger: "cron", runTimeoutMs: 30_000 })).toBe(30_000);
+  });
+
+  it("honors explicit cron run timeouts for local provider model calls", () => {
+    expect(
+      resolveLlmIdleTimeoutMs({
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { baseUrl: "http://127.0.0.1:11434" },
+      }),
+    ).toBe(600_000);
+  });
+
+  it.each([
+    ["ollama", "http://ollama-host:11434"],
+    ["ollama-beelink", "http://ollama-host:11434"],
+    ["lmstudio", "http://lmstudio-box:1234/v1"],
+    ["lmstudio-mac", "http://lmstudio-box:1234/v1"],
+    ["vllm", "http://vllm-rig:8000/v1"],
+    ["sglang", "http://sglang-rig:30000/v1"],
+  ])(
+    "honors explicit cron run timeouts for self-hosted provider %s hostname %s",
+    (provider, baseUrl) => {
+      expect(
+        resolveLlmIdleTimeoutMs({
+          trigger: "cron",
+          runTimeoutMs: 600_000,
+          model: { provider, baseUrl },
+        }),
+      ).toBe(600_000);
+    },
+  );
+
+  it("honors explicit cron run timeouts for explicit local host aliases", () => {
+    expect(
+      resolveLlmIdleTimeoutMs({
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { baseUrl: "http://host.docker.internal:11434" },
+      }),
+    ).toBe(600_000);
+  });
+
+  it("honors explicit cron run timeouts for custom local provider markers on bare hostnames", () => {
+    const cfg = {
+      models: {
+        providers: {
+          gpu: {
+            baseUrl: "http://gpu-box:8000/v1",
+            api: "openai-completions",
+            apiKey: "custom-local",
+            models: [],
+          },
+          "local-ollama": {
+            baseUrl: "http://ollama-box:11434",
+            api: "ollama",
+            apiKey: "ollama-local",
+            models: [],
+          },
+        },
+      },
+    } as unknown as OpenClawConfig;
+
+    expect(
+      resolveLlmIdleTimeoutMs({
+        cfg,
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { provider: "gpu", baseUrl: "http://gpu-box:8000/v1" },
+      }),
+    ).toBe(600_000);
+    expect(
+      resolveLlmIdleTimeoutMs({
+        cfg,
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { provider: "local-ollama", baseUrl: "http://ollama-box:11434" },
+      }),
+    ).toBe(600_000);
+  });
+
+  it("honors explicit cron run timeouts for provider-owned local services on bare hostnames", () => {
+    const cfg = {
+      models: {
+        providers: {
+          ds4: {
+            baseUrl: "http://ds4-box:8000/v1",
+            api: "openai-completions",
+            localService: {
+              command: "/opt/ds4/ds4-server",
+              healthUrl: "http://ds4-box:8000/v1/models",
+            },
+            models: [],
+          },
+        },
+      },
+    } as unknown as OpenClawConfig;
+
+    expect(
+      resolveLlmIdleTimeoutMs({
+        cfg,
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { provider: "ds4", baseUrl: "http://ds4-box:8000/v1" },
+      }),
+    ).toBe(600_000);
+  });
+
+  it.each([
+    ["openai", "openai/gpt-5.5", "http://api:8080/v1"],
+    ["custom-proxy", "custom-proxy/gpt-5.5", "http://gateway:4000/v1"],
+    ["ollama-cloud", "ollama-cloud/kimi-k2.6", "http://ollama-host:11434"],
+  ])(
+    "keeps the cron stall cap for cloud provider %s routed through single-label host %s",
+    (provider, id, baseUrl) => {
+      expect(
+        resolveLlmIdleTimeoutMs({
+          trigger: "cron",
+          runTimeoutMs: 600_000,
+          model: { provider, id, baseUrl },
+        }),
+      ).toBe(CRON_LLM_IDLE_TIMEOUT_MS);
+    },
+  );
+
+  it("keeps the cron stall cap for remote or cloud hostnames", () => {
+    expect(
+      resolveLlmIdleTimeoutMs({
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { provider: "openai", id: "openai/gpt-5.5", baseUrl: "https://api.openai.com/v1" },
+      }),
+    ).toBe(CRON_LLM_IDLE_TIMEOUT_MS);
+    expect(
+      resolveLlmIdleTimeoutMs({
+        trigger: "cron",
+        runTimeoutMs: 600_000,
+        model: { provider: "ollama", id: "ollama/gpt-oss:cloud", baseUrl: "http://ollama-host" },
+      }),
+    ).toBe(CRON_LLM_IDLE_TIMEOUT_MS);
   });
 
   it("disables the idle watchdog when an explicit run timeout disables timeouts", () => {

src/agents/embedded-agent-runner/run/llm-idle-timeout.ts

@@ -18,6 +18,16 @@ import type { EmbeddedRunTrigger } from "./params.js";
  * Default idle timeout for LLM streaming responses in milliseconds.
  */
 const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000;
+// Cron has its own outer watchdog; stream stalls must fail early enough for
+// the existing model fallback chain to try the next configured candidate.
+const CRON_LLM_IDLE_TIMEOUT_MS = 60_000;
+const LOCAL_PROVIDER_AUTH_MARKERS = new Set(["custom-local", "ollama-local"]);
+const SELF_HOSTED_PROVIDER_ID_PREFIXES = ["ollama", "lmstudio", "vllm", "sglang", "llama-cpp"];
+
+type IdleTimeoutProviderConfig = {
+  apiKey?: unknown;
+  localService?: unknown;
+};
 
 /**
  * Detects loopback / private-network / `.local` base URLs. Local providers
@@ -37,11 +47,9 @@ const DEFAULT_LLM_IDLE_TIMEOUT_MS = 120_000;
  *    matched, mirroring the SSRF-policy helper in
  *    `src/cron/isolated-agent/model-preflight.runtime.ts`.
  *  - DNS-resolved local aliases (e.g. an `/etc/hosts` entry mapping a custom
- *    hostname to a private IP) are not detected: classification keys on
- *    `URL.hostname` so resolution would have to happen here, and adding
- *    sync/async DNS to the watchdog hot path is disproportionate. Affected
- *    users can use the IP directly or set
- *    `models.providers.<id>.timeoutSeconds` explicitly.
+ *    hostname to a private IP) are not detected for the implicit watchdog opt-out:
+ *    classification keys on `URL.hostname` so resolution would have to happen
+ *    here, and adding sync/async DNS to the watchdog hot path is disproportionate.
  */
 function isLocalProviderBaseUrl(baseUrl: string): boolean {
   let host: string;
@@ -95,6 +103,82 @@ function isLocalProviderBaseUrl(baseUrl: string): boolean {
   );
 }
 
+function isExplicitLocalHostnameBaseUrl(baseUrl: string): boolean {
+  let host: string;
+  try {
+    host = new URL(baseUrl).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+
+  if (
+    host === "docker.orb.internal" ||
+    host === "host.docker.internal" ||
+    host === "host.orb.internal"
+  ) {
+    return true;
+  }
+  return false;
+}
+
+function isBareProviderHostnameBaseUrl(baseUrl: string): boolean {
+  let host: string;
+  try {
+    host = new URL(baseUrl).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+
+  if (host.includes(".") || host.includes(":")) {
+    return false;
+  }
+  return /^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/.test(host);
+}
+
+function isSelfHostedProviderId(provider: string | undefined): boolean {
+  const normalized = provider?.trim().toLowerCase();
+  if (!normalized || normalized === "ollama-cloud") {
+    return false;
+  }
+  return SELF_HOSTED_PROVIDER_ID_PREFIXES.some(
+    (prefix) => normalized === prefix || normalized.startsWith(`${prefix}-`),
+  );
+}
+
+function findConfiguredProviderConfig(
+  cfg: OpenClawConfig | undefined,
+  provider: string | undefined,
+): IdleTimeoutProviderConfig | undefined {
+  const normalizedProvider = provider?.trim().toLowerCase();
+  if (!normalizedProvider) {
+    return undefined;
+  }
+  const providers = cfg?.models?.providers as
+    | Record<string, IdleTimeoutProviderConfig | undefined>
+    | undefined;
+  const exact = providers?.[normalizedProvider];
+  if (exact) {
+    return exact;
+  }
+  return Object.entries(providers ?? {}).find(
+    ([key]) => key.trim().toLowerCase() === normalizedProvider,
+  )?.[1];
+}
+
+function hasLocalProviderAuthMarker(apiKey: unknown): boolean {
+  return typeof apiKey === "string" && LOCAL_PROVIDER_AUTH_MARKERS.has(apiKey.trim().toLowerCase());
+}
+
+function hasConfiguredLocalProviderSignal(params: {
+  cfg: OpenClawConfig | undefined;
+  provider: string | undefined;
+}): boolean {
+  const providerConfig = findConfiguredProviderConfig(params.cfg, params.provider);
+  return Boolean(
+    providerConfig?.localService || hasLocalProviderAuthMarker(providerConfig?.apiKey),
+  );
+}
+
 function isOllamaCloudModel(model: { id?: string; provider?: string } | undefined): boolean {
   const rawModelId = model?.id;
   if (typeof rawModelId !== "string") {
@@ -134,6 +218,22 @@ export function resolveLlmIdleTimeoutMs(params?: {
   const hasExplicitRunTimeout =
     typeof runTimeoutMs === "number" && Number.isFinite(runTimeoutMs) && runTimeoutMs > 0;
   const runTimeoutIsNoTimeout = hasExplicitRunTimeout && runTimeoutMs >= MAX_TIMER_TIMEOUT_MS;
+  const baseUrl = params?.model?.baseUrl;
+  const isLocalProvider =
+    typeof baseUrl === "string" && baseUrl.length > 0 && isLocalProviderBaseUrl(baseUrl);
+  const isLocalRuntimeModel = isLocalProvider && !isOllamaCloudModel(params?.model);
+  const isExplicitLocalHostnameRuntimeModel =
+    typeof baseUrl === "string" &&
+    baseUrl.length > 0 &&
+    isExplicitLocalHostnameBaseUrl(baseUrl) &&
+    !isOllamaCloudModel(params?.model);
+  const isSelfHostedHostnameRuntimeModel =
+    typeof baseUrl === "string" &&
+    baseUrl.length > 0 &&
+    isBareProviderHostnameBaseUrl(baseUrl) &&
+    (isSelfHostedProviderId(params?.model?.provider) ||
+      hasConfiguredLocalProviderSignal({ cfg: params?.cfg, provider: params?.model?.provider })) &&
+    !isOllamaCloudModel(params?.model);
   const timeoutBounds = [
     runTimeoutIsNoTimeout ? undefined : runTimeoutMs,
     hasExplicitRunTimeout ? undefined : agentTimeoutMs,
@@ -174,7 +274,14 @@ export function resolveLlmIdleTimeoutMs(params?: {
       return 0;
     }
     if (params?.trigger === "cron") {
-      return clampTimeoutMs(runTimeoutMs);
+      if (
+        isLocalRuntimeModel ||
+        isExplicitLocalHostnameRuntimeModel ||
+        isSelfHostedHostnameRuntimeModel
+      ) {
+        return clampTimeoutMs(runTimeoutMs);
+      }
+      return clampTimeoutMs(Math.min(runTimeoutMs, CRON_LLM_IDLE_TIMEOUT_MS));
     }
     return clampImplicitTimeoutMs(runTimeoutMs);
   }
@@ -190,10 +297,7 @@ export function resolveLlmIdleTimeoutMs(params?: {
   // baseUrl pointing at loopback / private-network / `.local`. Ollama cloud
   // models are still hosted remotely even when proxied through local Ollama, so
   // keep the cloud watchdog for `*:cloud` model ids.
-  const baseUrl = params?.model?.baseUrl;
-  const isLocalProvider =
-    typeof baseUrl === "string" && baseUrl.length > 0 && isLocalProviderBaseUrl(baseUrl);
-  if (isLocalProvider && !isOllamaCloudModel(params?.model)) {
+  if (isLocalRuntimeModel) {
     return 0;
   }