diff --git a/src/agents/model-fallback-attempt.ts b/src/agents/model-fallback-attempt.ts index 9fba742eb8ba..9ebf3c393aae 100644 --- a/src/agents/model-fallback-attempt.ts +++ b/src/agents/model-fallback-attempt.ts @@ -31,6 +31,7 @@ import { modelKey } from "./model-ref-shared.js"; import { isCliRuntimeAlias } from "./model-runtime-aliases.js"; import { isCliProvider } from "./model-selection-cli.js"; import { isAgentRunDirectAbortReason, isAgentRunRestartAbortReason } from "./run-termination.js"; +import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js"; import { runWithDeferredSessionSuspension, suspendSession, @@ -241,7 +242,11 @@ async function runFallbackCandidate(params: { : await run(); return { ok: true, result }; } catch (err) { - if (isCommandLaneTaskTimeoutError(err) || isAgentHarnessPreflightError(err)) { + if ( + isCommandLaneTaskTimeoutError(err) || + isAgentHarnessPreflightError(err) || + isSandboxProvisioningError(err) + ) { throw err; } const fallbackError = resolveModelFallbackError(err, { diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index 675cf57c8213..ce8a117f0701 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -41,6 +41,7 @@ import { createAgentRunRestartAbortError, resolveAgentRunErrorLifecycleFields, } from "./run-termination.js"; +import { toSandboxProvisioningError } from "./sandbox/provisioning-error.js"; import { resolveSessionSuspensionReason } from "./session-suspension.js"; import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js"; import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js"; @@ -1797,6 +1798,40 @@ describe("runWithModelFallback", () => { expect(run).toHaveBeenCalledTimes(1); }); + it("does not spend model fallbacks on sandbox provisioning failures", async () => { + const cfg = makeCfg({ + agents: { + defaults: { + model: { + primary: "openai/gpt-5.4", + fallbacks: ["anthropic/claude-sonnet-4-6", "openai/gpt-4.1-mini"], + }, + }, + }, + }); + const provisioningError = toSandboxProvisioningError( + new Error("Sandbox image not found: openclaw-sandbox:analyst. Build or pull it first."), + "docker", + ); + const run = vi.fn().mockRejectedValue(provisioningError); + const onError = vi.fn(); + const onFallbackStep = vi.fn(); + + await expect( + runWithModelFallback({ + cfg, + provider: "openai", + model: "gpt-5.4", + run, + onError, + onFallbackStep, + }), + ).rejects.toBe(provisioningError); + expect(run).toHaveBeenCalledTimes(1); + expect(onError).not.toHaveBeenCalled(); + expect(onFallbackStep).not.toHaveBeenCalled(); + }); + it("aborts fallback when a provider prompt error carries cleanup session takeover", async () => { const cfg = makeCfg({ agents: { diff --git a/src/agents/sandbox.resolveSandboxContext.test.ts b/src/agents/sandbox.resolveSandboxContext.test.ts index 8447d22239eb..6a084b6aa528 100644 --- a/src/agents/sandbox.resolveSandboxContext.test.ts +++ b/src/agents/sandbox.resolveSandboxContext.test.ts @@ -7,6 +7,7 @@ import type { OpenClawConfig } from "../config/config.js"; import type { SkillUsagePath } from "../skills/types.js"; import { registerSandboxBackend } from "./sandbox/backend.js"; import { ensureSandboxWorkspaceForSession, resolveSandboxContext } from "./sandbox/context.js"; +import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js"; const updateRegistryMock = vi.hoisted(() => vi.fn()); const readRegisteredSandboxRuntimeIdsMock = vi.hoisted(() => vi.fn(async () => [] as string[])); @@ -278,6 +279,202 @@ describe("resolveSandboxContext", () => { } }, 15_000); + it("types backend creation failures as sandbox provisioning errors", async () => { + const backendFailure = new Error("Sandbox image not found: missing:test"); + const restore = registerSandboxBackend("broken-backend", async () => { + throw backendFailure; + }); + try { + const cfg: OpenClawConfig = { + agents: { + defaults: { + sandbox: { + mode: "all", + backend: "broken-backend", + scope: "session", + workspaceAccess: "rw", + prune: { idleHours: 0, maxAgeDays: 0 }, + }, + }, + }, + }; + + const error = await resolveSandboxContext({ + config: cfg, + sessionKey: "agent:worker:broken-sandbox", + workspaceDir: await createSandboxFixtureDir("broken-sandbox"), + }).catch((caught: unknown) => caught); + + expect(isSandboxProvisioningError(error)).toBe(true); + expect(error).toMatchObject({ + name: "SandboxProvisioningError", + code: "sandbox_provisioning", + backendId: "broken-backend", + message: "Sandbox image not found: missing:test", + cause: backendFailure, + }); + } finally { + restore(); + } + }, 15_000); + + it("keeps sandbox registry failures inside the provisioning boundary", async () => { + const registryFailure = new Error("sandbox registry write failed"); + updateRegistryMock.mockRejectedValueOnce(registryFailure); + const restore = registerSandboxBackend("registry-failure-backend", async () => ({ + id: "registry-failure-backend", + runtimeId: "registry-failure-runtime", + runtimeLabel: "Registry Failure Runtime", + workdir: "/workspace", + buildExecSpec: async () => ({ + argv: ["registry-failure-backend", "exec"], + env: process.env, + stdinMode: "pipe-closed" as const, + }), + runShellCommand: async () => ({ + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + code: 0, + }), + })); + try { + const cfg: OpenClawConfig = { + agents: { + defaults: { + sandbox: { + mode: "all", + backend: "registry-failure-backend", + scope: "session", + workspaceAccess: "rw", + prune: { idleHours: 0, maxAgeDays: 0 }, + }, + }, + }, + }; + + const error = await resolveSandboxContext({ + config: cfg, + sessionKey: "agent:worker:registry-failure", + workspaceDir: await createSandboxFixtureDir("registry-failure"), + }).catch((caught: unknown) => caught); + + expect(isSandboxProvisioningError(error)).toBe(true); + expect(error).toMatchObject({ + backendId: "registry-failure-backend", + message: "sandbox registry write failed", + cause: registryFailure, + }); + } finally { + restore(); + } + }, 15_000); + + it("keeps sandbox browser startup failures inside the provisioning boundary", async () => { + const browserFailure = new Error("sandbox browser image missing"); + ensureSandboxBrowserMock.mockRejectedValueOnce(browserFailure); + const restore = registerSandboxBackend("browser-failure-backend", async () => ({ + id: "browser-failure-backend", + runtimeId: "browser-failure-runtime", + runtimeLabel: "Browser Failure Runtime", + workdir: "/workspace", + capabilities: { browser: true }, + buildExecSpec: async () => ({ + argv: ["browser-failure-backend", "exec"], + env: process.env, + stdinMode: "pipe-closed" as const, + }), + runShellCommand: async () => ({ + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + code: 0, + }), + })); + try { + const cfg: OpenClawConfig = { + agents: { + defaults: { + sandbox: { + mode: "all", + backend: "browser-failure-backend", + scope: "session", + workspaceAccess: "rw", + prune: { idleHours: 0, maxAgeDays: 0 }, + browser: { enabled: true }, + }, + }, + }, + }; + + const error = await resolveSandboxContext({ + config: cfg, + sessionKey: "agent:worker:browser-failure", + workspaceDir: await createSandboxFixtureDir("browser-failure"), + }).catch((caught: unknown) => caught); + + expect(isSandboxProvisioningError(error)).toBe(true); + expect(error).toMatchObject({ + backendId: "browser-failure-backend", + message: "sandbox browser image missing", + cause: browserFailure, + }); + } finally { + restore(); + } + }, 15_000); + + it("keeps filesystem bridge failures inside the provisioning boundary", async () => { + const bridgeFailure = new Error("sandbox filesystem bridge failed"); + const restore = registerSandboxBackend("bridge-failure-backend", async () => ({ + id: "bridge-failure-backend", + runtimeId: "bridge-failure-runtime", + runtimeLabel: "Bridge Failure Runtime", + workdir: "/workspace", + buildExecSpec: async () => ({ + argv: ["bridge-failure-backend", "exec"], + env: process.env, + stdinMode: "pipe-closed" as const, + }), + runShellCommand: async () => ({ + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + code: 0, + }), + createFsBridge: () => { + throw bridgeFailure; + }, + })); + try { + const cfg: OpenClawConfig = { + agents: { + defaults: { + sandbox: { + mode: "all", + backend: "bridge-failure-backend", + scope: "session", + workspaceAccess: "rw", + prune: { idleHours: 0, maxAgeDays: 0 }, + }, + }, + }, + }; + + const error = await resolveSandboxContext({ + config: cfg, + sessionKey: "agent:worker:bridge-failure", + workspaceDir: await createSandboxFixtureDir("bridge-failure"), + }).catch((caught: unknown) => caught); + + expect(isSandboxProvisioningError(error)).toBe(true); + expect(error).toMatchObject({ + backendId: "bridge-failure-backend", + message: "sandbox filesystem bridge failed", + cause: bridgeFailure, + }); + } finally { + restore(); + } + }, 15_000); + it("passes the resolved browser SSRF policy to sandbox browser setup", async () => { ensureSandboxBrowserMock.mockClear(); const restore = registerSandboxBackend("test-browser-backend", async () => ({ diff --git a/src/agents/sandbox/context.ts b/src/agents/sandbox/context.ts index 158ffb4a7722..9c87be08f7df 100644 --- a/src/agents/sandbox/context.ts +++ b/src/agents/sandbox/context.ts @@ -21,6 +21,7 @@ import { ensureSandboxBrowser } from "./browser.js"; import { resolveSandboxConfigForAgent } from "./config.js"; import { resolveSandboxDockerUser } from "./docker-user.js"; import { createSandboxFsBridge } from "./fs-bridge.js"; +import { toSandboxProvisioningError } from "./provisioning-error.js"; import { readRegisteredSandboxRuntimeIds, updateRegistry } from "./registry.js"; import { resolveSandboxRuntimeStatus } from "./runtime-status.js"; import { assertSshSandboxSecretOwnerAvailable } from "./secret-owner.js"; @@ -186,18 +187,21 @@ function resolveSandboxWorkspaceInfoWorkdir(params: { }); } -export async function resolveSandboxContext(params: { +type ResolveSandboxContextParams = { config?: OpenClawConfig; agentId?: string; execOverrides?: ExecPolicyOverrides; requireCurrentConfig?: boolean; sessionKey?: string; workspaceDir?: string; -}): Promise { - const resolved = resolveSandboxSession(params); - if (!resolved) { - return null; - } +}; + +type ResolvedSandboxSession = NonNullable>; + +async function resolveProvisionedSandboxContext( + params: ResolveSandboxContextParams, + resolved: ResolvedSandboxSession, +): Promise { const { rawSessionKey, cfg, runtime } = resolved; if (cfg.prune.idleHours !== 0 || cfg.prune.maxAgeDays !== 0) { @@ -324,6 +328,28 @@ export async function resolveSandboxContext(params: { return sandboxContext; } +export async function resolveSandboxContext(params: { + config?: OpenClawConfig; + agentId?: string; + execOverrides?: ExecPolicyOverrides; + requireCurrentConfig?: boolean; + sessionKey?: string; + workspaceDir?: string; +}): Promise { + const resolved = resolveSandboxSession(params); + if (!resolved) { + return null; + } + // Once a sandbox session is selected, every remaining step is local + // provisioning. Preserve that owner boundary across backend, browser, + // registry, and filesystem-bridge setup so model fallback never retries it. + try { + return await resolveProvisionedSandboxContext(params, resolved); + } catch (error) { + throw toSandboxProvisioningError(error, resolved.cfg.backend); + } +} + export async function ensureSandboxWorkspaceForSession(params: { config?: OpenClawConfig; sessionKey?: string; diff --git a/src/agents/sandbox/provisioning-error.test.ts b/src/agents/sandbox/provisioning-error.test.ts new file mode 100644 index 000000000000..53e486176dcf --- /dev/null +++ b/src/agents/sandbox/provisioning-error.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { isSandboxProvisioningError, toSandboxProvisioningError } from "./provisioning-error.js"; + +describe("sandbox provisioning errors", () => { + it("preserves an existing typed error", () => { + const error = toSandboxProvisioningError(new Error("missing image"), "docker"); + + expect(toSandboxProvisioningError(error, "other")).toBe(error); + }); + + it("recognizes provisioning failures through wrapper causes", () => { + const provisioningError = toSandboxProvisioningError( + new Error("backend unavailable"), + "docker", + ); + const wrapped = new Error("agent setup failed", { cause: provisioningError }); + + expect(isSandboxProvisioningError(wrapped)).toBe(true); + expect(isSandboxProvisioningError(new Error("provider failed"))).toBe(false); + }); +}); diff --git a/src/agents/sandbox/provisioning-error.ts b/src/agents/sandbox/provisioning-error.ts new file mode 100644 index 000000000000..6cf994d608f9 --- /dev/null +++ b/src/agents/sandbox/provisioning-error.ts @@ -0,0 +1,54 @@ +import { formatErrorMessage } from "../../infra/errors.js"; + +const SANDBOX_PROVISIONING_ERROR_CODE = "sandbox_provisioning"; + +/** Model-independent sandbox setup failure that must not consume model fallbacks. */ +class SandboxProvisioningError extends Error { + readonly code = SANDBOX_PROVISIONING_ERROR_CODE; + readonly backendId: string; + + constructor(message: string, params: { backendId: string; cause?: unknown }) { + super(message, { cause: params.cause }); + this.name = "SandboxProvisioningError"; + this.backendId = params.backendId; + } +} + +/** Preserve an existing typed failure or attach sandbox ownership to a backend setup error. */ +export function toSandboxProvisioningError(error: unknown, backendId: string) { + if (error instanceof SandboxProvisioningError) { + return error; + } + const message = + formatErrorMessage(error) || `Sandbox backend "${backendId}" provisioning failed.`; + return new SandboxProvisioningError(message, { backendId, cause: error }); +} + +/** Recognize the provisioning marker through ordinary error-wrapper cause chains. */ +export function isSandboxProvisioningError(error: unknown, seen: Set = new Set()): boolean { + if (error instanceof SandboxProvisioningError) { + return true; + } + if (!error || typeof error !== "object" || seen.has(error)) { + return false; + } + seen.add(error); + const candidate = error as { + name?: unknown; + code?: unknown; + cause?: unknown; + error?: unknown; + errors?: unknown; + }; + if ( + candidate.name === "SandboxProvisioningError" && + candidate.code === SANDBOX_PROVISIONING_ERROR_CODE + ) { + return true; + } + return [ + candidate.cause, + candidate.error, + ...(Array.isArray(candidate.errors) ? candidate.errors : []), + ].some((nested) => isSandboxProvisioningError(nested, seen)); +}