fix(sandbox): stop provisioning failures from exhausting fallbacks (#115481)

* fix(sandbox): stop fallback on provisioning failures

* fix(sandbox): preserve context SDK signature

* refactor(sandbox): keep provisioning class private
This commit is contained in:
Peter Steinberger
2026-07-28 22:06:52 -04:00
committed by GitHub
parent c092ec437c
commit dcffb9c9c2
6 changed files with 345 additions and 7 deletions
+6 -1
View File
@@ -31,6 +31,7 @@ import { modelKey } from "./model-ref-shared.js";
import { isCliRuntimeAlias } from "./model-runtime-aliases.js";
import { isCliProvider } from "./model-selection-cli.js";
import { isAgentRunDirectAbortReason, isAgentRunRestartAbortReason } from "./run-termination.js";
import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js";
import {
runWithDeferredSessionSuspension,
suspendSession,
@@ -241,7 +242,11 @@ async function runFallbackCandidate<T>(params: {
: await run();
return { ok: true, result };
} catch (err) {
if (isCommandLaneTaskTimeoutError(err) || isAgentHarnessPreflightError(err)) {
if (
isCommandLaneTaskTimeoutError(err) ||
isAgentHarnessPreflightError(err) ||
isSandboxProvisioningError(err)
) {
throw err;
}
const fallbackError = resolveModelFallbackError(err, {
+35
View File
@@ -41,6 +41,7 @@ import {
createAgentRunRestartAbortError,
resolveAgentRunErrorLifecycleFields,
} from "./run-termination.js";
import { toSandboxProvisioningError } from "./sandbox/provisioning-error.js";
import { resolveSessionSuspensionReason } from "./session-suspension.js";
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
@@ -1797,6 +1798,40 @@ describe("runWithModelFallback", () => {
expect(run).toHaveBeenCalledTimes(1);
});
it("does not spend model fallbacks on sandbox provisioning failures", async () => {
const cfg = makeCfg({
agents: {
defaults: {
model: {
primary: "openai/gpt-5.4",
fallbacks: ["anthropic/claude-sonnet-4-6", "openai/gpt-4.1-mini"],
},
},
},
});
const provisioningError = toSandboxProvisioningError(
new Error("Sandbox image not found: openclaw-sandbox:analyst. Build or pull it first."),
"docker",
);
const run = vi.fn().mockRejectedValue(provisioningError);
const onError = vi.fn();
const onFallbackStep = vi.fn();
await expect(
runWithModelFallback({
cfg,
provider: "openai",
model: "gpt-5.4",
run,
onError,
onFallbackStep,
}),
).rejects.toBe(provisioningError);
expect(run).toHaveBeenCalledTimes(1);
expect(onError).not.toHaveBeenCalled();
expect(onFallbackStep).not.toHaveBeenCalled();
});
it("aborts fallback when a provider prompt error carries cleanup session takeover", async () => {
const cfg = makeCfg({
agents: {
@@ -7,6 +7,7 @@ import type { OpenClawConfig } from "../config/config.js";
import type { SkillUsagePath } from "../skills/types.js";
import { registerSandboxBackend } from "./sandbox/backend.js";
import { ensureSandboxWorkspaceForSession, resolveSandboxContext } from "./sandbox/context.js";
import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js";
const updateRegistryMock = vi.hoisted(() => vi.fn());
const readRegisteredSandboxRuntimeIdsMock = vi.hoisted(() => vi.fn(async () => [] as string[]));
@@ -278,6 +279,202 @@ describe("resolveSandboxContext", () => {
}
}, 15_000);
it("types backend creation failures as sandbox provisioning errors", async () => {
const backendFailure = new Error("Sandbox image not found: missing:test");
const restore = registerSandboxBackend("broken-backend", async () => {
throw backendFailure;
});
try {
const cfg: OpenClawConfig = {
agents: {
defaults: {
sandbox: {
mode: "all",
backend: "broken-backend",
scope: "session",
workspaceAccess: "rw",
prune: { idleHours: 0, maxAgeDays: 0 },
},
},
},
};
const error = await resolveSandboxContext({
config: cfg,
sessionKey: "agent:worker:broken-sandbox",
workspaceDir: await createSandboxFixtureDir("broken-sandbox"),
}).catch((caught: unknown) => caught);
expect(isSandboxProvisioningError(error)).toBe(true);
expect(error).toMatchObject({
name: "SandboxProvisioningError",
code: "sandbox_provisioning",
backendId: "broken-backend",
message: "Sandbox image not found: missing:test",
cause: backendFailure,
});
} finally {
restore();
}
}, 15_000);
it("keeps sandbox registry failures inside the provisioning boundary", async () => {
const registryFailure = new Error("sandbox registry write failed");
updateRegistryMock.mockRejectedValueOnce(registryFailure);
const restore = registerSandboxBackend("registry-failure-backend", async () => ({
id: "registry-failure-backend",
runtimeId: "registry-failure-runtime",
runtimeLabel: "Registry Failure Runtime",
workdir: "/workspace",
buildExecSpec: async () => ({
argv: ["registry-failure-backend", "exec"],
env: process.env,
stdinMode: "pipe-closed" as const,
}),
runShellCommand: async () => ({
stdout: Buffer.alloc(0),
stderr: Buffer.alloc(0),
code: 0,
}),
}));
try {
const cfg: OpenClawConfig = {
agents: {
defaults: {
sandbox: {
mode: "all",
backend: "registry-failure-backend",
scope: "session",
workspaceAccess: "rw",
prune: { idleHours: 0, maxAgeDays: 0 },
},
},
},
};
const error = await resolveSandboxContext({
config: cfg,
sessionKey: "agent:worker:registry-failure",
workspaceDir: await createSandboxFixtureDir("registry-failure"),
}).catch((caught: unknown) => caught);
expect(isSandboxProvisioningError(error)).toBe(true);
expect(error).toMatchObject({
backendId: "registry-failure-backend",
message: "sandbox registry write failed",
cause: registryFailure,
});
} finally {
restore();
}
}, 15_000);
it("keeps sandbox browser startup failures inside the provisioning boundary", async () => {
const browserFailure = new Error("sandbox browser image missing");
ensureSandboxBrowserMock.mockRejectedValueOnce(browserFailure);
const restore = registerSandboxBackend("browser-failure-backend", async () => ({
id: "browser-failure-backend",
runtimeId: "browser-failure-runtime",
runtimeLabel: "Browser Failure Runtime",
workdir: "/workspace",
capabilities: { browser: true },
buildExecSpec: async () => ({
argv: ["browser-failure-backend", "exec"],
env: process.env,
stdinMode: "pipe-closed" as const,
}),
runShellCommand: async () => ({
stdout: Buffer.alloc(0),
stderr: Buffer.alloc(0),
code: 0,
}),
}));
try {
const cfg: OpenClawConfig = {
agents: {
defaults: {
sandbox: {
mode: "all",
backend: "browser-failure-backend",
scope: "session",
workspaceAccess: "rw",
prune: { idleHours: 0, maxAgeDays: 0 },
browser: { enabled: true },
},
},
},
};
const error = await resolveSandboxContext({
config: cfg,
sessionKey: "agent:worker:browser-failure",
workspaceDir: await createSandboxFixtureDir("browser-failure"),
}).catch((caught: unknown) => caught);
expect(isSandboxProvisioningError(error)).toBe(true);
expect(error).toMatchObject({
backendId: "browser-failure-backend",
message: "sandbox browser image missing",
cause: browserFailure,
});
} finally {
restore();
}
}, 15_000);
it("keeps filesystem bridge failures inside the provisioning boundary", async () => {
const bridgeFailure = new Error("sandbox filesystem bridge failed");
const restore = registerSandboxBackend("bridge-failure-backend", async () => ({
id: "bridge-failure-backend",
runtimeId: "bridge-failure-runtime",
runtimeLabel: "Bridge Failure Runtime",
workdir: "/workspace",
buildExecSpec: async () => ({
argv: ["bridge-failure-backend", "exec"],
env: process.env,
stdinMode: "pipe-closed" as const,
}),
runShellCommand: async () => ({
stdout: Buffer.alloc(0),
stderr: Buffer.alloc(0),
code: 0,
}),
createFsBridge: () => {
throw bridgeFailure;
},
}));
try {
const cfg: OpenClawConfig = {
agents: {
defaults: {
sandbox: {
mode: "all",
backend: "bridge-failure-backend",
scope: "session",
workspaceAccess: "rw",
prune: { idleHours: 0, maxAgeDays: 0 },
},
},
},
};
const error = await resolveSandboxContext({
config: cfg,
sessionKey: "agent:worker:bridge-failure",
workspaceDir: await createSandboxFixtureDir("bridge-failure"),
}).catch((caught: unknown) => caught);
expect(isSandboxProvisioningError(error)).toBe(true);
expect(error).toMatchObject({
backendId: "bridge-failure-backend",
message: "sandbox filesystem bridge failed",
cause: bridgeFailure,
});
} finally {
restore();
}
}, 15_000);
it("passes the resolved browser SSRF policy to sandbox browser setup", async () => {
ensureSandboxBrowserMock.mockClear();
const restore = registerSandboxBackend("test-browser-backend", async () => ({
+32 -6
View File
@@ -21,6 +21,7 @@ import { ensureSandboxBrowser } from "./browser.js";
import { resolveSandboxConfigForAgent } from "./config.js";
import { resolveSandboxDockerUser } from "./docker-user.js";
import { createSandboxFsBridge } from "./fs-bridge.js";
import { toSandboxProvisioningError } from "./provisioning-error.js";
import { readRegisteredSandboxRuntimeIds, updateRegistry } from "./registry.js";
import { resolveSandboxRuntimeStatus } from "./runtime-status.js";
import { assertSshSandboxSecretOwnerAvailable } from "./secret-owner.js";
@@ -186,18 +187,21 @@ function resolveSandboxWorkspaceInfoWorkdir(params: {
});
}
export async function resolveSandboxContext(params: {
type ResolveSandboxContextParams = {
config?: OpenClawConfig;
agentId?: string;
execOverrides?: ExecPolicyOverrides;
requireCurrentConfig?: boolean;
sessionKey?: string;
workspaceDir?: string;
}): Promise<SandboxContext | null> {
const resolved = resolveSandboxSession(params);
if (!resolved) {
return null;
}
};
type ResolvedSandboxSession = NonNullable<ReturnType<typeof resolveSandboxSession>>;
async function resolveProvisionedSandboxContext(
params: ResolveSandboxContextParams,
resolved: ResolvedSandboxSession,
): Promise<SandboxContext> {
const { rawSessionKey, cfg, runtime } = resolved;
if (cfg.prune.idleHours !== 0 || cfg.prune.maxAgeDays !== 0) {
@@ -324,6 +328,28 @@ export async function resolveSandboxContext(params: {
return sandboxContext;
}
export async function resolveSandboxContext(params: {
config?: OpenClawConfig;
agentId?: string;
execOverrides?: ExecPolicyOverrides;
requireCurrentConfig?: boolean;
sessionKey?: string;
workspaceDir?: string;
}): Promise<SandboxContext | null> {
const resolved = resolveSandboxSession(params);
if (!resolved) {
return null;
}
// Once a sandbox session is selected, every remaining step is local
// provisioning. Preserve that owner boundary across backend, browser,
// registry, and filesystem-bridge setup so model fallback never retries it.
try {
return await resolveProvisionedSandboxContext(params, resolved);
} catch (error) {
throw toSandboxProvisioningError(error, resolved.cfg.backend);
}
}
export async function ensureSandboxWorkspaceForSession(params: {
config?: OpenClawConfig;
sessionKey?: string;
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { isSandboxProvisioningError, toSandboxProvisioningError } from "./provisioning-error.js";
describe("sandbox provisioning errors", () => {
it("preserves an existing typed error", () => {
const error = toSandboxProvisioningError(new Error("missing image"), "docker");
expect(toSandboxProvisioningError(error, "other")).toBe(error);
});
it("recognizes provisioning failures through wrapper causes", () => {
const provisioningError = toSandboxProvisioningError(
new Error("backend unavailable"),
"docker",
);
const wrapped = new Error("agent setup failed", { cause: provisioningError });
expect(isSandboxProvisioningError(wrapped)).toBe(true);
expect(isSandboxProvisioningError(new Error("provider failed"))).toBe(false);
});
});
+54
View File
@@ -0,0 +1,54 @@
import { formatErrorMessage } from "../../infra/errors.js";
const SANDBOX_PROVISIONING_ERROR_CODE = "sandbox_provisioning";
/** Model-independent sandbox setup failure that must not consume model fallbacks. */
class SandboxProvisioningError extends Error {
readonly code = SANDBOX_PROVISIONING_ERROR_CODE;
readonly backendId: string;
constructor(message: string, params: { backendId: string; cause?: unknown }) {
super(message, { cause: params.cause });
this.name = "SandboxProvisioningError";
this.backendId = params.backendId;
}
}
/** Preserve an existing typed failure or attach sandbox ownership to a backend setup error. */
export function toSandboxProvisioningError(error: unknown, backendId: string) {
if (error instanceof SandboxProvisioningError) {
return error;
}
const message =
formatErrorMessage(error) || `Sandbox backend "${backendId}" provisioning failed.`;
return new SandboxProvisioningError(message, { backendId, cause: error });
}
/** Recognize the provisioning marker through ordinary error-wrapper cause chains. */
export function isSandboxProvisioningError(error: unknown, seen: Set<object> = new Set()): boolean {
if (error instanceof SandboxProvisioningError) {
return true;
}
if (!error || typeof error !== "object" || seen.has(error)) {
return false;
}
seen.add(error);
const candidate = error as {
name?: unknown;
code?: unknown;
cause?: unknown;
error?: unknown;
errors?: unknown;
};
if (
candidate.name === "SandboxProvisioningError" &&
candidate.code === SANDBOX_PROVISIONING_ERROR_CODE
) {
return true;
}
return [
candidate.cause,
candidate.error,
...(Array.isArray(candidate.errors) ? candidate.errors : []),
].some((nested) => isSandboxProvisioningError(nested, seen));
}