mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 04:15:48 -06:00
fix(sandbox): stop provisioning failures from exhausting fallbacks (#115481)
* fix(sandbox): stop fallback on provisioning failures * fix(sandbox): preserve context SDK signature * refactor(sandbox): keep provisioning class private
This commit is contained in:
committed by
GitHub
parent
c092ec437c
commit
dcffb9c9c2
@@ -31,6 +31,7 @@ import { modelKey } from "./model-ref-shared.js";
|
||||
import { isCliRuntimeAlias } from "./model-runtime-aliases.js";
|
||||
import { isCliProvider } from "./model-selection-cli.js";
|
||||
import { isAgentRunDirectAbortReason, isAgentRunRestartAbortReason } from "./run-termination.js";
|
||||
import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js";
|
||||
import {
|
||||
runWithDeferredSessionSuspension,
|
||||
suspendSession,
|
||||
@@ -241,7 +242,11 @@ async function runFallbackCandidate<T>(params: {
|
||||
: await run();
|
||||
return { ok: true, result };
|
||||
} catch (err) {
|
||||
if (isCommandLaneTaskTimeoutError(err) || isAgentHarnessPreflightError(err)) {
|
||||
if (
|
||||
isCommandLaneTaskTimeoutError(err) ||
|
||||
isAgentHarnessPreflightError(err) ||
|
||||
isSandboxProvisioningError(err)
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
const fallbackError = resolveModelFallbackError(err, {
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
createAgentRunRestartAbortError,
|
||||
resolveAgentRunErrorLifecycleFields,
|
||||
} from "./run-termination.js";
|
||||
import { toSandboxProvisioningError } from "./sandbox/provisioning-error.js";
|
||||
import { resolveSessionSuspensionReason } from "./session-suspension.js";
|
||||
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
|
||||
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
|
||||
@@ -1797,6 +1798,40 @@ describe("runWithModelFallback", () => {
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not spend model fallbacks on sandbox provisioning failures", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.4",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-6", "openai/gpt-4.1-mini"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const provisioningError = toSandboxProvisioningError(
|
||||
new Error("Sandbox image not found: openclaw-sandbox:analyst. Build or pull it first."),
|
||||
"docker",
|
||||
);
|
||||
const run = vi.fn().mockRejectedValue(provisioningError);
|
||||
const onError = vi.fn();
|
||||
const onFallbackStep = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
run,
|
||||
onError,
|
||||
onFallbackStep,
|
||||
}),
|
||||
).rejects.toBe(provisioningError);
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
expect(onError).not.toHaveBeenCalled();
|
||||
expect(onFallbackStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts fallback when a provider prompt error carries cleanup session takeover", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
|
||||
@@ -7,6 +7,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
||||
import type { SkillUsagePath } from "../skills/types.js";
|
||||
import { registerSandboxBackend } from "./sandbox/backend.js";
|
||||
import { ensureSandboxWorkspaceForSession, resolveSandboxContext } from "./sandbox/context.js";
|
||||
import { isSandboxProvisioningError } from "./sandbox/provisioning-error.js";
|
||||
|
||||
const updateRegistryMock = vi.hoisted(() => vi.fn());
|
||||
const readRegisteredSandboxRuntimeIdsMock = vi.hoisted(() => vi.fn(async () => [] as string[]));
|
||||
@@ -278,6 +279,202 @@ describe("resolveSandboxContext", () => {
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("types backend creation failures as sandbox provisioning errors", async () => {
|
||||
const backendFailure = new Error("Sandbox image not found: missing:test");
|
||||
const restore = registerSandboxBackend("broken-backend", async () => {
|
||||
throw backendFailure;
|
||||
});
|
||||
try {
|
||||
const cfg: OpenClawConfig = {
|
||||
agents: {
|
||||
defaults: {
|
||||
sandbox: {
|
||||
mode: "all",
|
||||
backend: "broken-backend",
|
||||
scope: "session",
|
||||
workspaceAccess: "rw",
|
||||
prune: { idleHours: 0, maxAgeDays: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const error = await resolveSandboxContext({
|
||||
config: cfg,
|
||||
sessionKey: "agent:worker:broken-sandbox",
|
||||
workspaceDir: await createSandboxFixtureDir("broken-sandbox"),
|
||||
}).catch((caught: unknown) => caught);
|
||||
|
||||
expect(isSandboxProvisioningError(error)).toBe(true);
|
||||
expect(error).toMatchObject({
|
||||
name: "SandboxProvisioningError",
|
||||
code: "sandbox_provisioning",
|
||||
backendId: "broken-backend",
|
||||
message: "Sandbox image not found: missing:test",
|
||||
cause: backendFailure,
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("keeps sandbox registry failures inside the provisioning boundary", async () => {
|
||||
const registryFailure = new Error("sandbox registry write failed");
|
||||
updateRegistryMock.mockRejectedValueOnce(registryFailure);
|
||||
const restore = registerSandboxBackend("registry-failure-backend", async () => ({
|
||||
id: "registry-failure-backend",
|
||||
runtimeId: "registry-failure-runtime",
|
||||
runtimeLabel: "Registry Failure Runtime",
|
||||
workdir: "/workspace",
|
||||
buildExecSpec: async () => ({
|
||||
argv: ["registry-failure-backend", "exec"],
|
||||
env: process.env,
|
||||
stdinMode: "pipe-closed" as const,
|
||||
}),
|
||||
runShellCommand: async () => ({
|
||||
stdout: Buffer.alloc(0),
|
||||
stderr: Buffer.alloc(0),
|
||||
code: 0,
|
||||
}),
|
||||
}));
|
||||
try {
|
||||
const cfg: OpenClawConfig = {
|
||||
agents: {
|
||||
defaults: {
|
||||
sandbox: {
|
||||
mode: "all",
|
||||
backend: "registry-failure-backend",
|
||||
scope: "session",
|
||||
workspaceAccess: "rw",
|
||||
prune: { idleHours: 0, maxAgeDays: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const error = await resolveSandboxContext({
|
||||
config: cfg,
|
||||
sessionKey: "agent:worker:registry-failure",
|
||||
workspaceDir: await createSandboxFixtureDir("registry-failure"),
|
||||
}).catch((caught: unknown) => caught);
|
||||
|
||||
expect(isSandboxProvisioningError(error)).toBe(true);
|
||||
expect(error).toMatchObject({
|
||||
backendId: "registry-failure-backend",
|
||||
message: "sandbox registry write failed",
|
||||
cause: registryFailure,
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("keeps sandbox browser startup failures inside the provisioning boundary", async () => {
|
||||
const browserFailure = new Error("sandbox browser image missing");
|
||||
ensureSandboxBrowserMock.mockRejectedValueOnce(browserFailure);
|
||||
const restore = registerSandboxBackend("browser-failure-backend", async () => ({
|
||||
id: "browser-failure-backend",
|
||||
runtimeId: "browser-failure-runtime",
|
||||
runtimeLabel: "Browser Failure Runtime",
|
||||
workdir: "/workspace",
|
||||
capabilities: { browser: true },
|
||||
buildExecSpec: async () => ({
|
||||
argv: ["browser-failure-backend", "exec"],
|
||||
env: process.env,
|
||||
stdinMode: "pipe-closed" as const,
|
||||
}),
|
||||
runShellCommand: async () => ({
|
||||
stdout: Buffer.alloc(0),
|
||||
stderr: Buffer.alloc(0),
|
||||
code: 0,
|
||||
}),
|
||||
}));
|
||||
try {
|
||||
const cfg: OpenClawConfig = {
|
||||
agents: {
|
||||
defaults: {
|
||||
sandbox: {
|
||||
mode: "all",
|
||||
backend: "browser-failure-backend",
|
||||
scope: "session",
|
||||
workspaceAccess: "rw",
|
||||
prune: { idleHours: 0, maxAgeDays: 0 },
|
||||
browser: { enabled: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const error = await resolveSandboxContext({
|
||||
config: cfg,
|
||||
sessionKey: "agent:worker:browser-failure",
|
||||
workspaceDir: await createSandboxFixtureDir("browser-failure"),
|
||||
}).catch((caught: unknown) => caught);
|
||||
|
||||
expect(isSandboxProvisioningError(error)).toBe(true);
|
||||
expect(error).toMatchObject({
|
||||
backendId: "browser-failure-backend",
|
||||
message: "sandbox browser image missing",
|
||||
cause: browserFailure,
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("keeps filesystem bridge failures inside the provisioning boundary", async () => {
|
||||
const bridgeFailure = new Error("sandbox filesystem bridge failed");
|
||||
const restore = registerSandboxBackend("bridge-failure-backend", async () => ({
|
||||
id: "bridge-failure-backend",
|
||||
runtimeId: "bridge-failure-runtime",
|
||||
runtimeLabel: "Bridge Failure Runtime",
|
||||
workdir: "/workspace",
|
||||
buildExecSpec: async () => ({
|
||||
argv: ["bridge-failure-backend", "exec"],
|
||||
env: process.env,
|
||||
stdinMode: "pipe-closed" as const,
|
||||
}),
|
||||
runShellCommand: async () => ({
|
||||
stdout: Buffer.alloc(0),
|
||||
stderr: Buffer.alloc(0),
|
||||
code: 0,
|
||||
}),
|
||||
createFsBridge: () => {
|
||||
throw bridgeFailure;
|
||||
},
|
||||
}));
|
||||
try {
|
||||
const cfg: OpenClawConfig = {
|
||||
agents: {
|
||||
defaults: {
|
||||
sandbox: {
|
||||
mode: "all",
|
||||
backend: "bridge-failure-backend",
|
||||
scope: "session",
|
||||
workspaceAccess: "rw",
|
||||
prune: { idleHours: 0, maxAgeDays: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const error = await resolveSandboxContext({
|
||||
config: cfg,
|
||||
sessionKey: "agent:worker:bridge-failure",
|
||||
workspaceDir: await createSandboxFixtureDir("bridge-failure"),
|
||||
}).catch((caught: unknown) => caught);
|
||||
|
||||
expect(isSandboxProvisioningError(error)).toBe(true);
|
||||
expect(error).toMatchObject({
|
||||
backendId: "bridge-failure-backend",
|
||||
message: "sandbox filesystem bridge failed",
|
||||
cause: bridgeFailure,
|
||||
});
|
||||
} finally {
|
||||
restore();
|
||||
}
|
||||
}, 15_000);
|
||||
|
||||
it("passes the resolved browser SSRF policy to sandbox browser setup", async () => {
|
||||
ensureSandboxBrowserMock.mockClear();
|
||||
const restore = registerSandboxBackend("test-browser-backend", async () => ({
|
||||
|
||||
@@ -21,6 +21,7 @@ import { ensureSandboxBrowser } from "./browser.js";
|
||||
import { resolveSandboxConfigForAgent } from "./config.js";
|
||||
import { resolveSandboxDockerUser } from "./docker-user.js";
|
||||
import { createSandboxFsBridge } from "./fs-bridge.js";
|
||||
import { toSandboxProvisioningError } from "./provisioning-error.js";
|
||||
import { readRegisteredSandboxRuntimeIds, updateRegistry } from "./registry.js";
|
||||
import { resolveSandboxRuntimeStatus } from "./runtime-status.js";
|
||||
import { assertSshSandboxSecretOwnerAvailable } from "./secret-owner.js";
|
||||
@@ -186,18 +187,21 @@ function resolveSandboxWorkspaceInfoWorkdir(params: {
|
||||
});
|
||||
}
|
||||
|
||||
export async function resolveSandboxContext(params: {
|
||||
type ResolveSandboxContextParams = {
|
||||
config?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
execOverrides?: ExecPolicyOverrides;
|
||||
requireCurrentConfig?: boolean;
|
||||
sessionKey?: string;
|
||||
workspaceDir?: string;
|
||||
}): Promise<SandboxContext | null> {
|
||||
const resolved = resolveSandboxSession(params);
|
||||
if (!resolved) {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
type ResolvedSandboxSession = NonNullable<ReturnType<typeof resolveSandboxSession>>;
|
||||
|
||||
async function resolveProvisionedSandboxContext(
|
||||
params: ResolveSandboxContextParams,
|
||||
resolved: ResolvedSandboxSession,
|
||||
): Promise<SandboxContext> {
|
||||
const { rawSessionKey, cfg, runtime } = resolved;
|
||||
|
||||
if (cfg.prune.idleHours !== 0 || cfg.prune.maxAgeDays !== 0) {
|
||||
@@ -324,6 +328,28 @@ export async function resolveSandboxContext(params: {
|
||||
return sandboxContext;
|
||||
}
|
||||
|
||||
export async function resolveSandboxContext(params: {
|
||||
config?: OpenClawConfig;
|
||||
agentId?: string;
|
||||
execOverrides?: ExecPolicyOverrides;
|
||||
requireCurrentConfig?: boolean;
|
||||
sessionKey?: string;
|
||||
workspaceDir?: string;
|
||||
}): Promise<SandboxContext | null> {
|
||||
const resolved = resolveSandboxSession(params);
|
||||
if (!resolved) {
|
||||
return null;
|
||||
}
|
||||
// Once a sandbox session is selected, every remaining step is local
|
||||
// provisioning. Preserve that owner boundary across backend, browser,
|
||||
// registry, and filesystem-bridge setup so model fallback never retries it.
|
||||
try {
|
||||
return await resolveProvisionedSandboxContext(params, resolved);
|
||||
} catch (error) {
|
||||
throw toSandboxProvisioningError(error, resolved.cfg.backend);
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensureSandboxWorkspaceForSession(params: {
|
||||
config?: OpenClawConfig;
|
||||
sessionKey?: string;
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isSandboxProvisioningError, toSandboxProvisioningError } from "./provisioning-error.js";
|
||||
|
||||
describe("sandbox provisioning errors", () => {
|
||||
it("preserves an existing typed error", () => {
|
||||
const error = toSandboxProvisioningError(new Error("missing image"), "docker");
|
||||
|
||||
expect(toSandboxProvisioningError(error, "other")).toBe(error);
|
||||
});
|
||||
|
||||
it("recognizes provisioning failures through wrapper causes", () => {
|
||||
const provisioningError = toSandboxProvisioningError(
|
||||
new Error("backend unavailable"),
|
||||
"docker",
|
||||
);
|
||||
const wrapped = new Error("agent setup failed", { cause: provisioningError });
|
||||
|
||||
expect(isSandboxProvisioningError(wrapped)).toBe(true);
|
||||
expect(isSandboxProvisioningError(new Error("provider failed"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
|
||||
const SANDBOX_PROVISIONING_ERROR_CODE = "sandbox_provisioning";
|
||||
|
||||
/** Model-independent sandbox setup failure that must not consume model fallbacks. */
|
||||
class SandboxProvisioningError extends Error {
|
||||
readonly code = SANDBOX_PROVISIONING_ERROR_CODE;
|
||||
readonly backendId: string;
|
||||
|
||||
constructor(message: string, params: { backendId: string; cause?: unknown }) {
|
||||
super(message, { cause: params.cause });
|
||||
this.name = "SandboxProvisioningError";
|
||||
this.backendId = params.backendId;
|
||||
}
|
||||
}
|
||||
|
||||
/** Preserve an existing typed failure or attach sandbox ownership to a backend setup error. */
|
||||
export function toSandboxProvisioningError(error: unknown, backendId: string) {
|
||||
if (error instanceof SandboxProvisioningError) {
|
||||
return error;
|
||||
}
|
||||
const message =
|
||||
formatErrorMessage(error) || `Sandbox backend "${backendId}" provisioning failed.`;
|
||||
return new SandboxProvisioningError(message, { backendId, cause: error });
|
||||
}
|
||||
|
||||
/** Recognize the provisioning marker through ordinary error-wrapper cause chains. */
|
||||
export function isSandboxProvisioningError(error: unknown, seen: Set<object> = new Set()): boolean {
|
||||
if (error instanceof SandboxProvisioningError) {
|
||||
return true;
|
||||
}
|
||||
if (!error || typeof error !== "object" || seen.has(error)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(error);
|
||||
const candidate = error as {
|
||||
name?: unknown;
|
||||
code?: unknown;
|
||||
cause?: unknown;
|
||||
error?: unknown;
|
||||
errors?: unknown;
|
||||
};
|
||||
if (
|
||||
candidate.name === "SandboxProvisioningError" &&
|
||||
candidate.code === SANDBOX_PROVISIONING_ERROR_CODE
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return [
|
||||
candidate.cause,
|
||||
candidate.error,
|
||||
...(Array.isArray(candidate.errors) ? candidate.errors : []),
|
||||
].some((nested) => isSandboxProvisioningError(nested, seen));
|
||||
}
|
||||
Reference in New Issue
Block a user