fix(cli): guard remaining embedded state writers (#121282)

This commit is contained in:
Peter Steinberger
2026-08-09 17:09:35 -07:00
committed by GitHub
parent 6efd013c25
commit 08507909ed
19 changed files with 773 additions and 120 deletions
+1 -1
View File
@@ -28,7 +28,7 @@ By default, the command creates and later removes a temporary state directory, a
Config is layered in three parts, entirely in memory: exec composes the run config and publishes it as this process's runtime config rather than writing a copy to disk. Exec defaults apply only where your config leaves a setting unset: workspace bootstrap files are skipped, the agent sandbox is off, the `coding` tool profile is selected, filesystem tools are restricted to `--cwd`, and exec runs under the full execution policy a headless turn needs. Anything your config sets wins over those defaults, so a configured sandbox, shell env, or tool profile is never downgraded, and exec host routing stays with the sandbox when your config enables one. The invocation itself always wins last: the run is scoped to `--cwd` and never bootstraps.
Use `--state-dir <dir>` to retain sessions and other run state. The directory must already exist and is never created or deleted by the command.
Use `--state-dir <dir>` to retain sessions and other run state. The directory must already exist and is never created or deleted by the command. A retained state directory requires exclusive ownership: exec refuses to start while a Gateway or another embedded writer owns it, then holds the state lock for the complete run. Omit `--state-dir` for isolated temporary state, or stop the Gateway first with `openclaw gateway stop`.
When exec uses the ambient or a pinned config, installed plugins continue to resolve from the operator's ordinary plugin roots while sessions and other run state use the ephemeral directory. In those modes, `--state-dir` controls run state only; it is not required for configured providers, channels, or harnesses supplied by installed plugins.
+2
View File
@@ -56,6 +56,8 @@ Options:
Probe rows can come from auth profiles, env credentials, or `models.json`. Probe status buckets: `ok`, `auth`, `rate_limit`, `billing`, `timeout`, `format`, `unknown`, `no_model`.
Direct `models status --probe` runs create temporary internal sessions in the selected agent's canonical database, so the command requires exclusive ownership of the configured state directory. Stop a running Gateway with `openclaw gateway stop` before probing; the command removes its internal sessions and releases the state lock when it finishes or is interrupted.
Probe detail/reason codes to expect when a probe never reaches a model call:
- `excluded_by_auth_order`: a stored profile exists, but explicit `auth.order.<provider>` omitted it, so probe reports the exclusion instead of trying it.
+4
View File
@@ -77,6 +77,10 @@ Aliases: `openclaw chat` and `openclaw terminal` invoke this command with
`agent:<id>:...`).
- Local mode uses the embedded agent runtime directly. Most local tools work,
but Gateway-only features are unavailable.
- Local mode requires exclusive ownership of the configured state directory. It
refuses to start while a Gateway or another embedded writer owns that state;
run without `--local` to use the active Gateway, or stop it first with
`openclaw gateway stop`.
- Local mode adds `/auth [provider]` to the TUI command surface.
- Plugin approval gates still apply in local mode: tools that require approval
prompt for a decision in the terminal, nothing is silently auto-approved.
@@ -104,7 +104,7 @@ describe("supervised gateway lock recovery", () => {
it("preserves an agent-embedded owner error under a supervisor", async () => {
const err = new GatewayLockError(
"another openclaw agent --local run is active (pid 123); lock timeout after 5000ms",
"another embedded OpenClaw state writer is active (pid 123); lock timeout after 5000ms",
);
const startLoop = vi.fn(async () => {
throw err;
+151
View File
@@ -0,0 +1,151 @@
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { acquireGatewayLock, type GatewayLockOptions } from "../infra/gateway-lock.js";
import type { RuntimeEnv } from "../runtime.js";
import { agentExecCommand } from "./agent-exec.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
function createRuntime() {
const error = vi.fn();
const runtime: RuntimeEnv = { log: vi.fn(), error, exit: vi.fn() };
return { runtime, error };
}
function successResult() {
return {
payloads: [{ text: "done" }],
meta: {
durationMs: 1,
agentMeta: { sessionId: "session-result", provider: "openai", model: "gpt-5.6-sol" },
},
};
}
function createGatewayLockOptions(
stateDir: string,
overrides: Partial<GatewayLockOptions> = {},
): GatewayLockOptions {
return {
allowInTests: true,
env: {
...process.env,
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
OPENCLAW_STATE_DIR: stateDir,
},
lockDir: path.join(stateDir, "gateway-locks"),
timeoutMs: 100,
...overrides,
};
}
function createSignalProcess() {
type SignalName = "SIGINT" | "SIGTERM";
const listeners = new Map<SignalName, Set<() => void>>();
const processLike = {
on(signal: SignalName, handler: () => void) {
const current = listeners.get(signal) ?? new Set<() => void>();
current.add(handler);
listeners.set(signal, current);
return processLike;
},
off(signal: SignalName, handler: () => void) {
listeners.get(signal)?.delete(handler);
return processLike;
},
};
return {
processLike,
emit(signal: SignalName) {
for (const handler of listeners.get(signal) ?? []) {
handler();
}
},
};
}
describe("agent exec retained-state ownership", () => {
it("refuses a state directory owned by a live Gateway", async () => {
const stateDir = tempDirs.make("openclaw-agent-exec-gateway-owner-");
const lockOptions = createGatewayLockOptions(stateDir, {
readProcessStartTime: () => 123_456,
});
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
expect(gatewayLock).not.toBeNull();
if (!gatewayLock) {
throw new Error("Expected live Gateway fixture lock");
}
const runAgent = vi.fn(async () => successResult());
const { runtime, error } = createRuntime();
try {
const result = await agentExecCommand("inspect", { stateDir }, runtime, {
gatewayLockOptions: lockOptions,
runAgent,
});
expect(result.exitCode).toBe(1);
expect(runAgent).not.toHaveBeenCalled();
expect(error).toHaveBeenCalledWith(
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Omit --state-dir to use isolated temporary state, or stop the Gateway first (openclaw gateway stop).`,
);
} finally {
await gatewayLock.release();
}
});
it("holds and releases the embedded state lock around the run", async () => {
const stateDir = tempDirs.make("openclaw-agent-exec-lock-owner-");
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
await agentExecCommand("inspect", { stateDir }, createRuntime().runtime, {
gatewayLockOptions: lockOptions,
runAgent: vi.fn(async () => {
const payload = JSON.parse(await fs.readFile(stateLockPath, "utf8")) as {
pid?: number;
role?: string;
};
expect(payload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
return successResult();
}),
});
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
});
it("releases the embedded state lock when SIGTERM aborts the run", async () => {
const stateDir = tempDirs.make("openclaw-agent-exec-signal-owner-");
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
const signals = createSignalProcess();
const { runtime } = createRuntime();
const runAgent = vi.fn(async (opts: Record<string, unknown>) => {
const signal = opts.abortSignal as AbortSignal;
return await new Promise<ReturnType<typeof successResult>>((_, reject) => {
signal.addEventListener(
"abort",
() => {
const error = new Error("agent exec aborted");
error.name = "AbortError";
reject(error);
},
{ once: true },
);
});
});
const run = agentExecCommand("inspect", { stateDir }, runtime, {
gatewayLockOptions: lockOptions,
process: signals.processLike,
runAgent,
});
await vi.waitFor(() => expect(runAgent).toHaveBeenCalledOnce());
signals.emit("SIGTERM");
await run;
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
expect(runtime.exit).toHaveBeenCalledWith(143, { resetStream: process.stderr });
});
});
+15
View File
@@ -1001,6 +1001,21 @@ describe("agent exec run config layering", () => {
expect(config.agents?.entries?.ops?.model).toBe("openai/gpt-5.6-sol");
});
it("drops an inherited session store so the invocation state dir owns the agent database", () => {
const config = buildExecRunConfig({
base: {
session: {
store: "/persistent/agents/{agentId}/sessions/sessions.json",
mainKey: "primary",
},
},
cwd: "/run/here",
});
expect(config.session?.store).toBeUndefined();
expect(config.session?.mainKey).toBe("primary");
});
it("drops an inherited harness cwd so --cwd wins", () => {
const config = buildExecRunConfig({
base: {
+52 -11
View File
@@ -8,9 +8,15 @@ import { readByteStreamWithLimit } from "@openclaw/media-core/read-byte-stream-w
import { findAgentRunTerminalOutcome } from "../agents/agent-run-terminal-error.js";
import type { EmbeddedAgentRunMeta } from "../agents/embedded-agent.js";
import { isExecutionIdentityCollectionEnabled } from "../audit/audit-config.js";
import { formatCliCommand } from "../cli/command-format.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { mergeDeep } from "../infra/deep-merge.js";
import type {
EmbeddedStateLockHandle,
EmbeddedStateSignalProcess,
} from "../infra/embedded-state-lock.js";
import { formatErrorMessage } from "../infra/errors.js";
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
import { writeRuntimeJson, writeRuntimeStdout, type RuntimeEnv } from "../runtime.js";
@@ -79,6 +85,8 @@ type AgentExecCommandResult = {
type AgentExecCommandDeps = {
stdin?: AsyncIterable<unknown>;
process?: EmbeddedStateSignalProcess;
gatewayLockOptions?: GatewayLockOptions;
runAgent?: (
opts: Record<string, unknown>,
runtime: RuntimeEnv,
@@ -284,23 +292,25 @@ function normalizeCodeMode(
* outrank whatever the resolved config says.
*/
/**
* Drops inherited per-agent location overrides, which outrank the facts this
* invocation owns. `agentDir` beats the state dir for session and transcript
* storage, so an ephemeral run would write state into the operator's persistent
* agent directory where deleting the temp state dir cannot reach it; a native
* harness `runtime.acp.cwd` beats `--cwd`, so the turn could edit the wrong
* repository. `agents.bindings[].acp.cwd` needs no equivalent because exec runs
* no channel, so no binding matches.
* Drops inherited state and workspace location overrides, which outrank the
* facts this invocation owns. `session.store` and `agentDir` can redirect state
* outside the invocation root, where its lock or temporary cleanup cannot own
* it; a native harness `runtime.acp.cwd` can make the turn edit the wrong repo.
* `agents.bindings[].acp.cwd` needs no equivalent because exec runs no channel,
* so no binding matches.
*/
function stripInheritedAgentLocations(base: OpenClawConfig): OpenClawConfig {
const entries = base.agents?.entries;
const { session, ...root } = base;
const { store: _store, ...sessionWithoutStore } = session ?? {};
const withoutSessionStore = session ? { ...root, session: sessionWithoutStore } : base;
const entries = withoutSessionStore.agents?.entries;
if (!entries) {
return base;
return withoutSessionStore;
}
return {
...base,
...withoutSessionStore,
agents: {
...base.agents,
...withoutSessionStore.agents,
entries: Object.fromEntries(
Object.entries(entries).map(([id, entry]) => {
const { agentDir: _agentDir, runtime, ...rest } = entry;
@@ -477,6 +487,10 @@ function setAgentExecEnvironment(params: { stateDir: string; cwd: string }): ()
};
}
function formatActiveGatewayExecRefusal(identity: GatewayLockIdentity): string {
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Omit --state-dir to use isolated temporary state, or stop the Gateway first (${formatCliCommand("openclaw gateway stop")}).`;
}
function isStructuredTimeoutError(error: unknown): boolean {
if (findAgentRunTerminalOutcome(error)?.status === "timeout") {
return true;
@@ -552,6 +566,12 @@ export async function agentExecCommand(
let runtimePaths: typeof import("../config/paths.js") | undefined;
let configIo: typeof import("../config/io.js") | undefined;
let stopLocalAuditWriter: (() => Promise<void>) | undefined;
let stateLock: EmbeddedStateLockHandle | null | undefined;
let signalBridge:
| ReturnType<
(typeof import("../infra/embedded-state-lock.js"))["createEmbeddedStateSignalBridge"]
>
| undefined;
try {
const prompt = await resolveAgentExecPrompt(
positionalMessage,
@@ -618,6 +638,16 @@ export async function agentExecCommand(
restoreEnvironment = setAgentExecEnvironment({ stateDir, cwd });
runtimePaths = await import("../config/paths.js");
runtimePaths.pinRuntimePaths();
if (opts.stateDir) {
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
await import("../infra/embedded-state-lock.js");
signalBridge = createEmbeddedStateSignalBridge(deps.process ?? process);
stateLock = await acquireEmbeddedStateLock({
options: deps.gatewayLockOptions,
signal: signalBridge.signal,
formatActiveGatewayRefusal: formatActiveGatewayExecRefusal,
});
}
// The runtime snapshot is the only in-process config cache (`clearConfigCache`
// is a no-op shim), so publishing the composed config here is what makes the
// run use it. Serializing it to a temporary file and repointing
@@ -668,6 +698,7 @@ export async function agentExecCommand(
cleanupBundleMcpOnRunEnd: true,
cleanupCliLiveSessionOnRunEnd: true,
oneShotCliRun: true,
abortSignal: signalBridge?.signal,
onModelFallbackExhausted: () => {
fallbackExhausted = true;
},
@@ -707,6 +738,9 @@ export async function agentExecCommand(
let cleanupError: unknown;
await stopLocalAuditWriter?.().catch(() => undefined);
await stateLock?.release().catch((error: unknown) => {
cleanupError ??= error;
});
const runCleanupStep = (step: () => void) => {
try {
step();
@@ -738,6 +772,13 @@ export async function agentExecCommand(
commandResult = { envelope, exitCode: exitCodeForEnvelope(envelope) };
}
const receivedSignal = signalBridge?.getReceivedSignal();
signalBridge?.dispose();
if (receivedSignal) {
runtime.exit(receivedSignal === "SIGINT" ? 130 : 143, { resetStream: process.stderr });
return commandResult;
}
writeAgentExecOutput(runtime, commandResult.envelope, opts.json === true);
return commandResult;
}
+1 -1
View File
@@ -541,7 +541,7 @@ describe("agentCliCommand", () => {
localGatewayLockOptions: { ...lockOptions, pollIntervalMs: 2, timeoutMs: 15 },
}),
).rejects.toThrow(
`another openclaw agent --local run is active (pid ${process.pid}); lock timeout after 15ms`,
`another embedded OpenClaw state writer is active (pid ${process.pid}); lock timeout after 15ms`,
);
expect(agentCommand).toHaveBeenCalledTimes(1);
+20 -59
View File
@@ -30,6 +30,11 @@ import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
import { ADMIN_SCOPE } from "../gateway/operator-scopes.js";
import { createAbortError } from "../infra/abort-signal.js";
import { readFileDescriptorBounded } from "../infra/boundary-file-read.js";
import {
createEmbeddedStateSignalBridge,
type EmbeddedStateSignal,
type EmbeddedStateSignalProcess,
} from "../infra/embedded-state-lock.js";
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
import { routeLogsToStderr } from "../logging/console.js";
@@ -96,11 +101,9 @@ type AgentDispatchOpts = Omit<AgentCliOpts, "messageFile"> & {
message: string;
};
type AgentCliSignal = "SIGINT" | "SIGTERM";
type AgentCliProcessLike = {
type AgentCliSignal = EmbeddedStateSignal;
type AgentCliProcessLike = EmbeddedStateSignalProcess & {
exitCode?: NodeJS.Process["exitCode"];
on(signal: AgentCliSignal, handler: () => void): unknown;
off(signal: AgentCliSignal, handler: () => void): unknown;
};
type AgentCliDeps = CliDeps & {
process?: AgentCliProcessLike;
@@ -113,7 +116,6 @@ type AgentGatewayCallIdentity = Pick<
type AgentSessionModule = typeof import("./agent/session.runtime.js");
type AgentSessionModuleLoader = () => Promise<AgentSessionModule>;
const AGENT_CLI_SIGNALS: readonly AgentCliSignal[] = ["SIGINT", "SIGTERM"];
const GATEWAY_ABORT_RETRY_DELAYS_MS = [50, 150, 300, 600] as const;
const GATEWAY_ABORT_REQUEST_TIMEOUT_MS = 2_000;
const AGENT_CLI_SIGNAL_EXIT_CODES: Record<AgentCliSignal, number> = {
@@ -138,9 +140,10 @@ const agentSessionModuleCache = createLazyPromiseLoader(() => agentSessionModule
const runtimeConfigModuleLoader = createLazyPromiseLoader(() => import("../config/io.js"), {
cacheRejections: true,
});
const gatewayLockModuleLoader = createLazyPromiseLoader(() => import("../infra/gateway-lock.js"), {
cacheRejections: true,
});
const embeddedStateLockModuleLoader = createLazyPromiseLoader(
() => import("../infra/embedded-state-lock.js"),
{ cacheRejections: true },
);
const replyPayloadModuleLoader = createLazyPromiseLoader(
() => import("openclaw/plugin-sdk/reply-payload"),
{ cacheRejections: true },
@@ -226,32 +229,12 @@ async function acquireEmbeddedAgentStateLock(
options: GatewayLockOptions | undefined,
signal: AbortSignal,
) {
const { acquireGatewayLock, GatewayLockError, readActiveGatewayLockIdentity } =
await gatewayLockModuleLoader.load();
const env = options?.env ?? process.env;
if (options?.allowInTests !== true && (env.VITEST || env.NODE_ENV === "test")) {
return null;
}
const activeGateway = await readActiveGatewayLockIdentity(options);
if (activeGateway) {
throw new GatewayLockError(formatActiveGatewayLocalRefusal(activeGateway));
}
try {
return await acquireGatewayLock({
...options,
role: "agent-embedded",
sleep: options?.sleep ?? (async (ms) => await delayMs(ms, signal)),
});
} catch (error) {
if (!(error instanceof GatewayLockError)) {
throw error;
}
const racedGateway = await readActiveGatewayLockIdentity(options);
if (racedGateway) {
throw new GatewayLockError(formatActiveGatewayLocalRefusal(racedGateway), error);
}
throw error;
}
const { acquireEmbeddedStateLock } = await embeddedStateLockModuleLoader.load();
return await acquireEmbeddedStateLock({
options,
signal,
formatActiveGatewayRefusal: formatActiveGatewayLocalRefusal,
});
}
const loadReplyPayloadModule = replyPayloadModuleLoader.load;
@@ -263,7 +246,7 @@ export const agentViaGatewayTesting = {
localAuditModuleLoader.clear();
agentSessionModuleCache.clear();
runtimeConfigModuleLoader.clear();
gatewayLockModuleLoader.clear();
embeddedStateLockModuleLoader.clear();
replyPayloadModuleLoader.clear();
agentSessionModuleLoader = defaultAgentSessionModuleLoader;
},
@@ -530,34 +513,12 @@ function readAcceptedRunContext(payload: unknown): {
}
function createAgentCliSignalBridge(processLike: AgentCliProcessLike = process) {
const controller = new AbortController();
let receivedSignal: AgentCliSignal | undefined;
const handlers = new Map<AgentCliSignal, () => void>();
const detachHandlers = () => {
for (const [signal, handler] of handlers) {
processLike.off(signal, handler);
}
handlers.clear();
};
for (const signal of AGENT_CLI_SIGNALS) {
const handler = () => {
receivedSignal = signal;
if (!controller.signal.aborted) {
// runtime.exit may bypass finally cleanup, so first-signal self-detach is load-bearing.
controller.abort();
detachHandlers();
}
};
handlers.set(signal, handler);
processLike.on(signal, handler);
}
const bridge = createEmbeddedStateSignalBridge(processLike);
return {
signal: controller.signal,
getReceivedSignal: () => receivedSignal,
...bridge,
setExitCode: (code: number) => {
processLike.exitCode = code;
},
dispose: detachHandlers,
};
}
+128
View File
@@ -1,10 +1,63 @@
// Model list probe tests cover runtime probing while listing configured models.
import fsSync from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures";
import { beforeAll, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { acquireGatewayLock, type GatewayLockOptions } from "../../infra/gateway-lock.js";
let probeModule: typeof import("./list.probe.js");
function createGatewayLockOptions(stateDir: string): GatewayLockOptions {
return {
allowInTests: true,
env: {
...process.env,
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
OPENCLAW_STATE_DIR: stateDir,
},
lockDir: path.join(stateDir, "gateway-locks"),
readProcessStartTime: () => 123_456,
timeoutMs: 100,
};
}
function createSignalProcess() {
type SignalName = "SIGINT" | "SIGTERM";
const listeners = new Map<SignalName, Set<() => void>>();
const processLike = {
on(signal: SignalName, handler: () => void) {
const current = listeners.get(signal) ?? new Set<() => void>();
current.add(handler);
listeners.set(signal, current);
return processLike;
},
off(signal: SignalName, handler: () => void) {
listeners.get(signal)?.delete(handler);
return processLike;
},
};
return {
processLike,
emit(signal: SignalName) {
for (const handler of listeners.get(signal) ?? []) {
handler();
}
},
};
}
async function withTempState<T>(run: (stateDir: string) => Promise<T>): Promise<T> {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-model-probe-lock-"));
try {
return await run(stateDir);
} finally {
await fs.rm(stateDir, { recursive: true, force: true });
}
}
describe("mapFailoverReasonToProbeStatus", () => {
beforeAll(async () => {
vi.doMock("../../agents/embedded-agent.js", () => {
@@ -42,6 +95,81 @@ describe("mapFailoverReasonToProbeStatus", () => {
});
describe("runAuthProbes", () => {
beforeAll(async () => {
probeModule ??= await import("./list.probe.js");
});
it("refuses direct CLI probes while a live Gateway owns canonical state", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
expect(gatewayLock).not.toBeNull();
if (!gatewayLock) {
throw new Error("Expected live Gateway fixture lock");
}
try {
await expect(
probeModule.withAuthProbeStateOwnership(
{ mode: "exclusive", gatewayLockOptions: lockOptions },
async () => undefined,
),
).rejects.toThrow(
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Stop the Gateway first (openclaw gateway stop), then rerun models status --probe.`,
);
} finally {
await gatewayLock.release();
}
});
});
it("holds and releases canonical state ownership around direct CLI probes", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
let observedPayload: { pid?: number; role?: string } | undefined;
await probeModule.withAuthProbeStateOwnership(
{ mode: "exclusive", gatewayLockOptions: lockOptions },
async () => {
observedPayload = JSON.parse(fsSync.readFileSync(stateLockPath, "utf8")) as {
pid?: number;
role?: string;
};
},
);
expect(observedPayload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("releases canonical state ownership when a direct CLI probe receives SIGTERM", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
const signals = createSignalProcess();
await probeModule.withAuthProbeStateOwnership(
{
mode: "exclusive",
gatewayLockOptions: lockOptions,
process: signals.processLike,
},
async (signal) => {
let markInterrupted!: () => void;
const interrupted = new Promise<void>((resolve) => {
markInterrupted = resolve;
});
signal?.addEventListener("abort", markInterrupted, { once: true });
signals.emit("SIGTERM");
await interrupted;
},
);
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("runs Codex auth probes through raw OpenClaw model-run mode", async () => {
const runEmbeddedAgent = vi.fn(
async (_params: {
+91 -38
View File
@@ -39,6 +39,7 @@ import { findNormalizedProviderValue, normalizeProviderId } from "../../agents/m
import { loadPreparedModelCatalog } from "../../agents/prepared-model-catalog.js";
import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";
import { resolveDefaultAgentWorkspaceDir } from "../../agents/workspace.js";
import { formatCliCommand } from "../../cli/command-format.js";
import { resolveStorePath } from "../../config/sessions/paths.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import {
@@ -46,6 +47,11 @@ import {
hasConfiguredSecretInput,
normalizeSecretInputString,
} from "../../config/types.secrets.js";
import type {
EmbeddedStateLockHandle,
EmbeddedStateSignalProcess,
} from "../../infra/embedded-state-lock.js";
import type { GatewayLockIdentity, GatewayLockOptions } from "../../infra/gateway-lock.js";
import { type SecretRefResolveCache, resolveSecretRefString } from "../../secrets/resolve.js";
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
import { disposeOpenClawAgentDatabaseByPath } from "../../state/openclaw-agent-db.js";
@@ -723,6 +729,7 @@ async function probeTarget(params: {
target: AuthProbeTarget;
timeoutMs: number;
maxTokens: number;
abortSignal?: AbortSignal;
}): Promise<AuthProbeResult> {
const { cfg, agentId, agentDir, workspaceDir, storePath, target, timeoutMs, maxTokens } = params;
// Marker credentials must be resolved by the runtime from config, but the
@@ -834,6 +841,7 @@ async function probeTarget(params: {
disableTools: true,
modelRun: true,
cleanupBundleMcpOnRunEnd: true,
abortSignal: params.abortSignal,
});
return buildResult("ok");
} catch (err) {
@@ -862,6 +870,7 @@ async function runTargetsWithConcurrency(params: {
maxTokens: number;
concurrency: number;
onProgress?: (update: { completed: number; total: number; label?: string }) => void;
abortSignal?: AbortSignal;
}): Promise<AuthProbeResult[]> {
const { cfg, targets, timeoutMs, maxTokens, onProgress } = params;
const concurrency = Math.max(1, Math.min(targets.length || 1, params.concurrency));
@@ -894,15 +903,55 @@ async function runTargetsWithConcurrency(params: {
target,
timeoutMs,
maxTokens,
abortSignal: params.abortSignal,
});
completed += 1;
onProgress?.({ completed, total: targets.length });
return result;
},
{ concurrency, stopOnError: true },
{
concurrency,
stopOnError: true,
...(params.abortSignal ? { signal: params.abortSignal } : {}),
},
);
}
function formatActiveGatewayModelsProbeRefusal(identity: GatewayLockIdentity): string {
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Stop the Gateway first (${formatCliCommand("openclaw gateway stop")}), then rerun models status --probe.`;
}
type AuthProbeStateOwnership = {
mode: "exclusive";
gatewayLockOptions?: GatewayLockOptions;
process?: EmbeddedStateSignalProcess;
};
/** Own canonical state only for direct CLI probes; Gateway RPC probes already run under its lock. */
export async function withAuthProbeStateOwnership<T>(
ownership: AuthProbeStateOwnership | undefined,
run: (signal?: AbortSignal) => Promise<T>,
): Promise<T> {
if (!ownership) {
return await run();
}
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
await import("../../infra/embedded-state-lock.js");
const signalBridge = createEmbeddedStateSignalBridge(ownership.process ?? process);
let stateLock: EmbeddedStateLockHandle | null | undefined;
try {
stateLock = await acquireEmbeddedStateLock({
options: ownership.gatewayLockOptions,
signal: signalBridge.signal,
formatActiveGatewayRefusal: formatActiveGatewayModelsProbeRefusal,
});
return await run(signalBridge.signal);
} finally {
await stateLock?.release();
signalBridge.dispose();
}
}
/** Runs all auth probes with bounded concurrency and returns a summary. */
export async function runAuthProbes(params: {
cfg: OpenClawConfig;
@@ -913,45 +962,49 @@ export async function runAuthProbes(params: {
modelCandidates: string[];
options: AuthProbeOptions;
onProgress?: (update: { completed: number; total: number; label?: string }) => void;
stateOwnership?: AuthProbeStateOwnership;
}): Promise<AuthProbeSummary> {
const startedAt = Date.now();
const plan = await buildProbeTargets({
cfg: params.cfg,
...(params.agentId ? { agentId: params.agentId } : {}),
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
providers: params.providers,
modelCandidates: params.modelCandidates,
options: params.options,
return await withAuthProbeStateOwnership(params.stateOwnership, async (abortSignal) => {
const startedAt = Date.now();
const plan = await buildProbeTargets({
cfg: params.cfg,
...(params.agentId ? { agentId: params.agentId } : {}),
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
providers: params.providers,
modelCandidates: params.modelCandidates,
options: params.options,
});
const totalTargets = plan.targets.length;
params.onProgress?.({ completed: 0, total: totalTargets });
const results = totalTargets
? await runTargetsWithConcurrency({
cfg: params.cfg,
agentId: params.agentId,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
targets: plan.targets,
timeoutMs: params.options.timeoutMs,
maxTokens: params.options.maxTokens,
concurrency: params.options.concurrency,
onProgress: params.onProgress,
abortSignal,
})
: [];
const finishedAt = Date.now();
return {
startedAt,
finishedAt,
durationMs: finishedAt - startedAt,
totalTargets,
options: params.options,
results: [...plan.results, ...results],
};
});
const totalTargets = plan.targets.length;
params.onProgress?.({ completed: 0, total: totalTargets });
const results = totalTargets
? await runTargetsWithConcurrency({
cfg: params.cfg,
agentId: params.agentId,
agentDir: params.agentDir,
workspaceDir: params.workspaceDir,
targets: plan.targets,
timeoutMs: params.options.timeoutMs,
maxTokens: params.options.maxTokens,
concurrency: params.options.concurrency,
onProgress: params.onProgress,
})
: [];
const finishedAt = Date.now();
return {
startedAt,
finishedAt,
durationMs: finishedAt - startedAt,
totalTargets,
options: params.options,
results: [...plan.results, ...results],
};
}
/** Formats probe latency for table output. */
@@ -1223,6 +1223,9 @@ export async function modelsStatusCommand(
concurrency: probeConcurrency,
maxTokens: probeMaxTokens,
},
// Direct CLI probes create hidden sessions in the canonical agent DB.
// Gateway RPC probes omit this because the Gateway already owns the lock.
stateOwnership: { mode: "exclusive" },
onProgress: update,
});
},
+100
View File
@@ -0,0 +1,100 @@
// Coordinates direct embedded state writers with the Gateway state-directory owner.
import { createAbortError } from "./abort-signal.js";
import type { GatewayLockIdentity, GatewayLockOptions } from "./gateway-lock.js";
export type EmbeddedStateSignal = "SIGINT" | "SIGTERM";
export type EmbeddedStateSignalProcess = {
on(signal: EmbeddedStateSignal, handler: () => void): unknown;
off(signal: EmbeddedStateSignal, handler: () => void): unknown;
};
export type EmbeddedStateLockHandle = {
release: () => Promise<void>;
};
const EMBEDDED_STATE_SIGNALS: readonly EmbeddedStateSignal[] = ["SIGINT", "SIGTERM"];
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
return Promise.reject(createAbortError("embedded state lock acquisition aborted"));
}
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
reject(createAbortError("embedded state lock acquisition aborted"));
};
signal?.addEventListener("abort", onAbort, { once: true });
});
}
/** Bridges process signals into embedded-run cancellation so lock cleanup can unwind. */
export function createEmbeddedStateSignalBridge(processLike: EmbeddedStateSignalProcess = process) {
const controller = new AbortController();
let receivedSignal: EmbeddedStateSignal | undefined;
const handlers = new Map<EmbeddedStateSignal, () => void>();
const dispose = () => {
for (const [signal, handler] of handlers) {
processLike.off(signal, handler);
}
handlers.clear();
};
for (const signal of EMBEDDED_STATE_SIGNALS) {
const handler = () => {
receivedSignal = signal;
if (!controller.signal.aborted) {
controller.abort();
dispose();
}
};
handlers.set(signal, handler);
processLike.on(signal, handler);
}
return {
signal: controller.signal,
getReceivedSignal: () => receivedSignal,
dispose,
};
}
/** Probe the Gateway owner first, then acquire the shared embedded-writer role. */
export async function acquireEmbeddedStateLock(params: {
options?: GatewayLockOptions;
signal?: AbortSignal;
formatActiveGatewayRefusal: (identity: GatewayLockIdentity) => string;
}): Promise<EmbeddedStateLockHandle | null> {
const { acquireGatewayLock, GatewayLockError, readActiveGatewayLockIdentity } =
await import("./gateway-lock.js");
const env = params.options?.env ?? process.env;
if (
params.options?.allowInTests !== true &&
(env.VITEST !== undefined || env.NODE_ENV === "test")
) {
return null;
}
const activeGateway = await readActiveGatewayLockIdentity(params.options);
if (activeGateway) {
throw new GatewayLockError(params.formatActiveGatewayRefusal(activeGateway));
}
try {
return await acquireGatewayLock({
...params.options,
role: "agent-embedded",
sleep: params.options?.sleep ?? (async (ms) => await abortableDelay(ms, params.signal)),
});
} catch (error) {
if (!(error instanceof GatewayLockError)) {
throw error;
}
const racedGateway = await readActiveGatewayLockIdentity(params.options);
if (racedGateway) {
throw new GatewayLockError(params.formatActiveGatewayRefusal(racedGateway), error);
}
throw error;
}
}
+1 -1
View File
@@ -174,7 +174,7 @@ describe("Gateway lock roles", () => {
timeoutMs: 15,
}),
).rejects.toThrow(
`another openclaw agent --local run is active (pid ${process.pid}); lock timeout after 15ms`,
`another embedded OpenClaw state writer is active (pid ${process.pid}); lock timeout after 15ms`,
);
} finally {
await lock.release();
+11 -3
View File
@@ -15,7 +15,12 @@ import { getFileLockProcessStartTime, isPidAlive } from "../shared/pid-alive.js"
import { safeParseJsonWithSchema } from "../utils/zod-parse.js";
import { sha256HexPrefix } from "./crypto-digest.js";
import { createFileLockManager } from "./file-lock-manager.js";
import { isGatewayArgv, isOpenClawCommandArgv, parseProcCmdline } from "./gateway-process-argv.js";
import {
isGatewayArgv,
isOpenClawArgv,
isOpenClawCommandArgv,
parseProcCmdline,
} from "./gateway-process-argv.js";
import { tryAcquireExclusiveSqliteCoordinator } from "./node-sqlite.js";
import {
readWindowsProcessArgsSync,
@@ -210,7 +215,10 @@ async function resolveGatewayOwnerStatus(
return "unknown";
}
if (role === "agent-embedded") {
return isOpenClawCommandArgv(args, "agent") && args.includes("--local") ? "alive" : "dead";
// The role covers every direct embedded surface (agent --local, agent exec,
// local TUI, and CLI model probes), so validate the owning OpenClaw process
// instead of baking one command spelling into stale-lock recovery.
return isOpenClawArgv(args) ? "alive" : "dead";
}
const command = role === "sqlite-maintenance" ? "doctor" : "skills";
return isOpenClawCommandArgv(args, command) ? "alive" : "dead";
@@ -557,7 +565,7 @@ async function acquireLockFile(
const ownerPid = lastPayload?.pid ? ` (pid ${lastPayload.pid})` : "";
const owner =
lastPayload?.role === "agent-embedded"
? `another openclaw agent --local run is active${ownerPid}`
? `another embedded OpenClaw state writer is active${ownerPid}`
: lastPayload?.role && lastPayload.role !== "gateway"
? `state directory is locked by ${lastPayload.role}${ownerPid}`
: `gateway already running${ownerPid}`;
+21 -1
View File
@@ -1,6 +1,11 @@
// Tests gateway process argv parsing for diagnostics.
import { describe, expect, it } from "vitest";
import { isGatewayArgv, isOpenClawCommandArgv, parseProcCmdline } from "./gateway-process-argv.js";
import {
isGatewayArgv,
isOpenClawArgv,
isOpenClawCommandArgv,
parseProcCmdline,
} from "./gateway-process-argv.js";
describe("parseProcCmdline", () => {
it("splits null-delimited argv and trims empty entries", () => {
@@ -72,3 +77,18 @@ describe("isOpenClawCommandArgv", () => {
expect(isOpenClawCommandArgv(["python", "doctor", "worker.py"], "doctor")).toBe(false);
});
});
describe("isOpenClawArgv", () => {
it.each([
["agent exec", ["openclaw", "agent", "exec", "task"]],
["local TUI", ["node", "/srv/openclaw/openclaw.mjs", "tui", "--local"]],
["models probe", ["openclaw", "models", "status", "--probe"]],
["bare local TUI", ["openclaw"]],
])("recognizes the %s embedded owner", (_label, argv) => {
expect(isOpenClawArgv(argv)).toBe(true);
});
it("rejects an unrelated process", () => {
expect(isOpenClawArgv(["python", "worker.py"])).toBe(false);
});
});
+6 -4
View File
@@ -19,18 +19,20 @@ export function parseProcCmdline(raw: string): string[] {
return normalizeStringEntries(raw.split("\0"));
}
export function isOpenClawCommandArgv(args: string[], command: string): boolean {
export function isOpenClawArgv(args: string[]): boolean {
const normalized = args.map(normalizeProcArg);
const exe = (normalized[0] ?? "").replace(/\.(bat|cmd|exe)$/i, "");
if (!normalized.includes(normalizeProcArg(command))) {
return false;
}
if (normalized.some((arg) => ENTRY_CANDIDATES.some((entry) => arg.endsWith(entry)))) {
return true;
}
return exe.endsWith("/openclaw") || exe === "openclaw";
}
export function isOpenClawCommandArgv(args: string[], command: string): boolean {
const normalizedCommand = normalizeProcArg(command);
return args.some((arg) => normalizeProcArg(arg) === normalizedCommand) && isOpenClawArgv(args);
}
export function isGatewayArgv(args: string[], opts?: { allowGatewayBinary?: boolean }): boolean {
const normalized = args.map(normalizeProcArg);
const exe = (normalized[0] ?? "").replace(/\.(bat|cmd|exe)$/i, "");
+126
View File
@@ -0,0 +1,126 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import { acquireGatewayLock, type GatewayLockOptions } from "../infra/gateway-lock.js";
import { withEmbeddedTuiStateLock } from "./tui.js";
function createGatewayLockOptions(stateDir: string): GatewayLockOptions {
return {
allowInTests: true,
env: {
...process.env,
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
OPENCLAW_STATE_DIR: stateDir,
},
lockDir: path.join(stateDir, "gateway-locks"),
readProcessStartTime: () => 123_456,
timeoutMs: 100,
};
}
function createSignalProcess() {
type SignalName = "SIGINT" | "SIGTERM";
const listeners = new Map<SignalName, Set<() => void>>();
const processLike = {
on(signal: SignalName, handler: () => void) {
const current = listeners.get(signal) ?? new Set<() => void>();
current.add(handler);
listeners.set(signal, current);
return processLike;
},
off(signal: SignalName, handler: () => void) {
listeners.get(signal)?.delete(handler);
return processLike;
},
};
return {
processLike,
emit(signal: SignalName) {
for (const handler of listeners.get(signal) ?? []) {
handler();
}
},
};
}
async function withTempState<T>(run: (stateDir: string) => Promise<T>): Promise<T> {
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-tui-state-lock-"));
try {
return await run(stateDir);
} finally {
await fs.rm(stateDir, { recursive: true, force: true });
}
}
describe("embedded TUI state ownership", () => {
it("refuses local startup while a live Gateway owns the state directory", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
expect(gatewayLock).not.toBeNull();
if (!gatewayLock) {
throw new Error("Expected live Gateway fixture lock");
}
const run = vi.fn(async () => undefined);
try {
await expect(
withEmbeddedTuiStateLock(run, { gatewayLockOptions: lockOptions }),
).rejects.toThrow(
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Run without --local to use it, or stop the Gateway first (openclaw gateway stop).`,
);
expect(run).not.toHaveBeenCalled();
} finally {
await gatewayLock.release();
}
});
});
it("holds and releases embedded state ownership for the local TUI lifetime", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
await withEmbeddedTuiStateLock(
async () => {
const payload = JSON.parse(await fs.readFile(stateLockPath, "utf8")) as {
pid?: number;
role?: string;
};
expect(payload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
},
{ gatewayLockOptions: lockOptions },
);
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("releases embedded state ownership when the local TUI receives SIGTERM", async () => {
await withTempState(async (stateDir) => {
const lockOptions = createGatewayLockOptions(stateDir);
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
const signals = createSignalProcess();
let markStarted!: () => void;
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
const run = withEmbeddedTuiStateLock(
async (signal) => {
markStarted();
return await new Promise<never>((_, reject) => {
signal.addEventListener("abort", () => reject(new Error("local TUI interrupted")), {
once: true,
});
});
},
{ gatewayLockOptions: lockOptions, process: signals.processLike },
);
await started;
signals.emit("SIGTERM");
await expect(run).rejects.toThrow("local TUI interrupted");
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
});
+39
View File
@@ -14,8 +14,11 @@ import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/s
import type { CommandEntry } from "../../packages/gateway-protocol/src/index.js";
import { resolveAgentIdByWorkspacePath, resolveDefaultAgentId } from "../agents/agent-scope.js";
import { normalizeThinkLevel } from "../auto-reply/thinking.shared.js";
import { formatCliCommand } from "../cli/command-format.js";
import { getRuntimeConfig, type OpenClawConfig } from "../config/config.js";
import { resolveCanonicalMainSessionKey } from "../config/sessions/main-session-key.js";
import type { EmbeddedStateSignalProcess } from "../infra/embedded-state-lock.js";
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
import { resolveCurrentOpenClawCliInvocation } from "../infra/openclaw-cli-invocation.js";
import { tryProcessCwd } from "../infra/safe-cwd.js";
import { registerUncaughtExceptionHandler } from "../infra/unhandled-rejections.js";
@@ -598,7 +601,43 @@ function resolveEmptySessionInfoDefaults(config: OpenClawConfig): SessionInfo {
};
}
function formatActiveGatewayTuiRefusal(identity: GatewayLockIdentity): string {
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Run without --local to use it, or stop the Gateway first (${formatCliCommand("openclaw gateway stop")}).`;
}
/** Hold canonical state ownership for the complete lifetime of a local TUI. */
export async function withEmbeddedTuiStateLock<T>(
run: (signal: AbortSignal) => Promise<T>,
deps: {
gatewayLockOptions?: GatewayLockOptions;
process?: EmbeddedStateSignalProcess;
} = {},
): Promise<T> {
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
await import("../infra/embedded-state-lock.js");
const signalBridge = createEmbeddedStateSignalBridge(deps.process ?? process);
let stateLock: Awaited<ReturnType<typeof acquireEmbeddedStateLock>> | undefined;
try {
stateLock = await acquireEmbeddedStateLock({
options: deps.gatewayLockOptions,
signal: signalBridge.signal,
formatActiveGatewayRefusal: formatActiveGatewayTuiRefusal,
});
return await run(signalBridge.signal);
} finally {
await stateLock?.release();
signalBridge.dispose();
}
}
export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
if (opts.local === true && opts.backend === undefined) {
return await withEmbeddedTuiStateLock(async () => await runTuiUnlocked(opts));
}
return await runTuiUnlocked(opts);
}
async function runTuiUnlocked(opts: RunTuiOptions): Promise<TuiResult> {
const isLocalMode = opts.local === true || opts.backend !== undefined;
const config = opts.config ?? getRuntimeConfig({ skipPluginValidation: !isLocalMode });
const cliInvocation = resolveCurrentOpenClawCliInvocation([]);