mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(cli): guard remaining embedded state writers (#121282)
This commit is contained in:
committed by
GitHub
parent
6efd013c25
commit
08507909ed
+1
-1
@@ -28,7 +28,7 @@ By default, the command creates and later removes a temporary state directory, a
|
||||
|
||||
Config is layered in three parts, entirely in memory: exec composes the run config and publishes it as this process's runtime config rather than writing a copy to disk. Exec defaults apply only where your config leaves a setting unset: workspace bootstrap files are skipped, the agent sandbox is off, the `coding` tool profile is selected, filesystem tools are restricted to `--cwd`, and exec runs under the full execution policy a headless turn needs. Anything your config sets wins over those defaults, so a configured sandbox, shell env, or tool profile is never downgraded, and exec host routing stays with the sandbox when your config enables one. The invocation itself always wins last: the run is scoped to `--cwd` and never bootstraps.
|
||||
|
||||
Use `--state-dir <dir>` to retain sessions and other run state. The directory must already exist and is never created or deleted by the command.
|
||||
Use `--state-dir <dir>` to retain sessions and other run state. The directory must already exist and is never created or deleted by the command. A retained state directory requires exclusive ownership: exec refuses to start while a Gateway or another embedded writer owns it, then holds the state lock for the complete run. Omit `--state-dir` for isolated temporary state, or stop the Gateway first with `openclaw gateway stop`.
|
||||
|
||||
When exec uses the ambient or a pinned config, installed plugins continue to resolve from the operator's ordinary plugin roots while sessions and other run state use the ephemeral directory. In those modes, `--state-dir` controls run state only; it is not required for configured providers, channels, or harnesses supplied by installed plugins.
|
||||
|
||||
|
||||
@@ -56,6 +56,8 @@ Options:
|
||||
|
||||
Probe rows can come from auth profiles, env credentials, or `models.json`. Probe status buckets: `ok`, `auth`, `rate_limit`, `billing`, `timeout`, `format`, `unknown`, `no_model`.
|
||||
|
||||
Direct `models status --probe` runs create temporary internal sessions in the selected agent's canonical database, so the command requires exclusive ownership of the configured state directory. Stop a running Gateway with `openclaw gateway stop` before probing; the command removes its internal sessions and releases the state lock when it finishes or is interrupted.
|
||||
|
||||
Probe detail/reason codes to expect when a probe never reaches a model call:
|
||||
|
||||
- `excluded_by_auth_order`: a stored profile exists, but explicit `auth.order.<provider>` omitted it, so probe reports the exclusion instead of trying it.
|
||||
|
||||
@@ -77,6 +77,10 @@ Aliases: `openclaw chat` and `openclaw terminal` invoke this command with
|
||||
`agent:<id>:...`).
|
||||
- Local mode uses the embedded agent runtime directly. Most local tools work,
|
||||
but Gateway-only features are unavailable.
|
||||
- Local mode requires exclusive ownership of the configured state directory. It
|
||||
refuses to start while a Gateway or another embedded writer owns that state;
|
||||
run without `--local` to use the active Gateway, or stop it first with
|
||||
`openclaw gateway stop`.
|
||||
- Local mode adds `/auth [provider]` to the TUI command surface.
|
||||
- Plugin approval gates still apply in local mode: tools that require approval
|
||||
prompt for a decision in the terminal, nothing is silently auto-approved.
|
||||
|
||||
@@ -104,7 +104,7 @@ describe("supervised gateway lock recovery", () => {
|
||||
|
||||
it("preserves an agent-embedded owner error under a supervisor", async () => {
|
||||
const err = new GatewayLockError(
|
||||
"another openclaw agent --local run is active (pid 123); lock timeout after 5000ms",
|
||||
"another embedded OpenClaw state writer is active (pid 123); lock timeout after 5000ms",
|
||||
);
|
||||
const startLoop = vi.fn(async () => {
|
||||
throw err;
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { acquireGatewayLock, type GatewayLockOptions } from "../infra/gateway-lock.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { agentExecCommand } from "./agent-exec.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function createRuntime() {
|
||||
const error = vi.fn();
|
||||
const runtime: RuntimeEnv = { log: vi.fn(), error, exit: vi.fn() };
|
||||
return { runtime, error };
|
||||
}
|
||||
|
||||
function successResult() {
|
||||
return {
|
||||
payloads: [{ text: "done" }],
|
||||
meta: {
|
||||
durationMs: 1,
|
||||
agentMeta: { sessionId: "session-result", provider: "openai", model: "gpt-5.6-sol" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createGatewayLockOptions(
|
||||
stateDir: string,
|
||||
overrides: Partial<GatewayLockOptions> = {},
|
||||
): GatewayLockOptions {
|
||||
return {
|
||||
allowInTests: true,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
},
|
||||
lockDir: path.join(stateDir, "gateway-locks"),
|
||||
timeoutMs: 100,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createSignalProcess() {
|
||||
type SignalName = "SIGINT" | "SIGTERM";
|
||||
const listeners = new Map<SignalName, Set<() => void>>();
|
||||
const processLike = {
|
||||
on(signal: SignalName, handler: () => void) {
|
||||
const current = listeners.get(signal) ?? new Set<() => void>();
|
||||
current.add(handler);
|
||||
listeners.set(signal, current);
|
||||
return processLike;
|
||||
},
|
||||
off(signal: SignalName, handler: () => void) {
|
||||
listeners.get(signal)?.delete(handler);
|
||||
return processLike;
|
||||
},
|
||||
};
|
||||
return {
|
||||
processLike,
|
||||
emit(signal: SignalName) {
|
||||
for (const handler of listeners.get(signal) ?? []) {
|
||||
handler();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("agent exec retained-state ownership", () => {
|
||||
it("refuses a state directory owned by a live Gateway", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-agent-exec-gateway-owner-");
|
||||
const lockOptions = createGatewayLockOptions(stateDir, {
|
||||
readProcessStartTime: () => 123_456,
|
||||
});
|
||||
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
|
||||
expect(gatewayLock).not.toBeNull();
|
||||
if (!gatewayLock) {
|
||||
throw new Error("Expected live Gateway fixture lock");
|
||||
}
|
||||
const runAgent = vi.fn(async () => successResult());
|
||||
const { runtime, error } = createRuntime();
|
||||
|
||||
try {
|
||||
const result = await agentExecCommand("inspect", { stateDir }, runtime, {
|
||||
gatewayLockOptions: lockOptions,
|
||||
runAgent,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(runAgent).not.toHaveBeenCalled();
|
||||
expect(error).toHaveBeenCalledWith(
|
||||
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Omit --state-dir to use isolated temporary state, or stop the Gateway first (openclaw gateway stop).`,
|
||||
);
|
||||
} finally {
|
||||
await gatewayLock.release();
|
||||
}
|
||||
});
|
||||
|
||||
it("holds and releases the embedded state lock around the run", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-agent-exec-lock-owner-");
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
|
||||
await agentExecCommand("inspect", { stateDir }, createRuntime().runtime, {
|
||||
gatewayLockOptions: lockOptions,
|
||||
runAgent: vi.fn(async () => {
|
||||
const payload = JSON.parse(await fs.readFile(stateLockPath, "utf8")) as {
|
||||
pid?: number;
|
||||
role?: string;
|
||||
};
|
||||
expect(payload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
|
||||
return successResult();
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("releases the embedded state lock when SIGTERM aborts the run", async () => {
|
||||
const stateDir = tempDirs.make("openclaw-agent-exec-signal-owner-");
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
const signals = createSignalProcess();
|
||||
const { runtime } = createRuntime();
|
||||
const runAgent = vi.fn(async (opts: Record<string, unknown>) => {
|
||||
const signal = opts.abortSignal as AbortSignal;
|
||||
return await new Promise<ReturnType<typeof successResult>>((_, reject) => {
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
const error = new Error("agent exec aborted");
|
||||
error.name = "AbortError";
|
||||
reject(error);
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
const run = agentExecCommand("inspect", { stateDir }, runtime, {
|
||||
gatewayLockOptions: lockOptions,
|
||||
process: signals.processLike,
|
||||
runAgent,
|
||||
});
|
||||
await vi.waitFor(() => expect(runAgent).toHaveBeenCalledOnce());
|
||||
signals.emit("SIGTERM");
|
||||
await run;
|
||||
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(runtime.exit).toHaveBeenCalledWith(143, { resetStream: process.stderr });
|
||||
});
|
||||
});
|
||||
@@ -1001,6 +1001,21 @@ describe("agent exec run config layering", () => {
|
||||
expect(config.agents?.entries?.ops?.model).toBe("openai/gpt-5.6-sol");
|
||||
});
|
||||
|
||||
it("drops an inherited session store so the invocation state dir owns the agent database", () => {
|
||||
const config = buildExecRunConfig({
|
||||
base: {
|
||||
session: {
|
||||
store: "/persistent/agents/{agentId}/sessions/sessions.json",
|
||||
mainKey: "primary",
|
||||
},
|
||||
},
|
||||
cwd: "/run/here",
|
||||
});
|
||||
|
||||
expect(config.session?.store).toBeUndefined();
|
||||
expect(config.session?.mainKey).toBe("primary");
|
||||
});
|
||||
|
||||
it("drops an inherited harness cwd so --cwd wins", () => {
|
||||
const config = buildExecRunConfig({
|
||||
base: {
|
||||
|
||||
+52
-11
@@ -8,9 +8,15 @@ import { readByteStreamWithLimit } from "@openclaw/media-core/read-byte-stream-w
|
||||
import { findAgentRunTerminalOutcome } from "../agents/agent-run-terminal-error.js";
|
||||
import type { EmbeddedAgentRunMeta } from "../agents/embedded-agent.js";
|
||||
import { isExecutionIdentityCollectionEnabled } from "../audit/audit-config.js";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { mergeDeep } from "../infra/deep-merge.js";
|
||||
import type {
|
||||
EmbeddedStateLockHandle,
|
||||
EmbeddedStateSignalProcess,
|
||||
} from "../infra/embedded-state-lock.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
|
||||
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
|
||||
import { writeRuntimeJson, writeRuntimeStdout, type RuntimeEnv } from "../runtime.js";
|
||||
|
||||
@@ -79,6 +85,8 @@ type AgentExecCommandResult = {
|
||||
|
||||
type AgentExecCommandDeps = {
|
||||
stdin?: AsyncIterable<unknown>;
|
||||
process?: EmbeddedStateSignalProcess;
|
||||
gatewayLockOptions?: GatewayLockOptions;
|
||||
runAgent?: (
|
||||
opts: Record<string, unknown>,
|
||||
runtime: RuntimeEnv,
|
||||
@@ -284,23 +292,25 @@ function normalizeCodeMode(
|
||||
* outrank whatever the resolved config says.
|
||||
*/
|
||||
/**
|
||||
* Drops inherited per-agent location overrides, which outrank the facts this
|
||||
* invocation owns. `agentDir` beats the state dir for session and transcript
|
||||
* storage, so an ephemeral run would write state into the operator's persistent
|
||||
* agent directory where deleting the temp state dir cannot reach it; a native
|
||||
* harness `runtime.acp.cwd` beats `--cwd`, so the turn could edit the wrong
|
||||
* repository. `agents.bindings[].acp.cwd` needs no equivalent because exec runs
|
||||
* no channel, so no binding matches.
|
||||
* Drops inherited state and workspace location overrides, which outrank the
|
||||
* facts this invocation owns. `session.store` and `agentDir` can redirect state
|
||||
* outside the invocation root, where its lock or temporary cleanup cannot own
|
||||
* it; a native harness `runtime.acp.cwd` can make the turn edit the wrong repo.
|
||||
* `agents.bindings[].acp.cwd` needs no equivalent because exec runs no channel,
|
||||
* so no binding matches.
|
||||
*/
|
||||
function stripInheritedAgentLocations(base: OpenClawConfig): OpenClawConfig {
|
||||
const entries = base.agents?.entries;
|
||||
const { session, ...root } = base;
|
||||
const { store: _store, ...sessionWithoutStore } = session ?? {};
|
||||
const withoutSessionStore = session ? { ...root, session: sessionWithoutStore } : base;
|
||||
const entries = withoutSessionStore.agents?.entries;
|
||||
if (!entries) {
|
||||
return base;
|
||||
return withoutSessionStore;
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
...withoutSessionStore,
|
||||
agents: {
|
||||
...base.agents,
|
||||
...withoutSessionStore.agents,
|
||||
entries: Object.fromEntries(
|
||||
Object.entries(entries).map(([id, entry]) => {
|
||||
const { agentDir: _agentDir, runtime, ...rest } = entry;
|
||||
@@ -477,6 +487,10 @@ function setAgentExecEnvironment(params: { stateDir: string; cwd: string }): ()
|
||||
};
|
||||
}
|
||||
|
||||
function formatActiveGatewayExecRefusal(identity: GatewayLockIdentity): string {
|
||||
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Omit --state-dir to use isolated temporary state, or stop the Gateway first (${formatCliCommand("openclaw gateway stop")}).`;
|
||||
}
|
||||
|
||||
function isStructuredTimeoutError(error: unknown): boolean {
|
||||
if (findAgentRunTerminalOutcome(error)?.status === "timeout") {
|
||||
return true;
|
||||
@@ -552,6 +566,12 @@ export async function agentExecCommand(
|
||||
let runtimePaths: typeof import("../config/paths.js") | undefined;
|
||||
let configIo: typeof import("../config/io.js") | undefined;
|
||||
let stopLocalAuditWriter: (() => Promise<void>) | undefined;
|
||||
let stateLock: EmbeddedStateLockHandle | null | undefined;
|
||||
let signalBridge:
|
||||
| ReturnType<
|
||||
(typeof import("../infra/embedded-state-lock.js"))["createEmbeddedStateSignalBridge"]
|
||||
>
|
||||
| undefined;
|
||||
try {
|
||||
const prompt = await resolveAgentExecPrompt(
|
||||
positionalMessage,
|
||||
@@ -618,6 +638,16 @@ export async function agentExecCommand(
|
||||
restoreEnvironment = setAgentExecEnvironment({ stateDir, cwd });
|
||||
runtimePaths = await import("../config/paths.js");
|
||||
runtimePaths.pinRuntimePaths();
|
||||
if (opts.stateDir) {
|
||||
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
|
||||
await import("../infra/embedded-state-lock.js");
|
||||
signalBridge = createEmbeddedStateSignalBridge(deps.process ?? process);
|
||||
stateLock = await acquireEmbeddedStateLock({
|
||||
options: deps.gatewayLockOptions,
|
||||
signal: signalBridge.signal,
|
||||
formatActiveGatewayRefusal: formatActiveGatewayExecRefusal,
|
||||
});
|
||||
}
|
||||
// The runtime snapshot is the only in-process config cache (`clearConfigCache`
|
||||
// is a no-op shim), so publishing the composed config here is what makes the
|
||||
// run use it. Serializing it to a temporary file and repointing
|
||||
@@ -668,6 +698,7 @@ export async function agentExecCommand(
|
||||
cleanupBundleMcpOnRunEnd: true,
|
||||
cleanupCliLiveSessionOnRunEnd: true,
|
||||
oneShotCliRun: true,
|
||||
abortSignal: signalBridge?.signal,
|
||||
onModelFallbackExhausted: () => {
|
||||
fallbackExhausted = true;
|
||||
},
|
||||
@@ -707,6 +738,9 @@ export async function agentExecCommand(
|
||||
|
||||
let cleanupError: unknown;
|
||||
await stopLocalAuditWriter?.().catch(() => undefined);
|
||||
await stateLock?.release().catch((error: unknown) => {
|
||||
cleanupError ??= error;
|
||||
});
|
||||
const runCleanupStep = (step: () => void) => {
|
||||
try {
|
||||
step();
|
||||
@@ -738,6 +772,13 @@ export async function agentExecCommand(
|
||||
commandResult = { envelope, exitCode: exitCodeForEnvelope(envelope) };
|
||||
}
|
||||
|
||||
const receivedSignal = signalBridge?.getReceivedSignal();
|
||||
signalBridge?.dispose();
|
||||
if (receivedSignal) {
|
||||
runtime.exit(receivedSignal === "SIGINT" ? 130 : 143, { resetStream: process.stderr });
|
||||
return commandResult;
|
||||
}
|
||||
|
||||
writeAgentExecOutput(runtime, commandResult.envelope, opts.json === true);
|
||||
return commandResult;
|
||||
}
|
||||
|
||||
@@ -541,7 +541,7 @@ describe("agentCliCommand", () => {
|
||||
localGatewayLockOptions: { ...lockOptions, pollIntervalMs: 2, timeoutMs: 15 },
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
`another openclaw agent --local run is active (pid ${process.pid}); lock timeout after 15ms`,
|
||||
`another embedded OpenClaw state writer is active (pid ${process.pid}); lock timeout after 15ms`,
|
||||
);
|
||||
expect(agentCommand).toHaveBeenCalledTimes(1);
|
||||
|
||||
|
||||
@@ -30,6 +30,11 @@ import { isGatewaySecretRefUnavailableError } from "../gateway/credentials.js";
|
||||
import { ADMIN_SCOPE } from "../gateway/operator-scopes.js";
|
||||
import { createAbortError } from "../infra/abort-signal.js";
|
||||
import { readFileDescriptorBounded } from "../infra/boundary-file-read.js";
|
||||
import {
|
||||
createEmbeddedStateSignalBridge,
|
||||
type EmbeddedStateSignal,
|
||||
type EmbeddedStateSignalProcess,
|
||||
} from "../infra/embedded-state-lock.js";
|
||||
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
|
||||
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
|
||||
import { routeLogsToStderr } from "../logging/console.js";
|
||||
@@ -96,11 +101,9 @@ type AgentDispatchOpts = Omit<AgentCliOpts, "messageFile"> & {
|
||||
message: string;
|
||||
};
|
||||
|
||||
type AgentCliSignal = "SIGINT" | "SIGTERM";
|
||||
type AgentCliProcessLike = {
|
||||
type AgentCliSignal = EmbeddedStateSignal;
|
||||
type AgentCliProcessLike = EmbeddedStateSignalProcess & {
|
||||
exitCode?: NodeJS.Process["exitCode"];
|
||||
on(signal: AgentCliSignal, handler: () => void): unknown;
|
||||
off(signal: AgentCliSignal, handler: () => void): unknown;
|
||||
};
|
||||
type AgentCliDeps = CliDeps & {
|
||||
process?: AgentCliProcessLike;
|
||||
@@ -113,7 +116,6 @@ type AgentGatewayCallIdentity = Pick<
|
||||
type AgentSessionModule = typeof import("./agent/session.runtime.js");
|
||||
type AgentSessionModuleLoader = () => Promise<AgentSessionModule>;
|
||||
|
||||
const AGENT_CLI_SIGNALS: readonly AgentCliSignal[] = ["SIGINT", "SIGTERM"];
|
||||
const GATEWAY_ABORT_RETRY_DELAYS_MS = [50, 150, 300, 600] as const;
|
||||
const GATEWAY_ABORT_REQUEST_TIMEOUT_MS = 2_000;
|
||||
const AGENT_CLI_SIGNAL_EXIT_CODES: Record<AgentCliSignal, number> = {
|
||||
@@ -138,9 +140,10 @@ const agentSessionModuleCache = createLazyPromiseLoader(() => agentSessionModule
|
||||
const runtimeConfigModuleLoader = createLazyPromiseLoader(() => import("../config/io.js"), {
|
||||
cacheRejections: true,
|
||||
});
|
||||
const gatewayLockModuleLoader = createLazyPromiseLoader(() => import("../infra/gateway-lock.js"), {
|
||||
cacheRejections: true,
|
||||
});
|
||||
const embeddedStateLockModuleLoader = createLazyPromiseLoader(
|
||||
() => import("../infra/embedded-state-lock.js"),
|
||||
{ cacheRejections: true },
|
||||
);
|
||||
const replyPayloadModuleLoader = createLazyPromiseLoader(
|
||||
() => import("openclaw/plugin-sdk/reply-payload"),
|
||||
{ cacheRejections: true },
|
||||
@@ -226,32 +229,12 @@ async function acquireEmbeddedAgentStateLock(
|
||||
options: GatewayLockOptions | undefined,
|
||||
signal: AbortSignal,
|
||||
) {
|
||||
const { acquireGatewayLock, GatewayLockError, readActiveGatewayLockIdentity } =
|
||||
await gatewayLockModuleLoader.load();
|
||||
const env = options?.env ?? process.env;
|
||||
if (options?.allowInTests !== true && (env.VITEST || env.NODE_ENV === "test")) {
|
||||
return null;
|
||||
}
|
||||
const activeGateway = await readActiveGatewayLockIdentity(options);
|
||||
if (activeGateway) {
|
||||
throw new GatewayLockError(formatActiveGatewayLocalRefusal(activeGateway));
|
||||
}
|
||||
try {
|
||||
return await acquireGatewayLock({
|
||||
...options,
|
||||
role: "agent-embedded",
|
||||
sleep: options?.sleep ?? (async (ms) => await delayMs(ms, signal)),
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof GatewayLockError)) {
|
||||
throw error;
|
||||
}
|
||||
const racedGateway = await readActiveGatewayLockIdentity(options);
|
||||
if (racedGateway) {
|
||||
throw new GatewayLockError(formatActiveGatewayLocalRefusal(racedGateway), error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const { acquireEmbeddedStateLock } = await embeddedStateLockModuleLoader.load();
|
||||
return await acquireEmbeddedStateLock({
|
||||
options,
|
||||
signal,
|
||||
formatActiveGatewayRefusal: formatActiveGatewayLocalRefusal,
|
||||
});
|
||||
}
|
||||
|
||||
const loadReplyPayloadModule = replyPayloadModuleLoader.load;
|
||||
@@ -263,7 +246,7 @@ export const agentViaGatewayTesting = {
|
||||
localAuditModuleLoader.clear();
|
||||
agentSessionModuleCache.clear();
|
||||
runtimeConfigModuleLoader.clear();
|
||||
gatewayLockModuleLoader.clear();
|
||||
embeddedStateLockModuleLoader.clear();
|
||||
replyPayloadModuleLoader.clear();
|
||||
agentSessionModuleLoader = defaultAgentSessionModuleLoader;
|
||||
},
|
||||
@@ -530,34 +513,12 @@ function readAcceptedRunContext(payload: unknown): {
|
||||
}
|
||||
|
||||
function createAgentCliSignalBridge(processLike: AgentCliProcessLike = process) {
|
||||
const controller = new AbortController();
|
||||
let receivedSignal: AgentCliSignal | undefined;
|
||||
const handlers = new Map<AgentCliSignal, () => void>();
|
||||
const detachHandlers = () => {
|
||||
for (const [signal, handler] of handlers) {
|
||||
processLike.off(signal, handler);
|
||||
}
|
||||
handlers.clear();
|
||||
};
|
||||
for (const signal of AGENT_CLI_SIGNALS) {
|
||||
const handler = () => {
|
||||
receivedSignal = signal;
|
||||
if (!controller.signal.aborted) {
|
||||
// runtime.exit may bypass finally cleanup, so first-signal self-detach is load-bearing.
|
||||
controller.abort();
|
||||
detachHandlers();
|
||||
}
|
||||
};
|
||||
handlers.set(signal, handler);
|
||||
processLike.on(signal, handler);
|
||||
}
|
||||
const bridge = createEmbeddedStateSignalBridge(processLike);
|
||||
return {
|
||||
signal: controller.signal,
|
||||
getReceivedSignal: () => receivedSignal,
|
||||
...bridge,
|
||||
setExitCode: (code: number) => {
|
||||
processLike.exitCode = code;
|
||||
},
|
||||
dispose: detachHandlers,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,10 +1,63 @@
|
||||
// Model list probe tests cover runtime probing while listing configured models.
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { acquireGatewayLock, type GatewayLockOptions } from "../../infra/gateway-lock.js";
|
||||
|
||||
let probeModule: typeof import("./list.probe.js");
|
||||
|
||||
function createGatewayLockOptions(stateDir: string): GatewayLockOptions {
|
||||
return {
|
||||
allowInTests: true,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
},
|
||||
lockDir: path.join(stateDir, "gateway-locks"),
|
||||
readProcessStartTime: () => 123_456,
|
||||
timeoutMs: 100,
|
||||
};
|
||||
}
|
||||
|
||||
function createSignalProcess() {
|
||||
type SignalName = "SIGINT" | "SIGTERM";
|
||||
const listeners = new Map<SignalName, Set<() => void>>();
|
||||
const processLike = {
|
||||
on(signal: SignalName, handler: () => void) {
|
||||
const current = listeners.get(signal) ?? new Set<() => void>();
|
||||
current.add(handler);
|
||||
listeners.set(signal, current);
|
||||
return processLike;
|
||||
},
|
||||
off(signal: SignalName, handler: () => void) {
|
||||
listeners.get(signal)?.delete(handler);
|
||||
return processLike;
|
||||
},
|
||||
};
|
||||
return {
|
||||
processLike,
|
||||
emit(signal: SignalName) {
|
||||
for (const handler of listeners.get(signal) ?? []) {
|
||||
handler();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function withTempState<T>(run: (stateDir: string) => Promise<T>): Promise<T> {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-model-probe-lock-"));
|
||||
try {
|
||||
return await run(stateDir);
|
||||
} finally {
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("mapFailoverReasonToProbeStatus", () => {
|
||||
beforeAll(async () => {
|
||||
vi.doMock("../../agents/embedded-agent.js", () => {
|
||||
@@ -42,6 +95,81 @@ describe("mapFailoverReasonToProbeStatus", () => {
|
||||
});
|
||||
|
||||
describe("runAuthProbes", () => {
|
||||
beforeAll(async () => {
|
||||
probeModule ??= await import("./list.probe.js");
|
||||
});
|
||||
|
||||
it("refuses direct CLI probes while a live Gateway owns canonical state", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
|
||||
expect(gatewayLock).not.toBeNull();
|
||||
if (!gatewayLock) {
|
||||
throw new Error("Expected live Gateway fixture lock");
|
||||
}
|
||||
try {
|
||||
await expect(
|
||||
probeModule.withAuthProbeStateOwnership(
|
||||
{ mode: "exclusive", gatewayLockOptions: lockOptions },
|
||||
async () => undefined,
|
||||
),
|
||||
).rejects.toThrow(
|
||||
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Stop the Gateway first (openclaw gateway stop), then rerun models status --probe.`,
|
||||
);
|
||||
} finally {
|
||||
await gatewayLock.release();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("holds and releases canonical state ownership around direct CLI probes", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
let observedPayload: { pid?: number; role?: string } | undefined;
|
||||
|
||||
await probeModule.withAuthProbeStateOwnership(
|
||||
{ mode: "exclusive", gatewayLockOptions: lockOptions },
|
||||
async () => {
|
||||
observedPayload = JSON.parse(fsSync.readFileSync(stateLockPath, "utf8")) as {
|
||||
pid?: number;
|
||||
role?: string;
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
expect(observedPayload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
|
||||
it("releases canonical state ownership when a direct CLI probe receives SIGTERM", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
const signals = createSignalProcess();
|
||||
|
||||
await probeModule.withAuthProbeStateOwnership(
|
||||
{
|
||||
mode: "exclusive",
|
||||
gatewayLockOptions: lockOptions,
|
||||
process: signals.processLike,
|
||||
},
|
||||
async (signal) => {
|
||||
let markInterrupted!: () => void;
|
||||
const interrupted = new Promise<void>((resolve) => {
|
||||
markInterrupted = resolve;
|
||||
});
|
||||
signal?.addEventListener("abort", markInterrupted, { once: true });
|
||||
signals.emit("SIGTERM");
|
||||
await interrupted;
|
||||
},
|
||||
);
|
||||
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
|
||||
it("runs Codex auth probes through raw OpenClaw model-run mode", async () => {
|
||||
const runEmbeddedAgent = vi.fn(
|
||||
async (_params: {
|
||||
|
||||
@@ -39,6 +39,7 @@ import { findNormalizedProviderValue, normalizeProviderId } from "../../agents/m
|
||||
import { loadPreparedModelCatalog } from "../../agents/prepared-model-catalog.js";
|
||||
import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";
|
||||
import { resolveDefaultAgentWorkspaceDir } from "../../agents/workspace.js";
|
||||
import { formatCliCommand } from "../../cli/command-format.js";
|
||||
import { resolveStorePath } from "../../config/sessions/paths.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
@@ -46,6 +47,11 @@ import {
|
||||
hasConfiguredSecretInput,
|
||||
normalizeSecretInputString,
|
||||
} from "../../config/types.secrets.js";
|
||||
import type {
|
||||
EmbeddedStateLockHandle,
|
||||
EmbeddedStateSignalProcess,
|
||||
} from "../../infra/embedded-state-lock.js";
|
||||
import type { GatewayLockIdentity, GatewayLockOptions } from "../../infra/gateway-lock.js";
|
||||
import { type SecretRefResolveCache, resolveSecretRefString } from "../../secrets/resolve.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import { disposeOpenClawAgentDatabaseByPath } from "../../state/openclaw-agent-db.js";
|
||||
@@ -723,6 +729,7 @@ async function probeTarget(params: {
|
||||
target: AuthProbeTarget;
|
||||
timeoutMs: number;
|
||||
maxTokens: number;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<AuthProbeResult> {
|
||||
const { cfg, agentId, agentDir, workspaceDir, storePath, target, timeoutMs, maxTokens } = params;
|
||||
// Marker credentials must be resolved by the runtime from config, but the
|
||||
@@ -834,6 +841,7 @@ async function probeTarget(params: {
|
||||
disableTools: true,
|
||||
modelRun: true,
|
||||
cleanupBundleMcpOnRunEnd: true,
|
||||
abortSignal: params.abortSignal,
|
||||
});
|
||||
return buildResult("ok");
|
||||
} catch (err) {
|
||||
@@ -862,6 +870,7 @@ async function runTargetsWithConcurrency(params: {
|
||||
maxTokens: number;
|
||||
concurrency: number;
|
||||
onProgress?: (update: { completed: number; total: number; label?: string }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<AuthProbeResult[]> {
|
||||
const { cfg, targets, timeoutMs, maxTokens, onProgress } = params;
|
||||
const concurrency = Math.max(1, Math.min(targets.length || 1, params.concurrency));
|
||||
@@ -894,15 +903,55 @@ async function runTargetsWithConcurrency(params: {
|
||||
target,
|
||||
timeoutMs,
|
||||
maxTokens,
|
||||
abortSignal: params.abortSignal,
|
||||
});
|
||||
completed += 1;
|
||||
onProgress?.({ completed, total: targets.length });
|
||||
return result;
|
||||
},
|
||||
{ concurrency, stopOnError: true },
|
||||
{
|
||||
concurrency,
|
||||
stopOnError: true,
|
||||
...(params.abortSignal ? { signal: params.abortSignal } : {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function formatActiveGatewayModelsProbeRefusal(identity: GatewayLockIdentity): string {
|
||||
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Stop the Gateway first (${formatCliCommand("openclaw gateway stop")}), then rerun models status --probe.`;
|
||||
}
|
||||
|
||||
type AuthProbeStateOwnership = {
|
||||
mode: "exclusive";
|
||||
gatewayLockOptions?: GatewayLockOptions;
|
||||
process?: EmbeddedStateSignalProcess;
|
||||
};
|
||||
|
||||
/** Own canonical state only for direct CLI probes; Gateway RPC probes already run under its lock. */
|
||||
export async function withAuthProbeStateOwnership<T>(
|
||||
ownership: AuthProbeStateOwnership | undefined,
|
||||
run: (signal?: AbortSignal) => Promise<T>,
|
||||
): Promise<T> {
|
||||
if (!ownership) {
|
||||
return await run();
|
||||
}
|
||||
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
|
||||
await import("../../infra/embedded-state-lock.js");
|
||||
const signalBridge = createEmbeddedStateSignalBridge(ownership.process ?? process);
|
||||
let stateLock: EmbeddedStateLockHandle | null | undefined;
|
||||
try {
|
||||
stateLock = await acquireEmbeddedStateLock({
|
||||
options: ownership.gatewayLockOptions,
|
||||
signal: signalBridge.signal,
|
||||
formatActiveGatewayRefusal: formatActiveGatewayModelsProbeRefusal,
|
||||
});
|
||||
return await run(signalBridge.signal);
|
||||
} finally {
|
||||
await stateLock?.release();
|
||||
signalBridge.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs all auth probes with bounded concurrency and returns a summary. */
|
||||
export async function runAuthProbes(params: {
|
||||
cfg: OpenClawConfig;
|
||||
@@ -913,45 +962,49 @@ export async function runAuthProbes(params: {
|
||||
modelCandidates: string[];
|
||||
options: AuthProbeOptions;
|
||||
onProgress?: (update: { completed: number; total: number; label?: string }) => void;
|
||||
stateOwnership?: AuthProbeStateOwnership;
|
||||
}): Promise<AuthProbeSummary> {
|
||||
const startedAt = Date.now();
|
||||
const plan = await buildProbeTargets({
|
||||
cfg: params.cfg,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
providers: params.providers,
|
||||
modelCandidates: params.modelCandidates,
|
||||
options: params.options,
|
||||
return await withAuthProbeStateOwnership(params.stateOwnership, async (abortSignal) => {
|
||||
const startedAt = Date.now();
|
||||
const plan = await buildProbeTargets({
|
||||
cfg: params.cfg,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
providers: params.providers,
|
||||
modelCandidates: params.modelCandidates,
|
||||
options: params.options,
|
||||
});
|
||||
|
||||
const totalTargets = plan.targets.length;
|
||||
params.onProgress?.({ completed: 0, total: totalTargets });
|
||||
|
||||
const results = totalTargets
|
||||
? await runTargetsWithConcurrency({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
targets: plan.targets,
|
||||
timeoutMs: params.options.timeoutMs,
|
||||
maxTokens: params.options.maxTokens,
|
||||
concurrency: params.options.concurrency,
|
||||
onProgress: params.onProgress,
|
||||
abortSignal,
|
||||
})
|
||||
: [];
|
||||
|
||||
const finishedAt = Date.now();
|
||||
|
||||
return {
|
||||
startedAt,
|
||||
finishedAt,
|
||||
durationMs: finishedAt - startedAt,
|
||||
totalTargets,
|
||||
options: params.options,
|
||||
results: [...plan.results, ...results],
|
||||
};
|
||||
});
|
||||
|
||||
const totalTargets = plan.targets.length;
|
||||
params.onProgress?.({ completed: 0, total: totalTargets });
|
||||
|
||||
const results = totalTargets
|
||||
? await runTargetsWithConcurrency({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
agentDir: params.agentDir,
|
||||
workspaceDir: params.workspaceDir,
|
||||
targets: plan.targets,
|
||||
timeoutMs: params.options.timeoutMs,
|
||||
maxTokens: params.options.maxTokens,
|
||||
concurrency: params.options.concurrency,
|
||||
onProgress: params.onProgress,
|
||||
})
|
||||
: [];
|
||||
|
||||
const finishedAt = Date.now();
|
||||
|
||||
return {
|
||||
startedAt,
|
||||
finishedAt,
|
||||
durationMs: finishedAt - startedAt,
|
||||
totalTargets,
|
||||
options: params.options,
|
||||
results: [...plan.results, ...results],
|
||||
};
|
||||
}
|
||||
|
||||
/** Formats probe latency for table output. */
|
||||
|
||||
@@ -1223,6 +1223,9 @@ export async function modelsStatusCommand(
|
||||
concurrency: probeConcurrency,
|
||||
maxTokens: probeMaxTokens,
|
||||
},
|
||||
// Direct CLI probes create hidden sessions in the canonical agent DB.
|
||||
// Gateway RPC probes omit this because the Gateway already owns the lock.
|
||||
stateOwnership: { mode: "exclusive" },
|
||||
onProgress: update,
|
||||
});
|
||||
},
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Coordinates direct embedded state writers with the Gateway state-directory owner.
|
||||
import { createAbortError } from "./abort-signal.js";
|
||||
import type { GatewayLockIdentity, GatewayLockOptions } from "./gateway-lock.js";
|
||||
|
||||
export type EmbeddedStateSignal = "SIGINT" | "SIGTERM";
|
||||
|
||||
export type EmbeddedStateSignalProcess = {
|
||||
on(signal: EmbeddedStateSignal, handler: () => void): unknown;
|
||||
off(signal: EmbeddedStateSignal, handler: () => void): unknown;
|
||||
};
|
||||
|
||||
export type EmbeddedStateLockHandle = {
|
||||
release: () => Promise<void>;
|
||||
};
|
||||
|
||||
const EMBEDDED_STATE_SIGNALS: readonly EmbeddedStateSignal[] = ["SIGINT", "SIGTERM"];
|
||||
|
||||
function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
return Promise.reject(createAbortError("embedded state lock acquisition aborted"));
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(createAbortError("embedded state lock acquisition aborted"));
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/** Bridges process signals into embedded-run cancellation so lock cleanup can unwind. */
|
||||
export function createEmbeddedStateSignalBridge(processLike: EmbeddedStateSignalProcess = process) {
|
||||
const controller = new AbortController();
|
||||
let receivedSignal: EmbeddedStateSignal | undefined;
|
||||
const handlers = new Map<EmbeddedStateSignal, () => void>();
|
||||
const dispose = () => {
|
||||
for (const [signal, handler] of handlers) {
|
||||
processLike.off(signal, handler);
|
||||
}
|
||||
handlers.clear();
|
||||
};
|
||||
for (const signal of EMBEDDED_STATE_SIGNALS) {
|
||||
const handler = () => {
|
||||
receivedSignal = signal;
|
||||
if (!controller.signal.aborted) {
|
||||
controller.abort();
|
||||
dispose();
|
||||
}
|
||||
};
|
||||
handlers.set(signal, handler);
|
||||
processLike.on(signal, handler);
|
||||
}
|
||||
return {
|
||||
signal: controller.signal,
|
||||
getReceivedSignal: () => receivedSignal,
|
||||
dispose,
|
||||
};
|
||||
}
|
||||
|
||||
/** Probe the Gateway owner first, then acquire the shared embedded-writer role. */
|
||||
export async function acquireEmbeddedStateLock(params: {
|
||||
options?: GatewayLockOptions;
|
||||
signal?: AbortSignal;
|
||||
formatActiveGatewayRefusal: (identity: GatewayLockIdentity) => string;
|
||||
}): Promise<EmbeddedStateLockHandle | null> {
|
||||
const { acquireGatewayLock, GatewayLockError, readActiveGatewayLockIdentity } =
|
||||
await import("./gateway-lock.js");
|
||||
const env = params.options?.env ?? process.env;
|
||||
if (
|
||||
params.options?.allowInTests !== true &&
|
||||
(env.VITEST !== undefined || env.NODE_ENV === "test")
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const activeGateway = await readActiveGatewayLockIdentity(params.options);
|
||||
if (activeGateway) {
|
||||
throw new GatewayLockError(params.formatActiveGatewayRefusal(activeGateway));
|
||||
}
|
||||
try {
|
||||
return await acquireGatewayLock({
|
||||
...params.options,
|
||||
role: "agent-embedded",
|
||||
sleep: params.options?.sleep ?? (async (ms) => await abortableDelay(ms, params.signal)),
|
||||
});
|
||||
} catch (error) {
|
||||
if (!(error instanceof GatewayLockError)) {
|
||||
throw error;
|
||||
}
|
||||
const racedGateway = await readActiveGatewayLockIdentity(params.options);
|
||||
if (racedGateway) {
|
||||
throw new GatewayLockError(params.formatActiveGatewayRefusal(racedGateway), error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -174,7 +174,7 @@ describe("Gateway lock roles", () => {
|
||||
timeoutMs: 15,
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
`another openclaw agent --local run is active (pid ${process.pid}); lock timeout after 15ms`,
|
||||
`another embedded OpenClaw state writer is active (pid ${process.pid}); lock timeout after 15ms`,
|
||||
);
|
||||
} finally {
|
||||
await lock.release();
|
||||
|
||||
@@ -15,7 +15,12 @@ import { getFileLockProcessStartTime, isPidAlive } from "../shared/pid-alive.js"
|
||||
import { safeParseJsonWithSchema } from "../utils/zod-parse.js";
|
||||
import { sha256HexPrefix } from "./crypto-digest.js";
|
||||
import { createFileLockManager } from "./file-lock-manager.js";
|
||||
import { isGatewayArgv, isOpenClawCommandArgv, parseProcCmdline } from "./gateway-process-argv.js";
|
||||
import {
|
||||
isGatewayArgv,
|
||||
isOpenClawArgv,
|
||||
isOpenClawCommandArgv,
|
||||
parseProcCmdline,
|
||||
} from "./gateway-process-argv.js";
|
||||
import { tryAcquireExclusiveSqliteCoordinator } from "./node-sqlite.js";
|
||||
import {
|
||||
readWindowsProcessArgsSync,
|
||||
@@ -210,7 +215,10 @@ async function resolveGatewayOwnerStatus(
|
||||
return "unknown";
|
||||
}
|
||||
if (role === "agent-embedded") {
|
||||
return isOpenClawCommandArgv(args, "agent") && args.includes("--local") ? "alive" : "dead";
|
||||
// The role covers every direct embedded surface (agent --local, agent exec,
|
||||
// local TUI, and CLI model probes), so validate the owning OpenClaw process
|
||||
// instead of baking one command spelling into stale-lock recovery.
|
||||
return isOpenClawArgv(args) ? "alive" : "dead";
|
||||
}
|
||||
const command = role === "sqlite-maintenance" ? "doctor" : "skills";
|
||||
return isOpenClawCommandArgv(args, command) ? "alive" : "dead";
|
||||
@@ -557,7 +565,7 @@ async function acquireLockFile(
|
||||
const ownerPid = lastPayload?.pid ? ` (pid ${lastPayload.pid})` : "";
|
||||
const owner =
|
||||
lastPayload?.role === "agent-embedded"
|
||||
? `another openclaw agent --local run is active${ownerPid}`
|
||||
? `another embedded OpenClaw state writer is active${ownerPid}`
|
||||
: lastPayload?.role && lastPayload.role !== "gateway"
|
||||
? `state directory is locked by ${lastPayload.role}${ownerPid}`
|
||||
: `gateway already running${ownerPid}`;
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
// Tests gateway process argv parsing for diagnostics.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { isGatewayArgv, isOpenClawCommandArgv, parseProcCmdline } from "./gateway-process-argv.js";
|
||||
import {
|
||||
isGatewayArgv,
|
||||
isOpenClawArgv,
|
||||
isOpenClawCommandArgv,
|
||||
parseProcCmdline,
|
||||
} from "./gateway-process-argv.js";
|
||||
|
||||
describe("parseProcCmdline", () => {
|
||||
it("splits null-delimited argv and trims empty entries", () => {
|
||||
@@ -72,3 +77,18 @@ describe("isOpenClawCommandArgv", () => {
|
||||
expect(isOpenClawCommandArgv(["python", "doctor", "worker.py"], "doctor")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isOpenClawArgv", () => {
|
||||
it.each([
|
||||
["agent exec", ["openclaw", "agent", "exec", "task"]],
|
||||
["local TUI", ["node", "/srv/openclaw/openclaw.mjs", "tui", "--local"]],
|
||||
["models probe", ["openclaw", "models", "status", "--probe"]],
|
||||
["bare local TUI", ["openclaw"]],
|
||||
])("recognizes the %s embedded owner", (_label, argv) => {
|
||||
expect(isOpenClawArgv(argv)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects an unrelated process", () => {
|
||||
expect(isOpenClawArgv(["python", "worker.py"])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,18 +19,20 @@ export function parseProcCmdline(raw: string): string[] {
|
||||
return normalizeStringEntries(raw.split("\0"));
|
||||
}
|
||||
|
||||
export function isOpenClawCommandArgv(args: string[], command: string): boolean {
|
||||
export function isOpenClawArgv(args: string[]): boolean {
|
||||
const normalized = args.map(normalizeProcArg);
|
||||
const exe = (normalized[0] ?? "").replace(/\.(bat|cmd|exe)$/i, "");
|
||||
if (!normalized.includes(normalizeProcArg(command))) {
|
||||
return false;
|
||||
}
|
||||
if (normalized.some((arg) => ENTRY_CANDIDATES.some((entry) => arg.endsWith(entry)))) {
|
||||
return true;
|
||||
}
|
||||
return exe.endsWith("/openclaw") || exe === "openclaw";
|
||||
}
|
||||
|
||||
export function isOpenClawCommandArgv(args: string[], command: string): boolean {
|
||||
const normalizedCommand = normalizeProcArg(command);
|
||||
return args.some((arg) => normalizeProcArg(arg) === normalizedCommand) && isOpenClawArgv(args);
|
||||
}
|
||||
|
||||
export function isGatewayArgv(args: string[], opts?: { allowGatewayBinary?: boolean }): boolean {
|
||||
const normalized = args.map(normalizeProcArg);
|
||||
const exe = (normalized[0] ?? "").replace(/\.(bat|cmd|exe)$/i, "");
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { acquireGatewayLock, type GatewayLockOptions } from "../infra/gateway-lock.js";
|
||||
import { withEmbeddedTuiStateLock } from "./tui.js";
|
||||
|
||||
function createGatewayLockOptions(stateDir: string): GatewayLockOptions {
|
||||
return {
|
||||
allowInTests: true,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
},
|
||||
lockDir: path.join(stateDir, "gateway-locks"),
|
||||
readProcessStartTime: () => 123_456,
|
||||
timeoutMs: 100,
|
||||
};
|
||||
}
|
||||
|
||||
function createSignalProcess() {
|
||||
type SignalName = "SIGINT" | "SIGTERM";
|
||||
const listeners = new Map<SignalName, Set<() => void>>();
|
||||
const processLike = {
|
||||
on(signal: SignalName, handler: () => void) {
|
||||
const current = listeners.get(signal) ?? new Set<() => void>();
|
||||
current.add(handler);
|
||||
listeners.set(signal, current);
|
||||
return processLike;
|
||||
},
|
||||
off(signal: SignalName, handler: () => void) {
|
||||
listeners.get(signal)?.delete(handler);
|
||||
return processLike;
|
||||
},
|
||||
};
|
||||
return {
|
||||
processLike,
|
||||
emit(signal: SignalName) {
|
||||
for (const handler of listeners.get(signal) ?? []) {
|
||||
handler();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function withTempState<T>(run: (stateDir: string) => Promise<T>): Promise<T> {
|
||||
const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-tui-state-lock-"));
|
||||
try {
|
||||
return await run(stateDir);
|
||||
} finally {
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("embedded TUI state ownership", () => {
|
||||
it("refuses local startup while a live Gateway owns the state directory", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const gatewayLock = await acquireGatewayLock({ ...lockOptions, port: 28789 });
|
||||
expect(gatewayLock).not.toBeNull();
|
||||
if (!gatewayLock) {
|
||||
throw new Error("Expected live Gateway fixture lock");
|
||||
}
|
||||
const run = vi.fn(async () => undefined);
|
||||
try {
|
||||
await expect(
|
||||
withEmbeddedTuiStateLock(run, { gatewayLockOptions: lockOptions }),
|
||||
).rejects.toThrow(
|
||||
`A Gateway is running for this state directory (pid ${process.pid}, port 28789). Run without --local to use it, or stop the Gateway first (openclaw gateway stop).`,
|
||||
);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await gatewayLock.release();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("holds and releases embedded state ownership for the local TUI lifetime", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
|
||||
await withEmbeddedTuiStateLock(
|
||||
async () => {
|
||||
const payload = JSON.parse(await fs.readFile(stateLockPath, "utf8")) as {
|
||||
pid?: number;
|
||||
role?: string;
|
||||
};
|
||||
expect(payload).toMatchObject({ pid: process.pid, role: "agent-embedded" });
|
||||
},
|
||||
{ gatewayLockOptions: lockOptions },
|
||||
);
|
||||
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
|
||||
it("releases embedded state ownership when the local TUI receives SIGTERM", async () => {
|
||||
await withTempState(async (stateDir) => {
|
||||
const lockOptions = createGatewayLockOptions(stateDir);
|
||||
const stateLockPath = path.join(lockOptions.lockDir!, "gateway.state.lock");
|
||||
const signals = createSignalProcess();
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
markStarted = resolve;
|
||||
});
|
||||
const run = withEmbeddedTuiStateLock(
|
||||
async (signal) => {
|
||||
markStarted();
|
||||
return await new Promise<never>((_, reject) => {
|
||||
signal.addEventListener("abort", () => reject(new Error("local TUI interrupted")), {
|
||||
once: true,
|
||||
});
|
||||
});
|
||||
},
|
||||
{ gatewayLockOptions: lockOptions, process: signals.processLike },
|
||||
);
|
||||
await started;
|
||||
signals.emit("SIGTERM");
|
||||
|
||||
await expect(run).rejects.toThrow("local TUI interrupted");
|
||||
await expect(fs.stat(stateLockPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,8 +14,11 @@ import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/s
|
||||
import type { CommandEntry } from "../../packages/gateway-protocol/src/index.js";
|
||||
import { resolveAgentIdByWorkspacePath, resolveDefaultAgentId } from "../agents/agent-scope.js";
|
||||
import { normalizeThinkLevel } from "../auto-reply/thinking.shared.js";
|
||||
import { formatCliCommand } from "../cli/command-format.js";
|
||||
import { getRuntimeConfig, type OpenClawConfig } from "../config/config.js";
|
||||
import { resolveCanonicalMainSessionKey } from "../config/sessions/main-session-key.js";
|
||||
import type { EmbeddedStateSignalProcess } from "../infra/embedded-state-lock.js";
|
||||
import type { GatewayLockIdentity, GatewayLockOptions } from "../infra/gateway-lock.js";
|
||||
import { resolveCurrentOpenClawCliInvocation } from "../infra/openclaw-cli-invocation.js";
|
||||
import { tryProcessCwd } from "../infra/safe-cwd.js";
|
||||
import { registerUncaughtExceptionHandler } from "../infra/unhandled-rejections.js";
|
||||
@@ -598,7 +601,43 @@ function resolveEmptySessionInfoDefaults(config: OpenClawConfig): SessionInfo {
|
||||
};
|
||||
}
|
||||
|
||||
function formatActiveGatewayTuiRefusal(identity: GatewayLockIdentity): string {
|
||||
return `A Gateway is running for this state directory (pid ${identity.pid}, port ${identity.port}). Run without --local to use it, or stop the Gateway first (${formatCliCommand("openclaw gateway stop")}).`;
|
||||
}
|
||||
|
||||
/** Hold canonical state ownership for the complete lifetime of a local TUI. */
|
||||
export async function withEmbeddedTuiStateLock<T>(
|
||||
run: (signal: AbortSignal) => Promise<T>,
|
||||
deps: {
|
||||
gatewayLockOptions?: GatewayLockOptions;
|
||||
process?: EmbeddedStateSignalProcess;
|
||||
} = {},
|
||||
): Promise<T> {
|
||||
const { acquireEmbeddedStateLock, createEmbeddedStateSignalBridge } =
|
||||
await import("../infra/embedded-state-lock.js");
|
||||
const signalBridge = createEmbeddedStateSignalBridge(deps.process ?? process);
|
||||
let stateLock: Awaited<ReturnType<typeof acquireEmbeddedStateLock>> | undefined;
|
||||
try {
|
||||
stateLock = await acquireEmbeddedStateLock({
|
||||
options: deps.gatewayLockOptions,
|
||||
signal: signalBridge.signal,
|
||||
formatActiveGatewayRefusal: formatActiveGatewayTuiRefusal,
|
||||
});
|
||||
return await run(signalBridge.signal);
|
||||
} finally {
|
||||
await stateLock?.release();
|
||||
signalBridge.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
if (opts.local === true && opts.backend === undefined) {
|
||||
return await withEmbeddedTuiStateLock(async () => await runTuiUnlocked(opts));
|
||||
}
|
||||
return await runTuiUnlocked(opts);
|
||||
}
|
||||
|
||||
async function runTuiUnlocked(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
const isLocalMode = opts.local === true || opts.backend !== undefined;
|
||||
const config = opts.config ?? getRuntimeConfig({ skipPluginValidation: !isLocalMode });
|
||||
const cliInvocation = resolveCurrentOpenClawCliInvocation([]);
|
||||
|
||||
Reference in New Issue
Block a user