feat(gateway): add cooperative host suspension (#103618)

* feat(gateway): add cooperative suspension preparation

* style: satisfy suspension lint checks

* test(gateway): reset work admission between shared suites

* fix(gateway): reject upgrades during suspension

* fix(gateway): preserve admitted work during suspension

* test(gateway): isolate suspension and restart state

* fix(gateway): close suspension false-ready gaps

* refactor(protocol): slim suspension declaration graph

* refactor(plugin-sdk): sever protocol registry edges

* fix(gateway): preserve admitted restart follow-ups

* fix(gateway): make suspension recovery fail closed

* fix(protocol): keep validation formatter re-export only

* test(gateway): simplify deferred fixture type

* style(gateway): clarify suspension entry name

* fix(gateway): retain detached work admission
This commit is contained in:
Peter Steinberger
2026-07-10 20:24:53 +01:00
committed by GitHub
parent fece8c9f54
commit 1bcc4c5e70
116 changed files with 7955 additions and 1070 deletions
@@ -425,6 +425,222 @@ public struct ErrorShape: Codable, Sendable {
}
}
public struct GatewaySuspendTaskBlocker: Codable, Sendable {
public let taskid: String
public let status: String
public let runtime: AnyCodable
public let runid: String?
public let label: String?
public let title: String?
public init(
taskid: String,
status: String,
runtime: AnyCodable,
runid: String? = nil,
label: String? = nil,
title: String? = nil)
{
self.taskid = taskid
self.status = status
self.runtime = runtime
self.runid = runid
self.label = label
self.title = title
}
private enum CodingKeys: String, CodingKey {
case taskid = "taskId"
case status
case runtime
case runid = "runId"
case label
case title
}
}
public struct GatewaySuspendBlocker: Codable, Sendable {
public let kind: AnyCodable
public let count: Int
public let message: String
public let task: GatewaySuspendTaskBlocker?
public init(
kind: AnyCodable,
count: Int,
message: String,
task: GatewaySuspendTaskBlocker? = nil)
{
self.kind = kind
self.count = count
self.message = message
self.task = task
}
private enum CodingKeys: String, CodingKey {
case kind
case count
case message
case task
}
}
public struct GatewaySuspendPrepareParams: Codable, Sendable {
public let requestid: String
public init(
requestid: String)
{
self.requestid = requestid
}
private enum CodingKeys: String, CodingKey {
case requestid = "requestId"
}
}
public struct GatewaySuspendPrepareBusyResult: Codable, Sendable {
public let status: String
public let reason: AnyCodable
public let retryafterms: Int
public let activecount: Int
public let blockers: [GatewaySuspendBlocker]
public init(
status: String,
reason: AnyCodable,
retryafterms: Int,
activecount: Int,
blockers: [GatewaySuspendBlocker])
{
self.status = status
self.reason = reason
self.retryafterms = retryafterms
self.activecount = activecount
self.blockers = blockers
}
private enum CodingKeys: String, CodingKey {
case status
case reason
case retryafterms = "retryAfterMs"
case activecount = "activeCount"
case blockers
}
}
public struct GatewaySuspendPrepareReadyResult: Codable, Sendable {
public let status: String
public let suspensionid: String
public let expiresatms: Int
public let activecount: Int
public let blockers: [GatewaySuspendBlocker]
public init(
status: String,
suspensionid: String,
expiresatms: Int,
activecount: Int,
blockers: [GatewaySuspendBlocker])
{
self.status = status
self.suspensionid = suspensionid
self.expiresatms = expiresatms
self.activecount = activecount
self.blockers = blockers
}
private enum CodingKeys: String, CodingKey {
case status
case suspensionid = "suspensionId"
case expiresatms = "expiresAtMs"
case activecount = "activeCount"
case blockers
}
}
public struct GatewaySuspendStatusParams: Codable, Sendable {
public let suspensionid: String
public init(
suspensionid: String)
{
self.suspensionid = suspensionid
}
private enum CodingKeys: String, CodingKey {
case suspensionid = "suspensionId"
}
}
public struct GatewaySuspendStatusRunningResult: Codable, Sendable {
public let status: String
public init(
status: String)
{
self.status = status
}
private enum CodingKeys: String, CodingKey {
case status
}
}
public struct GatewaySuspendStatusReadyResult: Codable, Sendable {
public let status: String
public let expiresatms: Int
public init(
status: String,
expiresatms: Int)
{
self.status = status
self.expiresatms = expiresatms
}
private enum CodingKeys: String, CodingKey {
case status
case expiresatms = "expiresAtMs"
}
}
public struct GatewaySuspendResumeParams: Codable, Sendable {
public let suspensionid: String
public init(
suspensionid: String)
{
self.suspensionid = suspensionid
}
private enum CodingKeys: String, CodingKey {
case suspensionid = "suspensionId"
}
}
public struct GatewaySuspendResumeResult: Codable, Sendable {
public let ok: Bool
public let status: String
public let resumed: Bool
public init(
ok: Bool,
status: String,
resumed: Bool)
{
self.ok = ok
self.status = status
self.resumed = resumed
}
private enum CodingKeys: String, CodingKey {
case ok
case status
case resumed
}
}
public struct EnvironmentSummary: Codable, Sendable {
public let id: String
public let type: String
@@ -9874,6 +10090,68 @@ public struct ShutdownEvent: Codable, Sendable {
}
}
public enum GatewaySuspendPrepareResult: Codable, Sendable {
case busy(GatewaySuspendPrepareBusyResult)
case ready(GatewaySuspendPrepareReadyResult)
private enum CodingKeys: String, CodingKey {
case discriminator = "status"
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let discriminator = try container.decode(String.self, forKey: .discriminator)
switch discriminator {
case "busy": self = try .busy(GatewaySuspendPrepareBusyResult(from: decoder))
case "ready": self = try .ready(GatewaySuspendPrepareReadyResult(from: decoder))
default:
throw DecodingError.dataCorruptedError(
forKey: .discriminator,
in: container,
debugDescription: "Unknown GatewaySuspendPrepareResult discriminator value"
)
}
}
public func encode(to encoder: Encoder) throws {
switch self {
case .busy(let value): try value.encode(to: encoder)
case .ready(let value): try value.encode(to: encoder)
}
}
}
public enum GatewaySuspendStatusResult: Codable, Sendable {
case running(GatewaySuspendStatusRunningResult)
case ready(GatewaySuspendStatusReadyResult)
private enum CodingKeys: String, CodingKey {
case discriminator = "status"
}
public init(from decoder: Decoder) throws {
let container = try decoder.container(keyedBy: CodingKeys.self)
let discriminator = try container.decode(String.self, forKey: .discriminator)
switch discriminator {
case "running": self = try .running(GatewaySuspendStatusRunningResult(from: decoder))
case "ready": self = try .ready(GatewaySuspendStatusReadyResult(from: decoder))
default:
throw DecodingError.dataCorruptedError(
forKey: .discriminator,
in: container,
debugDescription: "Unknown GatewaySuspendStatusResult discriminator value"
)
}
}
public func encode(to encoder: Encoder) throws {
switch self {
case .running(let value): try value.encode(to: encoder)
case .ready(let value): try value.encode(to: encoder)
}
}
}
public enum PluginCatalogInstallAction: Codable, Sendable {
case clawhub(PluginCatalogClawHubInstall)
case official(PluginCatalogOfficialInstall)
+1
View File
@@ -3339,6 +3339,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- Headings:
- H2: What is available today
- H2: Recommended path
- H2: Cooperative host suspension
- H2: App code vs plugin code
- H2: Related
+88
View File
@@ -50,6 +50,94 @@ terminal result. For durable conversation state, use the `sessions.*` methods.
For UI integrations, subscribe to Gateway events and render only the event
families your app understands.
## Cooperative host suspension
Hosting controllers that freeze or snapshot a running process can use the
host-neutral suspension handshake:
1. Stop admitting external ingress controlled by the host.
2. Call `gateway.suspend.prepare` with a stable, unique `requestId`.
3. If the response is `busy`, keep the process running and retry later.
4. If it is `ready`, save the returned `suspensionId`, then freeze or snapshot
the process before `expiresAtMs`.
5. After thaw, or if suspension is abandoned, call `gateway.suspend.resume`
with that `suspensionId` over the existing WebSocket or Admin HTTP control
path.
A prepared Gateway rejects new WebSocket handshakes. A WebSocket controller
must keep its authenticated connection open across the host operation. If that
cannot be guaranteed, enable and use the
[Admin HTTP RPC plugin](/plugins/admin-http-rpc) before preparing. If the
control path is lost, wait for the two-minute lease to expire before
reconnecting; expiry reopens admission automatically.
The RPC contract is:
- `gateway.suspend.prepare``operator.admin`; params
`{ "requestId": "stable-host-operation-id" }`
- `gateway.suspend.status``operator.read`; params
`{ "suspensionId": "id-from-prepare" }`
- `gateway.suspend.resume``operator.admin`; params
`{ "suspensionId": "id-from-prepare" }`
IDs are trimmed, must contain a non-whitespace character, and are limited to
128 characters. A busy prepare result has `status: "busy"`, `reason`,
`retryAfterMs`, `activeCount`, and `blockers`. A ready result has this shape:
```json
{
"status": "ready",
"suspensionId": "2c3f...",
"expiresAtMs": 1770000000000,
"activeCount": 0,
"blockers": []
}
```
Status returns `{"status":"running"}` or a ready result with `expiresAtMs`.
Resume returns `{"ok":true,"status":"running","resumed":true}`; repeating it
after a successful resume returns `resumed: false`.
A competing request ID or transient scheduler-resume failure returns retryable
`UNAVAILABLE` with `retryAfterMs`. During scheduler recovery, prepare, status,
and resume all return that error, the Gateway remains not-ready and
fail-closed, and the host must not freeze or snapshot it. OpenClaw retries the
scheduler automatically and reopens admission only after recovery succeeds. A
mismatched resume ID returns `INVALID_REQUEST`. Prepare shares the Gateway's
control-plane write budget of three attempts per minute; honor the returned
retry delay. WebSocket clients are bucketed by device and IP. Admin HTTP
controllers are bucketed by resolved client IP, so controllers behind one
proxy can share a budget.
Preparation is refuse-only: OpenClaw closes new root/session/command admission,
pauses automatic cron ticks, and inspects work synchronously. If anything is
active, it resumes the scheduler and reopens admission before returning
`busy`; it does not interrupt or drain that work. A ready lease lasts two
minutes. Repeating `prepare` with the same `requestId` renews it; expiry resumes
the scheduler before reopening admission.
Restart emission that becomes due during a ready lease waits until the lease
resumes; an in-flight restart makes preparation return `busy`.
While ready, `/healthz` remains live and `/readyz` returns `503`. Local or
authenticated readiness responses include `gateway-draining`; unauthenticated
remote probes receive only `{ "ready": false }`. The HTTP health probe,
suspension methods on existing WebSocket connections, and an already-enabled
Admin HTTP RPC route remain available. Other RPCs return retryable
`UNAVAILABLE`. Built-in HTTP user-work routes, including OpenAI-compatible
APIs, tool/session operations, node watches, and configured hooks, return
`503` with `error.code: "gateway_unavailable"`.
This handshake does not persist incoming messages, stop third-party channel
transports, or control the hosting platform. The host must fence its ingress
before preparation and remains responsible for wake, snapshot/freeze, and
stop. `activeCount` is the aggregate tracked-work count, while `blockers`
contains the non-zero category counts and bounded task details. This is not a
general process-quiescence barrier. Channel health, maintenance, cache refresh,
plugin-owned HTTP routes, and plugin-owned background work can remain active.
The hosting platform must freeze or snapshot the full process tree and its
filesystem consistently; unregistered work cannot be proven idle by this first
contract.
## App code vs plugin code
Use Gateway RPC when code lives outside OpenClaw:
+1
View File
@@ -348,6 +348,7 @@ methods. Treat this as feature discovery, not a full enumeration of
- `system-event` appends a system event and can update/broadcast presence context.
- `last-heartbeat` returns the latest persisted heartbeat event.
- `set-heartbeats` toggles heartbeat processing on the gateway.
- `gateway.suspend.prepare` creates a short cooperative-suspension lease only when tracked Gateway work is idle. `gateway.suspend.status` checks that lease, and `gateway.suspend.resume` releases it after thaw or an aborted host operation.
</Accordion>
+3 -3
View File
@@ -168,7 +168,7 @@ HTTP status follows the error code:
- discovery: `commands.list`
Returns the HTTP RPC method names allowed by this plugin.
- gateway: `health`, `status`, `logs.tail`, `usage.status`, `usage.cost`, `gateway.restart.request`
- gateway: `health`, `status`, `logs.tail`, `usage.status`, `usage.cost`, `gateway.restart.request`, `gateway.suspend.prepare`, `gateway.suspend.status`, `gateway.suspend.resume`
- config: `config.get`, `config.schema`, `config.schema.lookup`, `config.set`, `config.patch`, `config.apply`
- channels: `channels.status`, `channels.start`, `channels.stop`, `channels.logout`
- web: `web.login.start`, `web.login.wait`
@@ -209,11 +209,11 @@ Shared-token WebSocket clients without a trusted device identity cannot self-dec
`400 INVALID_REQUEST`
: The request body is not valid JSON, the `method` field is missing, or the method is not in the plugin allowlist.
: The request body is not valid JSON, the `method` field is missing, the method is not in the plugin allowlist, or a suspension resume ID does not match the active lease.
`503 UNAVAILABLE`
: The Gateway method handler is unavailable. Check Gateway logs and retry after the Gateway finishes startup.
: The Gateway method is starting, rate-limited, suspended, or waiting on a competing suspension/resume operation. Inspect `error.details` when present and honor `error.retryAfterMs` before retrying.
## Related
@@ -133,6 +133,27 @@ describe("admin-http-rpc plugin handler", () => {
},
);
it.each([
["gateway.suspend.prepare", { requestId: "host-request-1" }],
["gateway.suspend.status", { suspensionId: "suspension-1" }],
["gateway.suspend.resume", { suspensionId: "suspension-1" }],
] as const)("dispatches suspension method %s through Admin HTTP", async (method, params) => {
dispatchGatewayMethod.mockResolvedValueOnce({
ok: true,
payload: { status: "ok" },
});
const result = await invoke({ id: "suspension", method, params });
expect(dispatchGatewayMethod).toHaveBeenCalledWith(method, params);
expect(result.captured.statusCode).toBe(200);
expect(result.json).toEqual({
id: "suspension",
ok: true,
payload: { status: "ok" },
});
});
it("rejects methods outside the admin HTTP RPC allowlist", async () => {
const result = await invoke({ id: "bad", method: "sessions.send" });
+3
View File
@@ -10,6 +10,9 @@ const ADMIN_HTTP_RPC_ALLOWED_METHOD_GROUPS = {
"usage.status",
"usage.cost",
"gateway.restart.request",
"gateway.suspend.prepare",
"gateway.suspend.status",
"gateway.suspend.resume",
],
discovery: ["commands.list"],
config: [
+5 -7
View File
@@ -1,12 +1,5 @@
// Gateway Client module implements client behavior.
import { randomUUID } from "node:crypto";
import type {
ConnectParams,
ErrorShape,
EventFrame,
HelloOk,
RequestFrame,
} from "@openclaw/gateway-protocol";
import {
GATEWAY_CLIENT_MODES,
GATEWAY_CLIENT_NAMES,
@@ -22,8 +15,13 @@ import {
type ConnectErrorRecoveryAdvice,
} from "@openclaw/gateway-protocol/connect-error-details";
import {
type ConnectParams,
type ErrorShape,
type EventFrame,
type HelloOk,
isGatewayEventFrame,
isGatewayResponseFrame,
type RequestFrame,
} from "@openclaw/gateway-protocol/frame-guards";
import { resolveGatewayStartupRetryAfterMs } from "@openclaw/gateway-protocol/startup-unavailable";
import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION } from "@openclaw/gateway-protocol/version";
+10 -1
View File
@@ -1,4 +1,13 @@
import type { EventFrame, ResponseFrame } from "./schema/types.js";
import type { EventFrame, ResponseFrame } from "./schema/frames.js";
export type {
ConnectParams,
ErrorShape,
EventFrame,
GatewayFrame,
HelloOk,
RequestFrame,
ResponseFrame,
} from "./schema/frames.js";
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
@@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import {
validateGatewaySuspendPrepareParams,
validateGatewaySuspendPrepareResult,
validateGatewaySuspendResumeResult,
validateGatewaySuspendStatusResult,
} from "./index.js";
describe("gateway suspension protocol", () => {
it("keeps prepare params closed and bounded", () => {
expect(validateGatewaySuspendPrepareParams({ requestId: "host-request" })).toBe(true);
expect(validateGatewaySuspendPrepareParams({ requestId: " " })).toBe(false);
expect(validateGatewaySuspendPrepareParams({ requestId: "host-request", extra: true })).toBe(
false,
);
});
it("validates busy and ready prepare results", () => {
expect(
validateGatewaySuspendPrepareResult({
status: "busy",
reason: "active-work",
retryAfterMs: 20_000,
activeCount: 2,
blockers: [
{ kind: "queue", count: 1, message: "one queued operation" },
{
kind: "task",
count: 1,
message: "one active task",
task: { taskId: "task-1", status: "running", runtime: "subagent" },
},
],
}),
).toBe(true);
expect(
validateGatewaySuspendPrepareResult({
status: "ready",
suspensionId: "suspension-id",
expiresAtMs: 123,
activeCount: 0,
blockers: [],
}),
).toBe(true);
});
it("validates status and resume results", () => {
expect(validateGatewaySuspendStatusResult({ status: "running" })).toBe(true);
expect(validateGatewaySuspendStatusResult({ status: "ready", expiresAtMs: 123 })).toBe(true);
expect(
validateGatewaySuspendResumeResult({ ok: true, status: "running", resumed: false }),
).toBe(true);
expect(
validateGatewaySuspendResumeResult({
ok: true,
status: "running",
resumed: false,
warnings: [],
}),
).toBe(false);
});
it("keeps scheduler recovery on the error frame instead of success results", () => {
const recovering = {
status: "recovering",
reason: "scheduler-resume-failed",
retryAfterMs: 1_000,
};
expect(validateGatewaySuspendPrepareResult(recovering)).toBe(false);
expect(validateGatewaySuspendStatusResult(recovering)).toBe(false);
});
});
+60 -84
View File
@@ -9,6 +9,8 @@ export {
type ClawHubTrustErrorDetails,
} from "./clawhub-trust-error-details.js";
import { Compile, type Validator as TypeBoxValidator } from "typebox/compile";
import type { ValidationError } from "./validation-errors.js";
export { formatValidationErrors, type ValidationError } from "./validation-errors.js";
import {
type AgentEvent,
AgentEventSchema,
@@ -185,6 +187,26 @@ import {
UpdateStatusParamsSchema,
type ConnectParams,
ConnectParamsSchema,
type GatewaySuspendBlocker,
GatewaySuspendBlockerSchema,
type GatewaySuspendPrepareParams,
GatewaySuspendPrepareBusyResultSchema,
GatewaySuspendPrepareParamsSchema,
GatewaySuspendPrepareReadyResultSchema,
type GatewaySuspendPrepareResult,
GatewaySuspendPrepareResultSchema,
type GatewaySuspendResumeParams,
GatewaySuspendResumeParamsSchema,
type GatewaySuspendResumeResult,
GatewaySuspendResumeResultSchema,
type GatewaySuspendStatusParams,
GatewaySuspendStatusParamsSchema,
GatewaySuspendStatusReadyResultSchema,
type GatewaySuspendStatusResult,
GatewaySuspendStatusResultSchema,
GatewaySuspendStatusRunningResultSchema,
type GatewaySuspendTaskBlocker,
GatewaySuspendTaskBlockerSchema,
type CronAddParams,
CronAddParamsSchema,
type CronAddResult,
@@ -664,20 +686,6 @@ import {
WorktreesBranchesResultSchema,
} from "./schema.js";
/** Normalized validation error shape exposed by every protocol validator. */
export type ValidationError = {
/** Failed schema keyword, when the validator can report one. */
keyword?: string;
/** JSON-pointer path to the failing data location. */
instancePath?: string;
/** JSON-pointer path to the failing schema location. */
schemaPath?: string;
/** Validator-specific keyword parameters for richer diagnostics. */
params?: Record<string, unknown>;
/** Human-readable validation message. */
message?: string;
};
/** Runtime validator shape shared by gateway clients and server handlers. */
export type ProtocolValidator<T = unknown> = ((data: unknown) => data is T) & {
/** Last validation errors, matching Ajv-style caller expectations. */
@@ -728,6 +736,24 @@ function lazyCompile<T = unknown>(schema: unknown): ProtocolValidator<T> {
// constants so call sites can pair validation with the wire contract directly.
export const validateCommandsListParams = lazyCompile<CommandsListParams>(CommandsListParamsSchema);
export const validateConnectParams = lazyCompile<ConnectParams>(ConnectParamsSchema);
export const validateGatewaySuspendPrepareParams = lazyCompile<GatewaySuspendPrepareParams>(
GatewaySuspendPrepareParamsSchema,
);
export const validateGatewaySuspendPrepareResult = lazyCompile<GatewaySuspendPrepareResult>(
GatewaySuspendPrepareResultSchema,
);
export const validateGatewaySuspendStatusParams = lazyCompile<GatewaySuspendStatusParams>(
GatewaySuspendStatusParamsSchema,
);
export const validateGatewaySuspendStatusResult = lazyCompile<GatewaySuspendStatusResult>(
GatewaySuspendStatusResultSchema,
);
export const validateGatewaySuspendResumeParams = lazyCompile<GatewaySuspendResumeParams>(
GatewaySuspendResumeParamsSchema,
);
export const validateGatewaySuspendResumeResult = lazyCompile<GatewaySuspendResumeResult>(
GatewaySuspendResumeResultSchema,
);
export const validateRequestFrame = lazyCompile<RequestFrame>(RequestFrameSchema);
export const validateResponseFrame = lazyCompile<ResponseFrame>(ResponseFrameSchema);
export const validateEventFrame = lazyCompile<EventFrame>(EventFrameSchema);
@@ -1199,77 +1225,22 @@ export const validateWebLoginStartParams =
lazyCompile<WebLoginStartParams>(WebLoginStartParamsSchema);
export const validateWebLoginWaitParams = lazyCompile<WebLoginWaitParams>(WebLoginWaitParamsSchema);
function firstStringParam(value: unknown): string | undefined {
if (typeof value === "string" && value.trim()) {
return value;
}
if (Array.isArray(value)) {
return value.find(
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
);
}
return undefined;
}
/** Convert validator errors into compact operator-facing failure text. */
export function formatValidationErrors(errors: ValidationError[] | null | undefined) {
if (!errors?.length) {
return "unknown validation error";
}
const parts: string[] = [];
for (const err of errors) {
const keyword = typeof err?.keyword === "string" ? err.keyword : "";
const instancePath = typeof err?.instancePath === "string" ? err.instancePath : "";
if (keyword === "additionalProperties") {
const additionalProperty =
firstStringParam(err?.params?.additionalProperty) ??
firstStringParam(err?.params?.additionalProperties);
if (additionalProperty) {
const where = instancePath ? `at ${instancePath}` : "at root";
parts.push(`${where}: unexpected property '${additionalProperty}'`);
continue;
}
}
if (keyword === "required") {
const missingProperty =
firstStringParam(err?.params?.missingProperty) ??
firstStringParam(err?.params?.requiredProperties);
if (missingProperty) {
const where = instancePath ? `at ${instancePath}: ` : "";
parts.push(`${where}must have required property '${missingProperty}'`);
continue;
}
}
const failingKeyword =
typeof err?.params?.failingKeyword === "string" ? err.params.failingKeyword : "";
// TypeBox reports conditional required-property misses through if/then
// keywords, which otherwise hide the actionable missing-property context.
const message =
keyword === "then" || (keyword === "if" && failingKeyword === "then")
? "must have required conditional properties"
: typeof err?.message === "string" && err.message.trim()
? err.message
: "validation error";
const where = instancePath ? `at ${instancePath}: ` : "";
parts.push(`${where}${message}`);
}
// De-dupe while preserving order.
const unique = uniqueStrings(parts.filter((part) => part.trim()));
if (!unique.length) {
return "unknown validation error";
}
return unique.join("; ");
}
// Schema exports stay explicit to make additions/removals reviewable as public
// protocol surface changes.
export {
ConnectParamsSchema,
GatewaySuspendTaskBlockerSchema,
GatewaySuspendBlockerSchema,
GatewaySuspendPrepareBusyResultSchema,
GatewaySuspendPrepareParamsSchema,
GatewaySuspendPrepareReadyResultSchema,
GatewaySuspendPrepareResultSchema,
GatewaySuspendStatusReadyResultSchema,
GatewaySuspendStatusRunningResultSchema,
GatewaySuspendStatusParamsSchema,
GatewaySuspendStatusResultSchema,
GatewaySuspendResumeParamsSchema,
GatewaySuspendResumeResultSchema,
GATEWAY_SERVER_CAPS,
HelloOkSchema,
RequestFrameSchema,
@@ -1577,6 +1548,14 @@ export {
export type {
GatewayFrame,
ConnectParams,
GatewaySuspendTaskBlocker,
GatewaySuspendBlocker,
GatewaySuspendPrepareParams,
GatewaySuspendPrepareResult,
GatewaySuspendStatusParams,
GatewaySuspendStatusResult,
GatewaySuspendResumeParams,
GatewaySuspendResumeResult,
HelloOk,
RequestFrame,
ResponseFrame,
@@ -1869,9 +1848,6 @@ export type {
SessionsGroupsDeleteParams,
SessionsGroupsMutationResult,
};
function uniqueStrings(values: string[]): string[] {
return [...new Set(values)];
}
// The protocol package cannot import core session types. This local structural
// result mirrors the wire contract and keeps the package independent of src/.
+1
View File
@@ -20,6 +20,7 @@ export * from "./schema/environments.js";
export * from "./schema/exec-approvals.js";
export * from "./schema/devices.js";
export * from "./schema/frames.js";
export * from "./schema/gateway-suspend.js";
export * from "./schema/logs-chat.js";
export * from "./schema/nodes.js";
export * from "./schema/protocol-schemas.js";
@@ -1,5 +1,5 @@
// Gateway Protocol schema module defines protocol validation shapes.
import type { ErrorShape } from "./types.js";
import type { ErrorShape } from "./frames.js";
/** Gateway JSON-RPC style error codes shared by clients and server handlers. */
export const ErrorCodes = {
@@ -1,4 +1,5 @@
// Gateway Protocol schema module defines protocol validation shapes.
import type { Static } from "typebox";
import { Type } from "typebox";
import { GatewayClientIdSchema, GatewayClientModeSchema, NonEmptyString } from "./primitives.js";
import { SnapshotSchema, StateVersionSchema } from "./snapshot.js";
@@ -213,3 +214,13 @@ export const GatewayFrameSchema = Type.Union(
[RequestFrameSchema, ResponseFrameSchema, EventFrameSchema],
{ discriminator: "type" },
);
// Frame types are owner-local because they cross the public client/plugin SDK.
// Keeping them off the aggregate registry avoids retaining every RPC schema.
export type ConnectParams = Static<typeof ConnectParamsSchema>;
export type HelloOk = Static<typeof HelloOkSchema>;
export type ErrorShape = Static<typeof ErrorShapeSchema>;
export type RequestFrame = Static<typeof RequestFrameSchema>;
export type ResponseFrame = Static<typeof ResponseFrameSchema>;
export type EventFrame = Static<typeof EventFrameSchema>;
export type GatewayFrame = Static<typeof GatewayFrameSchema>;
@@ -0,0 +1,108 @@
// Gateway Protocol schemas for cooperative host suspension.
import { Type } from "typebox";
const SuspensionTokenSchema = Type.String({ minLength: 1, maxLength: 128, pattern: "\\S" });
const CountSchema = Type.Integer({ minimum: 0 });
export const GatewaySuspendTaskBlockerSchema = Type.Object(
{
taskId: Type.String(),
status: Type.Literal("running"),
runtime: Type.Union([
Type.Literal("subagent"),
Type.Literal("acp"),
Type.Literal("cli"),
Type.Literal("cron"),
]),
runId: Type.Optional(Type.String()),
label: Type.Optional(Type.String()),
title: Type.Optional(Type.String()),
},
{ additionalProperties: false },
);
export const GatewaySuspendBlockerSchema = Type.Object(
{
kind: Type.Union([
Type.Literal("queue"),
Type.Literal("reply"),
Type.Literal("embedded-run"),
Type.Literal("cron-run"),
Type.Literal("task"),
Type.Literal("root-request"),
Type.Literal("session-admission"),
Type.Literal("session-mutation"),
Type.Literal("chat-run"),
Type.Literal("queued-turn"),
Type.Literal("terminal-persistence"),
Type.Literal("terminal-session"),
]),
count: CountSchema,
message: Type.String(),
task: Type.Optional(GatewaySuspendTaskBlockerSchema),
},
{ additionalProperties: false },
);
export const GatewaySuspendPrepareParamsSchema = Type.Object(
{ requestId: SuspensionTokenSchema },
{ additionalProperties: false },
);
export const GatewaySuspendPrepareBusyResultSchema = Type.Object(
{
status: Type.Literal("busy"),
reason: Type.Union([Type.Literal("active-work"), Type.Literal("gateway-draining")]),
retryAfterMs: CountSchema,
activeCount: CountSchema,
blockers: Type.Array(GatewaySuspendBlockerSchema),
},
{ additionalProperties: false },
);
export const GatewaySuspendPrepareReadyResultSchema = Type.Object(
{
status: Type.Literal("ready"),
suspensionId: SuspensionTokenSchema,
expiresAtMs: CountSchema,
activeCount: CountSchema,
blockers: Type.Array(GatewaySuspendBlockerSchema),
},
{ additionalProperties: false },
);
export const GatewaySuspendPrepareResultSchema = Type.Union([
GatewaySuspendPrepareBusyResultSchema,
GatewaySuspendPrepareReadyResultSchema,
]);
export const GatewaySuspendStatusParamsSchema = Type.Object(
{ suspensionId: SuspensionTokenSchema },
{ additionalProperties: false },
);
export const GatewaySuspendStatusRunningResultSchema = Type.Object(
{ status: Type.Literal("running") },
{ additionalProperties: false },
);
export const GatewaySuspendStatusReadyResultSchema = Type.Object(
{ status: Type.Literal("ready"), expiresAtMs: CountSchema },
{ additionalProperties: false },
);
export const GatewaySuspendStatusResultSchema = Type.Union([
GatewaySuspendStatusRunningResultSchema,
GatewaySuspendStatusReadyResultSchema,
]);
export const GatewaySuspendResumeParamsSchema = GatewaySuspendStatusParamsSchema;
export const GatewaySuspendResumeResultSchema = Type.Object(
{
ok: Type.Literal(true),
status: Type.Literal("running"),
resumed: Type.Boolean(),
},
{ additionalProperties: false },
);
@@ -221,6 +221,20 @@ import {
ShutdownEventSchema,
TickEventSchema,
} from "./frames.js";
import {
GatewaySuspendBlockerSchema,
GatewaySuspendPrepareBusyResultSchema,
GatewaySuspendPrepareParamsSchema,
GatewaySuspendPrepareReadyResultSchema,
GatewaySuspendPrepareResultSchema,
GatewaySuspendResumeParamsSchema,
GatewaySuspendResumeResultSchema,
GatewaySuspendStatusReadyResultSchema,
GatewaySuspendStatusRunningResultSchema,
GatewaySuspendStatusParamsSchema,
GatewaySuspendStatusResultSchema,
GatewaySuspendTaskBlockerSchema,
} from "./gateway-suspend.js";
import {
ChatAbortedEventSchema,
ChatAbortParamsSchema,
@@ -420,6 +434,18 @@ export const ProtocolSchemas = {
StateVersion: StateVersionSchema,
Snapshot: SnapshotSchema,
ErrorShape: ErrorShapeSchema,
GatewaySuspendTaskBlocker: GatewaySuspendTaskBlockerSchema,
GatewaySuspendBlocker: GatewaySuspendBlockerSchema,
GatewaySuspendPrepareParams: GatewaySuspendPrepareParamsSchema,
GatewaySuspendPrepareBusyResult: GatewaySuspendPrepareBusyResultSchema,
GatewaySuspendPrepareReadyResult: GatewaySuspendPrepareReadyResultSchema,
GatewaySuspendPrepareResult: GatewaySuspendPrepareResultSchema,
GatewaySuspendStatusParams: GatewaySuspendStatusParamsSchema,
GatewaySuspendStatusRunningResult: GatewaySuspendStatusRunningResultSchema,
GatewaySuspendStatusReadyResult: GatewaySuspendStatusReadyResultSchema,
GatewaySuspendStatusResult: GatewaySuspendStatusResultSchema,
GatewaySuspendResumeParams: GatewaySuspendResumeParamsSchema,
GatewaySuspendResumeResult: GatewaySuspendResumeResultSchema,
// Environment and agent-facing control RPC payloads.
EnvironmentStatus: EnvironmentStatusSchema,
@@ -1,4 +1,5 @@
// Gateway Protocol schema module defines protocol validation shapes.
import type { Static } from "typebox";
import { Type } from "typebox";
import { PluginJsonValueSchema } from "./plugins.js";
import { NonEmptyString, SessionLabelString } from "./primitives.js";
@@ -406,6 +407,7 @@ export const SessionsPatchParamsSchema = Type.Object(
},
{ additionalProperties: false },
);
export type SessionsPatchParams = Static<typeof SessionsPatchParamsSchema>;
/** Updates or clears one plugin namespace value on a session record. */
export const SessionsPluginPatchParamsSchema = Type.Object(
+13 -12
View File
@@ -1,28 +1,30 @@
/**
* Static TypeScript types derived from the canonical gateway protocol schemas.
*
* Keep aliases wired through `ProtocolSchemas` so validators, runtime schemas,
* and exported compile-time types cannot drift apart.
* Owner-local schema modules export hot public types directly. The remaining
* aliases stay wired through `ProtocolSchemas` so validators, runtime schemas,
* and compile-time types cannot drift apart.
*/
import type { Static } from "typebox";
import { ProtocolSchemas } from "./protocol-schemas.js";
import type { ProtocolSchemas } from "./protocol-schemas.js";
/** Stable schema names registered in the protocol schema registry. */
type ProtocolSchemaName = keyof typeof ProtocolSchemas;
/** Inferred TypeScript type for a named TypeBox protocol schema. */
type SchemaType<TName extends ProtocolSchemaName> = Static<(typeof ProtocolSchemas)[TName]>;
/** Connection handshake, envelope, snapshot, and shared error wire types. */
export type ConnectParams = SchemaType<"ConnectParams">;
export type HelloOk = SchemaType<"HelloOk">;
export type RequestFrame = SchemaType<"RequestFrame">;
export type ResponseFrame = SchemaType<"ResponseFrame">;
export type EventFrame = SchemaType<"EventFrame">;
export type GatewayFrame = SchemaType<"GatewayFrame">;
/** Snapshot and shared state wire types. */
export type Snapshot = SchemaType<"Snapshot">;
export type PresenceEntry = SchemaType<"PresenceEntry">;
export type ErrorShape = SchemaType<"ErrorShape">;
export type StateVersion = SchemaType<"StateVersion">;
export type GatewaySuspendTaskBlocker = SchemaType<"GatewaySuspendTaskBlocker">;
export type GatewaySuspendBlocker = SchemaType<"GatewaySuspendBlocker">;
export type GatewaySuspendPrepareParams = SchemaType<"GatewaySuspendPrepareParams">;
export type GatewaySuspendPrepareResult = SchemaType<"GatewaySuspendPrepareResult">;
export type GatewaySuspendStatusParams = SchemaType<"GatewaySuspendStatusParams">;
export type GatewaySuspendStatusResult = SchemaType<"GatewaySuspendStatusResult">;
export type GatewaySuspendResumeParams = SchemaType<"GatewaySuspendResumeParams">;
export type GatewaySuspendResumeResult = SchemaType<"GatewaySuspendResumeResult">;
/** Environment status RPC payloads used by CLI and Control UI surfaces. */
export type EnvironmentStatus = SchemaType<"EnvironmentStatus">;
@@ -113,7 +115,6 @@ export type SessionsSendParams = SchemaType<"SessionsSendParams">;
export type SessionsMessagesSubscribeParams = SchemaType<"SessionsMessagesSubscribeParams">;
export type SessionsMessagesUnsubscribeParams = SchemaType<"SessionsMessagesUnsubscribeParams">;
export type SessionsAbortParams = SchemaType<"SessionsAbortParams">;
export type SessionsPatchParams = SchemaType<"SessionsPatchParams">;
export type SessionsPluginPatchParams = SchemaType<"SessionsPluginPatchParams">;
export type SessionsPluginPatchResult = SchemaType<"SessionsPluginPatchResult">;
export type SessionsResetParams = SchemaType<"SessionsResetParams">;
@@ -0,0 +1,76 @@
/** Normalized validation error shape exposed by every protocol validator. */
export type ValidationError = {
/** Failed schema keyword, when the validator can report one. */
keyword?: string;
/** JSON-pointer path to the failing data location. */
instancePath?: string;
/** JSON-pointer path to the failing schema location. */
schemaPath?: string;
/** Validator-specific keyword parameters for richer diagnostics. */
params?: Record<string, unknown>;
/** Human-readable validation message. */
message?: string;
};
function firstStringParam(value: unknown): string | undefined {
if (typeof value === "string" && value.trim()) {
return value;
}
if (Array.isArray(value)) {
return value.find(
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
);
}
return undefined;
}
/** Convert validator errors into compact operator-facing failure text. */
export function formatValidationErrors(errors: ValidationError[] | null | undefined) {
if (!errors?.length) {
return "unknown validation error";
}
const parts: string[] = [];
for (const err of errors) {
const keyword = typeof err?.keyword === "string" ? err.keyword : "";
const instancePath = typeof err?.instancePath === "string" ? err.instancePath : "";
if (keyword === "additionalProperties") {
const additionalProperty =
firstStringParam(err?.params?.additionalProperty) ??
firstStringParam(err?.params?.additionalProperties);
if (additionalProperty) {
const where = instancePath ? `at ${instancePath}` : "at root";
parts.push(`${where}: unexpected property '${additionalProperty}'`);
continue;
}
}
if (keyword === "required") {
const missingProperty =
firstStringParam(err?.params?.missingProperty) ??
firstStringParam(err?.params?.requiredProperties);
if (missingProperty) {
const where = instancePath ? `at ${instancePath}: ` : "";
parts.push(`${where}must have required property '${missingProperty}'`);
continue;
}
}
const failingKeyword =
typeof err?.params?.failingKeyword === "string" ? err.params.failingKeyword : "";
// TypeBox reports conditional required-property misses through if/then
// keywords, which otherwise hide the actionable missing-property context.
const message =
keyword === "then" || (keyword === "if" && failingKeyword === "then")
? "must have required conditional properties"
: typeof err?.message === "string" && err.message.trim()
? err.message
: "validation error";
const where = instancePath ? `at ${instancePath}: ` : "";
parts.push(`${where}${message}`);
}
const unique = [...new Set(parts.filter((part) => part.trim()))];
return unique.length > 0 ? unique.join("; ") : "unknown validation error";
}
+7
View File
@@ -49,6 +49,7 @@ const exportSet = new Set(exportedNames);
const requiredRuntimeShimEntries = ["compat.js", "root-alias.cjs"];
const forbiddenPublicDeclarationSpecifiers = ["@openclaw/llm-core"];
const FORBIDDEN_PUBLIC_PROTOCOL_REGISTRY_RE = /\bdeclare\s+const\s+ProtocolSchemas(?:\$\d+)?\b/u;
const RELATIVE_DECLARATION_SPECIFIER_RE = /\b(?:from|import)\s*(?:\(\s*)?["']([^"']+)["']/gu;
const requiredSubpathExports = {
"secret-input-runtime": [
@@ -140,6 +141,12 @@ while (declarationQueue.length > 0) {
}
declarationPaths.add(dtsPath);
const dtsContent = readFileSync(dtsPath, "utf8");
if (FORBIDDEN_PUBLIC_PROTOCOL_REGISTRY_RE.test(dtsContent)) {
console.error(
`FORBIDDEN PUBLIC DTS REGISTRY: ${relative(resolve(scriptDir, ".."), dtsPath)} retains ProtocolSchemas`,
);
missing += 1;
}
for (const match of dtsContent.matchAll(RELATIVE_DECLARATION_SPECIFIER_RE)) {
const specifier = match[1];
if (!specifier?.startsWith(".")) {
@@ -14,6 +14,11 @@ import {
registerAgentRunContext,
resetAgentRunContextForTest,
} from "../infra/agent-events.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import {
INTERNAL_RUNTIME_CONTEXT_BEGIN,
INTERNAL_RUNTIME_CONTEXT_END,
@@ -24,6 +29,7 @@ import {
markStartupOrphanedMainSessionsForRecovery,
recoverStartupOrphanedMainSessions,
recoverRestartAbortedMainSessions,
scheduleRestartAbortedMainSessionRecovery,
} from "./main-session-restart-recovery.js";
import type { SessionLockInspection } from "./session-write-lock.js";
@@ -36,10 +42,12 @@ let tmpDir: string;
beforeEach(async () => {
vi.clearAllMocks();
resetAgentRunContextForTest();
resetGatewayWorkAdmission();
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-main-restart-recovery-"));
});
afterEach(async () => {
resetGatewayWorkAdmission();
await fs.rm(tmpDir, { recursive: true, force: true });
});
@@ -1282,6 +1290,54 @@ describe("main-session-restart-recovery", () => {
expect(customStore["agent:main:main"]?.abortedLastRun).toBe(false);
});
it("admits each scheduled recovery attempt as independent root work", async () => {
const sessionsDir = await makeSessionsDir();
await writeStore(sessionsDir, {
"agent:main:main": {
sessionId: "main-session",
updatedAt: Date.now() - 10_000,
status: "running",
abortedLastRun: true,
pendingFinalDelivery: true,
pendingFinalDeliveryText: "interrupted response",
},
});
const suspensionRef: {
current: ReturnType<typeof tryBeginGatewaySuspendAdmission>;
} = { current: null };
vi.mocked(callGateway)
.mockImplementationOnce(async () => {
expect(getActiveGatewayRootWorkCount()).toBe(1);
suspensionRef.current = tryBeginGatewaySuspendAdmission(() => {});
expect(suspensionRef.current?.commit()).toBe(true);
throw new Error("retry after suspension");
})
.mockImplementationOnce(async () => {
expect(getActiveGatewayRootWorkCount()).toBe(1);
return { runId: "run-resumed" };
});
scheduleRestartAbortedMainSessionRecovery({
delayMs: 0,
maxRetries: 2,
stateDir: tmpDir,
});
await vi.waitFor(() => {
expect(callGateway).toHaveBeenCalledOnce();
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
expect(suspensionRef.current?.release()).toBe(true);
await vi.waitFor(() => {
expect(callGateway).toHaveBeenCalledTimes(2);
const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
expect(store["agent:main:main"]?.abortedLastRun).toBe(false);
});
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("fails marked sessions whose transcript tail cannot be resumed", async () => {
const sessionsDir = await makeSessionsDir();
await writeStore(sessionsDir, {
+12 -6
View File
@@ -26,6 +26,7 @@ import {
listAgentRunsForSession,
} from "../infra/agent-events.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { CommandLane } from "../process/lanes.js";
import {
isAcpSessionKey,
@@ -956,12 +957,17 @@ export function scheduleRestartAbortedMainSessionRecovery(
const startupRecoveryCutoffMs = Date.now();
const runRecoveryAttempt = (attempt: number, delay: number) => {
void recoverStartupOrphanedMainSessions({
cfg: params.cfg,
stateDir: params.stateDir,
resumedSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs,
})
// Delayed retries outlive startup; each attempt must independently block
// host suspension while it reads and rewrites recovery session state.
void runWithGatewayIndependentRootWorkAdmission(
async () =>
await recoverStartupOrphanedMainSessions({
cfg: params.cfg,
stateDir: params.stateDir,
resumedSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs,
}),
)
.then((result) => {
if (result.failed > 0 && attempt < maxRetries) {
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
+46 -1
View File
@@ -4,6 +4,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import * as sessions from "../config/sessions.js";
import * as gateway from "../gateway/call.js";
import * as sessionUtils from "../gateway/session-transcript-readers.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js";
import * as announceDelivery from "./subagent-announce-delivery.js";
import {
@@ -130,9 +135,11 @@ describe("subagent-orphan-recovery", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.clearAllMocks();
resetGatewayWorkAdmission();
});
afterEach(() => {
resetGatewayWorkAdmission();
vi.useRealTimers();
vi.restoreAllMocks();
});
@@ -630,7 +637,17 @@ describe("subagent-orphan-recovery", () => {
abortedLastRun: true,
},
});
vi.mocked(gateway.callGateway).mockRejectedValue(new Error("service restart"));
const admittedRootCounts: number[] = [];
vi.mocked(gateway.callGateway).mockImplementation(async () => {
admittedRootCounts.push(getActiveGatewayRootWorkCount());
throw new Error("service restart");
});
vi.mocked(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).mockImplementation(
async () => {
admittedRootCounts.push(getActiveGatewayRootWorkCount());
return 1;
},
);
const activeRuns = createActiveRuns(createTestRunRecord());
@@ -642,10 +659,13 @@ describe("subagent-orphan-recovery", () => {
await vi.advanceTimersByTimeAsync(1);
await Promise.resolve();
expect(getActiveGatewayRootWorkCount()).toBe(0);
await vi.advanceTimersByTimeAsync(2);
await Promise.resolve();
expect(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).toHaveBeenCalledOnce();
expect(admittedRootCounts).toEqual([1, 1, 1]);
expect(getActiveGatewayRootWorkCount()).toBe(0);
const finalizeParams = requireRecord(
firstCallParam(
vi.mocked(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).mock.calls,
@@ -658,4 +678,29 @@ describe("subagent-orphan-recovery", () => {
expect(finalizeParams.error).toContain("Automatic recovery failed after 2 attempts");
expect(finalizeParams.error).toContain("service restart");
});
it("waits for suspension to reopen before mutating an orphaned session", async () => {
mockSingleAbortedSession();
vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "resumed-run" });
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
scheduleOrphanRecovery({
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
delayMs: 1,
maxRetries: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(gateway.callGateway).not.toHaveBeenCalled();
expect(sessions.updateSessionStore).not.toHaveBeenCalled();
expect(getActiveGatewayRootWorkCount()).toBe(0);
expect(suspension?.release()).toBe(true);
await vi.advanceTimersByTimeAsync(0);
expect(gateway.callGateway).toHaveBeenCalledOnce();
expect(sessions.updateSessionStore).toHaveBeenCalledOnce();
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
+42 -42
View File
@@ -23,6 +23,7 @@ import { callGateway } from "../gateway/call.js";
import { readSessionMessagesAsync } from "../gateway/session-transcript-readers.js";
import { formatErrorMessage } from "../infra/errors.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { truncateUtf16Safe } from "../utils.js";
import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js";
import {
@@ -437,49 +438,48 @@ export function scheduleOrphanRecovery(params: {
const resumedSessionKeys = new Set<string>();
const attemptRecovery = (attempt: number, delay: number) => {
setTimeout(() => {
void recoverOrphanedSubagentSessions({
...params,
resumedSessionKeys,
})
.then((result) => {
if (result.failed > 0 && attempt < maxRetries) {
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
log.info(
`orphan recovery had ${result.failed} failure(s); retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
);
attemptRecovery(attempt + 1, nextDelay);
return;
}
if (result.failedRuns.length === 0) {
return;
}
const attempts = attempt + 1;
void Promise.allSettled(
result.failedRuns.map((run) =>
finalizeInterruptedSubagentRun({
runId: run.runId,
childSessionKey: run.childSessionKey,
error: buildRecoveryFailureMessage({
attempts,
error: run.error,
}),
}),
),
);
})
.catch((err: unknown) => {
if (attempt < maxRetries) {
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
log.warn(
`scheduled orphan recovery failed: ${String(err)}; retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
);
attemptRecovery(attempt + 1, nextDelay);
} else {
log.warn(
`scheduled orphan recovery failed after ${maxRetries} retries: ${String(err)}`,
);
}
// Every delayed/retry scan owns a fresh root lease. Keep terminal
// mutation in the same lease so suspension cannot become ready mid-attempt.
void runWithGatewayIndependentRootWorkAdmission(async () => {
const result = await recoverOrphanedSubagentSessions({
...params,
resumedSessionKeys,
});
if (result.failed > 0 && attempt < maxRetries) {
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
log.info(
`orphan recovery had ${result.failed} failure(s); retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
);
attemptRecovery(attempt + 1, nextDelay);
return;
}
if (result.failedRuns.length === 0) {
return;
}
const attempts = attempt + 1;
await Promise.allSettled(
result.failedRuns.map((run) =>
finalizeInterruptedSubagentRun({
runId: run.runId,
childSessionKey: run.childSessionKey,
error: buildRecoveryFailureMessage({
attempts,
error: run.error,
}),
}),
),
);
}).catch((err: unknown) => {
if (attempt < maxRetries) {
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
log.warn(
`scheduled orphan recovery failed: ${String(err)}; retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
);
attemptRecovery(attempt + 1, nextDelay);
} else {
log.warn(`scheduled orphan recovery failed after ${maxRetries} retries: ${String(err)}`);
}
});
}, delay).unref?.();
};
@@ -2,6 +2,11 @@
// detached task status, and resource retirement around child-run endings.
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CallGatewayOptions } from "../gateway/call.js";
import {
getActiveGatewayRootWorkCount,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
import {
buildAnnounceIdFromChildRun,
@@ -265,6 +270,7 @@ async function runNoReplyMirrorScenario(params: {
describe("subagent registry lifecycle hardening", () => {
beforeEach(() => {
resetGatewayWorkAdmission();
vi.clearAllMocks();
taskExecutorMocks.completeTaskRunByRunId.mockReset();
taskExecutorMocks.failTaskRunByRunId.mockReset();
@@ -276,6 +282,131 @@ describe("subagent registry lifecycle hardening", () => {
bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey.mockResolvedValue(true);
});
it("keeps task finalization, resource retirement, and announce cleanup root-admitted", async () => {
const entry = createRunEntry({ expectsCompletionMessage: true });
let releaseBrowserCleanup: (() => void) | undefined;
let releaseAnnounce: ((didAnnounce: boolean) => void) | undefined;
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
releaseBrowserCleanup = resolve;
}),
);
const runSubagentAnnounceFlow = vi.fn(
() =>
new Promise<boolean>((resolve) => {
releaseAnnounce = resolve;
}),
);
const controller = createLifecycleController({ entry, runSubagentAnnounceFlow });
const completion = controller.completeSubagentRun({
runId: entry.runId,
endedAt: 4_000,
outcome: { status: "ok" },
reason: SUBAGENT_ENDED_REASON_COMPLETE,
triggerCleanup: true,
});
await vi.waitFor(() =>
expect(
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd,
).toHaveBeenCalledOnce(),
);
expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledOnce();
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseBrowserCleanup?.();
await vi.waitFor(() => expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce());
await completion;
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseAnnounce?.(true);
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(entry.cleanupCompletedAt).toBeTypeOf("number");
});
it("keeps direct delete cleanup root-admitted until the gateway call settles", async () => {
const entry = createRunEntry({ cleanup: "delete", expectsCompletionMessage: false });
const runs = new Map([[entry.runId, entry]]);
let releaseDelete: (() => void) | undefined;
gatewayMocks.callGateway.mockImplementation((opts) => {
if (opts.method !== "sessions.delete") {
return Promise.resolve({});
}
return new Promise<Record<string, unknown>>((resolve) => {
releaseDelete = () => resolve({});
});
});
const controller = createLifecycleController({ entry, runs });
await controller.completeSubagentRun({
runId: entry.runId,
endedAt: 4_000,
outcome: { status: "ok" },
reason: SUBAGENT_ENDED_REASON_COMPLETE,
triggerCleanup: true,
});
await vi.waitFor(() => expect(releaseDelete).toBeTypeOf("function"));
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseDelete?.();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(runs.has(entry.runId)).toBe(false);
});
it("retries a cleanup handoff rejected by restart drain", async () => {
vi.useFakeTimers();
try {
const entry = createRunEntry({ expectsCompletionMessage: true });
let releaseBrowserCleanup: (() => void) | undefined;
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
releaseBrowserCleanup = resolve;
}),
);
const runSubagentAnnounceFlow = vi.fn(async () => true);
const resumeSubagentRun = vi.fn((runId: string) => {
controller.startSubagentAnnounceCleanupFlow(runId, entry);
});
const controller = createLifecycleController({
entry,
resumeSubagentRun,
runSubagentAnnounceFlow,
});
const completion = controller.completeSubagentRun({
runId: entry.runId,
endedAt: 4_000,
outcome: { status: "ok" },
reason: SUBAGENT_ENDED_REASON_COMPLETE,
triggerCleanup: true,
});
await vi.waitFor(() => expect(releaseBrowserCleanup).toBeTypeOf("function"));
markGatewayRestartDraining();
releaseBrowserCleanup?.();
await completion;
await vi.waitFor(() =>
expect(runtimeMocks.log).toHaveBeenCalledWith(
expect.stringContaining("subagent cleanup admission failed"),
),
);
expect(runSubagentAnnounceFlow).not.toHaveBeenCalled();
expect(entry.cleanupHandled).toBe(true);
resetGatewayWorkAdmission();
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce());
await vi.waitFor(() => expect(entry.cleanupCompletedAt).toBeTypeOf("number"));
expect(resumeSubagentRun).toHaveBeenCalledWith(entry.runId);
expect(getActiveGatewayRootWorkCount()).toBe(0);
} finally {
resetGatewayWorkAdmission();
vi.useRealTimers();
}
});
it("does not reject completion when task finalization throws", async () => {
const persist = vi.fn();
const persistOrThrow = vi.fn();
@@ -1985,6 +2116,53 @@ describe("subagent registry lifecycle hardening", () => {
});
});
it("keeps a late superseded-delivery retirement root-admitted", async () => {
const entry = createRunEntry({ expectsCompletionMessage: true, generation: 1 });
const runs = new Map([[entry.runId, entry]]);
let onDeliveryResult: ((delivery: SubagentAnnounceDeliveryResult) => void) | undefined;
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
async (announceParams) => {
onDeliveryResult = announceParams.onDeliveryResult;
return true;
},
);
let releaseRetirement = () => {};
const retirementPending = new Promise<void>((resolve) => {
releaseRetirement = resolve;
});
const retireSupersededRun = vi.fn(async () => {
await retirementPending;
});
const controller = createLifecycleController({
entry,
runs,
retireSupersededRun,
runSubagentAnnounceFlow,
});
await controller.completeSubagentRun({
runId: entry.runId,
endedAt: 4_000,
outcome: { status: "ok" },
reason: SUBAGENT_ENDED_REASON_COMPLETE,
triggerCleanup: true,
});
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
const newer = createRunEntry({
runId: "run-2",
childSessionKey: entry.childSessionKey,
generation: 2,
});
runs.set(newer.runId, newer);
onDeliveryResult?.({ delivered: false, path: "none" });
await vi.waitFor(() => expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry));
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseRetirement();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("finalizes terminal visible-send failures without scheduling completion retry", async () => {
const persist = vi.fn();
const entry = createRunEntry({
+274 -175
View File
@@ -8,6 +8,10 @@ import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import type { cleanupBrowserSessionsForLifecycleEnd } from "../browser-lifecycle-cleanup.js";
import type { callGateway as defaultCallGateway } from "../gateway/call.js";
import { formatErrorMessage, readErrorName } from "../infra/errors.js";
import {
isGatewayRestartDraining,
runWithGatewayIndependentRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import { defaultRuntime } from "../runtime.js";
import { emitSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js";
import { extractTextFromChatContent } from "../shared/chat-content.js";
@@ -215,13 +219,43 @@ export function createSubagentRegistryLifecycleController(params: {
};
};
const scheduleResumeSubagentRun = (runId: string, entry: SubagentRunRecord, delayMs: number) => {
const scheduleResumeSubagentRun = (
runId: string,
entry: SubagentRunRecord,
delayMs: number,
cleanupGeneration?: number,
) => {
const timer = setTimeout(() => {
scheduledResumeTimers.delete(timer);
if (params.runs.get(runId) !== entry) {
return;
}
params.resumeSubagentRun(runId);
void runWithGatewayIndependentRootWorkAdmission(async () => {
if (params.runs.get(runId) !== entry) {
return;
}
if (cleanupGeneration !== undefined) {
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
return;
}
entry.cleanupHandled = false;
params.persist();
}
params.resumedRuns.delete(runId);
params.resumeSubagentRun(runId);
}).catch((err: unknown) => {
defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`);
const current = params.runs.get(runId);
if (
isGatewayRestartDraining() &&
current === entry &&
typeof current.cleanupCompletedAt !== "number"
) {
scheduleResumeSubagentRun(
runId,
entry,
Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS),
cleanupGeneration,
);
}
});
}, delayMs);
timer.unref?.();
scheduledResumeTimers.add(timer);
@@ -234,6 +268,49 @@ export function createSubagentRegistryLifecycleController(params: {
scheduledResumeTimers.clear();
};
const runDetachedCleanupAttempt = (args: {
runId: string;
entry: SubagentRunRecord;
cleanupGeneration: number;
run: () => Promise<void>;
}) => {
// Completion makes the task projection non-blocking before delivery and
// cleanup finish. This independent lease bridges that handoff and owns the
// full detached attempt, including its final durable registry write.
void runWithGatewayIndependentRootWorkAdmission(async () => {
try {
await args.run();
} catch (err) {
defaultRuntime.log(
`[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`,
);
const current = params.runs.get(args.runId);
if (
!current ||
current.cleanupCompletedAt ||
!isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration)
) {
return;
}
current.cleanupHandled = false;
params.resumedRuns.delete(args.runId);
params.persist();
}
}).catch((err: unknown) => {
defaultRuntime.log(
`[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`,
);
if (isGatewayRestartDraining()) {
scheduleResumeSubagentRun(
args.runId,
args.entry,
MIN_ANNOUNCE_RETRY_DELAY_MS,
args.cleanupGeneration,
);
}
});
};
const maskRunId = (runId: string): string => {
const trimmed = runId.trim();
if (!trimmed) {
@@ -803,6 +880,22 @@ export function createSubagentRegistryLifecycleController(params: {
return true;
};
const retireSupersededCleanupInBackground = (
runId: string,
entry: SubagentRunRecord,
generation: number,
) => {
// Delivery callbacks are synchronous and may arrive after their announce
// attempt returns. Give the async retirement tail its own snapshot blocker.
void runWithGatewayIndependentRootWorkAdmission(async () => {
await retireSupersededCleanupIfNeeded(runId, entry, generation);
}).catch((error: unknown) => {
defaultRuntime.log(
`[warn] subagent superseded cleanup retirement failed (${runId}): ${String(error)}`,
);
});
};
const isTerminalCallbackCurrent = (
runId: string,
entry: SubagentRunRecord,
@@ -840,20 +933,17 @@ export function createSubagentRegistryLifecycleController(params: {
if (!beginSubagentCleanup(runId)) {
continue;
}
void finalizeResumedAnnounceGiveUp({
runDetachedCleanupAttempt({
runId,
entry,
reason: "expiry",
}).catch((error: unknown) => {
defaultRuntime.log(
`[warn] Subagent expiry finalize failed during deferred retry for run ${runId}: ${String(error)}`,
);
const current = params.runs.get(runId);
if (!current || current.cleanupCompletedAt) {
return;
}
current.cleanupHandled = false;
params.persist();
cleanupGeneration: cleanupGenerations.get(entry)!,
run: async () => {
await finalizeResumedAnnounceGiveUp({
runId,
entry,
reason: "expiry",
});
},
});
continue;
}
@@ -870,31 +960,46 @@ export function createSubagentRegistryLifecycleController(params: {
preserveTranscript?: boolean;
provisionalKill?: boolean;
}) => {
const runCleanupTail = (label: string, run: () => Promise<unknown>) => {
// These best-effort tails can outlive the durable registry transition,
// but they still mutate session-owned resources and must block snapshots.
void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => {
defaultRuntime.log(
`[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`,
);
});
};
if (!cleanupParams.preserveTranscript) {
void removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile);
runCleanupTail("transcript cleanup", async () => {
await removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile);
});
}
if (cleanupParams.entry.spawnMode !== "session") {
void retireSessionMcpRuntimeForSessionKey({
sessionKey: cleanupParams.entry.childSessionKey,
reason: "subagent-run-cleanup",
onError: (error, sessionId) => {
params.warn("failed to retire subagent bundle MCP runtime", {
error: buildSafeLifecycleErrorMeta(error),
sessionId,
runId: maskRunId(cleanupParams.runId),
childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey),
});
},
runCleanupTail("bundle MCP cleanup", async () => {
await retireSessionMcpRuntimeForSessionKey({
sessionKey: cleanupParams.entry.childSessionKey,
reason: "subagent-run-cleanup",
onError: (error, sessionId) => {
params.warn("failed to retire subagent bundle MCP runtime", {
error: buildSafeLifecycleErrorMeta(error),
sessionId,
runId: maskRunId(cleanupParams.runId),
childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey),
});
},
});
});
}
if (cleanupParams.cleanup === "delete") {
params.clearPendingLifecycleError(cleanupParams.runId);
if (!cleanupParams.provisionalKill) {
void params.notifyContextEngineSubagentEnded({
childSessionKey: cleanupParams.entry.childSessionKey,
reason: "deleted",
agentDir: cleanupParams.entry.agentDir,
workspaceDir: cleanupParams.entry.workspaceDir,
runCleanupTail("context-engine cleanup", async () => {
await params.notifyContextEngineSubagentEnded({
childSessionKey: cleanupParams.entry.childSessionKey,
reason: "deleted",
agentDir: cleanupParams.entry.agentDir,
workspaceDir: cleanupParams.entry.workspaceDir,
});
});
}
params.runs.delete(cleanupParams.runId);
@@ -903,11 +1008,13 @@ export function createSubagentRegistryLifecycleController(params: {
return;
}
if (!cleanupParams.provisionalKill) {
void params.notifyContextEngineSubagentEnded({
childSessionKey: cleanupParams.entry.childSessionKey,
reason: "completed",
agentDir: cleanupParams.entry.agentDir,
workspaceDir: cleanupParams.entry.workspaceDir,
runCleanupTail("context-engine cleanup", async () => {
await params.notifyContextEngineSubagentEnded({
childSessionKey: cleanupParams.entry.childSessionKey,
reason: "completed",
agentDir: cleanupParams.entry.agentDir,
workspaceDir: cleanupParams.entry.workspaceDir,
});
});
}
if (
@@ -1161,20 +1268,15 @@ export function createSubagentRegistryLifecycleController(params: {
return false;
}
const cleanupGeneration = cleanupGenerations.get(entry)!;
void finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
skipAnnounce: true,
}).catch((err: unknown) => {
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
const current = params.runs.get(runId);
if (
!current ||
current.cleanupCompletedAt ||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
) {
return;
}
current.cleanupHandled = false;
params.persist();
runDetachedCleanupAttempt({
runId,
entry,
cleanupGeneration,
run: async () => {
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
skipAnnounce: true,
});
},
});
return true;
}
@@ -1184,52 +1286,45 @@ export function createSubagentRegistryLifecycleController(params: {
const cleanupGeneration = cleanupGenerations.get(entry)!;
const skipRequesterDelivery = entry.suppressCompletionDelivery === true;
if (entry.expectsCompletionMessage === false || skipRequesterDelivery) {
void (async () => {
// This driver is detached. Yield once so synchronous successor
// registration can invalidate it before sessions.delete is submitted.
await Promise.resolve();
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
return;
}
if (cleanup === "delete") {
// This durable boundary prevents a late yield from reviving a run
// after deletion may already have reached the gateway.
entry.deleteCleanupDispatchedAt ??= Date.now();
params.persist();
await deleteSubagentSessionForCleanup({
callGateway: params.callGateway,
childSessionKey: entry.childSessionKey,
spawnMode: entry.spawnMode,
onError: (error) =>
params.warn("sessions.delete failed during subagent cleanup", {
error: buildSafeLifecycleErrorMeta(error),
runId: maskRunId(runId),
childSessionKey: maskSessionKey(entry.childSessionKey),
}),
runDetachedCleanupAttempt({
runId,
entry,
cleanupGeneration,
run: async () => {
// This driver is detached. Yield once so synchronous successor
// registration can invalidate it before sessions.delete is submitted.
await Promise.resolve();
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
return;
}
if (cleanup === "delete") {
// This durable boundary prevents a late yield from reviving a run
// after deletion may already have reached the gateway.
entry.deleteCleanupDispatchedAt ??= Date.now();
params.persist();
await deleteSubagentSessionForCleanup({
callGateway: params.callGateway,
childSessionKey: entry.childSessionKey,
spawnMode: entry.spawnMode,
onError: (error) =>
params.warn("sessions.delete failed during subagent cleanup", {
error: buildSafeLifecycleErrorMeta(error),
runId: maskRunId(runId),
childSessionKey: maskSessionKey(entry.childSessionKey),
}),
});
}
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
return;
}
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
skipAnnounce: true,
skipDeliveryStatus: true,
skipRequesterDelivery,
});
}
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
return;
}
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
skipAnnounce: true,
skipDeliveryStatus: true,
skipRequesterDelivery,
});
})().catch((err: unknown) => {
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
const current = params.runs.get(runId);
if (
!current ||
current.cleanupCompletedAt ||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
) {
return;
}
current.cleanupHandled = false;
params.persist();
},
});
return true;
}
@@ -1253,93 +1348,87 @@ export function createSubagentRegistryLifecycleController(params: {
if (!didAnnounce && latestDeliveryError) {
ensureDeliveryState(entry).lastError = latestDeliveryError;
}
void finalizeSubagentCleanup(
await finalizeSubagentCleanup(
runId,
cleanup,
didAnnounce || shouldCreditPriorDelivery,
cleanupGeneration,
).catch((err: unknown) => {
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
const current = params.runs.get(runId);
if (
!current ||
current.cleanupCompletedAt ||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
) {
return;
}
current.cleanupHandled = false;
params.persist();
});
);
};
void params
.runSubagentAnnounceFlow({
childSessionKey: pendingPayload.childSessionKey,
childRunId: pendingPayload.childRunId,
requesterSessionKey: pendingPayload.requesterSessionKey,
requesterOrigin,
requesterDisplayKey: pendingPayload.requesterDisplayKey,
task: pendingPayload.task,
timeoutMs: params.subagentAnnounceTimeoutMs,
cleanup,
roundOneReply: pendingPayload.frozenResultText ?? undefined,
fallbackReply: pendingPayload.fallbackFrozenResultText ?? undefined,
waitForCompletion: false,
startedAt: pendingPayload.startedAt,
endedAt: pendingPayload.endedAt,
label: pendingPayload.label,
outcome: pendingPayload.outcome,
spawnMode: pendingPayload.spawnMode,
expectsCompletionMessage: pendingPayload.expectsCompletionMessage,
wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true,
onBeforeDeleteChildSession:
cleanup === "delete"
? () => {
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
return false;
}
// Announce owns delete submission; fence late yields at the
// exact handoff instead of when cleanup merely starts.
entry.deleteCleanupDispatchedAt ??= Date.now();
params.persist();
return true;
const announceParams: Parameters<RunSubagentAnnounceFlow>[0] = {
childSessionKey: pendingPayload.childSessionKey,
childRunId: pendingPayload.childRunId,
requesterSessionKey: pendingPayload.requesterSessionKey,
requesterOrigin,
requesterDisplayKey: pendingPayload.requesterDisplayKey,
task: pendingPayload.task,
timeoutMs: params.subagentAnnounceTimeoutMs,
cleanup,
roundOneReply: pendingPayload.frozenResultText ?? undefined,
fallbackReply: pendingPayload.fallbackFrozenResultText ?? undefined,
waitForCompletion: false,
startedAt: pendingPayload.startedAt,
endedAt: pendingPayload.endedAt,
label: pendingPayload.label,
outcome: pendingPayload.outcome,
spawnMode: pendingPayload.spawnMode,
expectsCompletionMessage: pendingPayload.expectsCompletionMessage,
wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true,
onBeforeDeleteChildSession:
cleanup === "delete"
? () => {
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
return false;
}
: undefined,
onDeliveryResult: (delivery) => {
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
void retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
return;
}
recordAnnounceDeliveryResult(entry, delivery);
if (delivery.delivered) {
const deliveryState = ensureDeliveryState(entry);
if (deliveryState.lastError !== undefined) {
deliveryState.lastError = undefined;
// Announce owns delete submission; fence late yields at the
// exact handoff instead of when cleanup merely starts.
entry.deleteCleanupDispatchedAt ??= Date.now();
params.persist();
return true;
}
latestDeliveryError = undefined;
return;
}
if (delivery.path === "none") {
ensureDeliveryState(entry).lastDropReason = "sink_unavailable";
}
latestDeliveryError = formatAnnounceDeliveryError(delivery);
if (ensureDeliveryState(entry).lastError !== latestDeliveryError) {
ensureDeliveryState(entry).lastError = latestDeliveryError;
: undefined,
onDeliveryResult: (delivery) => {
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
retireSupersededCleanupInBackground(runId, entry, cleanupGeneration);
return;
}
recordAnnounceDeliveryResult(entry, delivery);
if (delivery.delivered) {
const deliveryState = ensureDeliveryState(entry);
if (deliveryState.lastError !== undefined) {
deliveryState.lastError = undefined;
params.persist();
}
},
})
.then((didAnnounce) => {
void finalizeAnnounceCleanup(didAnnounce);
})
.catch((error: unknown) => {
defaultRuntime.log(
`[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`,
);
void finalizeAnnounceCleanup(false);
});
latestDeliveryError = undefined;
return;
}
if (delivery.path === "none") {
ensureDeliveryState(entry).lastDropReason = "sink_unavailable";
}
latestDeliveryError = formatAnnounceDeliveryError(delivery);
if (ensureDeliveryState(entry).lastError !== latestDeliveryError) {
ensureDeliveryState(entry).lastError = latestDeliveryError;
params.persist();
}
},
};
runDetachedCleanupAttempt({
runId,
entry,
cleanupGeneration,
run: async () => {
let didAnnounce = false;
try {
didAnnounce = await params.runSubagentAnnounceFlow(announceParams);
} catch (error) {
defaultRuntime.log(
`[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`,
);
}
await finalizeAnnounceCleanup(didAnnounce);
},
});
return true;
};
@@ -1356,7 +1445,7 @@ export function createSubagentRegistryLifecycleController(params: {
completionSnapshot?: { resultText: string | null; capturedAt: number };
};
const completeSubagentRun = async (completeParams: CompleteSubagentRunParams) => {
const completeSubagentRunAttempt = async (completeParams: CompleteSubagentRunParams) => {
const releaseCompletionLock = await acquireTerminalCompletionLock(completeParams.runId);
let entry: SubagentRunRecord | undefined;
let terminalGeneration = 0;
@@ -1843,6 +1932,16 @@ export function createSubagentRegistryLifecycleController(params: {
startSubagentAnnounceCleanupFlow(completeParams.runId, entry);
};
const completeSubagentRun = async (completeParams: CompleteSubagentRunParams) => {
// Task finalization can make the run disappear from suspension blockers
// before browser/MCP retirement and cleanup delivery hand off. Own this
// entire transition as an independent root so that boundary stays atomic.
// Callers can detach while retaining parent ALS, so nesting is intentional.
await runWithGatewayIndependentRootWorkAdmission(async () => {
await completeSubagentRunAttempt(completeParams);
});
};
return {
clearScheduledResumeTimers,
completeCleanupBookkeeping,
+17 -6
View File
@@ -7,6 +7,7 @@ import { getRuntimeConfig } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { callGateway } from "../gateway/call.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import {
SUBAGENT_KILL_TASK_ERROR,
type DetachedTaskFindResult,
@@ -990,18 +991,28 @@ export function createSubagentRunManager(params: {
finalizeKilledTask(pending.entry, pending.endedAt);
}
for (const entry of entriesByChildSessionKey.values()) {
void persistSubagentSessionTiming(entry, {
isCurrentGeneration: () => currentRunOwnsSession(entry),
// Task finalization removes the suspension blocker before these session-owned
// writes finish. Join them under one independent root so snapshots stay atomic.
void runWithGatewayIndependentRootWorkAdmission(async () => {
await Promise.all([
persistSubagentSessionTiming(entry, {
isCurrentGeneration: () => currentRunOwnsSession(entry),
}).catch((err: unknown) => {
log.warn("failed to persist killed subagent session timing", {
err,
runId: entry.runId,
childSessionKey: entry.childSessionKey,
});
}),
shouldDeleteAttachments(entry) ? safeRemoveAttachmentsDir(entry) : Promise.resolve(),
]);
}).catch((err: unknown) => {
log.warn("failed to persist killed subagent session timing", {
log.warn("failed to run killed subagent cleanup tail", {
err,
runId: entry.runId,
childSessionKey: entry.childSessionKey,
});
});
if (shouldDeleteAttachments(entry)) {
void safeRemoveAttachmentsDir(entry);
}
params.completeCleanupBookkeeping({
runId: entry.runId,
entry,
+166
View File
@@ -11,6 +11,11 @@ import type {
SessionEntryPatchOptions,
} from "../config/sessions/session-accessor.js";
import type { AgentEventPayload } from "../infra/agent-events.js";
import {
getActiveGatewayRootWorkCount,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
import {
createRunningTaskRun,
@@ -255,6 +260,7 @@ describe("subagent registry seam flow", () => {
});
beforeEach(() => {
resetGatewayWorkAdmission();
resetDetachedTaskLifecycleRuntimeForTests();
vi.clearAllMocks();
vi.useFakeTimers();
@@ -311,12 +317,172 @@ describe("subagent registry seam flow", () => {
});
afterEach(() => {
resetGatewayWorkAdmission();
resetDetachedTaskLifecycleRuntimeForTests();
mod.testing.setDepsForTest();
mod.resetSubagentRegistryForTests({ persist: false });
vi.useRealTimers();
});
it("keeps a sweeper archive mutation root-admitted until deletion settles", async () => {
const now = Date.now();
let releaseDelete: (() => void) | undefined;
mocks.callGateway.mockImplementation((request: { method?: string }) => {
if (request.method !== "sessions.delete") {
return Promise.resolve({});
}
return new Promise<Record<string, unknown>>((resolve) => {
releaseDelete = () => resolve({});
});
});
mod.addSubagentRunForTests({
runId: "run-sweep-admission",
childSessionKey: "agent:main:subagent:sweep-admission",
requesterSessionKey: "agent:main:main",
requesterDisplayKey: "main",
task: "archive while suspension is possible",
cleanup: "delete",
createdAt: now - 10_000,
endedAt: now - 5_000,
cleanupCompletedAt: now - 4_000,
archiveAtMs: now - 1,
});
const sweep = mod.testing.runSweeperTickForTests();
await waitForFast(() => expect(releaseDelete).toBeTypeOf("function"));
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseDelete?.();
await sweep;
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("tracks missing-entry lifecycle result refresh until capture and persistence settle", async () => {
const childSessionKey = "agent:main:subagent:refresh-admission";
mocks.callGateway.mockImplementation(async (request: { method?: string }) =>
request.method === "agent.wait" ? { status: "pending" } : {},
);
mod.registerSubagentRun({
runId: "run-refresh-admission-old",
childSessionKey,
requesterSessionKey: "agent:main:main",
requesterDisplayKey: "main",
task: "capture replacement completion",
cleanup: "keep",
expectsCompletionMessage: true,
});
await waitForFast(() => expect(mocks.callGateway).toHaveBeenCalled());
const entry = mod.getSubagentRunByChildSessionKey(childSessionKey);
expect(entry).not.toBeNull();
Object.assign(entry ?? {}, {
endedAt: Date.now(),
outcome: { status: "ok" },
});
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
let finishCapture: ((value: string) => void) | undefined;
mocks.captureSubagentCompletionReply.mockImplementationOnce(
async () =>
await new Promise<string>((resolve) => {
finishCapture = resolve;
}),
);
mocks.persistSubagentRunsToDisk.mockClear();
const lifecycleHandler = mocks.onAgentEvent.mock.calls.at(-1)?.[0];
expect(lifecycleHandler).toBeTypeOf("function");
lifecycleHandler?.({
runId: "run-refresh-admission-new",
seq: 1,
stream: "lifecycle",
ts: Date.now(),
sessionKey: childSessionKey,
data: { phase: "end" },
});
await waitForFast(() => expect(finishCapture).toBeTypeOf("function"));
expect(getActiveGatewayRootWorkCount()).toBe(1);
expect(entry?.completion?.resultText).toBeUndefined();
finishCapture?.("replacement final reply");
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(entry?.completion?.resultText).toBe("replacement final reply");
expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalledOnce();
});
it("retries a terminal completion deferred by restart drain", async () => {
const now = Date.now();
const runId = "run-terminal-restart-retry";
mod.addSubagentRunForTests({
runId,
childSessionKey: "agent:main:subagent:terminal-restart-retry",
requesterSessionKey: "agent:main:main",
requesterDisplayKey: "main",
task: "deliver terminal completion after restart",
cleanup: "keep",
expectsCompletionMessage: true,
createdAt: now - 10_000,
startedAt: now - 9_000,
endedAt: now - 1_000,
endedReason: SUBAGENT_ENDED_REASON_ERROR,
outcome: { status: "error", error: "provider interrupted" },
});
markGatewayRestartDraining();
await expect(
mod.finalizeInterruptedSubagentRun({
runId,
error: "provider interrupted",
endedAt: now - 1_000,
}),
).resolves.toBe(1);
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
resetGatewayWorkAdmission();
await vi.advanceTimersByTimeAsync(1_000);
await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledOnce());
await waitForFast(() => {
const entry = mod
.listSubagentRunsForRequester("agent:main:main")
.find((candidate) => candidate.runId === runId);
expect(entry?.cleanupCompletedAt).toBeTypeOf("number");
});
});
it("keeps killed session timing root-admitted after task finalization", async () => {
let finishTiming: (() => void) | undefined;
mocks.patchSessionEntry.mockImplementationOnce(async () => {
await new Promise<void>((resolve) => {
finishTiming = resolve;
});
return null;
});
mocks.callGateway.mockImplementation(async (request: { method?: string }) =>
request.method === "agent.wait" ? { status: "pending" } : {},
);
const runId = "run-kill-tail-admission";
mod.registerSubagentRun({
runId,
childSessionKey: "agent:main:subagent:child",
requesterSessionKey: "agent:main:main",
requesterDisplayKey: "main",
task: "persist killed session state",
cleanup: "keep",
spawnMode: "session",
});
await waitForFast(() => expect(mocks.callGateway).toHaveBeenCalled());
expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1);
await waitForFast(() => expect(finishTiming).toBeTypeOf("function"));
expect(findTaskByRunIdForStatus(runId)).toMatchObject({ status: "cancelled" });
// Bookkeeping can launch additional tracked cleanup tails; the held timing
// write must keep at least one independent root visible until it settles.
expect(getActiveGatewayRootWorkCount()).toBeGreaterThan(0);
finishTiming?.();
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("lists active and pending-delivery child sessions for maintenance preservation", () => {
const now = Date.now();
mod.addSubagentRunForTests({
+147 -35
View File
@@ -11,6 +11,10 @@ import type { ContextEngine, SubagentEndReason } from "../context-engine/types.j
import { callGateway } from "../gateway/call.js";
import { getAgentRunContext, onAgentEvent } from "../infra/agent-events.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
isGatewayRestartDraining,
runWithGatewayIndependentRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import {
formatAbandonedLivenessError,
formatBlockedLivenessError,
@@ -225,6 +229,7 @@ let restoreAttempted = false;
const ORPHAN_RECOVERY_DEBOUNCE_MS = 1_000;
let lastOrphanRecoveryScheduleAt = 0;
const SUBAGENT_ANNOUNCE_TIMEOUT_MS = 120_000;
const GATEWAY_ADMISSION_RETRY_DELAY_MS = 1_000;
/**
* Embedded runs can emit transient lifecycle `error` events while provider/model
* retry is still in progress. Defer terminal error cleanup briefly so a
@@ -379,6 +384,8 @@ export function scheduleSubagentOrphanRecovery(params?: { delayMs?: number; maxR
lastOrphanRecoveryScheduleAt = now;
void import("./subagent-orphan-recovery.js").then(
({ scheduleOrphanRecovery }) => {
// This import only installs timers. Each delayed or retrying recovery
// attempt owns independent root admission inside the recovery module.
scheduleOrphanRecovery({
getActiveRuns: () => subagentRuns,
delayMs: params?.delayMs,
@@ -455,7 +462,10 @@ type CompleteSubagentRunParams = {
suppressSessionEffects?: boolean;
};
async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) {
async function completeSubagentRunWithRecoveryAttempt(
params: CompleteSubagentRunParams,
source: string,
) {
try {
await completeSubagentRun(params);
return;
@@ -506,6 +516,52 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams
resumeSubagentRun(params.runId);
}
function scheduleSubagentCompletionRetryAfterRestart(
params: CompleteSubagentRunParams,
source: string,
expectedEntry: SubagentRunRecord,
) {
const expectedGeneration = expectedEntry.generation;
const timer = setTimeout(() => {
resumeRetryTimers.delete(timer);
const current = subagentRuns.get(params.runId);
if (current !== expectedEntry || current.generation !== expectedGeneration) {
return;
}
void completeSubagentRunWithRecovery(params, source).catch((error: unknown) => {
log.warn("failed to retry subagent completion after gateway restart", {
source,
runId: params.runId,
error,
});
});
}, GATEWAY_ADMISSION_RETRY_DELAY_MS);
timer.unref?.();
resumeRetryTimers.add(timer);
}
async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) {
// Each controller attempt owns its terminal transition, while this outer
// lease closes the gap between failed attempts and fallback cleanup.
try {
await runWithGatewayIndependentRootWorkAdmission(async () => {
await completeSubagentRunWithRecoveryAttempt(params, source);
});
} catch (error) {
if (!isGatewayRestartDraining()) {
throw error;
}
log.warn("subagent completion deferred during gateway restart", {
source,
runId: params.runId,
});
const current = subagentRuns.get(params.runId);
if (current) {
scheduleSubagentCompletionRetryAfterRestart(params, source, current);
}
}
}
function completeSubagentRunInBackground(params: CompleteSubagentRunParams, source: string) {
void completeSubagentRunWithRecovery(params, source);
}
@@ -717,6 +773,61 @@ const {
startSubagentAnnounceCleanupFlow,
} = subagentLifecycleController;
function scheduleSubagentDeliveryResumeRetry(
runId: string,
scheduledEntry: SubagentRunRecord,
waitMs: number,
) {
const timer = setTimeout(() => {
resumeRetryTimers.delete(timer);
void runWithGatewayIndependentRootWorkAdmission(async () => {
if (subagentRuns.get(runId) !== scheduledEntry) {
resumedRuns.delete(runId);
return;
}
resumedRuns.delete(runId);
resumeSubagentRun(runId);
}).catch((error: unknown) => {
log.warn("failed to resume subagent delivery retry", { runId, error });
if (
isGatewayRestartDraining() &&
subagentRuns.get(runId) === scheduledEntry &&
typeof scheduledEntry.cleanupCompletedAt !== "number"
) {
scheduleSubagentDeliveryResumeRetry(
runId,
scheduledEntry,
Math.max(waitMs, GATEWAY_ADMISSION_RETRY_DELAY_MS),
);
return;
}
resumedRuns.delete(runId);
});
}, waitMs);
timer.unref?.();
resumeRetryTimers.add(timer);
}
function finalizeResumedAnnounceGiveUpInBackground(
runId: string,
entry: SubagentRunRecord,
reason: "retry-limit" | "expiry",
) {
void runWithGatewayIndependentRootWorkAdmission(async () => {
await finalizeResumedAnnounceGiveUp({ runId, entry, reason });
}).catch((error: unknown) => {
log.warn("failed to finalize exhausted subagent delivery", { runId, reason, error });
if (
isGatewayRestartDraining() &&
subagentRuns.get(runId) === entry &&
typeof entry.cleanupCompletedAt !== "number"
) {
scheduleSubagentDeliveryResumeRetry(runId, entry, GATEWAY_ADMISSION_RETRY_DELAY_MS);
resumedRuns.add(runId);
}
});
}
function resumeSubagentRun(runId: string) {
if (!runId || resumedRuns.has(runId)) {
return;
@@ -738,11 +849,7 @@ function resumeSubagentRun(runId: string) {
}
// Skip entries that have exhausted their retry budget or expired (#18264).
if (getDeliveryAttemptCount(entry) >= MAX_ANNOUNCE_RETRY_COUNT) {
void finalizeResumedAnnounceGiveUp({
runId,
entry,
reason: "retry-limit",
});
finalizeResumedAnnounceGiveUpInBackground(runId, entry, "retry-limit");
return;
}
if (
@@ -750,11 +857,7 @@ function resumeSubagentRun(runId: string) {
typeof entry.endedAt === "number" &&
Date.now() - entry.endedAt > ANNOUNCE_EXPIRY_MS
) {
void finalizeResumedAnnounceGiveUp({
runId,
entry,
reason: "expiry",
});
finalizeResumedAnnounceGiveUpInBackground(runId, entry, "expiry");
return;
}
@@ -764,17 +867,7 @@ function resumeSubagentRun(runId: string) {
const earliestRetryAt = (lastAttemptAt ?? 0) + delayMs;
if (entry.expectsCompletionMessage === true && lastAttemptAt && now < earliestRetryAt) {
const waitMs = Math.max(1, earliestRetryAt - now);
const scheduledEntry = entry;
const timer = setTimeout(() => {
resumeRetryTimers.delete(timer);
if (subagentRuns.get(runId) !== scheduledEntry) {
return;
}
resumedRuns.delete(runId);
resumeSubagentRun(runId);
}, waitMs);
timer.unref?.();
resumeRetryTimers.add(timer);
scheduleSubagentDeliveryResumeRetry(runId, entry, waitMs);
resumedRuns.add(runId);
return;
}
@@ -884,12 +977,20 @@ function startSweeper() {
async function runSubagentSweep() {
try {
await sweepSubagentRuns();
await runWithGatewayIndependentRootWorkAdmission(async () => {
await sweepSubagentRuns();
});
} catch (err) {
log.warn(`subagent run sweep failed: ${err instanceof Error ? err.message : String(err)}`);
}
}
function runSubagentSweepCleanupTail(runId: string, label: string, run: () => Promise<unknown>) {
void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => {
log.warn(`subagent sweep ${label} failed`, { runId, error });
});
}
function stopSweeper() {
if (!sweeper) {
return;
@@ -1342,11 +1443,13 @@ async function sweepSubagentRuns() {
now - entry.cleanupCompletedAt > SESSION_RUN_TTL_MS
) {
clearPendingLifecycleError(runId);
void notifyContextEngineSubagentEnded({
childSessionKey: entry.childSessionKey,
reason: "swept",
agentDir: entry.agentDir,
workspaceDir: entry.workspaceDir,
runSubagentSweepCleanupTail(runId, "context-engine cleanup", async () => {
await notifyContextEngineSubagentEnded({
childSessionKey: entry.childSessionKey,
reason: "swept",
agentDir: entry.agentDir,
workspaceDir: entry.workspaceDir,
});
});
subagentRuns.delete(runId);
mutated = true;
@@ -1382,11 +1485,13 @@ async function sweepSubagentRuns() {
mutated = true;
// Archive/purge is terminal for the run record; remove any retained attachments too.
await safeRemoveAttachmentsDir(entry);
void notifyContextEngineSubagentEnded({
childSessionKey: entry.childSessionKey,
reason: "swept",
agentDir: entry.agentDir,
workspaceDir: entry.workspaceDir,
runSubagentSweepCleanupTail(runId, "context-engine cleanup", async () => {
await notifyContextEngineSubagentEnded({
childSessionKey: entry.childSessionKey,
reason: "swept",
agentDir: entry.agentDir,
workspaceDir: entry.workspaceDir,
});
});
}
// Sweep orphaned pendingLifecycleError entries (absolute TTL).
@@ -1426,7 +1531,12 @@ function ensureListener() {
const entry = subagentRuns.get(evt.runId);
if (!entry) {
if (phase === "end" && typeof evt.sessionKey === "string") {
await refreshFrozenResultFromSession(evt.sessionKey);
const sessionKey = evt.sessionKey;
// A replacement generation can finish after its predecessor row is
// terminal. Keep capture + persistence inside the suspension fence.
await runWithGatewayIndependentRootWorkAdmission(async () => {
await refreshFrozenResultFromSession(sessionKey);
});
}
return;
}
@@ -1573,7 +1683,9 @@ const subagentRunManager = createSubagentRunManager({
resolveSubagentSessionStartedAt,
notifyContextEngineSubagentEnded,
completeCleanupBookkeeping,
completeSubagentRun,
completeSubagentRun: async (params) => {
await completeSubagentRunWithRecovery(params, "subagent-wait");
},
resolveSubagentTask: findSubagentTaskForRun,
});
@@ -1,9 +1,18 @@
// Tests queue drain restart behavior when follow-up runs chain together.
import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures";
import { describe, expect, it, vi } from "vitest";
import {
getActiveGatewayRootWorkCount,
isGatewaySubordinateWorkAdmissionClosed,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../../process/gateway-work-admission.js";
import { captureEnv, setTestEnvValue } from "../../test-utils/env.js";
import type { FollowupRun, QueueSettings } from "./queue.js";
import {
clearFollowupQueue,
clearSessionQueues,
enqueueFollowupRun,
FollowupRunDeferredError,
scheduleFollowupDrain,
@@ -18,6 +27,81 @@ import { getExistingFollowupQueue } from "./queue/state.js";
installQueueRuntimeErrorSilencer();
describe("followup queue drain restart after idle window", () => {
it("keeps a detached drain on a live root after its enqueue request returns", async () => {
resetGatewayWorkAdmission();
const key = `test-detached-drain-root-${Date.now()}`;
const settings: QueueSettings = { mode: "followup", debounceMs: 0, cap: 50 };
const parentReleased = createDeferred<void>();
const drained = createDeferred<void>();
const parent = tryBeginGatewayRootWorkAdmission();
if (!parent) {
throw new Error("expected parent Gateway work admission");
}
let suspensionStarted = false;
let subordinateAdmissionClosed: boolean | undefined;
let activeRootCountDuringDrain: number | undefined;
try {
await parent.run(async () => {
enqueueFollowupRun(key, createRun({ prompt: "detached" }), settings);
scheduleFollowupDrain(key, async () => {
await parentReleased.promise;
const suspension = tryBeginGatewaySuspendAdmission(() => {});
suspensionStarted = suspension !== null;
try {
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
activeRootCountDuringDrain = getActiveGatewayRootWorkCount();
} finally {
suspension?.rollback();
drained.resolve();
}
});
});
parent.release();
parentReleased.resolve();
await drained.promise;
expect(suspensionStarted).toBe(true);
expect(subordinateAdmissionClosed).toBe(false);
expect(activeRootCountDuringDrain).toBe(1);
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
} finally {
parent.release();
parentReleased.resolve();
clearSessionQueues([key]);
resetGatewayWorkAdmission();
}
});
it("releases a detached drain root when its queue is cleared during debounce", async () => {
resetGatewayWorkAdmission();
const env = captureEnv(["OPENCLAW_TEST_FAST"]);
setTestEnvValue("OPENCLAW_TEST_FAST", "0");
const key = `test-cleared-debounce-root-${Date.now()}`;
const settings: QueueSettings = { mode: "followup", debounceMs: 60_000, cap: 50 };
try {
enqueueFollowupRun(key, createRun({ prompt: "clear during debounce" }), settings);
scheduleFollowupDrain(key, async () => {});
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(1);
});
clearSessionQueues([key]);
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
} finally {
clearSessionQueues([key]);
env.restore();
resetGatewayWorkAdmission();
}
});
it("does not retain stale callbacks when scheduleFollowupDrain runs with an empty queue", async () => {
const key = `test-no-stale-callback-${Date.now()}`;
const settings: QueueSettings = { mode: "followup", debounceMs: 0, cap: 50 };
+9 -3
View File
@@ -9,6 +9,7 @@ import {
channelRouteCompactKey,
channelRouteDedupeKey,
} from "../../../plugin-sdk/channel-route.js";
import { runWithGatewayIndependentRootWorkContinuation } from "../../../process/gateway-work-admission.js";
import { defaultRuntime } from "../../../runtime.js";
import {
buildPersistedUserTurnMediaInputsFromFields,
@@ -1129,7 +1130,9 @@ export function scheduleFollowupDrain(
// Cache callback only when a drain actually starts. Avoid keeping stale
// callbacks around from finalize calls where no queue work is pending.
rememberFollowupDrainCallback(key, effectiveRunFollowup);
void (async () => {
// Queue drains outlive their enqueue request across debounce and retries.
// Give the detached chain its own root so inherited request admission cannot go stale.
void runWithGatewayIndependentRootWorkContinuation(async () => {
let retryDeferred = false;
try {
const collectState = { forceIndividualCollect: false };
@@ -1138,7 +1141,7 @@ export function scheduleFollowupDrain(
if (queue.items.length === 0 && queue.droppedCount === 0) {
break;
}
await waitForQueueDebounce(queue);
await waitForQueueDebounce(queue, queue.abortController.signal);
await dropAbortedFollowups(queue.items, effectiveRunFollowup);
if (queue.items.length === 0 && queue.droppedCount === 0) {
break;
@@ -1364,5 +1367,8 @@ export function scheduleFollowupDrain(
scheduleFollowupDrain(key, effectiveRunFollowup);
}
}
})();
}).catch((err: unknown) => {
queue.draining = false;
defaultRuntime.error?.(`followup queue drain admission failed for ${key}: ${String(err)}`);
});
}
@@ -5,6 +5,12 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi }
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { HookRunner } from "../../plugins/hooks.js";
import {
getActiveGatewayRootWorkCount,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
} from "../../process/gateway-work-admission.js";
import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js";
import { initSessionState } from "./session.js";
@@ -176,10 +182,12 @@ describe("session hook context wiring", () => {
});
beforeEach(() => {
resetGatewayWorkAdmission();
hookRunnerMocks.hasHooks.mockReset();
hookRunnerMocks.runSessionStart.mockReset();
hookRunnerMocks.runSessionEnd.mockReset();
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockClear();
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockResolvedValue(0);
sessionCleanupMocks.resetRegisteredAgentHarnessSessions.mockClear();
sessionCleanupMocks.retireSessionMcpRuntime.mockClear();
hookRunnerMocks.runSessionStart.mockResolvedValue(undefined);
@@ -190,6 +198,7 @@ describe("session hook context wiring", () => {
});
afterEach(() => {
resetGatewayWorkAdmission();
vi.restoreAllMocks();
});
@@ -246,6 +255,85 @@ describe("session hook context wiring", () => {
expectFields(startContext, { sessionId: startEvent?.sessionId });
});
it("keeps rollover hooks and browser cleanup root-admitted until they settle", async () => {
const releases: Array<() => void> = [];
const held = () =>
new Promise<void>((resolve) => {
releases.push(resolve);
});
hookRunnerMocks.runSessionEnd.mockImplementationOnce(held);
hookRunnerMocks.runSessionStart.mockImplementationOnce(held);
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockImplementationOnce(
() =>
new Promise<number>((resolve) => {
releases.push(() => resolve(0));
}),
);
const sessionKey = "agent:main:telegram:direct:held-rollover";
const { storePath } = await createStoredSession({
prefix: "openclaw-session-hook-held-rollover",
sessionKey,
sessionId: "old-held-session",
});
await initSessionState({
ctx: { Body: "/new", SessionKey: sessionKey },
cfg: { session: { store: storePath } } as OpenClawConfig,
commandAuthorized: true,
});
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(3));
await vi.waitFor(() => expect(releases).toHaveLength(3));
for (const release of releases) {
release();
}
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("hands rollover hooks off after restart drain closes admission", async () => {
const releases: Array<() => void> = [];
const held = () =>
new Promise<void>((resolve) => {
releases.push(resolve);
});
hookRunnerMocks.runSessionEnd.mockImplementationOnce(held);
hookRunnerMocks.runSessionStart.mockImplementationOnce(held);
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockImplementationOnce(
() =>
new Promise<number>((resolve) => {
releases.push(() => resolve(0));
}),
);
const sessionKey = "agent:main:telegram:direct:restart-handoff";
const { storePath } = await createStoredSession({
prefix: "openclaw-session-hook-restart-handoff",
sessionKey,
sessionId: "old-restart-session",
});
const admission = tryBeginGatewayRootWorkAdmission();
expect(admission).not.toBeNull();
await admission?.run(async () => {
markGatewayRestartDraining();
await initSessionState({
ctx: { Body: "/new", SessionKey: sessionKey },
cfg: { session: { store: storePath } } as OpenClawConfig,
commandAuthorized: true,
});
await vi.waitFor(() => expect(releases).toHaveLength(3));
expect(getActiveGatewayRootWorkCount()).toBe(4);
});
admission?.release();
expect(getActiveGatewayRootWorkCount()).toBe(3);
for (const release of releases) {
release();
}
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
});
it("marks explicit /reset rollovers with reason reset", async () => {
const sessionKey = "agent:main:telegram:direct:456";
const { storePath } = await createStoredSession({
@@ -6,6 +6,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import { loadSessionStore, type SessionEntry } from "../../config/sessions.js";
import type { HookRunner } from "../../plugins/hooks.js";
import {
getActiveGatewayRootWorkCount,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
} from "../../process/gateway-work-admission.js";
const hookRunnerMocks = vi.hoisted(() => ({
hasHooks: vi.fn<HookRunner["hasHooks"]>(),
@@ -46,6 +52,7 @@ function firstSessionStartCall() {
describe("session-updates lifecycle hooks", () => {
beforeEach(async () => {
resetGatewayWorkAdmission();
vi.resetModules();
vi.doMock("../../plugins/hook-runner-global.js", () => ({
getGlobalHookRunner: () =>
@@ -67,6 +74,7 @@ describe("session-updates lifecycle hooks", () => {
});
afterEach(async () => {
resetGatewayWorkAdmission();
vi.restoreAllMocks();
await Promise.all(
tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })),
@@ -109,6 +117,69 @@ describe("session-updates lifecycle hooks", () => {
expect(startContext?.agentId).toBe("main");
});
it("keeps compaction lifecycle hooks root-admitted until both settle", async () => {
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
const releases: Array<() => void> = [];
const heldHook = () =>
new Promise<void>((resolve) => {
releases.push(resolve);
});
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
await incrementCompactionCount({
cfg: { session: { store: storePath } } as OpenClawConfig,
sessionEntry: entry,
sessionStore,
sessionKey,
storePath,
newSessionId: "s2",
});
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(2));
await vi.waitFor(() => expect(releases).toHaveLength(2));
for (const release of releases) {
release();
}
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("hands compaction lifecycle hooks off after restart drain closes admission", async () => {
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
const releases: Array<() => void> = [];
const heldHook = () =>
new Promise<void>((resolve) => {
releases.push(resolve);
});
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
const admission = tryBeginGatewayRootWorkAdmission();
expect(admission).not.toBeNull();
await admission?.run(async () => {
markGatewayRestartDraining();
await incrementCompactionCount({
cfg: { session: { store: storePath } } as OpenClawConfig,
sessionEntry: entry,
sessionStore,
sessionKey,
storePath,
newSessionId: "s2",
});
await vi.waitFor(() => expect(releases).toHaveLength(2));
expect(getActiveGatewayRootWorkCount()).toBe(3);
});
admission?.release();
expect(getActiveGatewayRootWorkCount()).toBe(2);
for (const release of releases) {
release();
}
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
});
it("recreates a complete persisted row when compaction updates a missing store row", async () => {
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf-8");
+7 -2
View File
@@ -13,6 +13,7 @@ import {
import { resolveStableSessionEndTranscript } from "../../gateway/session-transcript-files.fs.js";
import { logVerbose } from "../../globals.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js";
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
import { getRemoteSkillEligibility } from "../../skills/runtime/remote.js";
import { resolveReusableWorkspaceSkillSnapshot } from "../../skills/runtime/session-snapshot.js";
@@ -96,7 +97,9 @@ function emitCompactionSessionLifecycleHooks(params: {
transcriptArchived: transcript.transcriptArchived,
nextSessionId: params.nextEntry.sessionId,
});
void hookRunner.runSessionEnd(payload.event, payload.context).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionEnd(payload.event, payload.context);
}).catch((err: unknown) => {
logVerbose(`session_end hook failed: ${String(err)}`);
});
}
@@ -108,7 +111,9 @@ function emitCompactionSessionLifecycleHooks(params: {
cfg: params.cfg,
resumedFrom: params.previousEntry.sessionId,
});
void hookRunner.runSessionStart(payload.event, payload.context).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionStart(payload.event, payload.context);
}).catch((err: unknown) => {
logVerbose(`session_start hook failed: ${String(err)}`);
});
}
+16 -7
View File
@@ -55,6 +55,7 @@ import { deliverSessionMaintenanceWarning } from "../../infra/session-maintenanc
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
import type { PluginHookSessionEndReason } from "../../plugins/hook-types.js";
import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js";
import {
buildAgentMainSessionKey,
isAcpSessionKey,
@@ -1062,11 +1063,15 @@ async function initSessionStateAttemptLocked(
// their transcript aliases main; cleanup must carry both exact keys.
const runtimePolicySessionKey =
resolveRuntimePolicySessionKey({ cfg, ctx: sessionCtxForState, sessionKey }) ?? sessionKey;
void cleanupBrowserSessionsForLifecycleEnd({
cfg,
sessionKeys: [previousSessionEntry.sessionId, sessionKey, runtimePolicySessionKey],
onWarn: (message) => log.warn(message),
onError: (error) => log.warn(`browser tab cleanup failed: ${String(error)}`),
void runWithGatewayIndependentRootWorkContinuation(async () => {
await cleanupBrowserSessionsForLifecycleEnd({
cfg,
sessionKeys: [previousSessionEntry.sessionId, sessionKey, runtimePolicySessionKey],
onWarn: (message) => log.warn(message),
onError: (error) => log.warn(`browser tab cleanup failed: ${String(error)}`),
});
}).catch((error: unknown) => {
log.warn(`browser tab cleanup admission failed: ${String(error)}`);
});
}
@@ -1108,7 +1113,9 @@ async function initSessionStateAttemptLocked(
transcriptArchived: previousSessionTranscript.transcriptArchived,
nextSessionId: effectiveSessionId,
});
void hookRunner.runSessionEnd(payload.event, payload.context).catch(() => {});
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionEnd(payload.event, payload.context);
}).catch(() => {});
}
}
@@ -1133,7 +1140,9 @@ async function initSessionStateAttemptLocked(
cfg,
resumedFrom: previousSessionEntry?.sessionId,
});
void hookRunner.runSessionStart(payload.event, payload.context).catch(() => {});
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionStart(payload.event, payload.context);
}).catch(() => {});
}
}
+1
View File
@@ -22,6 +22,7 @@ export {
markGatewaySigusr1RestartHandled,
peekGatewaySigusr1RestartReason,
resetGatewayRestartStateForInProcessRestart,
rollbackGatewayRestartSignalAdmission,
scheduleGatewaySigusr1Restart,
} from "../../infra/restart.js";
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
+44
View File
@@ -19,6 +19,7 @@ const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false);
const markGatewaySigusr1RestartHandled = vi.fn();
const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined);
const resetGatewayRestartStateForInProcessRestart = vi.fn();
const rollbackGatewayRestartSignalAdmission = vi.fn();
const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({
kind: "gateway-supervisor-restart-handoff" as const,
version: 1 as const,
@@ -129,6 +130,7 @@ vi.mock("../../infra/restart.js", () => ({
markGatewaySigusr1RestartHandled: () => markGatewaySigusr1RestartHandled(),
peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(),
resetGatewayRestartStateForInProcessRestart: () => resetGatewayRestartStateForInProcessRestart(),
rollbackGatewayRestartSignalAdmission: () => rollbackGatewayRestartSignalAdmission(),
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
@@ -492,6 +494,9 @@ describe("runGatewayLoop", () => {
expect(consumeGatewayRestartIntentPayloadSync).toHaveBeenCalledOnce();
expect(markGatewayDraining).toHaveBeenCalledOnce();
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
loadConfig.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
);
expect(waitForActiveTasks).toHaveBeenCalledWith(90_000);
expectRestartCloseCall(closeFirst, 90_000);
await startedSecond;
@@ -898,6 +903,9 @@ describe("runGatewayLoop", () => {
await startedSecond;
expect(start).toHaveBeenCalledTimes(2);
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
markGatewaySigusr1RestartHandled.mock.invocationCallOrder[0] ?? 0,
);
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
@@ -1477,6 +1485,9 @@ describe("runGatewayLoop", () => {
expect(consumeGatewaySigusr1RestartAuthorization).toHaveBeenCalledOnce();
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledOnce();
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
markGatewaySigusr1RestartHandled.mock.invocationCallOrder[0] ?? 0,
);
expect(markRestartAbortedMainSessions).toHaveBeenCalledWith({
cfg: {
gateway: {
@@ -2028,6 +2039,7 @@ describe("runGatewayLoop", () => {
"SIGUSR1 handler failed: lifecycle module corrupted",
);
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalled();
expect(rollbackGatewayRestartSignalAdmission).toHaveBeenCalledOnce();
expect(close).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledTimes(1);
@@ -2063,6 +2075,7 @@ describe("runGatewayLoop", () => {
// Restart token must be cleared so future SIGUSR1 restarts are not
// permanently coalesced as "already in-flight".
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalled();
expect(rollbackGatewayRestartSignalAdmission).toHaveBeenCalledOnce();
expect(close).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledTimes(1);
@@ -2070,6 +2083,37 @@ describe("runGatewayLoop", () => {
await expect(exited).resolves.toBe(0);
});
});
it("recloses restart admission after a failed SIGUSR1 handler rolls it back", async () => {
vi.clearAllMocks();
markGatewaySigusr1RestartHandled.mockImplementationOnce(() => {
throw new Error("restart token cleanup failed");
});
await withIsolatedSignals(async ({ captureSignal }) => {
const { close, start, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
const sigterm = captureSignal("SIGTERM");
sigusr1();
await waitForLoopCondition(
() => rollbackGatewayRestartSignalAdmission.mock.calls.length === 1,
"failed SIGUSR1 handler did not roll back restart admission",
);
sigusr1();
await waitForLoopCondition(
() => start.mock.calls.length === 2,
"second SIGUSR1 did not start a restart",
);
expect(close).toHaveBeenCalledTimes(1);
expect(markGatewayDraining).toHaveBeenCalledTimes(2);
sigterm();
await expect(exited).resolves.toBe(0);
});
});
});
describe("gateway discover routing helpers", () => {
+27 -18
View File
@@ -146,7 +146,7 @@ export async function runGatewayLoop(params: {
let activeRestartRequest: GatewayRunSignalRequest | null = null;
let forceActiveRestartExit: (() => void) | null = null;
let pendingStartupForceExitTimer: ReturnType<typeof setTimeout> | null = null;
let restartDrainingMarkPromise: Promise<void> | null = null;
let restartDrainingMarked = false;
let startupFailedWithoutServerHandle = false;
const processInstanceId = randomUUID();
const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild;
@@ -408,17 +408,15 @@ export async function runGatewayLoop(params: {
return DEFAULT_RESTART_DRAIN_TIMEOUT_MS;
}
};
const markRestartDraining = async () => {
if (!restartDrainingMarkPromise) {
restartDrainingMarkPromise = (async () => {
const { markGatewayDraining } = await loadGatewayLifecycleRuntimeModule();
markGatewayDraining();
})().catch((err: unknown) => {
restartDrainingMarkPromise = null;
throw err;
});
const markRestartDraining = () => {
if (restartDrainingMarked) {
return;
}
await restartDrainingMarkPromise;
// The lifecycle module is primed before listeners are installed. Keep this
// transition synchronous so an accepted signal cannot yield between token
// handling and closing process-wide root admission.
eagerLifecycleRuntime.markGatewayDraining();
restartDrainingMarked = true;
};
const runAcceptedRequest = (acceptedRequest: GatewayRunSignalRequest) => {
@@ -572,7 +570,7 @@ export async function runGatewayLoop(params: {
// Reject new enqueues immediately during the drain window so
// sessions get an explicit restart error instead of silent task loss.
await markRestartDraining();
markRestartDraining();
const activeTasks = getActiveTaskCount();
const activeRuns = getActiveEmbeddedRunCount();
activeTasksAtDrainStart = activeTasks;
@@ -736,8 +734,11 @@ export async function runGatewayLoop(params: {
gatewayLog.info(`received ${signal} during shutdown; ignoring`);
return;
}
shuttingDown = true;
const isRestart = action === "restart";
if (isRestart) {
markRestartDraining();
}
shuttingDown = true;
gatewayLog.info(`received ${signal}; ${isRestart ? "restarting" : "shutting down"}`);
if (isRestart) {
startGatewayRestartTrace("restart.signal.received", [
@@ -758,9 +759,6 @@ export async function runGatewayLoop(params: {
}
if (!server || !restartResolver) {
pendingStartupRequest = acceptedRequest;
void markRestartDraining().catch((err: unknown) => {
gatewayLog.warn(`failed to mark gateway draining for startup restart: ${String(err)}`);
});
armPendingStartupForceExitTimer();
return;
}
@@ -803,7 +801,9 @@ export async function runGatewayLoop(params: {
const restartIntent = consumeGatewayRestartIntentPayloadSync();
if (restartIntent) {
abortPendingChannelReloads();
if (consumeGatewaySigusr1RestartAuthorization()) {
const authorized = consumeGatewaySigusr1RestartAuthorization();
markRestartDraining();
if (authorized) {
markGatewaySigusr1RestartHandled();
}
request("restart", "SIGUSR1", restartIntent.reason ?? "gateway.restart", restartIntent);
@@ -836,6 +836,7 @@ export async function runGatewayLoop(params: {
abortPendingChannelReloads();
const sigusr1RestartIntent = consumeGatewaySigusr1RestartIntent();
const restartReason = peekGatewaySigusr1RestartReason();
markRestartDraining();
markGatewaySigusr1RestartHandled();
request(
"restart",
@@ -856,6 +857,14 @@ export async function runGatewayLoop(params: {
} catch {
// Best-effort: the eager reference itself is the recovery path.
}
try {
eagerLifecycleRuntime.rollbackGatewayRestartSignalAdmission();
// A later signal must repeat the synchronous close transition even if
// this handler failed after marking the one-way drain.
restartDrainingMarked = false;
} catch {
// Keep admission recovery independent from restart-token recovery.
}
});
};
@@ -908,7 +917,7 @@ export async function runGatewayLoop(params: {
let isFirstStart = true;
for (;;) {
await onIteration();
restartDrainingMarkPromise = null;
restartDrainingMarked = false;
startupStartedAt = Date.now();
let startupFailedBeforeServerHandle = false;
try {
+160
View File
@@ -1,5 +1,15 @@
// Every-job firing tests cover repeated schedule execution semantics.
import { describe, expect, it, vi } from "vitest";
import {
getGatewaySuspendStatus,
prepareGatewaySuspend,
resetGatewaySuspendCoordinatorForTest,
} from "../infra/gateway-suspend-coordinator.js";
import {
beginGatewayRestartSignalAdmission,
isGatewayWorkAdmissionClosed,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import { CronService } from "./service.js";
import {
createStartedCronServiceWithFinishedBarrier,
@@ -101,6 +111,156 @@ describe("CronService interval/cron jobs fire on time", () => {
await store.cleanup();
});
it("keeps a due timer frozen while scheduling is paused and fires it after resume", async () => {
const store = await makeStorePath();
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
storePath: store.storePath,
logger: noopLogger,
});
await cron.start();
const job = await cron.add({
name: "suspension pause check",
enabled: true,
schedule: { kind: "every", everyMs: 10_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "resumed-tick" },
});
cron.pauseScheduling();
await vi.advanceTimersByTimeAsync(10_005);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
const finishedRun = finished.waitForOk(job.id);
cron.resumeScheduling();
await vi.runOnlyPendingTimersAsync();
await finishedRun;
expectMainSystemEvent(enqueueSystemEvent, "resumed-tick", job.id);
cron.stop();
await store.cleanup();
});
it("rolls a failed scheduler resume back so a retry can rearm cron", async () => {
const store = await makeStorePath();
const logger = createNoopLogger();
const cron = new CronService({
storePath: store.storePath,
cronEnabled: true,
log: logger,
enqueueSystemEvent: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
await cron.start();
await cron.add({
name: "resume retry check",
enabled: true,
schedule: { kind: "every", everyMs: 10_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "resume" },
});
cron.pauseScheduling();
logger.debug.mockImplementationOnce(() => {
throw new Error("arm failed");
});
expect(() => cron.resumeScheduling()).toThrow("arm failed");
expect(vi.getTimerCount()).toBe(0);
expect(() => cron.resumeScheduling()).not.toThrow();
expect(vi.getTimerCount()).toBe(1);
cron.stop();
await store.cleanup();
});
it("keeps admission closed until a real cron scheduler resume retry succeeds", async () => {
const store = await makeStorePath();
const logger = createNoopLogger();
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
storePath: store.storePath,
logger,
});
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
try {
await cron.start();
const job = await cron.add({
name: "coordinator resume retry check",
enabled: true,
schedule: { kind: "every", everyMs: 10_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "recovered-tick" },
});
logger.debug.mockImplementationOnce(() => {
throw new Error("arm failed");
});
expect(
prepareGatewaySuspend({
requestId: "cron-resume-retry",
pauseScheduling: () => cron.pauseScheduling(),
resumeScheduling: () => cron.resumeScheduling(),
inspect: { getQueueSize: () => 1 },
}),
).toMatchObject({ status: "recovering" });
expect(isGatewayWorkAdmissionClosed()).toBe(true);
await vi.advanceTimersByTimeAsync(1_000);
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
expect(isGatewayWorkAdmissionClosed()).toBe(false);
const finishedRun = finished.waitForOk(job.id);
await vi.advanceTimersByTimeAsync(9_005);
await finishedRun;
expectMainSystemEvent(enqueueSystemEvent, "recovered-tick", job.id);
} finally {
cron.stop();
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
await store.cleanup();
}
});
it("keeps a due timer pending when restart signal admission rolls back", async () => {
const store = await makeStorePath();
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
storePath: store.storePath,
logger: noopLogger,
});
resetGatewayWorkAdmission();
try {
await cron.start();
const job = await cron.add({
name: "restart signal rollback check",
enabled: true,
schedule: { kind: "every", everyMs: 10_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "rollback-tick" },
});
const pendingSignal = beginGatewayRestartSignalAdmission();
const finishedRun = finished.waitForOk(job.id);
await vi.advanceTimersByTimeAsync(10_005);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(pendingSignal.rollback()).toBe(true);
await finishedRun;
expectMainSystemEvent(enqueueSystemEvent, "rollback-tick", job.id);
} finally {
cron.stop();
resetGatewayWorkAdmission();
await store.cleanup();
}
});
it("fires a cron-expression job when the timer fires a few ms late", async () => {
const store = await makeStorePath();
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
+2
View File
@@ -235,6 +235,8 @@ export function createMockCronStateForJobs(params: {
durableNextRunAtMsByJobId: new Map<string, number | undefined>(),
running: false,
stopped: false,
schedulingPaused: false,
schedulerStarted: false,
restartRecoveryPending: false,
pendingCatchupDeferralJobIds: new Set<string>(),
activeManualRunJobIds: new Set<string>(),
+56 -1
View File
@@ -20,19 +20,74 @@ export type { CronEvent, CronServiceDeps } from "./service/state.js";
/** Public cron service facade that owns mutable scheduler state and delegates to locked ops. */
export class CronService implements CronServiceContract {
private readonly state;
private startInProgress = 0;
private startState: { generation: number; promise: Promise<void> } | null = null;
private lifecycleGeneration = 0;
constructor(deps: CronServiceDeps) {
this.state = createCronServiceState(deps);
}
async start() {
await ops.start(this.state);
const generation = this.lifecycleGeneration;
const pending = this.startState;
if (pending) {
try {
await pending.promise;
} catch (err) {
if (pending.generation === generation) {
throw err;
}
}
if (pending.generation === generation) {
return;
}
await this.start();
return;
}
const promise = this.startOnce(generation);
this.startState = { generation, promise };
try {
await promise;
} finally {
if (this.startState?.promise === promise) {
this.startState = null;
}
}
}
private async startOnce(generation: number) {
this.startInProgress += 1;
this.state.schedulerStarted = false;
try {
await ops.start(this.state);
if (generation !== this.lifecycleGeneration) {
ops.stop(this.state);
return;
}
this.state.schedulerStarted = !this.state.stopped;
} finally {
this.startInProgress -= 1;
}
}
stop() {
this.lifecycleGeneration += 1;
ops.stop(this.state);
}
pauseScheduling() {
ops.pauseScheduling(this.state);
}
resumeScheduling() {
ops.resumeScheduling(this.state);
}
getSuspensionBlockerCount() {
return this.startInProgress;
}
async status() {
return await ops.status(this.state);
}
+26
View File
@@ -316,9 +316,35 @@ export async function start(state: CronServiceState) {
/** Stops the cron service timer without mutating persisted job state. */
export function stop(state: CronServiceState) {
state.stopped = true;
state.schedulerStarted = false;
stopTimer(state);
}
/** Temporarily stops automatic ticks without running startup recovery on resume. */
export function pauseScheduling(state: CronServiceState) {
state.schedulingPaused = true;
stopTimer(state);
}
export function resumeScheduling(state: CronServiceState) {
if (!state.schedulingPaused) {
return;
}
state.schedulingPaused = false;
if (!state.schedulerStarted) {
return;
}
try {
armTimer(state);
} catch (err) {
// armTimer can install a timer before a later dependency throws. Roll the
// whole transition back so a suspension retry cannot reopen without cron.
state.schedulingPaused = true;
stopTimer(state);
throw err;
}
}
/** Returns cron service status after a read-only maintenance pass. */
export async function status(state: CronServiceState) {
return await locked(state, async () => {
+4
View File
@@ -206,6 +206,8 @@ export type CronServiceState = {
timer: NodeJS.Timeout | null;
running: boolean;
stopped: boolean;
schedulingPaused: boolean;
schedulerStarted: boolean;
restartRecoveryPending: boolean;
/** Prevents maintenance reads from advancing deferred startup catch-up slots.
* Entries are removed when the deferred job runs or becomes irrelevant. */
@@ -234,6 +236,8 @@ export function createCronServiceState(deps: CronServiceDeps): CronServiceState
timer: null,
running: false,
stopped: false,
schedulingPaused: false,
schedulerStarted: false,
restartRecoveryPending: false,
pendingCatchupDeferralJobIds: new Set<string>(),
activeManualRunJobIds: new Set<string>(),
+28 -4
View File
@@ -10,6 +10,10 @@ import {
HEARTBEAT_SKIP_CRON_IN_PROGRESS,
isRetryableHeartbeatBusySkipReason,
} from "../../infra/heartbeat-wake.js";
import {
beginGatewayRootWorkAdmissionWhenOpen,
GatewayDrainingError,
} from "../../process/gateway-work-admission.js";
import {
DEFAULT_AGENT_ID,
normalizeAgentId,
@@ -1214,7 +1218,7 @@ export function armTimer(state: CronServiceState) {
clearTimeout(state.timer);
}
state.timer = null;
if (state.stopped) {
if (state.stopped || state.schedulingPaused) {
state.deps.log.debug({}, "cron: armTimer skipped - scheduler stopped");
return;
}
@@ -1276,7 +1280,7 @@ export function armTimer(state: CronServiceState) {
}
function armRunningRecheckTimer(state: CronServiceState) {
if (state.stopped) {
if (state.stopped || state.schedulingPaused) {
return;
}
if (state.timer) {
@@ -1289,9 +1293,29 @@ function armRunningRecheckTimer(state: CronServiceState) {
}, MAX_TIMER_DELAY_MS);
}
/** Handles one cron timer tick: load due jobs, reserve them, execute, persist, and re-arm. */
/** Handles one cron timer tick under the process-wide root work admission. */
export async function onTimer(state: CronServiceState) {
if (state.stopped) {
let admission;
try {
// A restart signal can be rejected after temporarily closing admission.
// Wait for that decision so the consumed timer is not silently lost.
admission = await beginGatewayRootWorkAdmissionWhenOpen();
} catch (err) {
if (err instanceof GatewayDrainingError) {
return;
}
throw err;
}
try {
await admission.run(async () => await onAdmittedTimer(state));
} finally {
admission.release();
}
}
/** Loads due jobs, reserves them, executes, persists, and re-arms. */
async function onAdmittedTimer(state: CronServiceState) {
if (state.stopped || state.schedulingPaused) {
return;
}
if (state.restartRecoveryPending) {
+34
View File
@@ -2,6 +2,7 @@
// abort fanout, history snapshots, and cleanup of buffered streaming state.
import { afterEach, describe, expect, it, vi } from "vitest";
import { isAgentRunRestartAbortReason } from "../agents/run-termination.js";
import { onAgentEvent } from "../infra/agent-events.js";
import {
abortChatRunById,
abortChatRunsForProvider,
@@ -307,6 +308,39 @@ describe("registerChatAbortController", () => {
});
describe("abortChatRunById", () => {
it("retains terminal persistence ownership observed during abort", () => {
const { runId, sessionKey, entry, ops } = createAbortRunFixture({});
let terminalEvents = 0;
const unsubscribe = onAgentEvent((event) => {
if (event.runId === runId && event.stream === "lifecycle" && event.data.phase === "end") {
terminalEvents += 1;
entry.projectSessionTerminalPending = true;
entry.projectSessionTerminalObservedAt = event.ts;
}
});
try {
const result = abortChatRunById(ops, { runId, sessionKey, stopReason: "user" });
expect(result).toEqual({ aborted: true });
expect(entry.controller.signal.aborted).toBe(true);
expect(entry.projectSessionActive).toBe(false);
expect(entry.registrationCleanupRequested).toBe(true);
expect(entry.projectSessionTerminalPending).toBe(true);
expect(entry.projectSessionTerminalObservedAt).toEqual(expect.any(Number));
expect(ops.chatAbortControllers.get(runId)).toBe(entry);
expect(abortChatRunById(ops, { runId, sessionKey, stopReason: "user" })).toEqual({
aborted: false,
});
expect(terminalEvents).toBe(1);
expect(ops.broadcast).toHaveBeenCalledOnce();
expect(ops.removeChatRun).toHaveBeenCalledOnce();
} finally {
unsubscribe();
}
});
it("broadcasts aborted payload with partial message when buffered text exists", () => {
const now = new Date("2026-01-02T03:04:05.000Z");
const { runId, sessionKey, entry, ops } = createAbortRunFixture({
+18 -1
View File
@@ -470,6 +470,9 @@ function resolveDefaultGlobalAgentId(ops: ChatAbortOps): string | undefined {
}
export function isChatAbortControllerEntryAbortable(entry: ChatAbortControllerEntry): boolean {
if (entry.controller.signal.aborted) {
return false;
}
try {
return entry.isAbortable?.(entry) !== false;
} catch {
@@ -521,8 +524,13 @@ export function abortChatRunById(
if (stopReason) {
active.abortStopReason = stopReason;
}
active.projectSessionActive = false;
// Reserve terminal ownership before abort listeners run; synchronous caller
// cleanup must not erase the entry before Gateway observes the event below.
active.projectSessionTerminalPending = true;
active.projectSessionTerminalObservedAt = undefined;
active.registrationCleanupRequested = true;
active.controller.abort(createChatAbortSignalReason(stopReason));
removeChatAbortControllerEntry(ops.chatAbortControllers, runId, active);
ops.clearChatRunState(runId);
const removed = ops.removeChatRun(runId, runId, sessionKey);
if (active.controlUiVisible !== false) {
@@ -551,6 +559,15 @@ export function abortChatRunById(
endedAt: Date.now(),
},
});
// Gateway listeners synchronously stamp the terminal observation. Keep the
// entry as suspension-visible ownership until its persistence write settles.
if (
ops.chatAbortControllers.get(runId) === active &&
active.projectSessionTerminalObservedAt === undefined &&
!active.projectSessionTerminalPersistence
) {
removeChatAbortControllerEntry(ops.chatAbortControllers, runId, active);
}
ops.agentRunSeq.delete(runId);
if (removed?.clientRunId) {
ops.agentRunSeq.delete(removed.clientRunId);
+50 -1
View File
@@ -16,6 +16,11 @@ import {
resetPluginRuntimeStateForTest,
setActivePluginRegistry,
} from "../plugins/runtime.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
runWithGatewayIndependentRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import {
getSkillsSnapshotVersion,
resetSkillsRefreshStateForTest,
@@ -723,6 +728,8 @@ function createReloaderHarness(
promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise<boolean>;
initialPluginInstallRecords?: Record<string, PluginInstallRecord>;
readPluginInstallRecords?: () => Promise<Record<string, PluginInstallRecord>>;
runTransaction?: <T>(run: () => Promise<T>) => Promise<T>;
onRestart?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
} = {},
) {
const watcher = createWatcherMock();
@@ -735,7 +742,9 @@ function createReloaderHarness(
async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {},
);
const onHotReload = vi.fn(async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {});
const onRestart = vi.fn((_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {});
const onRestart = vi.fn(
options.onRestart ?? ((_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}),
);
let writeListener: ((event: ConfigWriteNotification) => void) | null = null;
const subscribeToWrites = vi.fn((listener: (event: ConfigWriteNotification) => void) => {
writeListener = listener;
@@ -765,6 +774,7 @@ function createReloaderHarness(
onNoopConfigCommit,
onHotReload,
onRestart,
...(options.runTransaction ? { runTransaction: options.runTransaction } : {}),
log,
watchPath: "/tmp/openclaw.json",
});
@@ -816,10 +826,12 @@ function getOnlyPromoteSnapshotCall(promoteSnapshot: {
describe("startGatewayConfigReloader", () => {
beforeEach(() => {
resetGatewayWorkAdmission();
vi.useFakeTimers();
});
afterEach(() => {
resetGatewayWorkAdmission();
vi.useRealTimers();
vi.restoreAllMocks();
});
@@ -932,6 +944,43 @@ describe("startGatewayConfigReloader", () => {
await harness.reloader.stop();
});
it("keeps restart preparation inside the accepted config root", async () => {
let releaseRestart = () => {};
let noteRestartStarted = () => {};
const restartStarted = new Promise<void>((resolve) => {
noteRestartStarted = resolve;
});
const restartPending = new Promise<void>((resolve) => {
releaseRestart = resolve;
});
const initialConfig: OpenClawConfig = {
gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } },
};
const nextConfig: OpenClawConfig = {
gateway: { reload: { debounceMs: 0 }, terminal: { enabled: false } },
};
const harness = createReloaderHarness(
async () => makeSnapshot({ config: nextConfig, hash: "restart-root" }),
{
initialConfig,
runTransaction: runWithGatewayIndependentRootWorkAdmission,
onRestart: async () => {
noteRestartStarted();
await restartPending;
},
},
);
harness.watcher.emit("change");
await vi.runOnlyPendingTimersAsync();
await restartStarted;
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseRestart();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
await harness.reloader.stop();
});
it("does not notify lifecycle owners when reload mode ignores the change", async () => {
const initialConfig: OpenClawConfig = {
gateway: { reload: { mode: "off", debounceMs: 0 }, terminal: { enabled: true } },
+36 -22
View File
@@ -116,6 +116,8 @@ export function startGatewayConfigReloader(opts: {
onNoopConfigCommit: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
/** Keeps one accepted config transaction inside the Gateway work fence. */
runTransaction?: <T>(run: () => Promise<T>) => Promise<T>;
promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise<boolean>;
initialPluginInstallRecords?: PluginInstallRecords;
readPluginInstallRecords?: () => Promise<PluginInstallRecords>;
@@ -164,21 +166,21 @@ export function startGatewayConfigReloader(opts: {
const schedule = () => {
scheduleAfter(settings.debounceMs);
};
const queueRestart = (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => {
const queueRestart = async (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => {
if (restartQueued) {
return;
}
restartQueued = true;
void (async () => {
try {
await opts.onRestart(plan, nextConfig);
} catch (err) {
// Restart checks can fail (for example unresolved SecretRefs). Keep the
// reloader alive and allow a future change to retry restart scheduling.
restartQueued = false;
opts.log.error(`config restart failed: ${String(err)}`);
}
})();
try {
// Restart preparation reads secrets and can mutate auth/runtime state.
// Keep it inside the accepted config transaction instead of detaching it.
await opts.onRestart(plan, nextConfig);
} catch (err) {
// Restart checks can fail (for example unresolved SecretRefs). Keep the
// reloader alive and allow a future change to retry restart scheduling.
restartQueued = false;
opts.log.error(`config restart failed: ${String(err)}`);
}
};
const handleMissingSnapshot = (snapshot: ConfigFileSnapshot): boolean => {
@@ -297,12 +299,12 @@ export function startGatewayConfigReloader(opts: {
restartReasons: [...plan.restartReasons, followUp.reason],
};
await opts.onConfigChange?.(restartPlan, nextConfig);
queueRestart(restartPlan, nextConfig);
await queueRestart(restartPlan, nextConfig);
return;
}
if (settings.mode === "restart") {
await opts.onConfigChange?.({ ...plan, restartGateway: true }, nextConfig);
queueRestart(plan, nextConfig);
await queueRestart(plan, nextConfig);
return;
}
if (plan.restartGateway) {
@@ -315,7 +317,7 @@ export function startGatewayConfigReloader(opts: {
return;
}
await opts.onConfigChange?.(plan, nextConfig);
queueRestart(plan, nextConfig);
await queueRestart(plan, nextConfig);
return;
}
@@ -335,6 +337,14 @@ export function startGatewayConfigReloader(opts: {
}
};
const runAcceptedTransaction = async (run: () => Promise<void>) => {
if (opts.runTransaction) {
await opts.runTransaction(run);
return;
}
await run();
};
const promoteAcceptedInProcessWrite = async (persistedHash: string) => {
if (!opts.promoteSnapshot) {
return;
@@ -368,12 +378,14 @@ export function startGatewayConfigReloader(opts: {
const pendingWrite = pendingInProcessConfig;
pendingInProcessConfig = null;
missingConfigRetries = 0;
await applySnapshot(
pendingWrite.config,
pendingWrite.compareConfig,
pendingWrite.afterWrite,
);
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
await runAcceptedTransaction(async () => {
await applySnapshot(
pendingWrite.config,
pendingWrite.compareConfig,
pendingWrite.afterWrite,
);
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
});
return;
}
const snapshot = await opts.readSnapshot();
@@ -390,8 +402,10 @@ export function startGatewayConfigReloader(opts: {
handleInvalidSnapshot(snapshot);
return;
}
await applySnapshot(snapshot.config, snapshot.sourceConfig);
await promoteAcceptedSnapshot(snapshot, "valid-config");
await runAcceptedTransaction(async () => {
await applySnapshot(snapshot.config, snapshot.sourceConfig);
await promoteAcceptedSnapshot(snapshot, "valid-config");
});
} catch (err) {
opts.log.error(`config reload failed: ${String(err)}`);
} finally {
+38 -4
View File
@@ -289,7 +289,9 @@ describe("createCronExitWatchers", () => {
// The orphaned child is killed and the job never fires.
expect(fake.runCancels.length).toBe(1);
expect(fireOnExit).not.toHaveBeenCalled();
expect(w.activeJobIds()).toEqual([]);
expect(w.activeJobIds()).toEqual(["job-a"]);
fake.runs[0].deferred.resolve({ exitCode: null, reason: "manual-cancel" });
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
});
it("does not arm a watcher for time-based or disabled jobs", async () => {
@@ -325,8 +327,8 @@ describe("createCronExitWatchers", () => {
expect(supervisor.spawn).toHaveBeenCalledTimes(1);
});
it("cancels the watcher when the job is removed from the set", async () => {
const { supervisor, cancelled } = makeFakeSupervisor();
it("keeps a cancelled watcher blocking until the supervised child settles", async () => {
const { supervisor, cancelled, runs } = makeFakeSupervisor();
const w = createCronExitWatchers({
getProcessSupervisor: () => supervisor as never,
persistCompletion: vi.fn(async () => {}),
@@ -337,7 +339,10 @@ describe("createCronExitWatchers", () => {
await flush();
w.reconcile([]);
expect(cancelled).toContain("cron-exit:job-a");
expect(w.activeJobIds()).toEqual([]);
expect(w.activeJobIds()).toEqual(["job-a"]);
runs[0].deferred.resolve({ exitCode: null, reason: "manual-cancel" });
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
});
it("does not fire a job whose watcher was cancelled before exit", async () => {
@@ -357,6 +362,35 @@ describe("createCronExitWatchers", () => {
expect(fireOnExit).not.toHaveBeenCalled();
});
it("retains a blocker and suppresses stale fire when removed during terminal persistence", async () => {
const { supervisor, runs } = makeFakeSupervisor();
let releasePersist = () => {};
const persistCompletion = vi.fn(
() =>
new Promise<void>((resolve) => {
releasePersist = resolve;
}),
);
const fireOnExit = vi.fn(async () => {});
const w = createCronExitWatchers({
getProcessSupervisor: () => supervisor as never,
persistCompletion,
fireOnExit,
logger: noopLogger,
});
w.reconcile([onExitJob("job-a")]);
await flush();
runs[0].deferred.resolve({ exitCode: 0, reason: "exit" });
await vi.waitFor(() => expect(persistCompletion).toHaveBeenCalledOnce());
w.reconcile([]);
expect(w.activeJobIds()).toEqual(["job-a"]);
releasePersist();
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
expect(fireOnExit).not.toHaveBeenCalled();
});
it("is one-shot: a fired job is not re-armed on a later reconcile", async () => {
const { supervisor, runs } = makeFakeSupervisor();
const w = createCronExitWatchers({
+58 -5
View File
@@ -79,17 +79,32 @@ export function createCronExitWatchers(params: {
job: OnExitCronJob;
run: ManagedRun | undefined;
fired: boolean;
terminalPersisting: boolean;
cancelled: boolean;
lifecycleSettled: boolean;
command: string;
cwd: string | undefined;
};
const active = new Map<string, WatcherSlot>();
// A cancelled child can keep running until the supervisor observes exit.
// Retain those slots separately so replacement arms can own the job id while
// suspension still sees every predecessor that is settling.
const settlingCancelledSlots = new Set<WatcherSlot>();
const cancel = (jobId: string) => {
const slot = active.get(jobId);
if (!slot) {
return;
}
active.delete(jobId);
slot.cancelled = true;
if (!slot.lifecycleSettled) {
settlingCancelledSlots.add(slot);
}
// Terminal persistence is user-visible state. Keep the slot as a suspend
// blocker until that write settles even when hot reload cancels the watcher.
if (!slot.terminalPersisting) {
active.delete(jobId);
}
// Cancel an already-spawned child; an in-flight spawn (run undefined) is
// killed by the arm() ownership check once it resolves.
slot.run?.cancel("manual-cancel");
@@ -106,7 +121,17 @@ export function createCronExitWatchers(params: {
const armToken: object = {};
// Reserve the slot synchronously so a concurrent cancel/replace can observe
// and act on this arm before the child is spawned.
const slot: WatcherSlot = { armToken, job, run: undefined, fired: false, command, cwd };
const slot: WatcherSlot = {
armToken,
job,
run: undefined,
fired: false,
terminalPersisting: false,
cancelled: false,
lifecycleSettled: false,
command,
cwd,
};
active.set(job.id, slot);
const owns = () => active.get(job.id) === slot && slot.armToken === armToken;
void (async () => {
@@ -136,8 +161,14 @@ export function createCronExitWatchers(params: {
}
if (!owns()) {
// Cancelled or re-armed (changed command/cwd) while the spawn was in
// flight — kill this now-orphaned child instead of leaking it.
// flight — kill this now-orphaned child instead of leaking it. Wait for
// supervisor settlement so suspension cannot snapshot a live child.
run.cancel("manual-cancel");
try {
await run.wait();
} catch {
// The watcher was already cancelled; settlement, not outcome, matters.
}
return;
}
slot.run = run;
@@ -165,6 +196,7 @@ export function createCronExitWatchers(params: {
{ jobId: job.id, exitCode: exit.exitCode, reason: exit.reason },
"cron-exit: watched command exited; firing job",
);
slot.terminalPersisting = true;
// Persist the terminal one-shot state BEFORE firing. FAIL CLOSED: if the
// store write fails we do NOT wake — waking without a persisted terminal
// state would let a gateway restart re-arm and re-run the command.
@@ -180,6 +212,13 @@ export function createCronExitWatchers(params: {
);
return;
}
slot.terminalPersisting = false;
if (!owns() || slot.cancelled) {
if (active.get(job.id) === slot) {
active.delete(job.id);
}
return;
}
slot.fired = true;
try {
await params.fireOnExit(slot.job, {
@@ -196,7 +235,13 @@ export function createCronExitWatchers(params: {
"cron-exit: fireOnExit after exit failed",
);
}
})();
})().finally(() => {
slot.lifecycleSettled = true;
settlingCancelledSlots.delete(slot);
if (slot.cancelled && active.get(job.id) === slot) {
active.delete(job.id);
}
});
};
const reconcile = (jobs: CronJob[]) => {
@@ -236,6 +281,14 @@ export function createCronExitWatchers(params: {
reconcile,
cancel,
cancelAll,
activeJobIds: () => Array.from(active.keys()),
activeJobIds: () =>
Array.from(
new Set([
...Array.from(active.entries())
.filter(([, slot]) => !slot.fired)
.map(([jobId]) => jobId),
...Array.from(settlingCancelledSlots, (slot) => slot.job.id),
]),
),
};
}
+4 -2
View File
@@ -3,7 +3,6 @@
import { loadManifestModelCatalog } from "../agents/model-catalog.js";
import type { CliDeps } from "../cli/deps.types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { CronServiceContract } from "../cron/service-contract.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
getPluginRuntimeGatewayRequestScope,
@@ -12,6 +11,7 @@ import {
import { NodeRegistry } from "./node-registry.js";
import type { ChannelRuntimeSnapshot } from "./server-channel-runtime.types.js";
import { createChatRunEntry, type ChatRunEntry } from "./server-chat-state.js";
import type { GatewayCronServiceContract } from "./server-cron.js";
import type { GatewayRequestContext } from "./server-methods/types.js";
// Embedded/local agent calls need enough GatewayRequestContext to reuse server
@@ -26,11 +26,13 @@ function cronUnavailable(): never {
throw new Error("Cron is unavailable in local embedded agent gateway context.");
}
const unavailableCron: CronServiceContract = {
const unavailableCron: GatewayCronServiceContract = {
start: async () => {
cronUnavailable();
},
stop: () => {},
pauseScheduling: () => {},
resumeScheduling: () => {},
status: async () => cronUnavailable(),
list: async () => cronUnavailable(),
listPage: async () => cronUnavailable(),
+9
View File
@@ -288,6 +288,15 @@ export const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
// Session PR chips read the session's own checkout metadata, matching the
// sessions.files.* trusted-operator read domain.
{ name: "controlUi.sessionPullRequests", scope: "operator.read" },
{
name: "gateway.suspend.prepare",
scope: "operator.admin",
startup: true,
controlPlaneWrite: true,
},
{ name: "gateway.suspend.status", scope: "operator.read" },
// Resume is the safety escape hatch and must not sit behind write-rate limiting.
{ name: "gateway.suspend.resume", scope: "operator.admin" },
] as const;
const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap<string, CoreGatewayMethodSpec> = new Map(
+57
View File
@@ -0,0 +1,57 @@
// Covers server-local chat, cron watcher, queued-turn, and terminal blockers.
import { describe, expect, it, vi } from "vitest";
import { createGatewayServerActiveWorkInspectors } from "./server-active-work.js";
import type { GatewayRequestContext } from "./server-methods/shared-types.js";
vi.mock("../cron/active-jobs.js", () => ({
getActiveCronJobCount: vi.fn(() => 2),
}));
vi.mock("../tasks/cron-task-cancel.js", () => ({
getSuspensionVisibleCronTaskRunCount: vi.fn(() => 4),
}));
function controller(aborted = false): AbortController {
const value = new AbortController();
if (aborted) {
value.abort();
}
return value;
}
describe("gateway server active work inspectors", () => {
it("filters completed chat entries while retaining persistence and watcher blockers", () => {
const context = {
cron: { getSuspensionBlockerCount: () => 1 },
chatAbortControllers: new Map([
["active", { controller: controller() }],
["aborted", { controller: controller(true) }],
[
"persisting",
{
controller: controller(),
registrationCleanupRequested: true,
controlUiVisible: true,
projectSessionTerminalPending: true,
},
],
]),
chatQueuedTurns: new Map([
["queued", { controller: controller() }],
["cancelled", { controller: controller(true) }],
]),
terminalSessions: { size: 2 },
} as unknown as Pick<
GatewayRequestContext,
"chatAbortControllers" | "chatQueuedTurns" | "cron" | "terminalSessions"
>;
const inspectors = createGatewayServerActiveWorkInspectors(context);
expect(inspectors.getCronRuns?.()).toBe(5);
expect(inspectors.getChatRuns?.()).toBe(1);
expect(inspectors.getQueuedTurns?.()).toBe(1);
expect(inspectors.getTerminalPersistence?.()).toBe(1);
expect(inspectors.getTerminalSessions?.()).toBe(2);
});
});
+35
View File
@@ -0,0 +1,35 @@
// Adapts server-local chat and terminal state to the shared activity inspector.
import { getActiveCronJobCount } from "../cron/active-jobs.js";
import type { GatewayActiveWorkInspectors } from "../infra/gateway-active-work.js";
import { getSuspensionVisibleCronTaskRunCount } from "../tasks/cron-task-cancel.js";
import type { GatewayRequestContext } from "./server-methods/shared-types.js";
export function createGatewayServerActiveWorkInspectors(
context: Pick<
GatewayRequestContext,
"chatAbortControllers" | "chatQueuedTurns" | "cron" | "terminalSessions"
>,
): Partial<GatewayActiveWorkInspectors> {
return {
getCronRuns: () =>
Math.max(getActiveCronJobCount(), getSuspensionVisibleCronTaskRunCount()) +
(context.cron.getSuspensionBlockerCount?.() ?? 0),
getChatRuns: () =>
Array.from(context.chatAbortControllers.values()).filter(
(entry) => !entry.controller.signal.aborted && entry.registrationCleanupRequested !== true,
).length,
getQueuedTurns: () =>
Array.from(context.chatQueuedTurns.values()).filter(
(entry) => !entry.controller.signal.aborted,
).length,
getTerminalPersistence: () =>
Array.from(context.chatAbortControllers.values()).filter(
(entry) =>
entry.controlUiVisible !== false &&
entry.projectSessionTerminalPersisted !== true &&
(entry.projectSessionTerminalPending === true ||
entry.projectSessionTerminalPersistence !== undefined),
).length,
getTerminalSessions: () => context.terminalSessions?.size ?? 0,
};
}
+131 -4
View File
@@ -4,8 +4,7 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CliDeps } from "../cli/deps.types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { CronServiceContract } from "../cron/service-contract.js";
import type { GatewayCronState } from "./server-cron.js";
import type { GatewayCronServiceContract, GatewayCronState } from "./server-cron.js";
const hoisted = vi.hoisted(() => {
let state: unknown;
@@ -23,6 +22,14 @@ vi.mock("./server-cron.js", () => ({
const { createLazyGatewayCronState } = await import("./server-cron-lazy.js");
function deferred() {
let resolve = () => {};
const promise = new Promise<void>((done) => {
resolve = done;
});
return { promise, resolve };
}
describe("createLazyGatewayCronState", () => {
beforeEach(() => {
vi.unstubAllEnvs();
@@ -111,6 +118,26 @@ describe("createLazyGatewayCronState", () => {
expect(cron["start"]).toHaveBeenCalledTimes(2);
});
it("restarts after stop interrupts an in-flight startup", async () => {
const finishFirstStart = deferred();
const cron = createCronService();
cron.start = vi
.fn()
.mockImplementationOnce(async () => await finishFirstStart.promise)
.mockResolvedValueOnce(undefined);
hoisted.setState(createCronState(cron));
const lazy = createLazyGatewayCronState(createParams());
const firstStart = lazy.cron.start();
await vi.waitFor(() => expect(cron["start"]).toHaveBeenCalledOnce());
lazy.cron.stop();
const restarted = lazy.cron.start();
finishFirstStart.resolve();
await Promise.all([firstStart, restarted]);
expect(cron["start"]).toHaveBeenCalledTimes(2);
});
it("keeps synchronous wake non-blocking before the cron service is loaded", async () => {
const cron = createCronService();
hoisted.setState(createCronState(cron));
@@ -134,6 +161,104 @@ describe("createLazyGatewayCronState", () => {
expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled();
});
it("does not arm a read-loaded scheduler when suspension ends", async () => {
const cron = createCronService();
hoisted.setState(createCronState(cron));
const lazy = createLazyGatewayCronState(createParams());
lazy.cron.pauseScheduling();
expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled();
await lazy.cron.status();
expect(cron["pauseScheduling"]).toHaveBeenCalledOnce();
lazy.cron.resumeScheduling();
expect(cron["resumeScheduling"]).not.toHaveBeenCalled();
await lazy.cron.start();
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
lazy.cron.pauseScheduling();
lazy.cron.resumeScheduling();
expect(cron["resumeScheduling"]).toHaveBeenCalledTimes(2);
});
it("waits to start while scheduling is paused", async () => {
const cron = createCronService();
hoisted.setState(createCronState(cron));
const lazy = createLazyGatewayCronState(createParams());
lazy.cron.pauseScheduling();
const startPromise = lazy.cron.start();
await vi.waitFor(() => expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1));
expect(cron["start"]).not.toHaveBeenCalled();
lazy.cron.resumeScheduling();
await startPromise;
expect(cron["start"]).toHaveBeenCalledOnce();
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
});
it("keeps in-flight startup as a blocker until startup settles", async () => {
const finishStart = deferred();
const cron = createCronService();
cron.start = vi.fn(async () => await finishStart.promise);
hoisted.setState(createCronState(cron));
const lazy = createLazyGatewayCronState(createParams());
const startPromise = lazy.cron.start();
await vi.waitFor(() => expect(cron["start"]).toHaveBeenCalledOnce());
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
lazy.cron.pauseScheduling();
lazy.cron.resumeScheduling();
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
finishStart.resolve();
await startPromise;
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
});
it("resumes a paused scheduler while exit watchers are reconciling", async () => {
const reconcileStarted = deferred();
const finishReconcile = deferred();
const cron = createCronService();
hoisted.setState({
...createCronState(cron),
reconcileExitWatchers: vi.fn(async () => {
reconcileStarted.resolve();
await finishReconcile.promise;
}),
});
const lazy = createLazyGatewayCronState(createParams());
const startPromise = lazy.cron.start();
await reconcileStarted.promise;
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
lazy.cron.pauseScheduling();
lazy.cron.resumeScheduling();
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
finishReconcile.resolve();
await startPromise;
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
});
it("allows startup to retry after the underlying service rejects", async () => {
const cron = createCronService();
cron.start = vi
.fn()
.mockRejectedValueOnce(new Error("startup failed"))
.mockResolvedValueOnce(undefined);
hoisted.setState(createCronState(cron));
const lazy = createLazyGatewayCronState(createParams());
await expect(lazy.cron.start()).rejects.toThrow("startup failed");
await lazy.cron.start();
expect(cron["start"]).toHaveBeenCalledTimes(2);
});
it("does not reconcile exit watchers when cron is disabled", async () => {
const cron = createCronService();
const reconcileExitWatchers = vi.fn(async () => {});
@@ -161,7 +286,7 @@ function createParams(overrides: Partial<OpenClawConfig> = {}) {
};
}
function createCronState(cron: CronServiceContract): GatewayCronState {
function createCronState(cron: GatewayCronServiceContract): GatewayCronState {
return {
cron,
storePath: "/tmp/openclaw-cron.json",
@@ -169,10 +294,12 @@ function createCronState(cron: CronServiceContract): GatewayCronState {
} as GatewayCronState;
}
function createCronService(): CronServiceContract {
function createCronService(): GatewayCronServiceContract {
return {
start: vi.fn(async () => undefined),
stop: vi.fn(),
pauseScheduling: vi.fn(),
resumeScheduling: vi.fn(),
status: vi.fn(async () => ({ enabled: true }) as never),
list: vi.fn(async () => [] as never),
listPage: vi.fn(async () => ({ items: [], total: 0 }) as never),
+147 -21
View File
@@ -2,10 +2,9 @@
// Defers scheduler startup until cron is touched by runtime or API handlers.
import type { CliDeps } from "../cli/deps.types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { CronServiceContract } from "../cron/service-contract.js";
import { resolveCronJobsStorePath } from "../cron/store.js";
import { createLazyPromiseLoader } from "../shared/lazy-runtime.js";
import type { GatewayCronState } from "./server-cron.js";
import type { GatewayCronServiceContract, GatewayCronState } from "./server-cron.js";
type LazyGatewayCronParams = {
cfg: OpenClawConfig;
@@ -15,7 +14,12 @@ type LazyGatewayCronParams = {
type LoadedGatewayCronState = {
state: GatewayCronState;
started: boolean;
phase: "idle" | "starting" | "started" | "stopped";
startPromise: Promise<void> | null;
startGeneration: number | null;
schedulingPaused: boolean;
underlyingStartInFlight: boolean;
underlyingStarted: boolean;
};
/** Creates a cron state proxy that imports the real cron service on first use. */
@@ -24,13 +28,40 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
let loaded: LoadedGatewayCronState | null = null;
let stopped = false;
let lifecycleGeneration = 0;
let schedulingPaused = false;
const schedulingResumeWaiters = new Set<() => void>();
const releaseSchedulingResumeWaiters = () => {
const waiters = Array.from(schedulingResumeWaiters);
schedulingResumeWaiters.clear();
for (const resolve of waiters) {
resolve();
}
};
const waitForSchedulingResume = async () => {
if (!schedulingPaused) {
return;
}
await new Promise<void>((resolve) => {
schedulingResumeWaiters.add(resolve);
});
};
const cronStateLoader = createLazyPromiseLoader(
() =>
import("./server-cron.js").then(({ buildGatewayCronService }) => {
loaded = {
state: buildGatewayCronService(params),
started: false,
phase: "idle",
startPromise: null,
startGeneration: null,
schedulingPaused: false,
underlyingStartInFlight: false,
underlyingStarted: false,
};
if (schedulingPaused) {
loaded.state.cron.pauseScheduling();
loaded.schedulingPaused = true;
}
return loaded;
}),
{ cacheRejections: true },
@@ -45,35 +76,106 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
return await cronStateLoader.load();
};
const cron: CronServiceContract = {
const cron: GatewayCronServiceContract = {
async start() {
stopped = false;
const generation = lifecycleGeneration;
const startCancelled = () => stopped || generation !== lifecycleGeneration;
const resolved = await load();
if (stopped) {
const hasStarted = () => resolved.phase === "started";
if (startCancelled()) {
return;
}
if (resolved.started) {
if (hasStarted()) {
return;
}
resolved.started = true;
await resolved.state.cron.start();
// Arm on-exit watchers for jobs loaded from the store at startup (no
// change event fires for already-persisted jobs).
if (resolved.state.cronEnabled) {
await resolved.state.reconcileExitWatchers?.();
if (resolved.startPromise) {
const pendingGeneration = resolved.startGeneration;
try {
await resolved.startPromise;
} catch (err) {
if (pendingGeneration === generation) {
throw err;
}
}
if (startCancelled() || hasStarted()) {
return;
}
if (pendingGeneration !== generation) {
await cron.start();
return;
}
}
// If stop raced the lazy import/start path, immediately stop the loaded
// scheduler so shutdown does not leave a background loop alive.
if (stopped && resolved.started) {
resolved.started = false;
resolved.state.cron.stop();
resolved.state.stopExitWatchers?.();
resolved.phase = "starting";
resolved.startGeneration = generation;
const startPromise = (async () => {
await waitForSchedulingResume();
if (startCancelled()) {
resolved.phase = "stopped";
return;
}
if (resolved.schedulingPaused) {
resolved.state.cron.resumeScheduling();
resolved.schedulingPaused = false;
}
resolved.underlyingStartInFlight = true;
try {
await resolved.state.cron.start();
resolved.underlyingStarted = true;
} catch (err) {
resolved.underlyingStarted = false;
resolved.phase = startCancelled() ? "stopped" : "idle";
throw err;
} finally {
resolved.underlyingStartInFlight = false;
}
if (startCancelled()) {
resolved.phase = "stopped";
resolved.underlyingStarted = false;
resolved.state.cron.stop();
resolved.state.stopExitWatchers?.();
return;
}
if (schedulingPaused) {
resolved.state.cron.pauseScheduling();
resolved.schedulingPaused = true;
}
// Arm on-exit watchers for jobs loaded from the store at startup (no
// change event fires for already-persisted jobs).
try {
if (resolved.state.cronEnabled) {
await resolved.state.reconcileExitWatchers?.();
}
} catch (err) {
resolved.phase = startCancelled() ? "stopped" : "started";
throw err;
}
if (startCancelled()) {
resolved.phase = "stopped";
resolved.underlyingStarted = false;
resolved.state.cron.stop();
resolved.state.stopExitWatchers?.();
return;
}
resolved.phase = "started";
})();
resolved.startPromise = startPromise;
try {
await startPromise;
} finally {
if (resolved.startPromise === startPromise) {
resolved.startPromise = null;
resolved.startGeneration = null;
}
}
},
stop() {
stopped = true;
lifecycleGeneration += 1;
releaseSchedulingResumeWaiters();
if (loaded) {
loaded.started = false;
loaded.phase = "stopped";
loaded.underlyingStarted = false;
loaded.state.cron.stop();
loaded.state.stopExitWatchers?.();
return;
@@ -87,13 +189,37 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
if (!stopped) {
return;
}
resolved.started = false;
resolved.phase = "stopped";
resolved.underlyingStarted = false;
resolved.state.cron.stop();
resolved.state.stopExitWatchers?.();
})
.catch(() => {});
}
},
pauseScheduling() {
schedulingPaused = true;
if (loaded) {
loaded.state.cron.pauseScheduling();
loaded.schedulingPaused = true;
}
},
resumeScheduling() {
schedulingPaused = false;
releaseSchedulingResumeWaiters();
if (
loaded &&
loaded.schedulingPaused &&
(loaded.underlyingStarted || loaded.underlyingStartInFlight)
) {
loaded.state.cron.resumeScheduling();
loaded.schedulingPaused = false;
}
},
getSuspensionBlockerCount() {
const loadedBlockers = loaded?.state.cron.getSuspensionBlockerCount?.() ?? 0;
return loaded?.phase === "starting" ? Math.max(1, loadedBlockers) : loadedBlockers;
},
async status() {
return await (await load()).state.cron.status();
},
+188 -2
View File
@@ -1,13 +1,20 @@
// Cron notification tests protect completion-delivery warning behavior,
// including URL redaction for invalid webhook destinations.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { CliDeps } from "../cli/deps.types.js";
import { makeCronJob } from "../cron/delivery.test-helpers.js";
import type { CronJob } from "../cron/types.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
const mocks = vi.hoisted(() => ({
fetchWithSsrFGuard: vi.fn(async (_request: unknown) => ({ release: vi.fn() })),
sendFailureNotificationAnnounce: vi.fn(),
sendCronAnnouncePayloadStrict: vi.fn(),
}));
vi.mock("../infra/net/fetch-guard.js", () => ({
@@ -19,10 +26,14 @@ vi.mock("../cron/delivery.js", async (importOriginal) => {
return {
...actual,
sendFailureNotificationAnnounce: mocks.sendFailureNotificationAnnounce,
sendCronAnnouncePayloadStrict: mocks.sendCronAnnouncePayloadStrict,
};
});
import { dispatchGatewayCronFinishedNotifications } from "./server-cron-notifications.js";
import {
dispatchGatewayCronFinishedNotifications,
sendGatewayCronFailureAlert,
} from "./server-cron-notifications.js";
function requireRecord(value: unknown, label: string): Record<string, unknown> {
if (!value || typeof value !== "object") {
@@ -44,9 +55,184 @@ function webhookRequestBody() {
return JSON.parse(init.body);
}
function createVoidDeferred(): { promise: Promise<void>; resolve: () => void } {
let resolve = () => {};
const promise = new Promise<void>((resolvePromise) => {
resolve = resolvePromise;
});
return { promise, resolve };
}
function createWebhookJob(delivery: NonNullable<CronJob["delivery"]>): CronJob {
return {
id: "cron-notification-admission",
name: "notification admission",
enabled: true,
createdAtMs: 1,
updatedAtMs: 1,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "isolated",
wakeMode: "next-heartbeat",
payload: { kind: "agentTurn", message: "hello" },
delivery,
state: {},
};
}
describe("dispatchGatewayCronFinishedNotifications", () => {
beforeEach(() => {
resetGatewayWorkAdmission();
vi.clearAllMocks();
mocks.fetchWithSsrFGuard.mockImplementation(async () => ({ release: vi.fn() }));
mocks.sendFailureNotificationAnnounce.mockResolvedValue(undefined);
mocks.sendCronAnnouncePayloadStrict.mockResolvedValue(undefined);
});
afterEach(() => {
resetGatewayWorkAdmission();
});
it("independently admits detached completion webhook delivery", async () => {
const deferred = createVoidDeferred();
mocks.fetchWithSsrFGuard.mockImplementationOnce(async () => {
await deferred.promise;
return { release: vi.fn() };
});
const job = createWebhookJob({
mode: "webhook",
to: "https://example.invalid/cron",
});
const parentAdmission = tryBeginGatewayRootWorkAdmission();
expect(parentAdmission).not.toBeNull();
if (!parentAdmission) {
throw new Error("expected parent Gateway work admission");
}
try {
await parentAdmission.run(async () => {
dispatchGatewayCronFinishedNotifications({
evt: { jobId: job.id, action: "finished", status: "ok", summary: "done" },
job,
deps: {} as CliDeps,
logger: { warn: vi.fn() },
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
});
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
expect(getActiveGatewayRootWorkCount()).toBe(2);
});
} finally {
parentAdmission.release();
}
expect(getActiveGatewayRootWorkCount()).toBe(1);
deferred.resolve();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("independently admits immediate failure alerts", async () => {
const deferred = createVoidDeferred();
mocks.sendCronAnnouncePayloadStrict.mockImplementationOnce(async () => {
await deferred.promise;
});
const job = createWebhookJob({ mode: "announce", channel: "discord", to: "channel:ops" });
const delivery = sendGatewayCronFailureAlert({
deps: {} as CliDeps,
logger: { warn: vi.fn() },
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
job,
text: "cron failed",
channel: "discord",
to: "channel:ops",
mode: "announce",
});
await vi.waitFor(() => expect(mocks.sendCronAnnouncePayloadStrict).toHaveBeenCalledOnce());
expect(getActiveGatewayRootWorkCount()).toBe(1);
deferred.resolve();
await delivery;
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("defers detached completion delivery while suspension is prepared", async () => {
const job = createWebhookJob({
mode: "webhook",
to: "https://example.invalid/cron",
});
const suspensionAdmission = tryBeginGatewaySuspendAdmission(() => {});
expect(suspensionAdmission?.commit()).toBe(true);
dispatchGatewayCronFinishedNotifications({
evt: { jobId: job.id, action: "finished", status: "ok", summary: "done" },
job,
deps: {} as CliDeps,
logger: { warn: vi.fn() },
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
});
await Promise.resolve();
expect(mocks.fetchWithSsrFGuard).not.toHaveBeenCalled();
expect(getActiveGatewayRootWorkCount()).toBe(0);
expect(suspensionAdmission?.release()).toBe(true);
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
});
it("independently admits failure destination webhook delivery", async () => {
const deferred = createVoidDeferred();
mocks.fetchWithSsrFGuard.mockImplementationOnce(async () => {
await deferred.promise;
return { release: vi.fn() };
});
const job = createWebhookJob({
mode: "announce",
channel: "last",
failureDestination: {
mode: "webhook",
to: "https://example.invalid/failure",
},
});
dispatchGatewayCronFinishedNotifications({
evt: { jobId: job.id, action: "finished", status: "error", error: "boom" },
job,
deps: {} as CliDeps,
logger: { warn: vi.fn() },
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
});
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
expect(getActiveGatewayRootWorkCount()).toBe(1);
deferred.resolve();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("independently admits failure destination announce delivery", async () => {
const deferred = createVoidDeferred();
mocks.sendFailureNotificationAnnounce.mockImplementationOnce(() => deferred.promise);
const job = createWebhookJob({
mode: "announce",
channel: "last",
failureDestination: {
mode: "announce",
channel: "telegram",
to: "-1001234567890",
},
});
dispatchGatewayCronFinishedNotifications({
evt: { jobId: job.id, action: "finished", status: "error", error: "boom" },
job,
deps: {} as CliDeps,
logger: { warn: vi.fn() },
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
});
await vi.waitFor(() => expect(mocks.sendFailureNotificationAnnounce).toHaveBeenCalledTimes(1));
expect(getActiveGatewayRootWorkCount()).toBe(1);
deferred.resolve();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
});
it("redacts invalid completion webhook targets in warnings", () => {
+111 -69
View File
@@ -21,6 +21,7 @@ import { normalizeHttpWebhookUrl } from "../cron/webhook-url.js";
import { formatErrorMessage } from "../infra/errors.js";
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
import { SsrFBlockedError } from "../infra/net/ssrf.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
const CRON_WEBHOOK_TIMEOUT_MS = 10_000;
@@ -38,6 +39,19 @@ type CronWebhookTarget = {
source: "delivery" | "completionDestination";
};
type CronFailureAlertParams = {
deps: CliDeps;
logger: CronLogger;
resolveCronAgent: CronAgentResolver;
webhookToken?: unknown;
job: CronJob;
text: string;
channel: CronMessageChannel;
to?: string;
mode?: "announce" | "webhook";
accountId?: string;
};
function redactWebhookUrl(url: string): string {
try {
const parsed = new URL(url);
@@ -232,19 +246,30 @@ async function postCronWebhook(params: {
}
}
/** Sends the immediate failure alert for cron jobs that failed before normal completion delivery. */
export async function sendGatewayCronFailureAlert(params: {
deps: CliDeps;
/** Detached sends outlive cron ticks; own roots block mid-delivery suspension snapshots. */
function dispatchDetachedCronNotification(params: {
jobId: string;
logger: CronLogger;
resolveCronAgent: CronAgentResolver;
webhookToken?: unknown;
job: CronJob;
text: string;
channel: CronMessageChannel;
to?: string;
mode?: "announce" | "webhook";
accountId?: string;
}): Promise<void> {
deliver: () => Promise<void>;
}): void {
void runWithGatewayIndependentRootWorkAdmission(params.deliver).catch((err: unknown) => {
params.logger.warn(
{ jobId: params.jobId, err: formatErrorMessage(err) },
"cron: detached notification delivery failed",
);
});
}
/** Sends the immediate failure alert for cron jobs that failed before normal completion delivery. */
export async function sendGatewayCronFailureAlert(params: CronFailureAlertParams): Promise<void> {
await runWithGatewayIndependentRootWorkAdmission(async () => {
await sendGatewayCronFailureAlertUnderAdmission(params);
});
}
async function sendGatewayCronFailureAlertUnderAdmission(
params: CronFailureAlertParams,
): Promise<void> {
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
const webhookToken = normalizeOptionalString(params.webhookToken);
@@ -355,17 +380,20 @@ export function dispatchGatewayCronFinishedNotifications(params: {
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
// Completion notification fanout is best-effort; the cron service has
// already recorded the run result and must not wait on slow webhooks.
void (async () => {
await postCronWebhook({
webhookUrl: webhookTarget.url,
webhookToken,
payload,
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
blockedLog: "cron: webhook delivery blocked by SSRF guard",
failedLog: "cron: webhook delivery failed",
logger: params.logger,
});
})();
dispatchDetachedCronNotification({
jobId: params.evt.jobId,
logger: params.logger,
deliver: () =>
postCronWebhook({
webhookUrl: webhookTarget.url,
webhookToken,
payload,
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
blockedLog: "cron: webhook delivery blocked by SSRF guard",
failedLog: "cron: webhook delivery failed",
logger: params.logger,
}),
});
}
}
@@ -393,9 +421,10 @@ function dispatchCronFailureDestinationNotifications(params: {
return;
}
const failureDest = resolveFailureDestination(params.job, params.globalFailureDestination);
const deliverySessionKey = resolveCronDeliverySessionKey(params.job);
const failurePayload = buildCronFailureWebhookPayload({ evt: params.evt, job: params.job });
const job = params.job;
const failureDest = resolveFailureDestination(job, params.globalFailureDestination);
const deliverySessionKey = resolveCronDeliverySessionKey(job);
const failurePayload = buildCronFailureWebhookPayload({ evt: params.evt, job });
if (failureDest) {
if (failureDest.mode === "webhook" && failureDest.to) {
@@ -403,17 +432,20 @@ function dispatchCronFailureDestinationNotifications(params: {
if (webhookUrl) {
// Failure destinations mirror completion webhooks: notify in the
// background and log failures without rewriting the cron event result.
void (async () => {
await postCronWebhook({
webhookUrl,
webhookToken: params.webhookToken,
payload: failurePayload,
logContext: { jobId: params.evt.jobId },
blockedLog: "cron: failure destination webhook blocked by SSRF guard",
failedLog: "cron: failure destination webhook failed",
logger: params.logger,
});
})();
dispatchDetachedCronNotification({
jobId: params.evt.jobId,
logger: params.logger,
deliver: () =>
postCronWebhook({
webhookUrl,
webhookToken: params.webhookToken,
payload: failurePayload,
logContext: { jobId: params.evt.jobId },
blockedLog: "cron: failure destination webhook blocked by SSRF guard",
failedLog: "cron: failure destination webhook failed",
logger: params.logger,
}),
});
} else {
params.logger.warn(
{
@@ -427,44 +459,54 @@ function dispatchCronFailureDestinationNotifications(params: {
}
if (failureDest.mode === "announce") {
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
void sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
params.job.id,
{
channel: failureDest.channel,
to: failureDest.to,
accountId: failureDest.accountId,
sessionKey: deliverySessionKey,
// A configured failure route is already explicit; keep the cron run
// session only for context, not for reattaching the primary topic.
inheritSessionThread: false,
},
`⚠️ ${failurePayload.message}`,
);
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
dispatchDetachedCronNotification({
jobId: job.id,
logger: params.logger,
deliver: () =>
sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
job.id,
{
channel: failureDest.channel,
to: failureDest.to,
accountId: failureDest.accountId,
sessionKey: deliverySessionKey,
// A configured failure route is already explicit; keep the cron run
// session only for context, not for reattaching the primary topic.
inheritSessionThread: false,
},
`⚠️ ${failurePayload.message}`,
),
});
}
return;
}
const primaryPlan = resolveCronDeliveryPlan(params.job);
const primaryPlan = resolveCronDeliveryPlan(job);
if (primaryPlan.mode !== "announce" || !primaryPlan.requested) {
return;
}
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
void sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
params.job.id,
{
channel: primaryPlan.channel,
to: primaryPlan.to,
accountId: primaryPlan.accountId,
sessionKey: deliverySessionKey,
},
`⚠️ ${failurePayload.message}`,
);
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
dispatchDetachedCronNotification({
jobId: job.id,
logger: params.logger,
deliver: () =>
sendFailureNotificationAnnounce(
params.deps,
runtimeConfig,
agentId,
job.id,
{
channel: primaryPlan.channel,
to: primaryPlan.to,
accountId: primaryPlan.accountId,
sessionKey: deliverySessionKey,
},
`⚠️ ${failurePayload.message}`,
),
});
}
+37 -1
View File
@@ -6,6 +6,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import type { CliDeps } from "../cli/deps.js";
import type { OpenClawConfig } from "../config/config.js";
import { SsrFBlockedError } from "../infra/net/ssrf.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import { createDeferred } from "../test-utils/deferred.js";
type RunCronIsolatedAgentTurnMock = (params: {
@@ -435,7 +439,6 @@ describe("buildGatewayCronService", () => {
vi.setSystemTime(new Date("2026-07-10T12:00:00.000Z"));
const cfg = createCronConfig("server-cron-hook-scheduled");
loadConfigMock.mockReturnValue(cfg);
const state = buildGatewayCronService({
cfg,
deps: {} as CliDeps,
@@ -486,6 +489,39 @@ describe("buildGatewayCronService", () => {
}
});
it("keeps detached cron_changed hooks root-admitted until they settle", async () => {
resetGatewayWorkAdmission();
const deferred = createDeferred();
runCronChangedMock.mockImplementationOnce(async () => await deferred.promise);
const cfg = createCronConfig("server-cron-hook-admission");
loadConfigMock.mockReturnValue(cfg);
const state = buildGatewayCronService({
cfg,
deps: {} as CliDeps,
broadcast: () => {},
});
try {
await state.cron.add({
name: "held hook",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "next-heartbeat",
payload: { kind: "systemEvent", text: "hello" },
});
await vi.waitFor(() => expect(runCronChangedMock).toHaveBeenCalledTimes(1));
expect(getActiveGatewayRootWorkCount()).toBe(1);
deferred.resolve();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
} finally {
deferred.resolve();
state.cron.stop();
resetGatewayWorkAdmission();
}
});
it("cron_changed removed events include the deleted job snapshot", async () => {
const cfg = createCronConfig("server-cron-hook-removed");
loadConfigMock.mockReturnValue(cfg);
+76 -40
View File
@@ -49,6 +49,7 @@ import type {
PluginHookGatewayCronService,
PluginHookGatewayContext,
} from "../plugins/hook-types.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { getProcessSupervisor } from "../process/supervisor/index.js";
import {
normalizeAgentId,
@@ -63,8 +64,16 @@ import {
sendGatewayCronFailureAlert,
} from "./server-cron-notifications.js";
export type GatewayCronServiceContract = CronServiceContract & {
/** Temporarily disarm ticks without running startup recovery on resume. */
pauseScheduling(): void;
resumeScheduling(): void;
/** Scheduler-owned work not represented by active cron run markers. */
getSuspensionBlockerCount?(): number;
};
export type GatewayCronState = {
cron: CronServiceContract;
cron: GatewayCronServiceContract;
storePath: string;
cronEnabled: boolean;
reconcileExitWatchers?: () => Promise<void>;
@@ -371,7 +380,11 @@ export function buildGatewayCronService(params: {
config: getRuntimeConfig(),
getCron: () => cron as PluginHookGatewayCronService,
};
void hookRunner.runCronChanged(evt, hookCtx).catch((err: unknown) => {
// Hook execution is detached from the cron mutation/tick that emitted it.
// Keep the whole plugin callback visible until its user-state effects settle.
void runWithGatewayIndependentRootWorkAdmission(async () => {
await hookRunner.runCronChanged(evt, hookCtx);
}).catch((err: unknown) => {
cronLogger.warn(
{ err: formatErrorMessage(err), jobId: evt.jobId },
"cron_changed hook failed",
@@ -383,12 +396,19 @@ export function buildGatewayCronService(params: {
const exitWatchersRef: { current: ReturnType<typeof createCronExitWatchers> | undefined } = {
current: undefined,
};
let exitWatcherReconciliations = 0;
let exitWatcherGeneration = 0;
const reconcileExitWatchers = async () => {
if (!exitWatchersRef.current) {
return;
}
const generation = exitWatcherGeneration;
exitWatcherReconciliations += 1;
try {
if (!exitWatchersRef.current) {
return;
}
const result = await cron.list({ includeDisabled: true });
if (generation !== exitWatcherGeneration) {
return;
}
const jobs: CronJob[] = Array.isArray(result) ? result : (result as { jobs: CronJob[] }).jobs;
reconcileCronExitWatchers({
cronEnabled,
@@ -397,6 +417,8 @@ export function buildGatewayCronService(params: {
});
} catch (err) {
cronLogger.warn({ err: String(err) }, "cron-exit: reconcile failed");
} finally {
exitWatcherReconciliations -= 1;
}
};
@@ -728,33 +750,35 @@ export function buildGatewayCronService(params: {
globalFailureDestination: params.cfg.cron?.failureDestination,
});
void appendCronRunLog({
storePath,
entry: {
ts: Date.now(),
jobId: evt.jobId,
action: "finished",
status: evt.status,
error: evt.error,
summary: evt.summary,
diagnostics: evt.diagnostics,
delivered: evt.delivered,
deliveryStatus: evt.deliveryStatus,
deliveryError: evt.deliveryError,
failureNotificationDelivery: evt.failureNotificationDelivery,
delivery: evt.delivery,
sessionId: evt.sessionId,
sessionKey: evt.sessionKey,
runId: evt.runId,
runAtMs: evt.runAtMs,
durationMs: evt.durationMs,
nextRunAtMs: evt.nextRunAtMs,
triggerFired: evt.triggerFired,
model: evt.model,
provider: evt.provider,
usage: evt.usage,
},
opts: { keepLines: runLogPrune.keepLines },
void runWithGatewayIndependentRootWorkAdmission(async () => {
await appendCronRunLog({
storePath,
entry: {
ts: Date.now(),
jobId: evt.jobId,
action: "finished",
status: evt.status,
error: evt.error,
summary: evt.summary,
diagnostics: evt.diagnostics,
delivered: evt.delivered,
deliveryStatus: evt.deliveryStatus,
deliveryError: evt.deliveryError,
failureNotificationDelivery: evt.failureNotificationDelivery,
delivery: evt.delivery,
sessionId: evt.sessionId,
sessionKey: evt.sessionKey,
runId: evt.runId,
runAtMs: evt.runAtMs,
durationMs: evt.durationMs,
nextRunAtMs: evt.nextRunAtMs,
triggerFired: evt.triggerFired,
model: evt.model,
provider: evt.provider,
usage: evt.usage,
},
opts: { keepLines: runLogPrune.keepLines },
});
}).catch((err: unknown) => {
cronLogger.warn(
{ err: String(err), storePath, jobId: evt.jobId },
@@ -767,19 +791,31 @@ export function buildGatewayCronService(params: {
exitWatchersRef.current = createCronExitWatchers({
getProcessSupervisor,
persistCompletion: async (jobId) => {
await cron.update(jobId, { enabled: false });
},
fireOnExit: (job, exit) =>
fireOnExitJob(job, exit, {
run: (jobId, payload) => cron.run(jobId, "force", payload ? { payload } : undefined),
persistCompletion: async (jobId) =>
await runWithGatewayIndependentRootWorkAdmission(async () => {
await cron.update(jobId, { enabled: false });
}),
fireOnExit: (job, exit) =>
runWithGatewayIndependentRootWorkAdmission(async () =>
fireOnExitJob(job, exit, {
run: (jobId, payload) => cron.run(jobId, "force", payload ? { payload } : undefined),
}),
),
logger: cronLogger,
});
const getCronSuspensionBlockerCount = cron.getSuspensionBlockerCount.bind(cron);
cron.getSuspensionBlockerCount = () =>
getCronSuspensionBlockerCount() +
exitWatcherReconciliations +
(exitWatchersRef.current?.activeJobIds().length ?? 0);
const stopExitWatchers = () => {
exitWatcherGeneration += 1;
exitWatchersRef.current?.cancelAll();
};
const stopCron = cron.stop.bind(cron);
cron.stop = () => {
stopCron();
exitWatchersRef.current?.cancelAll();
stopExitWatchers();
};
return {
@@ -787,6 +823,6 @@ export function buildGatewayCronService(params: {
storePath,
cronEnabled,
reconcileExitWatchers,
stopExitWatchers: () => exitWatchersRef.current?.cancelAll(),
stopExitWatchers,
};
}
+167 -1
View File
@@ -1,6 +1,18 @@
// Server HTTP probe tests cover readiness, health, disabled compat routes, and
// auth handling through the in-memory HTTP harness.
import type { IncomingMessage, ServerResponse } from "node:http";
import { describe, expect, it, vi } from "vitest";
import {
prepareGatewaySuspend,
resetGatewaySuspendCoordinatorForTest,
resumeGatewaySuspend,
} from "../infra/gateway-suspend-coordinator.js";
import { isGatewayDraining } from "../process/command-queue.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import type { ChannelManager } from "./server-channels.js";
import {
AUTH_TOKEN,
AUTH_NONE,
@@ -9,7 +21,7 @@ import {
dispatchRequest,
withGatewayServer,
} from "./server-http.test-harness.js";
import type { ReadinessChecker } from "./server/readiness.js";
import { createReadinessChecker, type ReadinessChecker } from "./server/readiness.js";
import { withTempConfig } from "./test-temp-config.js";
type GatewayServerHarness = Parameters<typeof dispatchRequest>[0];
@@ -44,6 +56,160 @@ describe("gateway OpenAI-compatible disabled HTTP routes", () => {
});
describe("gateway probe endpoints", () => {
it("keeps liveness green while a prepared suspension lease makes readiness red", async () => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
const channelManager = {
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
getAutostartSuppression: () => null,
} as unknown as ChannelManager;
const getReadiness = createReadinessChecker({
channelManager,
startedAt: Date.now(),
getGatewayDraining: isGatewayDraining,
cacheTtlMs: 0,
});
try {
await withGatewayServer({
prefix: "probe-suspension-lease",
resolvedAuth: AUTH_NONE,
overrides: { getReadiness, openAiChatCompletionsEnabled: true },
run: async (server) => {
const prepared = prepareGatewaySuspend({
requestId: "request-readiness-probe",
pauseScheduling: vi.fn(),
resumeScheduling: vi.fn(),
createSuspensionId: () => "suspension-readiness-probe",
inspect: {
getQueueSize: () => 0,
getPendingReplies: () => 0,
getEmbeddedRuns: () => 0,
getCronRuns: () => 0,
getActiveTasks: () => 0,
getTaskBlockers: () => [],
getRootRequests: () => 0,
getSessionAdmissions: () => 0,
getSessionMutations: () => 0,
getChatRuns: () => 0,
getQueuedTurns: () => 0,
getTerminalPersistence: () => 0,
getTerminalSessions: () => 0,
},
});
if (prepared.status !== "ready") {
throw new Error(`expected prepared suspension, received ${prepared.status}`);
}
const health = await sendGatewayRequest(server, { path: "/healthz" });
expect(health.res.statusCode).toBe(200);
expect(JSON.parse(health.getBody())).toEqual({ ok: true, status: "live" });
const suspendedReadiness = await sendGatewayRequest(server, { path: "/readyz" });
expect(suspendedReadiness.res.statusCode).toBe(503);
expect(JSON.parse(suspendedReadiness.getBody())).toMatchObject({
ready: false,
failing: ["gateway-draining"],
});
const blockedChat = await sendGatewayRequest(server, {
path: "/v1/chat/completions",
method: "POST",
});
expect(blockedChat.res.statusCode).toBe(503);
expect(JSON.parse(blockedChat.getBody())).toMatchObject({
error: { code: "gateway_unavailable" },
});
expect(resumeGatewaySuspend(prepared.suspensionId)).toEqual({
ok: true,
status: "running",
resumed: true,
});
const resumedReadiness = await sendGatewayRequest(server, { path: "/readyz" });
expect(resumedReadiness.res.statusCode).toBe(200);
expect(JSON.parse(resumedReadiness.getBody())).toMatchObject({
ready: true,
failing: [],
});
},
});
} finally {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
}
});
it("keeps in-flight core HTTP work visible to suspension preparation", async () => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
let releaseWatch = () => {};
let markWatchStarted = () => {};
const watchStarted = new Promise<void>((resolve) => {
markWatchStarted = resolve;
});
const heldWatch = new Promise<void>((resolve) => {
releaseWatch = resolve;
});
const handleWatchNodeRequest = vi.fn(async (_req: IncomingMessage, res: ServerResponse) => {
markWatchStarted();
await heldWatch;
res.statusCode = 200;
res.end("ok");
return true;
});
try {
await withGatewayServer({
prefix: "probe-http-work-admission",
resolvedAuth: AUTH_NONE,
overrides: { handleWatchNodeRequest },
run: async (server) => {
const request = createRequest({ path: "/api/nodes/watch/node-1" });
const response = createResponse();
const pendingRequest = dispatchRequest(server, request, response.res);
await watchStarted;
expect(getActiveGatewayRootWorkCount()).toBe(1);
const prepared = prepareGatewaySuspend({
requestId: "request-http-work",
pauseScheduling: vi.fn(),
resumeScheduling: vi.fn(),
inspect: {
getQueueSize: () => 0,
getPendingReplies: () => 0,
getEmbeddedRuns: () => 0,
getCronRuns: () => 0,
getActiveTasks: () => 0,
getTaskBlockers: () => [],
getSessionAdmissions: () => 0,
getSessionMutations: () => 0,
getChatRuns: () => 0,
getQueuedTurns: () => 0,
getTerminalPersistence: () => 0,
getTerminalSessions: () => 0,
},
});
expect(prepared).toMatchObject({
status: "busy",
reason: "active-work",
activeCount: 1,
});
releaseWatch();
await pendingRequest;
expect(response.res.statusCode).toBe(200);
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
},
});
} finally {
releaseWatch();
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
}
});
it("returns detailed readiness payload for local /ready requests", async () => {
const getReadiness: ReadinessChecker = () => ({
ready: true,
+116 -47
View File
@@ -16,6 +16,10 @@ import {
createDiagnosticTraceContext,
runWithDiagnosticTraceContext,
} from "../infra/diagnostic-trace-context.js";
import {
isGatewayWorkAdmissionClosed,
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
import { resolveAssistantIdentity } from "./assistant-identity.js";
import type { AuthRateLimiter } from "./auth-rate-limit.js";
@@ -52,6 +56,7 @@ type PluginHttpRequestHandler = (
gatewayAuthSatisfied?: boolean;
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
gatewayRequestOperatorScopes?: readonly string[];
gatewayRequestClientIp?: string;
},
) => Promise<boolean>;
@@ -66,6 +71,7 @@ type PluginHttpUpgradeHandler = (
gatewayAuthSatisfied?: boolean;
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
gatewayRequestOperatorScopes?: readonly string[];
gatewayRequestClientIp?: string;
},
) => Promise<boolean>;
@@ -348,6 +354,35 @@ export async function runGatewayHttpRequestStages(
return false;
}
/** Runs one core HTTP user-work route under the same root fence as Gateway RPCs. */
export async function runWithGatewayHttpWorkAdmission(
res: ServerResponse,
run: () => Promise<boolean> | boolean,
): Promise<boolean> {
const admission = tryBeginGatewayRootWorkAdmission();
if (!admission) {
res.statusCode = 503;
res.setHeader("Content-Type", "application/json; charset=utf-8");
res.setHeader("Cache-Control", "no-store");
res.setHeader("Retry-After", "1");
res.end(
JSON.stringify({
error: {
message: "Gateway is temporarily unavailable while suspending or restarting",
type: "service_unavailable",
code: "gateway_unavailable",
},
}),
);
return true;
}
try {
return await admission.run(async () => await run());
} finally {
admission.release();
}
}
function buildPluginRequestStages(params: {
req: IncomingMessage;
res: ServerResponse;
@@ -364,6 +399,11 @@ function buildPluginRequestStages(params: {
if (!params.handlePluginRequest) {
return [];
}
const requestClientIp = resolveRequestClientIp(
params.req,
params.trustedProxies,
params.allowRealIpFallback,
);
let pluginGatewayAuthSatisfied = false;
let pluginGatewayRequestAuth: AuthorizedGatewayHttpRequest | undefined;
let pluginRequestOperatorScopes: string[] | undefined;
@@ -422,6 +462,7 @@ function buildPluginRequestStages(params: {
gatewayAuthSatisfied: pluginGatewayAuthSatisfied,
gatewayRequestAuth: pluginGatewayRequestAuth,
gatewayRequestOperatorScopes: pluginRequestOperatorScopes,
gatewayRequestClientIp: requestClientIp,
}) ?? false
);
},
@@ -559,94 +600,112 @@ export function createGatewayHttpServer(opts: {
if (opts.handleWatchNodeRequest && scopedRequestPath.startsWith("/api/nodes/watch/")) {
requestStages.push({
name: "watch-node",
run: () => opts.handleWatchNodeRequest?.(req, res) ?? false,
run: () =>
runWithGatewayHttpWorkAdmission(
res,
() => opts.handleWatchNodeRequest?.(req, res) ?? false,
),
});
}
if (openAiCompatEnabled && isOpenAiModelsPath(scopedRequestPath)) {
requestStages.push({
name: "models",
run: async () =>
(await getModelsHttpModule()).handleOpenAiModelsHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getModelsHttpModule()).handleOpenAiModelsHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (openAiCompatEnabled && isEmbeddingsPath(scopedRequestPath)) {
requestStages.push({
name: "embeddings",
run: async () =>
(await getEmbeddingsHttpModule()).handleOpenAiEmbeddingsHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getEmbeddingsHttpModule()).handleOpenAiEmbeddingsHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (isToolsInvokePath(scopedRequestPath)) {
requestStages.push({
name: "tools-invoke",
run: async () =>
(await getToolsInvokeHttpModule()).handleToolsInvokeHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getToolsInvokeHttpModule()).handleToolsInvokeHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (isSessionKillPath(scopedRequestPath)) {
requestStages.push({
name: "sessions-kill",
run: async () =>
(await getSessionKillHttpModule()).handleSessionKillHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getSessionKillHttpModule()).handleSessionKillHttpRequest(req, res, {
auth: resolvedAuthValue,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (isSessionHistoryPath(scopedRequestPath)) {
requestStages.push({
name: "sessions-history",
run: async () =>
(await getSessionHistoryHttpModule()).handleSessionHistoryHttpRequest(req, res, {
auth: resolvedAuthValue,
getResolvedAuth,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getSessionHistoryHttpModule()).handleSessionHistoryHttpRequest(req, res, {
auth: resolvedAuthValue,
getResolvedAuth,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (openResponsesEnabled && isOpenResponsesPath(scopedRequestPath)) {
requestStages.push({
name: "openresponses",
run: async () =>
(await getOpenResponsesHttpModule()).handleOpenResponsesHttpRequest(req, res, {
auth: resolvedAuthValue,
config: openResponsesConfig,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getOpenResponsesHttpModule()).handleOpenResponsesHttpRequest(req, res, {
auth: resolvedAuthValue,
config: openResponsesConfig,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (openAiChatCompletionsEnabled && isOpenAiChatCompletionsPath(scopedRequestPath)) {
requestStages.push({
name: "openai",
run: async () =>
(await getOpenAiHttpModule()).handleOpenAiHttpRequest(req, res, {
auth: resolvedAuthValue,
config: openAiChatCompletionsConfig,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
await runWithGatewayHttpWorkAdmission(res, async () =>
(await getOpenAiHttpModule()).handleOpenAiHttpRequest(req, res, {
auth: resolvedAuthValue,
config: openAiChatCompletionsConfig,
trustedProxies,
allowRealIpFallback,
rateLimiter,
}),
),
});
}
if (
@@ -843,6 +902,7 @@ export function attachGatewayUpgradeHandler(opts: {
const configSnapshot = getRuntimeConfig();
const trustedProxies = configSnapshot.gateway?.trustedProxies ?? [];
const allowRealIpFallback = configSnapshot.gateway?.allowRealIpFallback === true;
const requestClientIp = resolveRequestClientIp(req, trustedProxies, allowRealIpFallback);
const scopedNodeCapability = normalizePluginNodeCapabilityScopedUrl(req.url ?? "/");
if (scopedNodeCapability.malformedScopedPath) {
writeUpgradeAuthFailure(socket, { ok: false, reason: "unauthorized" });
@@ -916,12 +976,21 @@ export function attachGatewayUpgradeHandler(opts: {
gatewayAuthSatisfied: pluginGatewayAuthSatisfied,
gatewayRequestAuth: pluginGatewayRequestAuth,
gatewayRequestOperatorScopes: pluginGatewayRequestOperatorScopes,
gatewayRequestClientIp: requestClientIp,
})
) {
return;
}
}
const preauthBudgetKey = resolveRequestClientIp(req, trustedProxies, allowRealIpFallback);
// Plugin-owned upgrade routes have already had the opportunity to claim the socket.
// Core Gateway upgrades must stop at the HTTP boundary so a client cannot hold an
// untracked pre-connect socket after suspension or restart admission closes.
if (isGatewayWorkAdmissionClosed()) {
writeUpgradeServiceUnavailable(socket, "Gateway websocket admission closed");
socket.destroy();
return;
}
const preauthBudgetKey = requestClientIp;
if (wss.listenerCount("connection") === 0) {
writeUpgradeServiceUnavailable(socket, "Gateway websocket handlers unavailable");
socket.destroy();
@@ -3,6 +3,10 @@
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { isRetryableGatewayStartupUnavailableError } from "../../packages/gateway-protocol/src/startup-unavailable.js";
import {
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import {
testing as controlPlaneRateLimitTesting,
resolveControlPlaneRateLimitKey,
@@ -16,12 +20,14 @@ const noWebchat = () => false;
describe("gateway control-plane write rate limit", () => {
beforeEach(() => {
controlPlaneRateLimitTesting.resetControlPlaneRateLimitState();
resetGatewayWorkAdmission();
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-02-19T00:00:00.000Z"));
});
afterEach(() => {
vi.useRealTimers();
resetGatewayWorkAdmission();
controlPlaneRateLimitTesting.resetControlPlaneRateLimitState();
});
@@ -145,6 +151,57 @@ describe("gateway control-plane write rate limit", () => {
expect(handlerCalls).toHaveBeenCalledTimes(4);
});
it("does not consume the write budget for requests refused during suspension", async () => {
const handlerCalls = vi.fn();
const handler: GatewayRequestHandler = (opts) => {
handlerCalls(opts);
opts.respond(true, undefined, undefined);
};
const context = buildContext();
const client = buildClient();
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
for (let attempt = 0; attempt < 4; attempt += 1) {
const refused = await runRequest({ method: "config.patch", context, client, handler });
expect(respondCall(refused)[2]).toMatchObject({
code: "UNAVAILABLE",
details: { reason: "gateway-suspending" },
});
}
expect(suspension?.release()).toBe(true);
const allowed = await runRequest({ method: "config.patch", context, client, handler });
expect(allowed).toHaveBeenCalledWith(true, undefined, undefined);
expect(handlerCalls).toHaveBeenCalledOnce();
});
it("keeps suspension preparation rate-limited while admission is closed", async () => {
const handlerCalls = vi.fn();
const handler: GatewayRequestHandler = (opts) => {
handlerCalls(opts);
opts.respond(true, undefined, undefined);
};
const context = buildContext();
const client = buildClient();
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
const blocked = await runRequest({
method: "gateway.suspend.prepare",
context,
client,
handler,
});
expect(handlerCalls).toHaveBeenCalledTimes(3);
expect(respondCall(blocked)[2]).toMatchObject({ code: "UNAVAILABLE", retryable: true });
expect(suspension?.release()).toBe(true);
});
it.each(STARTUP_UNAVAILABLE_GATEWAY_METHODS)(
"blocks startup-gated method %s before dispatch with a retryable startup error",
async (method) => {
@@ -0,0 +1,273 @@
// Proves dispatcher root-work accounting and fail-closed suspension behavior.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
resetGatewaySuspendCoordinatorForTest,
resumeGatewaySuspend,
} from "../infra/gateway-suspend-coordinator.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import {
createGatewayMethodRegistry,
createPluginGatewayMethodDescriptor,
} from "./methods/registry.js";
import { handleGatewayRequest } from "./server-methods.js";
import { suspendHandlers } from "./server-methods/suspend.js";
import type { GatewayRequestHandler } from "./server-methods/types.js";
function deferred() {
let resolve = () => {};
const promise = new Promise<void>((done) => {
resolve = done;
});
return { promise, resolve };
}
function dispatch(params: {
method: string;
scope: "operator.read" | "operator.write" | "operator.admin";
handler: GatewayRequestHandler;
requestParams?: Record<string, unknown>;
context?: Parameters<typeof handleGatewayRequest>[0]["context"];
}) {
const respond = vi.fn();
const methodRegistry = createGatewayMethodRegistry([
createPluginGatewayMethodDescriptor({
pluginId: "suspend-proof",
name: params.method,
handler: params.handler,
scope: params.scope,
}),
]);
const request = handleGatewayRequest({
req: {
type: "req",
id: `request-${params.method}`,
method: params.method,
params: params.requestParams ?? {},
},
respond,
client: {
connId: "conn-suspend-proof",
connect: {
role: "operator",
scopes: [params.scope],
client: { id: "cli", version: "test", platform: "linux", mode: "cli" },
minProtocol: 1,
maxProtocol: 1,
},
},
isWebchatConnect: () => false,
context:
params.context ??
({ logGateway: { warn: vi.fn() } } as unknown as Parameters<
typeof handleGatewayRequest
>[0]["context"]),
methodRegistry,
});
return { request, respond };
}
beforeEach(() => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
});
afterEach(() => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
});
describe("gateway request suspension admission", () => {
it("keeps preparation busy while a previously admitted handler is active", async () => {
const started = deferred();
const finish = deferred();
const handler = vi.fn<GatewayRequestHandler>(async ({ respond }) => {
started.resolve();
await finish.promise;
respond(true, { ok: true });
});
const active = dispatch({
method: "suspend-proof.run",
scope: "operator.write",
handler,
});
await started.promise;
expect(getActiveGatewayRootWorkCount()).toBe(1);
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(getActiveGatewayRootWorkCount()).toBe(1);
expect(suspension?.rollback()).toBe(true);
finish.resolve();
await active.request;
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("reports a concurrent root as busy then excludes its own prepare request", async () => {
const started = deferred();
const finish = deferred();
const active = dispatch({
method: "suspend-proof.concurrent",
scope: "operator.write",
handler: async ({ respond }) => {
started.resolve();
await finish.promise;
respond(true, { ok: true });
},
});
await started.promise;
const prepareHandler = suspendHandlers["gateway.suspend.prepare"];
expect(prepareHandler).toBeTypeOf("function");
if (!prepareHandler) {
throw new Error("expected gateway suspension prepare handler");
}
const cron = {
pauseScheduling: vi.fn(),
resumeScheduling: vi.fn(),
getSuspensionBlockerCount: vi.fn(() => 0),
};
const context = {
cron,
logGateway: { warn: vi.fn() },
chatAbortControllers: new Map(),
chatQueuedTurns: new Map(),
} as unknown as Parameters<typeof handleGatewayRequest>[0]["context"];
const busy = dispatch({
method: "gateway.suspend.prepare",
scope: "operator.admin",
handler: prepareHandler,
requestParams: { requestId: "request-concurrent-root" },
context,
});
await busy.request;
expect(busy.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({
status: "busy",
reason: "active-work",
activeCount: 1,
blockers: expect.arrayContaining([
expect.objectContaining({ kind: "root-request", count: 1 }),
]),
}),
);
finish.resolve();
await active.request;
const ready = dispatch({
method: "gateway.suspend.prepare",
scope: "operator.admin",
handler: prepareHandler,
requestParams: { requestId: "request-own-root-excluded" },
context,
});
await ready.request;
expect(ready.respond).toHaveBeenCalledWith(
true,
expect.objectContaining({
status: "ready",
activeCount: 0,
blockers: [],
}),
);
const readyPayload = ready.respond.mock.calls[0]?.[1] as { suspensionId?: string } | undefined;
expect(readyPayload?.suspensionId).toBeTypeOf("string");
expect(resumeGatewaySuspend(readyPayload?.suspensionId ?? "missing")).toMatchObject({
ok: true,
resumed: true,
});
});
it("rejects new read and write handlers outside the suspension allowlist", async () => {
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
const writeHandler = vi.fn<GatewayRequestHandler>();
const blocked = dispatch({
method: "suspend-proof.write",
scope: "operator.write",
handler: writeHandler,
});
await blocked.request;
expect(writeHandler).not.toHaveBeenCalled();
expect(blocked.respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
retryable: true,
details: expect.objectContaining({ reason: "gateway-suspending" }),
}),
);
const readHandler = vi.fn<GatewayRequestHandler>(({ respond }) => {
respond(true, { state: "visible" });
});
const allowed = dispatch({
method: "suspend-proof.read",
scope: "operator.read",
handler: readHandler,
});
await allowed.request;
expect(readHandler).not.toHaveBeenCalled();
expect(allowed.respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({ code: "UNAVAILABLE", retryable: true }),
);
suspension?.release();
});
it("keeps suspension status reachable while prepared", async () => {
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
const handler = vi.fn<GatewayRequestHandler>(({ respond }) => {
respond(true, { ok: true });
});
const status = dispatch({
method: "gateway.suspend.status",
scope: "operator.read",
handler,
});
await status.request;
expect(handler).toHaveBeenCalledOnce();
expect(status.respond).toHaveBeenCalledWith(true, { ok: true });
suspension?.release();
});
it("rejects suspension preparation nested inside another root request", async () => {
const root = tryBeginGatewayRootWorkAdmission();
expect(root?.ownsRoot).toBe(true);
const handler = vi.fn<GatewayRequestHandler>();
await root?.run(async () => {
const nested = dispatch({
method: "gateway.suspend.prepare",
scope: "operator.admin",
handler,
});
await nested.request;
expect(nested.respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
retryable: true,
details: expect.objectContaining({ reason: "nested-gateway-request" }),
}),
);
});
root?.release();
expect(handler).not.toHaveBeenCalled();
});
});
+114 -27
View File
@@ -7,6 +7,11 @@ import {
} from "../../packages/gateway-protocol/src/startup-unavailable.js";
import { getPluginRegistryState } from "../plugins/runtime-state.js";
import { withPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js";
import {
getGatewaySuspendAdmissionPhase,
isGatewayRestartDraining,
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import { formatControlPlaneActor, resolveControlPlaneActor } from "./control-plane-audit.js";
import { consumeControlPlaneWriteBudget } from "./control-plane-rate-limit.js";
import {
@@ -190,6 +195,10 @@ const loadRestartHandlers = lazyHandlerModule(
() => import("./server-methods/restart.js"),
(module) => module.restartHandlers,
);
const loadSuspendHandlers = lazyHandlerModule(
() => import("./server-methods/suspend.js"),
(module) => module.suspendHandlers,
);
const loadSendHandlers = lazyHandlerModule(
() => import("./server-methods/send.js"),
(module) => module.sendHandlers,
@@ -306,6 +315,16 @@ function authorizeGatewayMethod(
return null;
}
const SUSPEND_CONTROL_METHODS = new Set([
"gateway.suspend.prepare",
"gateway.suspend.status",
"gateway.suspend.resume",
]);
function isGatewayMethodAllowedDuringSuspension(method: string): boolean {
return SUSPEND_CONTROL_METHODS.has(method);
}
export const coreGatewayHandlers: GatewayRequestHandlers = {
...createLazyCoreHandlers({
methods: ["connect"],
@@ -665,6 +684,10 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
methods: ["gateway.restart.request", "gateway.restart.preflight"],
loadHandlers: loadRestartHandlers,
}),
...createLazyCoreHandlers({
methods: ["gateway.suspend.prepare", "gateway.suspend.status", "gateway.suspend.resume"],
loadHandlers: loadSuspendHandlers,
}),
...createLazyCoreHandlers({
methods: ["message.action", "send", "poll"],
loadHandlers: loadSendHandlers,
@@ -781,33 +804,40 @@ export async function handleGatewayRequest(
);
return;
}
if (methodRegistry.isControlPlaneWrite(req.method)) {
const budget = consumeControlPlaneWriteBudget({ client });
if (!budget.allowed) {
// Control-plane writes mutate gateway-wide state; rate limit before handler lookup so
// plugin and aux write methods share the same protection.
const actor = resolveControlPlaneActor(client);
context.logGateway.warn(
`control-plane write rate-limited method=${req.method} ${formatControlPlaneActor(actor)} retryAfterMs=${budget.retryAfterMs} key=${budget.key}`,
);
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
`rate limit exceeded for ${req.method}; retry after ${Math.ceil(budget.retryAfterMs / 1000)}s`,
{
retryable: true,
retryAfterMs: budget.retryAfterMs,
details: {
method: req.method,
limit: "3 per 60s",
},
},
),
);
return;
const rejectRateLimitedControlPlaneWrite = (): boolean => {
if (!methodRegistry.isControlPlaneWrite(req.method)) {
return false;
}
const budget = consumeControlPlaneWriteBudget({ client });
if (budget.allowed) {
return false;
}
const actor = resolveControlPlaneActor(client);
context.logGateway.warn(
`control-plane write rate-limited method=${req.method} ${formatControlPlaneActor(actor)} retryAfterMs=${budget.retryAfterMs} key=${budget.key}`,
);
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
`rate limit exceeded for ${req.method}; retry after ${Math.ceil(budget.retryAfterMs / 1000)}s`,
{
retryable: true,
retryAfterMs: budget.retryAfterMs,
details: {
method: req.method,
limit: "3 per 60s",
},
},
),
);
return true;
};
const isSuspendPrepare = req.method === "gateway.suspend.prepare";
if (isSuspendPrepare && rejectRateLimitedControlPlaneWrite()) {
// Preparation must stay protected even before it owns the root admission that it closes.
return;
}
const handler = methodRegistry.getHandler(req.method) as GatewayRequestHandler | undefined;
if (!handler) {
@@ -818,6 +848,50 @@ export async function handleGatewayRequest(
);
return;
}
const rootWorkAdmission = tryBeginGatewayRootWorkAdmission();
if (
req.method === "gateway.suspend.prepare" &&
rootWorkAdmission &&
!rootWorkAdmission.ownsRoot
) {
respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "gateway suspension cannot begin from a nested request", {
retryable: true,
retryAfterMs: 1_000,
details: { method: req.method, reason: "nested-gateway-request" },
}),
);
return;
}
if (!rootWorkAdmission && !isGatewayMethodAllowedDuringSuspension(req.method)) {
const restartDraining = isGatewayRestartDraining();
respond(
false,
undefined,
errorShape(
ErrorCodes.UNAVAILABLE,
`${req.method} unavailable during gateway ${restartDraining ? "restart" : "suspension"}`,
{
retryable: true,
retryAfterMs: 1_000,
details: {
method: req.method,
reason: restartDraining ? "gateway-restarting" : "gateway-suspending",
phase: getGatewaySuspendAdmissionPhase(),
},
},
),
);
return;
}
if (!isSuspendPrepare && rejectRateLimitedControlPlaneWrite()) {
// A closed admission must reject first so refused writes do not exhaust the controller's
// budget and strand it behind rate limiting after suspension resumes.
rootWorkAdmission?.release();
return;
}
const invokeHandler = () =>
handler({
req,
@@ -831,5 +905,18 @@ export async function handleGatewayRequest(
// subagent methods (e.g. context engine tools spawning sub-agents
// during tool execution) can dispatch back into the gateway.
// The scope also carries caller identity into plugin-owned gateway methods.
await withPluginRuntimeGatewayRequestScope({ context, client, isWebchatConnect }, invokeHandler);
const invokeWithRequestScope = async () =>
await withPluginRuntimeGatewayRequestScope(
{ context, client, isWebchatConnect },
invokeHandler,
);
if (!rootWorkAdmission) {
await invokeWithRequestScope();
return;
}
try {
await rootWorkAdmission.run(invokeWithRequestScope);
} finally {
rootWorkAdmission.release();
}
}
+7
View File
@@ -116,6 +116,7 @@ import {
} from "../../infra/voicewake-routing.js";
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
import type { PluginHookSessionEndReason } from "../../plugins/hook-types.js";
import { retainGatewayRootWorkAdmissionContinuation } from "../../process/gateway-work-admission.js";
import {
classifySessionKeyShape,
isAcpSessionKey,
@@ -3543,9 +3544,12 @@ export const agentHandlers: GatewayRequestHandlers = {
if (!activeRunAbort.registered) {
activeGatewayWorkAdmission.release();
}
let releaseGatewayRootContinuation: (() => void) | undefined;
const cleanupAdmittedRun: typeof activeRunAbort.cleanup = (options) => {
activeRunAbort.cleanup(options);
activeGatewayWorkAdmission.release();
releaseGatewayRootContinuation?.();
releaseGatewayRootContinuation = undefined;
};
if (activeRunAbort.registered) {
retainEmbeddedAgentRunAbortabilityForRunId(runId);
@@ -3644,6 +3648,9 @@ export const agentHandlers: GatewayRequestHandlers = {
// is scheduled out of this request handler so immediate agent.wait calls
// can reach the gateway before the pre-turn runner monopolizes the loop.
gatewayAdmissionTransferred = true;
// Reserve the detached run before this request releases its root. Otherwise
// its inherited ALS context becomes retired and rejects subordinate work.
releaseGatewayRootContinuation = retainGatewayRootWorkAdmissionContinuation() ?? undefined;
void activeGatewayWorkAdmission.run(async () => {
await yieldAfterAgentAcceptedAck();
+26 -8
View File
@@ -104,6 +104,10 @@ import { deleteMediaBuffer, MEDIA_MAX_BYTES, type SavedMedia } from "../../media
import { createChannelMessageReplyPipeline } from "../../plugin-sdk/channel-outbound.js";
import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";
import { isPluginOwnedSessionBindingRecord } from "../../plugins/conversation-binding.js";
import {
retainGatewayRootWorkAdmissionContinuation,
runWithGatewayIndependentRootWorkContinuation,
} from "../../process/gateway-work-admission.js";
import { normalizeAgentId, scopeLegacySessionKeyToAgent } from "../../routing/session-key.js";
import { normalizeInputProvenance, type InputProvenance } from "../../sessions/input-provenance.js";
import { resolveSendPolicy } from "../../sessions/send-policy.js";
@@ -4237,9 +4241,12 @@ export const chatHandlers: GatewayRequestHandlers = {
});
return;
}
let releaseGatewayRootContinuation: (() => void) | undefined;
const cleanupAdmittedRun: typeof activeRunAbort.cleanup = (options) => {
activeRunAbort.cleanup(options);
gatewayWorkAdmission?.release();
releaseGatewayRootContinuation?.();
releaseGatewayRootContinuation = undefined;
};
claimAgentRunContext(clientRunId, {
sessionKey,
@@ -4407,7 +4414,7 @@ export const chatHandlers: GatewayRequestHandlers = {
const chatSendAckedAtMs = chatSendTiming?.ackedAtMs ?? performance.now();
const titleSource = stripInlineDirectiveTagsForDisplay(rawMessage).text;
if (isDashboardSessionTitleCandidate({ sessionKey, userMessage: titleSource })) {
void (async () => {
void runWithGatewayIndependentRootWorkContinuation(async () => {
const titleEntry =
entry?.sessionId === admittedSessionId
? entry
@@ -4432,7 +4439,7 @@ export const chatHandlers: GatewayRequestHandlers = {
reason: "chat.title",
});
}
})().catch((err: unknown) => {
}).catch((err: unknown) => {
context.logGateway.warn(
`dashboard session title generation failed: ${formatForLog(err)}`,
);
@@ -4790,6 +4797,9 @@ export const chatHandlers: GatewayRequestHandlers = {
}
emitServerTiming("first-assistant-event", undefined, dispatchStartedAtMs);
};
// Reserve the detached dispatch before this request releases its root. Otherwise
// its inherited ALS context becomes retired and rejects queued/session work.
releaseGatewayRootContinuation = retainGatewayRootWorkAdmissionContinuation() ?? undefined;
void gatewayWorkAdmission
.run(() =>
measureDiagnosticsTimelineSpan(
@@ -5839,17 +5849,19 @@ export const chatHandlers: GatewayRequestHandlers = {
});
})
.finally(() => {
const dispatchError = pendingDispatchLifecycleError;
// Reserve error projection before cleanup retires the dispatch root. Restart
// drain may already reject fresh roots, but this accepted request must finish.
const releaseDispatchErrorRoot = dispatchError
? retainGatewayRootWorkAdmissionContinuation()
: null;
cleanupAdmittedRun();
clearAgentRunContext(clientRunId, lifecycleGeneration);
context.removeChatRun(clientRunId, clientRunId, sessionKey);
if (!pendingDispatchLifecycleError) {
if (!dispatchError) {
return;
}
const persistDispatchLifecycleError = async () => {
const dispatchError = pendingDispatchLifecycleError;
if (!dispatchError) {
return;
}
const hasActiveRun = hasTrackedActiveSessionRun({
context,
requestedKey: rawSessionKey,
@@ -5888,7 +5900,13 @@ export const chatHandlers: GatewayRequestHandlers = {
);
}
};
void persistDispatchLifecycleError();
void persistDispatchLifecycleError()
.catch((continuationErr: unknown) => {
context.logGateway.warn(
`webchat session lifecycle continuation failed: ${formatForLog(continuationErr)}`,
);
})
.finally(() => releaseDispatchErrorRoot?.());
});
} catch (err) {
cleanupAdmittedRun({ force: true });
+4 -2
View File
@@ -4,9 +4,11 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe
import {
ErrorCodes,
errorShape,
} from "../../../packages/gateway-protocol/src/schema/error-codes.js";
import {
formatValidationErrors,
} from "../../../packages/gateway-protocol/src/index.js";
import type { ValidationError } from "../../../packages/gateway-protocol/src/index.js";
type ValidationError,
} from "../../../packages/gateway-protocol/src/validation-errors.js";
export { safeParseJson } from "../server-json.js";
import { formatForLog } from "../ws-log.js";
import type { RespondFn } from "./types.js";
+3 -3
View File
@@ -4,12 +4,11 @@ import type {
ConnectParams,
ErrorShape,
RequestFrame,
} from "../../../packages/gateway-protocol/src/index.js";
} from "../../../packages/gateway-protocol/src/schema/frames.js";
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
import type { CliDeps } from "../../cli/deps.types.js";
import type { HealthSummary } from "../../commands/health.types.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { CronServiceContract } from "../../cron/service-contract.js";
import type { PluginApprovalRequestPayload } from "../../infra/plugin-approvals.js";
import type { createSubsystemLogger } from "../../logging/subsystem.js";
import type { WizardSession } from "../../wizard/session.js";
@@ -28,6 +27,7 @@ import type {
ChatRunEntry,
ChatRunRegistration,
} from "../server-chat-state.js";
import type { GatewayCronServiceContract } from "../server-cron.js";
import type { DedupeEntry } from "../server-shared.js";
import type { GatewayEventLoopHealth } from "../server/event-loop-health.js";
import type { TerminalLaunchResolution } from "../terminal/launch.js";
@@ -68,7 +68,7 @@ export type RespondFn = (
/** Runtime services and mutable gateway state available to request handlers. */
export type GatewayRequestContext = {
deps: CliDeps;
cron: CronServiceContract;
cron: GatewayCronServiceContract;
cronStorePath: string;
getRuntimeConfig: () => OpenClawConfig;
resolveTerminalLaunchPolicy: (agentId?: string) => TerminalLaunchResolution;
+176
View File
@@ -0,0 +1,176 @@
// Covers suspension RPC validation and coordinator response mapping.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { suspendHandlers } from "./suspend.js";
const coordinator = vi.hoisted(() => ({
prepare: vi.fn(),
status: vi.fn(),
resume: vi.fn(),
}));
vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({
prepareGatewaySuspend: coordinator.prepare,
getGatewaySuspendStatus: coordinator.status,
resumeGatewaySuspend: coordinator.resume,
}));
vi.mock("../server-active-work.js", () => ({
createGatewayServerActiveWorkInspectors: vi.fn(() => ({ getChatRuns: vi.fn(() => 0) })),
}));
function invoke(method: keyof typeof suspendHandlers, params: unknown) {
const respond = vi.fn();
const pauseScheduling = vi.fn();
const resumeScheduling = vi.fn();
const warn = vi.fn();
const handler = suspendHandlers[method];
return Promise.resolve(
handler({
params,
respond,
context: {
cron: { pauseScheduling, resumeScheduling },
logGateway: { warn },
chatAbortControllers: new Map(),
chatQueuedTurns: new Map(),
},
} as unknown as Parameters<typeof handler>[0]),
).then(() => ({ respond, pauseScheduling, resumeScheduling }));
}
beforeEach(() => {
vi.clearAllMocks();
});
describe("gateway suspend handlers", () => {
it("validates the closed prepare params shape", async () => {
const { respond } = await invoke("gateway.suspend.prepare", {
requestId: "request-1",
extra: true,
});
expect(coordinator.prepare).not.toHaveBeenCalled();
expect(respond).toHaveBeenCalledWith(false, undefined, {
code: "INVALID_REQUEST",
message: "invalid gateway.suspend.prepare params",
});
});
it("wires prepare to scheduler pause/resume and returns busy or ready", async () => {
coordinator.prepare.mockReturnValueOnce({
status: "busy",
reason: "active-work",
activeCount: 1,
blockers: [{ kind: "queue", count: 1, message: "busy" }],
});
const { respond, pauseScheduling, resumeScheduling } = await invoke("gateway.suspend.prepare", {
requestId: "request-1",
});
expect(coordinator.prepare).toHaveBeenCalledWith(
expect.objectContaining({
requestId: "request-1",
pauseScheduling: expect.any(Function),
resumeScheduling: expect.any(Function),
}),
);
const options = coordinator.prepare.mock.calls[0]?.[0];
options.pauseScheduling();
options.resumeScheduling();
expect(pauseScheduling).toHaveBeenCalledOnce();
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(respond).toHaveBeenCalledWith(
true,
expect.objectContaining({ status: "busy", reason: "active-work" }),
);
});
it("maps a competing prepared lease to retryable unavailable", async () => {
coordinator.prepare.mockReturnValueOnce({ status: "conflict", expiresAtMs: Date.now() + 5000 });
const { respond } = await invoke("gateway.suspend.prepare", { requestId: "request-2" });
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
details: expect.objectContaining({ reason: "gateway-suspension-conflict" }),
retryable: true,
}),
);
});
it("maps prepare and status recovery to the same retryable unavailable error", async () => {
const recovering = {
status: "recovering",
reason: "scheduler-resume-failed",
retryAfterMs: 1_000,
};
coordinator.prepare.mockReturnValueOnce(recovering);
coordinator.status.mockReturnValueOnce(recovering);
const prepared = await invoke("gateway.suspend.prepare", { requestId: "request-recovery" });
const status = await invoke("gateway.suspend.status", { suspensionId: "stale-id" });
for (const respond of [prepared.respond, status.respond]) {
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
message: "gateway scheduler recovery is pending",
retryable: true,
retryAfterMs: 1_000,
details: { reason: "scheduler-resume-failed" },
}),
);
}
});
it("keeps resume idempotent and rejects a mismatched active lease", async () => {
coordinator.resume.mockReturnValueOnce({ ok: false, reason: "suspension-mismatch" });
const mismatch = await invoke("gateway.suspend.resume", {
suspensionId: "suspension-wrong",
});
expect(mismatch.respond).toHaveBeenCalledWith(false, undefined, {
code: "INVALID_REQUEST",
message: "gateway suspension id does not match",
});
coordinator.resume.mockReturnValueOnce({
ok: true,
status: "running",
resumed: false,
});
const resumed = await invoke("gateway.suspend.resume", { suspensionId: "suspension-1" });
expect(resumed.respond).toHaveBeenCalledWith(true, {
ok: true,
status: "running",
resumed: false,
});
});
it("returns retryable unavailable when scheduler resume needs retry", async () => {
coordinator.resume.mockReturnValueOnce({
ok: false,
reason: "scheduler-resume-failed",
retryAfterMs: 1_000,
});
const { respond } = await invoke("gateway.suspend.resume", {
suspensionId: "suspension-1",
});
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({
code: "UNAVAILABLE",
message: "gateway scheduler recovery is pending",
retryable: true,
retryAfterMs: 1_000,
details: { reason: "scheduler-resume-failed" },
}),
);
});
});
+107
View File
@@ -0,0 +1,107 @@
// Gateway RPC handlers for cooperative, host-neutral process suspension.
import {
ErrorCodes,
errorShape,
validateGatewaySuspendPrepareParams,
validateGatewaySuspendResumeParams,
validateGatewaySuspendStatusParams,
} from "../../../packages/gateway-protocol/src/index.js";
import {
getGatewaySuspendStatus,
prepareGatewaySuspend,
resumeGatewaySuspend,
} from "../../infra/gateway-suspend-coordinator.js";
import { createGatewayServerActiveWorkInspectors } from "../server-active-work.js";
import type { GatewayRequestHandlers } from "./types.js";
function invalidParams(method: string) {
return errorShape(ErrorCodes.INVALID_REQUEST, `invalid ${method} params`);
}
function schedulerRecoveryError(retryAfterMs: number) {
return errorShape(ErrorCodes.UNAVAILABLE, "gateway scheduler recovery is pending", {
retryable: true,
retryAfterMs,
details: { reason: "scheduler-resume-failed" },
});
}
export const suspendHandlers: GatewayRequestHandlers = {
"gateway.suspend.prepare": async ({ respond, params, context }) => {
if (!validateGatewaySuspendPrepareParams(params)) {
respond(false, undefined, invalidParams("gateway.suspend.prepare"));
return;
}
const requestId = params.requestId.trim();
const result = prepareGatewaySuspend({
requestId,
pauseScheduling: () => context.cron.pauseScheduling(),
resumeScheduling: () => context.cron.resumeScheduling(),
inspect: createGatewayServerActiveWorkInspectors(context),
warn: (message) => context.logGateway.warn(message),
});
if (result.status === "conflict") {
respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "another gateway suspension is already prepared", {
retryable: true,
retryAfterMs: Math.max(0, result.expiresAtMs - Date.now()),
details: { reason: "gateway-suspension-conflict", expiresAtMs: result.expiresAtMs },
}),
);
return;
}
if (result.status === "recovering") {
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
return;
}
respond(true, result);
},
"gateway.suspend.status": async ({ respond, params }) => {
if (!validateGatewaySuspendStatusParams(params)) {
respond(false, undefined, invalidParams("gateway.suspend.status"));
return;
}
const suspensionId = params.suspensionId.trim();
const result = getGatewaySuspendStatus(suspensionId);
if (result.status === "conflict") {
respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "a different gateway suspension is prepared", {
retryable: true,
retryAfterMs: Math.max(0, result.expiresAtMs - Date.now()),
details: { reason: "gateway-suspension-conflict", expiresAtMs: result.expiresAtMs },
}),
);
return;
}
if (result.status === "recovering") {
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
return;
}
respond(true, result);
},
"gateway.suspend.resume": async ({ respond, params }) => {
if (!validateGatewaySuspendResumeParams(params)) {
respond(false, undefined, invalidParams("gateway.suspend.resume"));
return;
}
const suspensionId = params.suspensionId.trim();
const result = resumeGatewaySuspend(suspensionId);
if (!result.ok) {
if (result.reason === "scheduler-resume-failed") {
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
return;
}
respond(
false,
undefined,
errorShape(ErrorCodes.INVALID_REQUEST, "gateway suspension id does not match"),
);
return;
}
respond(true, result);
},
};
+1 -1
View File
@@ -9,7 +9,7 @@ import {
GATEWAY_CLIENT_IDS,
GATEWAY_CLIENT_MODES,
} from "../../packages/gateway-protocol/src/client-info.js";
import type { ErrorShape } from "../../packages/gateway-protocol/src/index.js";
import type { ErrorShape } from "../../packages/gateway-protocol/src/schema/frames.js";
import { PROTOCOL_VERSION } from "../../packages/gateway-protocol/src/version.js";
import { normalizeModelRef, parseModelRef } from "../agents/model-selection.js";
import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
+52 -3
View File
@@ -4,11 +4,20 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { ConfigWriteNotification } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { consumeGatewaySigusr1RestartIntent } from "../infra/restart.js";
import {
consumeGatewaySigusr1RestartIntent,
markGatewaySigusr1RestartHandled,
testing as restartTesting,
} from "../infra/restart.js";
import {
pinActivePluginChannelRegistry,
releasePinnedPluginChannelRegistry,
} from "../plugins/runtime.js";
import {
isGatewayWorkAdmissionClosed,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js";
import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot } from "../secrets/runtime.js";
import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js";
@@ -449,6 +458,48 @@ describe("gateway hot reload model state", () => {
});
describe("gateway restart deferral preflight", () => {
it("holds root admission across an immediate config-reload restart signal", () => {
restartTesting.resetSigusr1State();
resetGatewayWorkAdmission();
const signalSpy = vi.fn();
process.once("SIGUSR1", signalSpy);
const { requestGatewayRestart } = createReloadHandlersForTest();
try {
expect(
requestGatewayRestart(
{
changedPaths: ["gateway.port"],
restartGateway: true,
restartReasons: ["gateway.port"],
hotReasons: [],
reloadHooks: false,
restartGmailWatcher: false,
restartCron: false,
restartHeartbeat: false,
restartHealthMonitor: false,
reloadPlugins: false,
restartChannels: new Set(),
disposeMcpRuntimes: false,
noopPaths: [],
},
{},
),
).toBe(true);
expect(signalSpy).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
markGatewaySigusr1RestartHandled();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
process.removeListener("SIGUSR1", signalSpy);
restartTesting.resetSigusr1State();
resetGatewayWorkAdmission();
}
});
it("defers channel hot reload until active embedded work drains", async () => {
const previousSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS;
const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS;
@@ -815,7 +866,6 @@ describe("gateway restart deferral preflight", () => {
});
it("logs active task run ids before waiting and when forcing after timeout", async () => {
const restartTesting = (await import("../infra/restart.js")).testing;
restartTesting.resetSigusr1State();
const logReload = { info: vi.fn(), warn: vi.fn() };
const { requestGatewayRestart } = createReloadHandlersForTest(logReload);
@@ -902,7 +952,6 @@ describe("gateway restart deferral preflight", () => {
});
it("uses the default restart deferral timeout when config omits deferralTimeoutMs", async () => {
const restartTesting = (await import("../infra/restart.js")).testing;
restartTesting.resetSigusr1State();
const { requestGatewayRestart } = createReloadHandlersForTest();
hoisted.activeTaskCount.value = 1;
+7 -2
View File
@@ -23,11 +23,12 @@ import type { HeartbeatRunner } from "../infra/heartbeat-runner.js";
import { resetDirectoryCache } from "../infra/outbound/target-resolver.js";
import {
deferGatewayRestartUntilIdle,
emitGatewayRestart,
emitGatewayRestartWithSignalAdmission,
resolveGatewayRestartDeferralTimeoutMs,
setGatewaySigusr1RestartPolicy,
} from "../infra/restart.js";
import { getTotalQueueSize } from "../process/command-queue.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import {
clearSecretsRuntimeSnapshot,
getActiveSecretsRuntimeSnapshot,
@@ -713,7 +714,10 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
}
// No active operations or pending replies, restart immediately
params.logReload.warn(`config change requires gateway restart (${reasons})`);
const emitted = emitGatewayRestart();
// The managed reloader owns independent root admission until onRestart
// returns. Extend that fence across signal delivery until the run loop
// atomically promotes it to one-way restart drain.
const emitted = emitGatewayRestartWithSignalAdmission();
if (!emitted) {
params.logReload.info("gateway restart already scheduled; skipping duplicate signal");
}
@@ -778,6 +782,7 @@ export function startManagedGatewayConfigReloader(
initialConfig: params.initialConfig,
initialCompareConfig: params.initialCompareConfig,
initialInternalWriteHash: params.initialInternalWriteHash,
runTransaction: runWithGatewayIndependentRootWorkAdmission,
readSnapshot: params.readSnapshot,
promoteSnapshot: async (snapshot, _reason) => await params.promoteSnapshot(snapshot),
subscribeToWrites: params.subscribeToWrites,
@@ -158,6 +158,7 @@ const mocks = vi.hoisted(() => {
recordInboundSessionAndDispatchReply: vi.fn(
async (_params: RecordInboundSessionAndDispatchReplyParams) => {},
),
logDebug: vi.fn(),
logInfo: vi.fn(),
logWarn: vi.fn(),
logError: vi.fn(),
@@ -283,6 +284,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
vi.mock("../logging/subsystem.js", () => {
const logger = {
debug: mocks.logDebug,
info: mocks.logInfo,
warn: mocks.logWarn,
error: mocks.logError,
@@ -305,6 +307,12 @@ const {
refreshLatestUpdateRestartSentinel,
scheduleRestartSentinelWake,
} = await import("./server-restart-sentinel.js");
const {
getActiveGatewayRootWorkCount,
getGatewaySuspendAdmissionPhase,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} = await import("../process/gateway-work-admission.js");
function expectRecordFields(
record: unknown,
@@ -371,12 +379,15 @@ function expectContinuationDispatchFields(
describe("scheduleRestartSentinelWake", () => {
afterEach(() => {
resetGatewayWorkAdmission();
vi.useRealTimers();
});
beforeEach(() => {
resetGatewayWorkAdmission();
vi.useRealTimers();
mocks.queuedSessionDelivery = null;
mocks.readRestartSentinel.mockReset();
mocks.readRestartSentinel.mockResolvedValue({
version: 1,
payload: {
@@ -476,6 +487,47 @@ describe("scheduleRestartSentinelWake", () => {
expect(mocks.logWarn).not.toHaveBeenCalled();
});
it("defers pending update retries until suspension resumes", async () => {
vi.useFakeTimers();
const pendingPayload: RestartSentinelPayload = {
kind: "update",
status: "skipped",
ts: 123,
stats: {
mode: "git",
handoffId: "handoff-1",
reason: "managed-service-handoff-started",
},
};
let finishRetryRead: ((value: RestartSentinel | null) => void) | undefined;
const retryRead = new Promise<RestartSentinel | null>((resolve) => {
finishRetryRead = resolve;
});
mocks.readRestartSentinel
.mockResolvedValueOnce({ version: 1, payload: pendingPayload })
.mockImplementationOnce(async () => (await retryRead) as RestartSentinel);
await scheduleRestartSentinelWake({ deps: {} as never });
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
await vi.advanceTimersByTimeAsync(1);
expect(getGatewaySuspendAdmissionPhase()).toBe("prepared");
expect(mocks.readRestartSentinel).toHaveBeenCalledTimes(1);
expect(getActiveGatewayRootWorkCount()).toBe(0);
expect(suspension?.release()).toBe(true);
await vi.waitFor(() => {
expect(mocks.readRestartSentinel).toHaveBeenCalledTimes(2);
});
expect(getActiveGatewayRootWorkCount()).toBe(1);
finishRetryRead?.(null);
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
it("retries outbound delivery once and logs a warning without dropping the agent wake", async () => {
vi.useFakeTimers();
mocks.deliverOutboundPayloads
+6 -3
View File
@@ -41,6 +41,7 @@ import { isPendingControlPlaneUpdateRestartSentinel } from "../infra/update-cont
import { createSubsystemLogger } from "../logging/subsystem.js";
import { stringifyRouteThreadId } from "../plugin-sdk/channel-route.js";
import type { OutboundReplyPayload } from "../plugin-sdk/reply-payload.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import {
deliveryContextFromSession,
mergeDeliveryContext,
@@ -468,9 +469,11 @@ async function loadRestartSentinelStartupTask(params: {
const attempt = params.attempt ?? 0;
if (attempt < CONTROL_PLANE_UPDATE_PENDING_MAX_ATTEMPTS) {
const timer = setTimeout(() => {
void scheduleRestartSentinelWakeAttempt({
deps: params.deps,
attempt: attempt + 1,
void runWithGatewayIndependentRootWorkAdmission(async () => {
await scheduleRestartSentinelWakeAttempt({
deps: params.deps,
attempt: attempt + 1,
});
}).catch((err: unknown) => {
log.warn(`restart sentinel pending update retry failed: ${formatErrorMessage(err)}`);
});
+47 -42
View File
@@ -5,6 +5,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { isVitestRuntimeEnv } from "../infra/env.js";
import { startHeartbeatRunner, type HeartbeatRunner } from "../infra/heartbeat-runner.js";
import type { PluginMetadataRegistryView } from "../plugins/plugin-metadata-snapshot.types.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { isGatewayModelPricingEnabled } from "./model-pricing-config.js";
import type { startGatewayMaintenanceTimers } from "./server-maintenance.js";
import {
@@ -30,10 +31,10 @@ export function startGatewayCronWithLogging(params: {
afterStart?: () => Promise<void>;
logCron: { error: (message: string) => void };
}): void {
void params.cron
.start()
.then(() => params.afterStart?.())
.catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`));
void runWithGatewayIndependentRootWorkAdmission(async () => {
await params.cron.start();
await params.afterStart?.();
}).catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`));
}
function clearGatewayMaintenanceHandles(maintenance: GatewayMaintenanceHandles | null): void {
@@ -100,38 +101,42 @@ export function scheduleGatewayPostReadyMaintenance(params: {
if (params.isClosing()) {
return;
}
void runGatewayPostReadyMaintenance({
startMaintenance: async () => {
if (params.isClosing()) {
return null;
}
const maintenance = await params.startMaintenance();
if (params.isClosing()) {
// Maintenance can allocate intervals before shutdown is observed; clear them here
// instead of handing live timers to a closing gateway.
clearGatewayMaintenanceHandles(maintenance);
return null;
}
return maintenance;
},
applyMaintenance: (maintenance) => {
if (params.isClosing()) {
clearGatewayMaintenanceHandles(maintenance);
return;
}
params.applyMaintenance(maintenance);
},
shouldStartCron: () => !params.isClosing() && params.shouldStartCron(),
markCronStartHandled: params.markCronStartHandled,
cron: params.cron,
logCron: params.logCron,
log: params.log,
recordPostReadyMemory: () => {
if (!params.isClosing()) {
params.recordPostReadyMemory();
}
},
});
void runWithGatewayIndependentRootWorkAdmission(async () =>
runGatewayPostReadyMaintenance({
startMaintenance: async () => {
if (params.isClosing()) {
return null;
}
const maintenance = await params.startMaintenance();
if (params.isClosing()) {
// Maintenance can allocate intervals before shutdown is observed; clear them here
// instead of handing live timers to a closing gateway.
clearGatewayMaintenanceHandles(maintenance);
return null;
}
return maintenance;
},
applyMaintenance: (maintenance) => {
if (params.isClosing()) {
clearGatewayMaintenanceHandles(maintenance);
return;
}
params.applyMaintenance(maintenance);
},
shouldStartCron: () => !params.isClosing() && params.shouldStartCron(),
markCronStartHandled: params.markCronStartHandled,
cron: params.cron,
logCron: params.logCron,
log: params.log,
recordPostReadyMemory: () => {
if (!params.isClosing()) {
params.recordPostReadyMemory();
}
},
}),
).catch((err: unknown) =>
params.log.warn(`gateway post-ready maintenance deferred task failed: ${String(err)}`),
);
}, params.delayMs);
timer.unref?.();
return timer;
@@ -143,7 +148,7 @@ function recoverPendingOutboundDeliveries(params: {
}): void {
// Recovery is best-effort background work; startup must continue even if outbound modules fail
// to import or queued delivery replay fails.
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { recoverPendingDeliveries } = await import("../infra/outbound/delivery-queue.js");
const { deliverOutboundPayloadsInternal } = await import("../infra/outbound/deliver.js");
const logRecovery = params.log.child("delivery-recovery");
@@ -152,7 +157,7 @@ function recoverPendingOutboundDeliveries(params: {
log: logRecovery,
cfg: params.cfg,
});
})().catch((err: unknown) => params.log.error(`Delivery recovery failed: ${String(err)}`));
}).catch((err: unknown) => params.log.error(`Delivery recovery failed: ${String(err)}`));
}
function recoverPendingSessionDeliveries(params: {
@@ -163,7 +168,7 @@ function recoverPendingSessionDeliveries(params: {
// Delay session continuation recovery so the gateway has time to publish ready state and
// request routing before replaying restart-sentinel deliveries.
const timer = setTimeout(() => {
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { recoverPendingRestartContinuationDeliveries } =
await import("./server-restart-sentinel.js");
const logRecovery = params.log.child("session-delivery-recovery");
@@ -172,7 +177,7 @@ function recoverPendingSessionDeliveries(params: {
log: logRecovery,
maxEnqueuedAt: params.maxEnqueuedAt,
});
})().catch((err: unknown) =>
}).catch((err: unknown) =>
params.log.error(`Session delivery recovery failed: ${String(err)}`),
);
}, 1_250);
@@ -191,7 +196,7 @@ function startGatewayModelPricingRefreshOnDemand(params: {
let stopRefresh: (() => void) | undefined;
// Import pricing refresh lazily; many gateway starts never use model-pricing metadata.
// The stopped flag closes the race where shutdown happens before the import resolves.
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { startGatewayModelPricingRefresh } = await import("./model-pricing-cache.js");
if (stopped) {
return;
@@ -204,7 +209,7 @@ function startGatewayModelPricingRefreshOnDemand(params: {
stopRefresh();
stopRefresh = undefined;
}
})().catch((err: unknown) =>
}).catch((err: unknown) =>
params.log.error(`Model pricing refresh failed to start: ${String(err)}`),
);
return () => {
+21 -15
View File
@@ -30,7 +30,11 @@ import {
createToolEventRecipientRegistry,
} from "./server-chat-state.js";
import { MAX_PREAUTH_PAYLOAD_BYTES } from "./server-constants.js";
import { attachGatewayUpgradeHandler, createGatewayHttpServer } from "./server-http.js";
import {
attachGatewayUpgradeHandler,
createGatewayHttpServer,
runWithGatewayHttpWorkAdmission,
} from "./server-http.js";
import type { GatewayRequestContext } from "./server-methods/types.js";
import type { DedupeEntry } from "./server-shared.js";
import type { HookClientIpConfig, HooksRequestHandler } from "./server/hooks-request-handler.js";
@@ -153,20 +157,22 @@ export async function createGatewayRuntimeState(params: {
if (url.pathname !== basePath && !url.pathname.startsWith(`${basePath}/`)) {
return false;
}
if (!loadedHooksRequestHandler) {
// Hooks are cold for most gateway starts; create the handler only after a request
// matches the configured base path so startup avoids importing hook runtime code.
const { createGatewayHooksRequestHandler } = await import("./server/hooks.js");
loadedHooksRequestHandler = createGatewayHooksRequestHandler({
deps: params.deps,
getHooksConfig: params.hooksConfig,
getClientIpConfig: params.getHookClientIpConfig,
bindHost: params.bindHost,
port: params.port,
logHooks: params.logHooks,
});
}
return await loadedHooksRequestHandler(req, res);
return await runWithGatewayHttpWorkAdmission(res, async () => {
if (!loadedHooksRequestHandler) {
// Hooks are cold for most gateway starts; create the handler only after a request
// matches the configured base path so startup avoids importing hook runtime code.
const { createGatewayHooksRequestHandler } = await import("./server/hooks.js");
loadedHooksRequestHandler = createGatewayHooksRequestHandler({
deps: params.deps,
getHooksConfig: params.hooksConfig,
getClientIpConfig: params.getHookClientIpConfig,
bindHost: params.bindHost,
port: params.port,
logHooks: params.logHooks,
});
}
return await loadedHooksRequestHandler(req, res);
});
};
let loadedPluginRequestHandler: GatewayPluginRequestHandler | null = null;
+1 -1
View File
@@ -1,6 +1,6 @@
// Gateway shared request-state types.
// Defines cached dedupe entries for idempotent Gateway method calls.
import type { ErrorShape } from "../../packages/gateway-protocol/src/index.js";
import type { ErrorShape } from "../../packages/gateway-protocol/src/schema/frames.js";
export const PENDING_CHAT_SEND_DEDUPE_PREFIX = "pending-chat:";
+47 -1
View File
@@ -11,6 +11,10 @@ import type {
PluginHookGatewayStartEvent,
} from "../plugins/hook-types.js";
import type { PluginServicesHandle } from "../plugins/services.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { withEnvAsync } from "../test-utils/env.js";
@@ -40,7 +44,8 @@ const hoisted = vi.hoisted(() => {
skipped: 0,
}));
const scheduleRestartAbortedMainSessionRecovery = vi.fn();
const scheduleRestartSentinelWake = vi.fn();
const scheduleRestartSentinelWake =
vi.fn<typeof import("./server-restart-sentinel.js").scheduleRestartSentinelWake>();
const refreshLatestUpdateRestartSentinel = vi.fn<
typeof import("./server-restart-sentinel.js").refreshLatestUpdateRestartSentinel
>(async () => null);
@@ -301,6 +306,7 @@ function firstGatewayStartCall(
describe("startGatewayPostAttachRuntime", () => {
beforeEach(() => {
resetGatewayWorkAdmission();
closeOpenClawStateDatabaseForTest();
vi.stubEnv("OPENCLAW_SKIP_CHANNELS", "0");
vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", "0");
@@ -364,6 +370,7 @@ describe("startGatewayPostAttachRuntime", () => {
});
afterEach(() => {
resetGatewayWorkAdmission();
closeOpenClawStateDatabaseForTest();
vi.useRealTimers();
vi.unstubAllEnvs();
@@ -371,6 +378,10 @@ describe("startGatewayPostAttachRuntime", () => {
it("re-enables startup-gated methods after post-attach sidecars start", async () => {
const unavailableGatewayMethods = new Set<string>(["chat.history", "models.list"]);
const methodsAtRecoveryRegistration: string[][] = [];
hoisted.scheduleRestartAbortedMainSessionRecovery.mockImplementationOnce(() => {
methodsAtRecoveryRegistration.push([...unavailableGatewayMethods]);
});
const onSidecarsReady = vi.fn();
const log = { info: vi.fn(), warn: vi.fn() };
@@ -399,6 +410,7 @@ describe("startGatewayPostAttachRuntime", () => {
expect(hoisted.scheduleRestartAbortedMainSessionRecovery).toHaveBeenCalledWith({
cfg: { hooks: { internal: { enabled: false } } },
});
expect(methodsAtRecoveryRegistration).toStrictEqual([["chat.history", "models.list"]]);
expect(hoisted.startGatewayMemoryBackend).not.toHaveBeenCalled();
});
@@ -430,6 +442,29 @@ describe("startGatewayPostAttachRuntime", () => {
expect(events).toEqual(["sidecars", "returned", "sentinel"]);
});
it("keeps delayed restart sentinel recovery admitted until wake work completes", async () => {
vi.useFakeTimers();
let finishWake: (() => void) | undefined;
const wake = new Promise<void>((resolve) => {
finishWake = resolve;
});
hoisted.scheduleRestartSentinelWake.mockReturnValueOnce(wake);
testing.scheduleRestartSentinelWakeAfterReady({
deps: {} as never,
log: { warn: vi.fn() },
});
await vi.advanceTimersByTimeAsync(750);
expect(hoisted.scheduleRestartSentinelWake).toHaveBeenCalledOnce();
expect(getActiveGatewayRootWorkCount()).toBe(1);
finishWake?.();
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
it("starts sidecars while startup logging is still pending", async () => {
const events: string[] = [];
let finishStartupLog: (() => void) | undefined;
@@ -2050,6 +2085,13 @@ describe("startGatewayPostAttachRuntime", () => {
it("dispatches registered gateway startup internal hooks without configured hook packs", async () => {
vi.useFakeTimers();
hoisted.hasInternalHookListeners.mockReturnValue(true);
let releaseHook = () => {};
hoisted.triggerInternalHook.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
releaseHook = resolve;
}),
);
const cfg = {} as never;
const deps = {} as never;
@@ -2088,7 +2130,11 @@ describe("startGatewayPostAttachRuntime", () => {
},
);
expect(hoisted.triggerInternalHook).toHaveBeenCalledWith(hoisted.startupHookEvent);
expect(getActiveGatewayRootWorkCount()).toBe(1);
releaseHook();
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
} finally {
releaseHook();
vi.useRealTimers();
}
});
+120 -99
View File
@@ -16,6 +16,7 @@ import type { loadOpenClawPlugins } from "../plugins/loader.js";
import { getPluginModuleLoaderStats } from "../plugins/plugin-module-loader-cache.js";
import type { PluginRegistry } from "../plugins/registry.js";
import type { PluginServicesHandle } from "../plugins/services.js";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
import {
GATEWAY_EVENT_UPDATE_AVAILABLE,
@@ -157,13 +158,12 @@ function scheduleGatewayMemoryBackend(params: {
return;
}
const start = () => {
void import("./server-startup-memory.js")
.then(({ startGatewayMemoryBackend }) =>
startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }),
)
.catch((err: unknown) => {
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
});
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { startGatewayMemoryBackend } = await import("./server-startup-memory.js");
await startGatewayMemoryBackend({ cfg: params.cfg, log: params.log });
}).catch((err: unknown) => {
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
});
};
if (params.policy.mode === "immediate") {
setImmediate(start);
@@ -181,12 +181,10 @@ function schedulePostAttachUpdateSentinelRefresh(params: {
>;
}): void {
const handle = setImmediate(() => {
void measureStartup(params.startupTrace, "post-attach.update-sentinel", async () => {
try {
void runWithGatewayIndependentRootWorkAdmission(async () => {
await measureStartup(params.startupTrace, "post-attach.update-sentinel", async () => {
await params.refreshLatestUpdateRestartSentinel();
} catch (err) {
params.log.warn(`restart sentinel refresh failed: ${String(err)}`);
}
});
}).catch((err: unknown) => {
params.log.warn(`restart sentinel refresh failed: ${String(err)}`);
});
@@ -210,39 +208,41 @@ function scheduleProviderAuthStatePrewarm(params: {
let pendingRewarmReason: string | undefined;
const isStopped = () => stopped;
const delayMs = params.delayMs ?? PROVIDER_AUTH_PREWARM_START_DELAY_MS;
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const [{ setAuthProfileFailureHook }, { clearCurrentProviderAuthState }] = await Promise.all([
import("../agents/auth-profiles/failure-hook.js"),
import("../agents/model-provider-auth-state.js"),
]);
const loadProviderAuthWarmModule = () => import("../agents/model-provider-auth.js");
const runRewarm = async (reason: string) => {
if (isStopped()) {
return;
}
const cfg = params.getConfig();
rewarmInFlight = true;
try {
const { warmCurrentProviderAuthStateOffMainThread } = await loadProviderAuthWarmModule();
const metrics = await measureProviderAuthWarm(() =>
warmCurrentProviderAuthStateOffMainThread(cfg, { isCancelled: isStopped }),
);
await runWithGatewayIndependentRootWorkAdmission(async () => {
if (isStopped()) {
return;
}
params.log.info(
`provider auth state re-warmed (${reason}) ${formatProviderAuthWarmMetrics(metrics)}`,
);
} catch (err) {
params.log.warn(`provider auth state rewarm failed: ${String(err)}`);
} finally {
rewarmInFlight = false;
const nextReason = pendingRewarmReason;
pendingRewarmReason = undefined;
if (nextReason && !isStopped()) {
scheduleAuthMapRewarm(nextReason);
const cfg = params.getConfig();
rewarmInFlight = true;
try {
const { warmCurrentProviderAuthStateOffMainThread } = await loadProviderAuthWarmModule();
const metrics = await measureProviderAuthWarm(() =>
warmCurrentProviderAuthStateOffMainThread(cfg, { isCancelled: isStopped }),
);
if (isStopped()) {
return;
}
params.log.info(
`provider auth state re-warmed (${reason}) ${formatProviderAuthWarmMetrics(metrics)}`,
);
} catch (err) {
params.log.warn(`provider auth state rewarm failed: ${String(err)}`);
} finally {
rewarmInFlight = false;
const nextReason = pendingRewarmReason;
pendingRewarmReason = undefined;
if (nextReason && !isStopped()) {
scheduleAuthMapRewarm(nextReason);
}
}
}
});
};
const scheduleAuthMapRewarm = (reason: string) => {
// Collapse repeated auth-profile failures into one rewarm turn while a
@@ -279,7 +279,7 @@ function scheduleProviderAuthStatePrewarm(params: {
}
startupTimer = setTimeout(
() => {
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
if (isStopped()) {
return;
}
@@ -294,14 +294,14 @@ function scheduleProviderAuthStatePrewarm(params: {
params.log.info(
`provider auth state pre-warmed ${formatProviderAuthWarmMetrics(metrics)}`,
);
})().catch((err: unknown) => {
}).catch((err: unknown) => {
params.log.warn(`provider auth state pre-warm failed: ${String(err)}`);
});
},
Math.max(0, delayMs),
);
startupTimer.unref?.();
})().catch((err: unknown) => {
}).catch((err: unknown) => {
params.log.warn(`provider auth state pre-warm setup failed: ${String(err)}`);
});
return {
@@ -335,26 +335,28 @@ function scheduleAgentRuntimePluginPrewarm(params: {
timer = setTimeout(
() => {
timer = undefined;
void measureStartup(params.startupTrace, "post-ready.agent-runtime-plugins", async () => {
if (isStopped()) {
return;
}
const started = performance.now();
const { ensureRuntimePluginsLoaded } = await import("../agents/runtime-plugins.js");
const cfg = params.getConfig();
if (isStopped()) {
return;
}
ensureRuntimePluginsLoaded({
config: cfg,
workspaceDir: params.workspaceDir,
allowGatewaySubagentBinding: true,
void runWithGatewayIndependentRootWorkAdmission(async () => {
await measureStartup(params.startupTrace, "post-ready.agent-runtime-plugins", async () => {
if (isStopped()) {
return;
}
const started = performance.now();
const { ensureRuntimePluginsLoaded } = await import("../agents/runtime-plugins.js");
const cfg = params.getConfig();
if (isStopped()) {
return;
}
ensureRuntimePluginsLoaded({
config: cfg,
workspaceDir: params.workspaceDir,
allowGatewaySubagentBinding: true,
});
if (!isStopped()) {
params.log.info(
`agent runtime plugins pre-warmed in ${(performance.now() - started).toFixed(0)}ms`,
);
}
});
if (!isStopped()) {
params.log.info(
`agent runtime plugins pre-warmed in ${(performance.now() - started).toFixed(0)}ms`,
);
}
}).catch((err: unknown) => {
params.log.warn(`agent runtime plugin pre-warm failed: ${String(err)}`);
});
@@ -387,9 +389,11 @@ function schedulePostReadySidecarTask(params: {
if (isStopped()) {
return;
}
void measureStartup(params.startupTrace, params.name, () =>
params.run(isStopped, abortController.signal),
).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
await measureStartup(params.startupTrace, params.name, () =>
params.run(isStopped, abortController.signal),
);
}).catch((err: unknown) => {
params.log.warn(`${params.name} failed after gateway ready: ${String(err)}`);
});
});
@@ -406,6 +410,20 @@ function schedulePostReadySidecarTask(params: {
};
}
function scheduleRestartSentinelWakeAfterReady(params: {
deps: CliDeps;
log: { warn: (msg: string) => void };
}): void {
setTimeout(() => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { scheduleRestartSentinelWake } = await loadGatewayRestartSentinelModule();
await scheduleRestartSentinelWake({ deps: params.deps });
}).catch((err: unknown) => {
params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`);
});
}, 750);
}
type CleanStaleLockFiles = typeof import("../agents/session-write-lock.js").cleanStaleLockFiles;
type MarkRestartAbortedMainSessionsFromLocks =
typeof import("../agents/main-session-restart-recovery.js").markRestartAbortedMainSessionsFromLocks;
@@ -779,19 +797,22 @@ export async function startGatewaySidecars(params: {
// Run startup hooks after sidecar startup has yielded once so gateway bind
// and channel startup are not delayed by hook handlers.
setTimeout(() => {
void loadInternalHooksModule().then(({ createInternalHookEvent, triggerInternalHook }) => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const { createInternalHookEvent, triggerInternalHook } = await loadInternalHooksModule();
const hookEvent = createInternalHookEvent("gateway", "startup", "gateway:startup", {
cfg: params.cfg,
deps: params.deps,
workspaceDir: params.defaultWorkspaceDir,
});
void triggerInternalHook(hookEvent);
await triggerInternalHook(hookEvent);
}).catch((err: unknown) => {
params.logHooks.warn(`gateway startup hook failed: ${String(err)}`);
});
}, 250);
}
if (params.cfg.acp?.enabled) {
void (async () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const ready = await measureStartup(params.startupTrace, "sidecars.acp.runtime-ready", () =>
waitForAcpRuntimeBackendReady({ backendId: params.cfg.acp?.backend }),
);
@@ -815,7 +836,7 @@ export async function startGatewaySidecars(params: {
`acp startup identity reconcile (renderer=${ACP_SESSION_IDENTITY_RENDERER_VERSION}): checked=${result.checked} resolved=${result.resolved} failed=${result.failed}`,
);
});
})().catch((err: unknown) => {
}).catch((err: unknown) => {
params.log.warn(`acp startup identity reconcile failed: ${String(err)}`);
});
}
@@ -864,15 +885,7 @@ export async function startGatewaySidecars(params: {
if (!(await hasRestartSentinelFast())) {
return;
}
setTimeout(() => {
void loadGatewayRestartSentinelModule()
.then(({ scheduleRestartSentinelWake }) =>
scheduleRestartSentinelWake({ deps: params.deps }),
)
.catch((err: unknown) => {
params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`);
});
}, 750);
scheduleRestartSentinelWakeAfterReady({ deps: params.deps, log: params.log });
},
});
@@ -1025,16 +1038,19 @@ function createDeferredGatewayUpdateCheck(params: {
if (stopped) {
return;
}
void measureStartup(params.startupTrace, "post-attach.update-check", () =>
params.runtimeDeps.scheduleGatewayUpdateCheck({
cfg: params.cfg,
log: params.log,
isNixMode: params.isNixMode,
onUpdateAvailableChange: (updateAvailable) => {
const payload: GatewayUpdateAvailableEventPayload = { updateAvailable };
params.broadcast(GATEWAY_EVENT_UPDATE_AVAILABLE, payload, { dropIfSlow: true });
},
}),
void runWithGatewayIndependentRootWorkAdmission(
async () =>
await measureStartup(params.startupTrace, "post-attach.update-check", () =>
params.runtimeDeps.scheduleGatewayUpdateCheck({
cfg: params.cfg,
log: params.log,
isNixMode: params.isNixMode,
onUpdateAvailableChange: (updateAvailable) => {
const payload: GatewayUpdateAvailableEventPayload = { updateAvailable };
params.broadcast(GATEWAY_EVENT_UPDATE_AVAILABLE, payload, { dropIfSlow: true });
},
}),
),
)
.then((nextStop) => {
if (stopped) {
@@ -1259,9 +1275,6 @@ export async function startGatewayPostAttachRuntime(
loaderStatsAfter.sourceTransformFallbacks - loaderStatsBefore.sourceTransformFallbacks,
],
]);
for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) {
params.unavailableGatewayMethods.delete(method);
}
try {
const { scheduleRestartAbortedMainSessionRecovery } =
await loadMainSessionRestartRecoveryModule();
@@ -1269,6 +1282,11 @@ export async function startGatewayPostAttachRuntime(
} catch (err) {
params.log.warn(`main-session restart recovery failed to schedule: ${String(err)}`);
}
// Capture the orphan-recovery cutoff before new startup-gated agent
// work can create sessions that the recovery scan must leave alone.
for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) {
params.unavailableGatewayMethods.delete(method);
}
if (!pluginServicesReported) {
reportPluginServices(result.pluginServices);
}
@@ -1343,19 +1361,21 @@ export async function startGatewayPostAttachRuntime(
const hookRunner = await runtimeDeps.getGlobalHookRunner();
if (hookRunner?.hasHooks("gateway_start")) {
const { withPluginHttpRouteRegistry } = await import("../plugins/http-registry.js");
void withPluginHttpRouteRegistry(sidecarsResult.pluginRegistry, () =>
hookRunner.runGatewayStart(
{ port: params.port },
{
port: params.port,
config: params.gatewayPluginConfigAtStart,
workspaceDir: params.defaultWorkspaceDir,
getCron: () =>
params.getCronService?.() ??
(params.deps.cron as PluginHookGatewayCronService | undefined),
},
),
).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
await withPluginHttpRouteRegistry(sidecarsResult.pluginRegistry, () =>
hookRunner.runGatewayStart(
{ port: params.port },
{
port: params.port,
config: params.gatewayPluginConfigAtStart,
workspaceDir: params.defaultWorkspaceDir,
getCron: () =>
params.getCronService?.() ??
(params.deps.cron as PluginHookGatewayCronService | undefined),
},
),
);
}).catch((err: unknown) => {
params.log.warn(`gateway_start hook failed: ${String(err)}`);
});
}
@@ -1398,6 +1418,7 @@ export const testing = {
cleanupStaleSessionLocks,
scheduleProviderAuthStatePrewarm,
schedulePrimaryModelPrewarm,
scheduleRestartSentinelWakeAfterReady,
shouldSkipStartupModelPrewarm,
stopPostReadySidecarsAfterCloseStarted,
};
@@ -4,6 +4,11 @@
import fs from "node:fs/promises";
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
import type { ChannelPlugin } from "../channels/plugins/types.js";
import {
getActiveGatewayRootWorkCount,
isGatewaySubordinateWorkAdmissionClosed,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import {
createChannelTestPluginBase,
createDirectOutboundTestAdapter,
@@ -215,6 +220,42 @@ describe("gateway server agent", () => {
testState.allowFrom = undefined;
});
test("keeps accepted detached agent work on its retained request root", async () => {
await setTestSessionStore({
entries: {
main: {
sessionId: "sess-agent-detached-root",
updatedAt: Date.now(),
},
},
});
let subordinateAdmissionClosed: boolean | undefined;
vi.mocked(agentCommand).mockImplementationOnce(async () => {
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension).not.toBeNull();
try {
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
} finally {
suspension?.rollback();
}
});
const res = await rpcReq(gatewaySuite.ws, "agent", {
message: "prove detached root transfer",
sessionKey: "main",
idempotencyKey: "idem-agent-detached-root",
});
expect(res.ok).toBe(true);
expect(res.payload?.status).toBe("accepted");
await vi.waitFor(() => {
expect(subordinateAdmissionClosed).toBe(false);
});
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
test("agent marks implicit delivery when lastTo is stale", async () => {
testState.allowFrom = ["+436769770569"];
await setTestSessionStore({
@@ -6,9 +6,18 @@ import path from "node:path";
import { beforeEach, describe, expect, test, vi } from "vitest";
import { WebSocket } from "ws";
import { emitAgentEvent, registerAgentRunContext } from "../infra/agent-events.js";
import {
getActiveGatewayRootWorkCount,
isGatewaySubordinateWorkAdmissionClosed,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import { extractFirstTextBlock } from "../shared/chat-message-content.js";
import { createDeferred } from "../test-utils/deferred.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js";
import * as sessionLifecycleState from "./session-lifecycle-state.js";
import {
connectOk,
dispatchInboundMessageMock,
@@ -201,6 +210,64 @@ describe("gateway server chat", () => {
);
});
test("keeps started chat dispatch on its retained request root", async () => {
await withMainSessionStore(async () => {
let subordinateAdmissionClosed: boolean | undefined;
dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => {
await new Promise<void>((resolve) => setTimeout(resolve, 0));
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension).not.toBeNull();
try {
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
} finally {
suspension?.rollback();
}
const [params] = args as [
{
dispatcher: {
sendFinalReply: (payload: { text: string }) => boolean;
markComplete: () => void;
waitForIdle: () => Promise<void>;
getQueuedCounts: () => { final: number; block: number; tool: number };
};
},
];
params.dispatcher.sendFinalReply({ text: "detached root stayed live" });
params.dispatcher.markComplete();
await params.dispatcher.waitForIdle();
return {
queuedFinal: true,
counts: params.dispatcher.getQueuedCounts(),
};
});
const finalPromise = onceMessage(
ws,
(message) =>
message.type === "event" &&
message.event === "chat" &&
message.payload?.state === "final" &&
message.payload?.runId === "idem-chat-detached-root",
8_000,
);
const res = await rpcReq(ws, "chat.send", {
sessionKey: "main",
message: "prove detached root transfer",
idempotencyKey: "idem-chat-detached-root",
});
expect(res.ok).toBe(true);
expect(res.payload?.status).toBe("started");
await vi.waitFor(() => {
expect(subordinateAdmissionClosed).toBe(false);
});
await finalPromise;
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
});
const waitForAgentRunOk = async (runId: string, timeoutMs = 1_000) => {
const res = await rpcReq(ws, "agent.wait", {
runId,
@@ -714,7 +781,7 @@ describe("gateway server chat", () => {
});
});
test("marks a running webchat session failed when dispatch rejects before a reply", async () => {
test("marks a running webchat session failed when restart drain overlaps dispatch rejection", async () => {
await withMainSessionStore(async (dir) => {
await writeSessionStore({
entries: {
@@ -729,34 +796,72 @@ describe("gateway server chat", () => {
});
const subscribeRes = await rpcReq(ws, "sessions.subscribe", {});
expect(subscribeRes.ok).toBe(true);
dispatchInboundMessageMock.mockRejectedValueOnce(new Error("provider rejected request"));
const errorPromise = onceMessage(
ws,
(o) =>
o.type === "event" &&
o.event === "chat" &&
o.payload?.state === "error" &&
o.payload?.runId === "idem-dispatch-error-1",
8_000,
);
const sessionChangedPromise = onceMessage(
ws,
(o) =>
o.type === "event" &&
o.event === "sessions.changed" &&
o.payload?.reason === "chat.dispatch-error" &&
o.payload?.sessionKey === "agent:main:main",
8_000,
);
const res = await rpcReq(ws, "chat.send", {
sessionKey: "main",
message: "run: pwd",
idempotencyKey: "idem-dispatch-error-1",
});
expect(res.ok).toBe(true);
await errorPromise;
const sessionChanged = await sessionChangedPromise;
const rejectDispatch = createDeferred<void>();
const releasePersistence = createDeferred<void>();
let dispatchStarted = false;
let persistenceEntered = false;
const persistLifecycleEvent = sessionLifecycleState.persistGatewaySessionLifecycleEvent;
const persistSpy = vi
.spyOn(sessionLifecycleState, "persistGatewaySessionLifecycleEvent")
.mockImplementation(async (params) => {
persistenceEntered = true;
await releasePersistence.promise;
await persistLifecycleEvent(params);
});
const sessionChanged = await (async () => {
try {
dispatchInboundMessageMock.mockImplementationOnce(async () => {
dispatchStarted = true;
await rejectDispatch.promise;
throw new Error("provider rejected request");
});
const errorPromise = onceMessage(
ws,
(o) =>
o.type === "event" &&
o.event === "chat" &&
o.payload?.state === "error" &&
o.payload?.runId === "idem-dispatch-error-1",
8_000,
);
const sessionChangedPromise = onceMessage(
ws,
(o) =>
o.type === "event" &&
o.event === "sessions.changed" &&
o.payload?.reason === "chat.dispatch-error" &&
o.payload?.sessionKey === "agent:main:main",
8_000,
);
const res = await rpcReq(ws, "chat.send", {
sessionKey: "main",
message: "run: pwd",
idempotencyKey: "idem-dispatch-error-1",
});
expect(res.ok).toBe(true);
await vi.waitFor(() => {
expect(dispatchStarted).toBe(true);
});
markGatewayRestartDraining();
rejectDispatch.resolve();
await errorPromise;
await vi.waitFor(() => {
expect(persistenceEntered).toBe(true);
});
expect(getActiveGatewayRootWorkCount()).toBe(1);
releasePersistence.resolve();
const changed = await sessionChangedPromise;
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
return changed;
} finally {
rejectDispatch.resolve();
releasePersistence.resolve();
persistSpy.mockRestore();
resetGatewayWorkAdmission();
}
})();
expectRecordFields(sessionChanged.payload, {
sessionId: "sess-main",
status: "failed",
@@ -10,6 +10,7 @@ import {
resetDiagnosticEventsForTest,
type DiagnosticEventPayload,
} from "../infra/diagnostic-events.js";
import { tryBeginGatewaySuspendAdmission } from "../process/gateway-work-admission.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import type { ResolvedGatewayAuth } from "./auth.js";
import { MAX_PREAUTH_PAYLOAD_BYTES } from "./server-constants.js";
@@ -144,6 +145,22 @@ describe("gateway pre-auth hardening", () => {
}
});
it("rejects core websocket upgrades while suspension admission is closed", async () => {
const harness = await createGatewaySuiteHarness();
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
try {
await expect(requestUpgradeRejection(harness.port)).resolves.toEqual({
status: 503,
body: "Gateway websocket admission closed",
});
} finally {
suspension?.release();
await harness.close();
}
});
it("closes idle unauthenticated sockets after the handshake timeout", async () => {
setEnvForTest("OPENCLAW_TEST_HANDSHAKE_TIMEOUT_MS", "200");
@@ -445,6 +445,9 @@ describe("gateway update.run", () => {
await vi.waitFor(() => {
expect(updateMock).toHaveBeenCalledOnce();
}, FAST_WAIT_OPTS);
await vi.waitFor(() => {
expect(sigusr1).toHaveBeenCalled();
}, FAST_WAIT_OPTS);
} finally {
process.off("SIGUSR1", sigusr1);
}
@@ -11,6 +11,7 @@ import {
setActivePluginRegistry,
} from "../../plugins/runtime.js";
import { getPluginRuntimeGatewayRequestScope } from "../../plugins/runtime/gateway-request-scope.js";
import { resolveControlPlaneRateLimitKey } from "../control-plane-rate-limit.js";
import { ExecApprovalManager } from "../exec-approval-manager.js";
import type { AuthorizedGatewayHttpRequest } from "../http-utils.js";
import { authorizeOperatorScopesForMethod, CLI_DEFAULT_OPERATOR_SCOPES } from "../method-scopes.js";
@@ -248,6 +249,60 @@ describe("plugin HTTP route runtime scopes", () => {
});
});
it("threads resolved HTTP client IP into the shared control-plane rate identity", async () => {
const observed: Array<{ clientIp?: string; connId?: string; rateLimitKey: string }> = [];
const handler = createPluginRequestHandler({
routes: [
createRoute({
path: SECURE_HOOK_PATH,
auth: "gateway",
gatewayRuntimeScopeSurface: "trusted-operator",
gatewayMethodDispatchAllowed: true,
handler: async () => {
const client = getPluginRuntimeGatewayRequestScope()?.client;
observed.push({
...(client?.clientIp ? { clientIp: client.clientIp } : {}),
...(client?.connId ? { connId: client.connId } : {}),
rateLimitKey: resolveControlPlaneRateLimitKey(client ?? null),
});
return true;
},
}),
],
});
for (const clientIp of ["203.0.113.10", "203.0.113.10", "203.0.113.11"]) {
const { handled, res } = await dispatchPluginRequest(handler, {
path: SECURE_HOOK_PATH,
authContext: {
gatewayAuthSatisfied: true,
gatewayRequestAuth: { authMethod: "token", trustDeclaredOperatorScopes: false },
gatewayRequestClientIp: clientIp,
},
});
expect(handled).toBe(true);
expect(res.statusCode).toBe(200);
}
expect(observed).toEqual([
{
clientIp: "203.0.113.10",
connId: "plugin-http:203.0.113.10",
rateLimitKey: "unknown-device|203.0.113.10",
},
{
clientIp: "203.0.113.10",
connId: "plugin-http:203.0.113.10",
rateLimitKey: "unknown-device|203.0.113.10",
},
{
clientIp: "203.0.113.11",
connId: "plugin-http:203.0.113.11",
rateLimitKey: "unknown-device|203.0.113.11",
},
]);
});
it("uses server-local routes and gateway context when the active registry belongs to another gateway", async () => {
const serverAContext = { label: "server-a" } as unknown as GatewayRequestContext;
const serverBContext = { label: "server-b" } as unknown as GatewayRequestContext;
+12 -1
View File
@@ -46,8 +46,11 @@ function resolvePluginRoutePathContextForRequest(
function createPluginRouteRuntimeClient(
scopes: readonly string[],
clientIp: string | undefined,
): GatewayRequestOptions["client"] {
return {
connId: `plugin-http:${clientIp ?? "unknown"}`,
...(clientIp ? { clientIp } : {}),
connect: {
minProtocol: PROTOCOL_VERSION,
maxProtocol: PROTOCOL_VERSION,
@@ -71,6 +74,7 @@ function writeUpgradeUnauthorized(socket: Duplex) {
type PluginRouteRuntimeDispatchContext = {
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
gatewayRequestOperatorScopes?: readonly string[];
gatewayRequestClientIp?: string;
};
function getMissingPluginRouteRuntimeContext(
@@ -92,6 +96,7 @@ function createPluginRouteRuntimeScope(params: {
gatewayRequestContext?: GatewayRequestContext;
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
gatewayRequestOperatorScopes?: readonly string[];
gatewayRequestClientIp?: string;
}): PluginRouteRuntimeScope {
const runtimeScopes =
params.route.auth !== "gateway"
@@ -103,7 +108,10 @@ function createPluginRouteRuntimeScope(params: {
"trusted-operator",
)
: params.gatewayRequestOperatorScopes!;
const runtimeClient = createPluginRouteRuntimeClient(runtimeScopes);
const runtimeClient = createPluginRouteRuntimeClient(
runtimeScopes,
params.gatewayRequestClientIp,
);
return {
...(params.gatewayRequestContext ? { context: params.gatewayRequestContext } : {}),
client: runtimeClient,
@@ -120,6 +128,7 @@ export type PluginRouteDispatchContext = {
gatewayAuthSatisfied?: boolean;
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
gatewayRequestOperatorScopes?: readonly string[];
gatewayRequestClientIp?: string;
};
export type PluginHttpRequestHandler = (
@@ -189,6 +198,7 @@ export function createGatewayPluginRequestHandler(params: {
gatewayRequestContext,
gatewayRequestAuth,
gatewayRequestOperatorScopes,
gatewayRequestClientIp: dispatchContext?.gatewayRequestClientIp,
}),
async () => route.handler(req, res),
);
@@ -265,6 +275,7 @@ export function createGatewayPluginUpgradeHandler(params: {
gatewayRequestContext,
gatewayRequestAuth,
gatewayRequestOperatorScopes,
gatewayRequestClientIp: dispatchContext?.gatewayRequestClientIp,
}),
async () => route.handleUpgrade?.(req, socket, head),
);
@@ -0,0 +1,229 @@
// WebSocket connect suspension tests cover root admission before handshake mutations.
import type { IncomingMessage } from "node:http";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { WebSocket } from "ws";
import { PROTOCOL_VERSION } from "../../../../packages/gateway-protocol/src/index.js";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../../../process/gateway-work-admission.js";
import type { GatewayRequestContext } from "../../server-methods/types.js";
const { incrementPresenceVersionMock, loadConfigMock, upsertPresenceMock } = vi.hoisted(() => ({
incrementPresenceVersionMock: vi.fn(() => 2),
loadConfigMock: vi.fn(() => ({ gateway: { auth: { mode: "none" } } })),
upsertPresenceMock: vi.fn(),
}));
vi.mock("../../../config/config.js", () => ({
getRuntimeConfig: loadConfigMock,
loadConfig: loadConfigMock,
}));
vi.mock("../../../config/io.js", () => ({
getRuntimeConfig: loadConfigMock,
}));
vi.mock("../../../infra/system-presence.js", () => ({
upsertPresence: upsertPresenceMock,
}));
vi.mock("../health-state.js", () => ({
buildGatewaySnapshot: vi.fn(() => ({
presence: [],
health: {},
stateVersion: { presence: 1, health: 1 },
uptimeMs: 1,
sessionDefaults: {
defaultAgentId: "main",
mainKey: "main",
mainSessionKey: "main",
scope: "per-sender",
},
})),
getHealthCache: vi.fn(() => null),
getHealthVersion: vi.fn(() => 1),
incrementPresenceVersion: incrementPresenceVersionMock,
}));
import { attachGatewayWsMessageHandler } from "./message-handler.js";
function createLogger() {
return {
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
};
}
function attachHarness(params: { deferSocketSend?: boolean } = {}) {
let onMessage: ((data: string) => void) | undefined;
let finishSocketSend: (() => void) | undefined;
let client: unknown = null;
const socketSend = vi.fn((_payload: string, callback?: (error?: Error) => void) => {
if (params.deferSocketSend) {
finishSocketSend = () => callback?.();
return;
}
callback?.();
});
const socket = {
_receiver: {},
send: socketSend,
on: vi.fn((event: string, handler: (data: string) => void) => {
if (event === "message") {
onMessage = handler;
}
return socket;
}),
} as unknown as WebSocket;
const close = vi.fn();
const setClient = vi.fn((next: unknown) => {
client = next;
return true;
});
attachGatewayWsMessageHandler({
socket,
upgradeReq: {
headers: { host: "127.0.0.1:19001" },
socket: { localAddress: "127.0.0.1", remoteAddress: "127.0.0.1" },
} as unknown as IncomingMessage,
connId: "suspension-connect",
remoteAddr: "127.0.0.1",
localAddr: "127.0.0.1",
requestHost: "127.0.0.1:19001",
connectNonce: "suspension-connect-nonce",
getResolvedAuth: () => ({ mode: "none", allowTailscale: false }),
gatewayMethods: [],
events: [],
extraHandlers: {},
buildRequestContext: () => ({}) as GatewayRequestContext,
refreshHealthSnapshot: vi.fn(async () => ({}) as never),
send: vi.fn(),
close,
isClosed: vi.fn(() => false),
clearHandshakeTimer: vi.fn(),
getClient: () => client as never,
setClient: setClient as never,
setHandshakeState: vi.fn(),
advanceHandshakePhase: vi.fn(),
setCloseCause: vi.fn(),
setLastFrameMeta: vi.fn(),
originCheckMetrics: { hostHeaderFallbackAccepted: 0 },
logGateway: createLogger() as never,
logHealth: createLogger() as never,
logWsControl: createLogger() as never,
});
if (!onMessage) {
throw new Error("expected websocket message handler");
}
return {
close,
finishSocketSend: () => finishSocketSend?.(),
get client() {
return client;
},
sendConnect: () =>
onMessage?.(
JSON.stringify({
type: "req",
id: "connect-1",
method: "connect",
params: {
minProtocol: PROTOCOL_VERSION,
maxProtocol: PROTOCOL_VERSION,
client: {
id: "gateway-client",
version: "dev",
platform: "test",
mode: "backend",
},
role: "operator",
scopes: [],
caps: [],
},
}),
),
setClient,
socketSend,
};
}
beforeEach(() => {
resetGatewayWorkAdmission();
vi.clearAllMocks();
});
afterEach(resetGatewayWorkAdmission);
describe("WebSocket connect suspension admission", () => {
it.each(["preparing", "prepared"] as const)(
"rejects a validated connect while suspension is %s before session mutations",
async (phase) => {
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension).not.toBeNull();
if (phase === "prepared") {
expect(suspension?.commit()).toBe(true);
}
const harness = attachHarness();
harness.sendConnect();
await vi.waitFor(() => {
expect(harness.socketSend).toHaveBeenCalledOnce();
});
const response = JSON.parse(harness.socketSend.mock.calls[0]?.[0] ?? "{}") as {
error?: {
code?: string;
retryable?: boolean;
retryAfterMs?: number;
details?: Record<string, unknown>;
};
};
expect(response.error).toMatchObject({
code: "UNAVAILABLE",
retryable: true,
retryAfterMs: 1_000,
details: {
method: "connect",
reason: "gateway-suspending",
phase,
},
});
expect(harness.client).toBeNull();
expect(harness.setClient).not.toHaveBeenCalled();
expect(upsertPresenceMock).not.toHaveBeenCalled();
expect(incrementPresenceVersionMock).not.toHaveBeenCalled();
await vi.waitFor(() => {
expect(harness.close).toHaveBeenCalledWith(1013, "gateway suspension in progress");
});
if (phase === "prepared") {
suspension?.release();
} else {
suspension?.rollback();
}
},
);
it("keeps an accepted handshake visible as root work until hello is sent", async () => {
const harness = attachHarness({ deferSocketSend: true });
harness.sendConnect();
await vi.waitFor(() => {
expect(harness.socketSend).toHaveBeenCalledOnce();
});
expect(getActiveGatewayRootWorkCount()).toBe(1);
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension).not.toBeNull();
expect(getActiveGatewayRootWorkCount()).toBe(1);
expect(suspension?.rollback()).toBe(true);
harness.finishSocketSend();
await vi.waitFor(() => {
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
});
});
@@ -92,6 +92,12 @@ import { loadVoiceWakeConfig } from "../../../infra/voicewake.js";
import { rawDataToString } from "../../../infra/ws.js";
import { logRejectedLargePayload } from "../../../logging/diagnostic-payload.js";
import type { createSubsystemLogger } from "../../../logging/subsystem.js";
import {
getGatewaySuspendAdmissionPhase,
isGatewayRestartDraining,
runWithGatewayIndependentRootWorkAdmission,
tryBeginGatewayRootWorkAdmission,
} from "../../../process/gateway-work-admission.js";
import {
BOOTSTRAP_HANDOFF_OPERATOR_SCOPES,
isPairingSetupBootstrapProfile,
@@ -188,6 +194,8 @@ import { isUnauthorizedRoleError, UnauthorizedFloodGuard } from "./unauthorized-
type SubsystemLogger = ReturnType<typeof createSubsystemLogger>;
const DEVICE_SIGNATURE_SKEW_MS = 2 * 60 * 1000;
const GATEWAY_WORK_ADMISSION_RETRY_AFTER_MS = 1_000;
const GATEWAY_WORK_ADMISSION_CLOSE_CODE = 1013;
const DEVICE_CREDENTIAL_INVALIDATING_METHODS = new Set([
"device.pair.remove",
"device.token.rotate",
@@ -662,6 +670,11 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
close(4001, `client invalidated: ${reason}`);
return true;
};
const runDetachedConnectWork = (run: () => Promise<void>, onError: (error: unknown) => void) => {
// Connect-triggered mutations outlive hello-ok. Give each tail its own
// root lease so suspension cannot report ready while one is still active.
void runWithGatewayIndependentRootWorkAdmission(run).catch(onError);
};
const handleMessage = async (data: RawData) => {
if (isClosed()) {
@@ -2179,10 +2192,16 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
nodeIdsForPairing.add(instanceIdLocal);
}
for (const nodeId of nodeIdsForPairing) {
void updatePairedNodeMetadata(nodeId, {
lastConnectedAtMs: nodeSession.connectedAtMs,
}).catch((err: unknown) =>
logGateway.warn(`failed to record last connect for ${nodeId}: ${formatForLog(err)}`),
runDetachedConnectWork(
async () => {
await updatePairedNodeMetadata(nodeId, {
lastConnectedAtMs: nodeSession.connectedAtMs,
});
},
(err) =>
logGateway.warn(
`failed to record last connect for ${nodeId}: ${formatForLog(err)}`,
),
);
}
recordRemoteNodeInfo({
@@ -2194,42 +2213,48 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
commands: nodeSession.commands,
remoteIp: nodeSession.remoteIp,
});
void refreshRemoteNodeBins({
nodeId: nodeSession.nodeId,
platform: nodeSession.platform,
deviceFamily: nodeSession.deviceFamily,
commands: nodeSession.commands,
cfg: getRuntimeConfig(),
// The node socket is registered before macOS app command handlers finish warming.
// Delay only the connect-time probe; later skill refreshes use the live session.
readinessDelayMs: 5_000,
}).catch((err: unknown) =>
logGateway.warn(
`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
),
runDetachedConnectWork(
async () => {
await refreshRemoteNodeBins({
nodeId: nodeSession.nodeId,
platform: nodeSession.platform,
deviceFamily: nodeSession.deviceFamily,
commands: nodeSession.commands,
cfg: getRuntimeConfig(),
// The node socket is registered before macOS app command handlers finish warming.
// Delay only the connect-time probe; later skill refreshes use the live session.
readinessDelayMs: 5_000,
});
},
(err) =>
logGateway.warn(
`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
),
);
void loadVoiceWakeConfig()
.then((cfg) => {
runDetachedConnectWork(
async () => {
const cfg = await loadVoiceWakeConfig();
context.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.changed", {
triggers: cfg.triggers,
});
})
.catch((err: unknown) =>
},
(err) =>
logGateway.warn(
`voicewake snapshot failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
),
);
void loadVoiceWakeRoutingConfig()
.then((routing) => {
);
runDetachedConnectWork(
async () => {
const routing = await loadVoiceWakeRoutingConfig();
context.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.routing.changed", {
config: routing,
});
})
.catch((err: unknown) =>
},
(err) =>
logGateway.warn(
`voicewake routing snapshot failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
),
);
);
}
const snapshot = buildGatewaySnapshot({
@@ -2510,8 +2535,80 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
}
};
const rejectConnectForClosedAdmission = async (data: RawData): Promise<boolean> => {
if (isClosed() || getRawDataByteLength(data) > MAX_PREAUTH_PAYLOAD_BYTES) {
return false;
}
let parsed: unknown;
try {
parsed = JSON.parse(rawDataToString(data));
} catch {
return false;
}
if (
!validateRequestFrame(parsed) ||
parsed.method !== "connect" ||
!validateConnectParams(parsed.params)
) {
return false;
}
const restartDraining = isGatewayRestartDraining();
const reason = restartDraining ? "gateway-restarting" : "gateway-suspending";
const operation = restartDraining ? "restart" : "suspension";
const phase = getGatewaySuspendAdmissionPhase();
setLastFrameMeta({ type: "req", method: "connect", id: parsed.id });
setHandshakeState("failed");
setCloseCause(reason, {
method: "connect",
phase,
});
await sendFrame({
type: "res",
id: parsed.id,
ok: false,
error: errorShape(ErrorCodes.UNAVAILABLE, `connect unavailable during gateway ${operation}`, {
retryable: true,
retryAfterMs: GATEWAY_WORK_ADMISSION_RETRY_AFTER_MS,
details: {
method: "connect",
reason,
phase,
},
}),
}).catch(() => {});
queueMicrotask(() =>
close(GATEWAY_WORK_ADMISSION_CLOSE_CODE, `gateway ${operation} in progress`),
);
return true;
};
const handleIncomingMessage = async (data: RawData) => {
if (getClient()) {
await handleMessage(data);
return;
}
const admission = tryBeginGatewayRootWorkAdmission();
if (!admission) {
if (await rejectConnectForClosedAdmission(data)) {
return;
}
// Malformed pre-auth frames still use the established validation and
// close path; only a validated connect can cross into mutable work.
await handleMessage(data);
return;
}
try {
await admission.run(() => handleMessage(data));
} finally {
admission.release();
}
};
socket.on("message", (data) => {
void runWithDiagnosticTraceContext(createDiagnosticTraceContext(), () => handleMessage(data));
void runWithDiagnosticTraceContext(createDiagnosticTraceContext(), () =>
handleIncomingMessage(data),
);
});
}
+1 -1
View File
@@ -1,6 +1,6 @@
// Gateway WebSocket client types describe authenticated client state retained by the server.
import type { WebSocket } from "ws";
import type { ConnectParams } from "../../../packages/gateway-protocol/src/index.js";
import type { ConnectParams } from "../../../packages/gateway-protocol/src/schema/frames.js";
import type { AgentRuntimeIdentity } from "../agent-runtime-identity-token.js";
import type { PluginNodeCapabilityClient } from "../plugin-node-capability.js";
+7 -2
View File
@@ -50,6 +50,7 @@ import { getSessionBindingService } from "../infra/outbound/session-binding-serv
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
import { runPluginHostCleanup } from "../plugins/host-hook-cleanup.js";
import { getActivePluginRegistry } from "../plugins/runtime.js";
import { runWithGatewayIndependentRootWorkContinuation } from "../process/gateway-work-admission.js";
import {
isSubagentSessionKey,
normalizeAgentId,
@@ -188,7 +189,9 @@ export function emitGatewaySessionEndPluginHook(params: {
nextSessionId: params.nextSessionId,
nextSessionKey: params.nextSessionKey,
});
void hookRunner.runSessionEnd(payload.event, payload.context).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionEnd(payload.event, payload.context);
}).catch((err: unknown) => {
logVerbose(`session_end hook failed: ${String(err)}`);
});
}
@@ -231,7 +234,9 @@ export function emitGatewaySessionStartPluginHook(params: {
cfg: params.cfg,
resumedFrom: params.resumedFrom,
});
void hookRunner.runSessionStart(payload.event, payload.context).catch((err: unknown) => {
void runWithGatewayIndependentRootWorkContinuation(async () => {
await hookRunner.runSessionStart(payload.event, payload.context);
}).catch((err: unknown) => {
logVerbose(`session_start hook failed: ${String(err)}`);
});
}
+18
View File
@@ -26,10 +26,13 @@ import {
getPairedDevice,
requestDevicePairing,
} from "../infra/device-pairing.js";
import { resetGatewaySuspendCoordinatorForTest } from "../infra/gateway-suspend-coordinator.js";
import { __testing as restartTesting } from "../infra/restart.js";
import { drainSystemEvents, peekSystemEvents } from "../infra/system-events.js";
import { rawDataToString } from "../infra/ws.js";
import { resetLogger, setLoggerOverride } from "../logging.js";
import { clearGatewaySubagentRuntime } from "../plugins/runtime/gateway-bindings.js";
import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js";
import {
DEFAULT_AGENT_ID,
normalizeMainKey,
@@ -250,9 +253,22 @@ function applyGatewaySkipEnv() {
: "openclaw-test-no-bundled-extensions";
}
function resetGatewayLifecycleTestState(options: { preserveRuntimeBindings: boolean }): void {
// Resume a held scheduler before hard admission reset invalidates and forgets its
// lease. Then cancel restart timers and retire their module-local signal lease.
resetGatewaySuspendCoordinatorForTest();
if (options.preserveRuntimeBindings) {
restartTesting.resetSigusr1TransientState();
} else {
restartTesting.resetSigusr1State();
}
resetGatewayWorkAdmission();
}
async function resetGatewayTestState(options: { uniqueConfigRoot: boolean }) {
// Some tests intentionally use fake timers; ensure they don't leak into gateway suites.
vi.useRealTimers();
resetGatewayLifecycleTestState({ preserveRuntimeBindings: false });
setLoggerOverride({ level: "silent", consoleLevel: "silent" });
if (!tempHome) {
throw new Error("resetGatewayTestState called before temp home was initialized");
@@ -369,6 +385,7 @@ async function resetGatewayTestState(options: { uniqueConfigRoot: boolean }) {
async function cleanupGatewayTestHome(options: { restoreEnv: boolean }) {
vi.useRealTimers();
resetGatewayLifecycleTestState({ preserveRuntimeBindings: activeSuiteGatewayServerCount > 0 });
clearGatewaySubagentRuntime();
resetLogger();
resetTaskRegistryForTests({ persist: false });
@@ -395,6 +412,7 @@ async function cleanupGatewayTestHome(options: { restoreEnv: boolean }) {
async function resetGatewayTestRuntimeOnly() {
vi.useRealTimers();
resetGatewayLifecycleTestState({ preserveRuntimeBindings: true });
setLoggerOverride({ level: "silent", consoleLevel: "silent" });
applyGatewaySkipEnv();
delete process.env.OPENCLAW_GATEWAY_TOKEN;
+1 -1
View File
@@ -5,7 +5,7 @@
* like command processing, session lifecycle, etc.
*/
import type { SessionsPatchParams } from "../../packages/gateway-protocol/src/schema.js";
import type { SessionsPatchParams } from "../../packages/gateway-protocol/src/schema/sessions.js";
import type { WorkspaceBootstrapFile } from "../agents/workspace.js";
import type { CliDeps } from "../cli/outbound-send-deps.js";
import type { SessionEntry } from "../config/sessions/types.js";
+1 -1
View File
@@ -1,6 +1,6 @@
// Runs the gateway-backed runtime that delivers native approval events.
import { readConnectErrorDetailCode } from "../../packages/gateway-protocol/src/connect-error-details.js";
import type { EventFrame } from "../../packages/gateway-protocol/src/index.js";
import type { EventFrame } from "../../packages/gateway-protocol/src/schema/frames.js";
import { startGatewayClientWhenEventLoopReady } from "../gateway/client-start-readiness.js";
import type { GatewayClient, GatewayReconnectPausedInfo } from "../gateway/client.js";
import { isApprovalMethod } from "../gateway/method-scopes.js";
+188
View File
@@ -0,0 +1,188 @@
// Collects process activity shared by restart and host-suspension decisions.
import { getActiveEmbeddedRunCount } from "../agents/embedded-agent-runner/run-state.js";
import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js";
import { getActiveCronJobCount } from "../cron/active-jobs.js";
import { getTotalQueueSize } from "../process/command-queue.js";
import { getActiveGatewayRootWorkCount } from "../process/gateway-work-admission.js";
import {
getActiveSessionLifecycleMutationCount,
getActiveSessionWorkAdmissionCount,
} from "../sessions/session-lifecycle-admission.js";
import { getSuspensionVisibleCronTaskRunCount } from "../tasks/cron-task-cancel.js";
import { getInspectableActiveTaskRestartBlockers } from "../tasks/task-registry.maintenance.js";
import {
type ActiveTaskRestartBlocker,
formatActiveTaskRestartBlocker,
} from "../tasks/task-restart-blocker.js";
export type GatewayActiveWorkCounts = {
queueSize: number;
pendingReplies: number;
embeddedRuns: number;
cronRuns: number;
activeTasks: number;
rootRequests: number;
sessionAdmissions: number;
sessionMutations: number;
chatRuns: number;
queuedTurns: number;
terminalPersistence: number;
terminalSessions: number;
/** Compatibility aggregate. Categories can overlap; use individual counts for diagnostics. */
totalActive: number;
};
export type GatewayActiveWorkBlocker = {
kind:
| "queue"
| "reply"
| "embedded-run"
| "cron-run"
| "task"
| "root-request"
| "session-admission"
| "session-mutation"
| "chat-run"
| "queued-turn"
| "terminal-persistence"
| "terminal-session";
count: number;
message: string;
task?: ActiveTaskRestartBlocker;
};
export type GatewayActiveWorkSnapshot = {
idle: boolean;
counts: GatewayActiveWorkCounts;
blockers: GatewayActiveWorkBlocker[];
};
export type GatewayActiveWorkInspectors = {
getQueueSize: () => number;
getPendingReplies: () => number;
getEmbeddedRuns: () => number;
getCronRuns: () => number;
getActiveTasks: () => number;
getTaskBlockers: () => ActiveTaskRestartBlocker[];
getRootRequests: () => number;
getSessionAdmissions: () => number;
getSessionMutations: () => number;
getChatRuns: () => number;
getQueuedTurns: () => number;
getTerminalPersistence: () => number;
getTerminalSessions: () => number;
};
const defaultInspectors: GatewayActiveWorkInspectors = {
getQueueSize: getTotalQueueSize,
getPendingReplies: getTotalPendingReplies,
getEmbeddedRuns: getActiveEmbeddedRunCount,
getCronRuns: () => Math.max(getActiveCronJobCount(), getSuspensionVisibleCronTaskRunCount()),
getActiveTasks: () => getInspectableActiveTaskRestartBlockers().length,
getTaskBlockers: getInspectableActiveTaskRestartBlockers,
getRootRequests: () => getActiveGatewayRootWorkCount({ excludeCurrent: true }),
getSessionAdmissions: getActiveSessionWorkAdmissionCount,
getSessionMutations: getActiveSessionLifecycleMutationCount,
getChatRuns: () => 0,
getQueuedTurns: () => 0,
getTerminalPersistence: () => 0,
getTerminalSessions: () => 0,
};
function normalizeCount(value: number): number {
return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
}
export function createGatewayActiveWorkSnapshot(
inspectors: Partial<GatewayActiveWorkInspectors> = {},
): GatewayActiveWorkSnapshot {
const resolved = { ...defaultInspectors, ...inspectors };
const counts: GatewayActiveWorkCounts = {
queueSize: normalizeCount(resolved.getQueueSize()),
pendingReplies: normalizeCount(resolved.getPendingReplies()),
embeddedRuns: normalizeCount(resolved.getEmbeddedRuns()),
cronRuns: normalizeCount(resolved.getCronRuns()),
activeTasks: normalizeCount(resolved.getActiveTasks()),
rootRequests: normalizeCount(resolved.getRootRequests()),
sessionAdmissions: normalizeCount(resolved.getSessionAdmissions()),
sessionMutations: normalizeCount(resolved.getSessionMutations()),
chatRuns: normalizeCount(resolved.getChatRuns()),
queuedTurns: normalizeCount(resolved.getQueuedTurns()),
terminalPersistence: normalizeCount(resolved.getTerminalPersistence()),
terminalSessions: normalizeCount(resolved.getTerminalSessions()),
totalActive: 0,
};
counts.totalActive = Object.entries(counts).reduce(
(total, [key, count]) => (key === "totalActive" ? total : total + count),
0,
);
const blockers: GatewayActiveWorkBlocker[] = [];
const add = (count: number, kind: GatewayActiveWorkBlocker["kind"], message: string) => {
if (count > 0) {
blockers.push({ kind, count, message });
}
};
add(counts.queueSize, "queue", `${counts.queueSize} queued or active operation(s)`);
add(
counts.pendingReplies,
"reply",
`${counts.pendingReplies} pending reply delivery operation(s)`,
);
add(counts.embeddedRuns, "embedded-run", `${counts.embeddedRuns} active embedded run(s)`);
add(counts.cronRuns, "cron-run", `${counts.cronRuns} active cron run(s)`);
add(counts.rootRequests, "root-request", `${counts.rootRequests} active gateway request(s)`);
add(
counts.sessionAdmissions,
"session-admission",
`${counts.sessionAdmissions} admitted session turn(s)`,
);
add(
counts.sessionMutations,
"session-mutation",
`${counts.sessionMutations} active session lifecycle mutation(s)`,
);
add(counts.chatRuns, "chat-run", `${counts.chatRuns} active chat run(s)`);
add(counts.queuedTurns, "queued-turn", `${counts.queuedTurns} queued chat turn(s)`);
add(
counts.terminalPersistence,
"terminal-persistence",
`${counts.terminalPersistence} pending terminal session write(s)`,
);
add(
counts.terminalSessions,
"terminal-session",
`${counts.terminalSessions} open terminal session(s)`,
);
if (counts.activeTasks > 0) {
const taskBlockers = resolved.getTaskBlockers();
if (taskBlockers.length === 0) {
blockers.push({
kind: "task",
count: counts.activeTasks,
message: `${counts.activeTasks} active background task run(s)`,
});
} else {
const shownTaskBlockers = taskBlockers.slice(0, 8);
for (const task of shownTaskBlockers) {
blockers.push({
kind: "task",
count: 1,
message: formatActiveTaskRestartBlocker(task),
task,
});
}
const omitted = counts.activeTasks - shownTaskBlockers.length;
if (omitted > 0) {
blockers.push({
kind: "task",
count: omitted,
message: `${omitted} additional active background task run(s)`,
});
}
}
}
return { idle: counts.totalActive === 0, counts, blockers };
}
@@ -0,0 +1,404 @@
// Covers atomic refuse-only suspension preparation, renewal, and release.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
isGatewayWorkAdmissionClosed,
markGatewayRestartDraining,
resetGatewayWorkAdmission,
} from "../process/gateway-work-admission.js";
import type { GatewayActiveWorkInspectors } from "./gateway-active-work.js";
import {
GATEWAY_SUSPEND_TTL_MS,
getGatewaySuspendStatus,
prepareGatewaySuspend,
resetGatewaySuspendCoordinatorForTest,
resumeGatewaySuspend,
} from "./gateway-suspend-coordinator.js";
function inspectors(
overrides: Partial<GatewayActiveWorkInspectors> = {},
): GatewayActiveWorkInspectors {
return {
getQueueSize: () => 0,
getPendingReplies: () => 0,
getEmbeddedRuns: () => 0,
getCronRuns: () => 0,
getActiveTasks: () => 0,
getTaskBlockers: () => [],
getRootRequests: () => 0,
getSessionAdmissions: () => 0,
getSessionMutations: () => 0,
getChatRuns: () => 0,
getQueuedTurns: () => 0,
getTerminalPersistence: () => 0,
getTerminalSessions: () => 0,
...overrides,
};
}
beforeEach(() => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
});
afterEach(() => {
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
});
describe("gateway suspend coordinator", () => {
it("test reset resumes a held scheduler before admission is cleared", () => {
const resumeScheduling = vi.fn(() => {
expect(isGatewayWorkAdmissionClosed()).toBe(true);
});
expect(
prepareGatewaySuspend({
requestId: "request-test-reset",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
}),
).toMatchObject({ status: "ready" });
resetGatewaySuspendCoordinatorForTest();
resetGatewayWorkAdmission();
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
it("reopens admission in the same turn when active work refuses preparation", () => {
const events: string[] = [];
const result = prepareGatewaySuspend({
requestId: "request-busy",
pauseScheduling: () => events.push("pause"),
resumeScheduling: () => events.push("resume"),
inspect: inspectors({
getQueueSize: () => {
events.push("inspect");
return 1;
},
}),
});
expect(result.status).toBe("busy");
expect(events).toEqual(["pause", "inspect", "resume"]);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
it("keeps admission closed until a failed busy rollback resumes scheduling", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi
.fn()
.mockImplementationOnce(() => {
throw new Error("timer unavailable");
})
.mockImplementationOnce(() => {});
const first = prepareGatewaySuspend({
requestId: "request-busy-resume-retry",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors({ getQueueSize: () => 1 }),
});
expect(first).toEqual({
status: "recovering",
reason: "scheduler-resume-failed",
retryAfterMs: 1_000,
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(getGatewaySuspendStatus("stale-id")).toEqual(first);
expect(resumeGatewaySuspend("stale-id")).toEqual({
ok: false,
reason: "scheduler-resume-failed",
retryAfterMs: 1_000,
});
expect(
prepareGatewaySuspend({
requestId: "request-before-scheduler-resume",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
}),
).toEqual(first);
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledTimes(2);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
expect(
prepareGatewaySuspend({
requestId: "request-after-scheduler-resume",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
createSuspensionId: () => "suspension-after-scheduler-resume",
}),
).toMatchObject({
status: "ready",
suspensionId: "suspension-after-scheduler-resume",
});
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledTimes(2);
} finally {
vi.useRealTimers();
}
});
it("cancels scheduler recovery when restart supersedes suspension", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi.fn(() => {
throw new Error("timer unavailable");
});
expect(
prepareGatewaySuspend({
requestId: "request-recovery-restart",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors({ getQueueSize: () => 1 }),
}),
).toMatchObject({ status: "recovering" });
markGatewayRestartDraining();
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
} finally {
vi.useRealTimers();
}
});
it("owns recovery when inspection fails before admission commits", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi
.fn()
.mockImplementationOnce(() => {
throw new Error("timer unavailable");
})
.mockImplementationOnce(() => {});
const result = prepareGatewaySuspend({
requestId: "request-inspection-failure",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors({
getQueueSize: () => {
throw new Error("inspection failed");
},
}),
});
expect(result).toMatchObject({ status: "recovering" });
expect(isGatewayWorkAdmissionClosed()).toBe(true);
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledTimes(2);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("renews one ready lease and resumes only with the matching id", () => {
const resumeScheduling = vi.fn();
const first = prepareGatewaySuspend({
requestId: "request-ready",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
nowMs: () => 1_000,
createSuspensionId: () => "suspension-1",
});
expect(first).toMatchObject({
status: "ready",
suspensionId: "suspension-1",
expiresAtMs: 1_000 + GATEWAY_SUSPEND_TTL_MS,
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(getGatewaySuspendStatus("suspension-1")).toEqual({
status: "ready",
expiresAtMs: 1_000 + GATEWAY_SUSPEND_TTL_MS,
});
const renewed = prepareGatewaySuspend({
requestId: "request-ready",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors({ getQueueSize: () => 99 }),
nowMs: () => 2_000,
});
expect(renewed).toMatchObject({
status: "ready",
suspensionId: "suspension-1",
expiresAtMs: 2_000 + GATEWAY_SUSPEND_TTL_MS,
});
expect(
prepareGatewaySuspend({
requestId: "request-other",
pauseScheduling: vi.fn(),
resumeScheduling,
}).status,
).toBe("conflict");
expect(resumeGatewaySuspend("wrong-id")).toEqual({
ok: false,
reason: "suspension-mismatch",
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(resumeGatewaySuspend("suspension-1")).toEqual({
ok: true,
status: "running",
resumed: true,
});
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
expect(resumeGatewaySuspend("suspension-1")).toEqual({
ok: true,
status: "running",
resumed: false,
});
});
it("lets restart supersede a suspension without reopening its scheduler", () => {
const resumeScheduling = vi.fn();
const result = prepareGatewaySuspend({
requestId: "request-restart",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
createSuspensionId: () => "suspension-restart",
});
expect(result.status).toBe("ready");
markGatewayRestartDraining();
expect(getGatewaySuspendStatus("suspension-restart")).toEqual({ status: "running" });
expect(resumeScheduling).not.toHaveBeenCalled();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
});
it("exposes scheduler recovery after a ready lease cannot resume", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi
.fn()
.mockImplementationOnce(() => {
throw new Error("timer unavailable");
})
.mockImplementationOnce(() => {});
prepareGatewaySuspend({
requestId: "request-resume-retry",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
createSuspensionId: () => "suspension-resume-retry",
});
expect(resumeGatewaySuspend("suspension-resume-retry")).toMatchObject({
ok: false,
reason: "scheduler-resume-failed",
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(getGatewaySuspendStatus("suspension-resume-retry")).toMatchObject({
status: "recovering",
});
expect(
prepareGatewaySuspend({
requestId: "request-resume-retry",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
}),
).toMatchObject({ status: "recovering" });
expect(resumeGatewaySuspend("suspension-resume-retry")).toMatchObject({
ok: false,
reason: "scheduler-resume-failed",
});
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledTimes(2);
expect(getGatewaySuspendStatus("suspension-resume-retry")).toEqual({ status: "running" });
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("auto-resumes an abandoned ready lease at expiry", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi.fn();
prepareGatewaySuspend({
requestId: "request-expiry",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
createSuspensionId: () => "suspension-expiry",
});
vi.advanceTimersByTime(GATEWAY_SUSPEND_TTL_MS);
expect(getGatewaySuspendStatus("suspension-expiry")).toEqual({ status: "running" });
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("enters recovery when lease expiry cannot resume the scheduler", () => {
vi.useFakeTimers();
try {
const resumeScheduling = vi
.fn()
.mockImplementationOnce(() => {
throw new Error("timer unavailable");
})
.mockImplementationOnce(() => {});
prepareGatewaySuspend({
requestId: "request-expiry-recovery",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
createSuspensionId: () => "suspension-expiry-recovery",
});
vi.advanceTimersByTime(GATEWAY_SUSPEND_TTL_MS);
expect(getGatewaySuspendStatus("suspension-expiry-recovery")).toMatchObject({
status: "recovering",
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
vi.advanceTimersByTime(1_000);
expect(resumeScheduling).toHaveBeenCalledTimes(2);
expect(getGatewaySuspendStatus("suspension-expiry-recovery")).toEqual({
status: "running",
});
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("expires synchronously when timer delivery is delayed", () => {
let nowMs = 10_000;
const resumeScheduling = vi.fn();
prepareGatewaySuspend({
requestId: "request-delayed-expiry",
pauseScheduling: vi.fn(),
resumeScheduling,
inspect: inspectors(),
nowMs: () => nowMs,
createSuspensionId: () => "suspension-delayed-expiry",
});
nowMs += GATEWAY_SUSPEND_TTL_MS;
expect(getGatewaySuspendStatus("suspension-delayed-expiry")).toEqual({ status: "running" });
expect(resumeScheduling).toHaveBeenCalledOnce();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
});
+415
View File
@@ -0,0 +1,415 @@
// Coordinates an atomic, refuse-only host suspension preparation lease.
import { randomUUID } from "node:crypto";
import type {
GatewaySuspendPrepareResult as GatewaySuspendPrepareWireResult,
GatewaySuspendResumeResult as GatewaySuspendResumeWireResult,
GatewaySuspendStatusResult as GatewaySuspendStatusWireResult,
} from "../../packages/gateway-protocol/src/index.js";
import { tryBeginGatewaySuspendAdmission } from "../process/gateway-work-admission.js";
import { resolveGlobalSingleton } from "../shared/global-singleton.js";
import {
createGatewayActiveWorkSnapshot,
type GatewayActiveWorkInspectors,
type GatewayActiveWorkSnapshot,
} from "./gateway-active-work.js";
export const GATEWAY_SUSPEND_TTL_MS = 2 * 60_000;
export const GATEWAY_SUSPEND_RETRY_AFTER_MS = 20_000;
const GATEWAY_SCHEDULER_RECOVERY_RETRY_MS = 1_000;
type GatewaySchedulerRecoveryResult = {
status: "recovering";
reason: "scheduler-resume-failed";
retryAfterMs: number;
};
export type GatewaySuspendPrepareResult =
| GatewaySuspendPrepareWireResult
| { status: "conflict"; expiresAtMs: number }
| GatewaySchedulerRecoveryResult;
export type GatewaySuspendStatusResult =
| GatewaySuspendStatusWireResult
| { status: "conflict"; expiresAtMs: number }
| GatewaySchedulerRecoveryResult;
export type GatewaySuspendResumeResult =
| GatewaySuspendResumeWireResult
| { ok: false; reason: "suspension-mismatch" }
| { ok: false; reason: "scheduler-resume-failed"; retryAfterMs: number };
type GatewaySuspendCoordinatorEntryBase = {
owner: object;
resumeScheduling: () => void;
reopenAdmission: () => boolean;
warn?: (message: string) => void;
timer?: ReturnType<typeof setTimeout>;
};
type HeldGatewaySuspension = GatewaySuspendCoordinatorEntryBase & {
kind: "held";
requestId: string;
suspensionId: string;
expiresAtMs: number;
snapshot: GatewayActiveWorkSnapshot;
nowMs: () => number;
};
type GatewaySchedulerRecovery = GatewaySuspendCoordinatorEntryBase & {
kind: "recovering";
};
type GatewaySuspendCoordinatorEntry = HeldGatewaySuspension | GatewaySchedulerRecovery;
type GatewaySuspendCoordinatorState = {
current: GatewaySuspendCoordinatorEntry | null;
};
const COORDINATOR_STATE = resolveGlobalSingleton(
Symbol.for("openclaw.gatewaySuspendCoordinatorState"),
(): GatewaySuspendCoordinatorState => ({
current: null,
}),
);
function schedulerRecoveryResult(): GatewaySchedulerRecoveryResult {
return {
status: "recovering",
reason: "scheduler-resume-failed",
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
};
}
function clearEntryTimer(entry: GatewaySuspendCoordinatorEntry): void {
if (entry.timer) {
clearTimeout(entry.timer);
entry.timer = undefined;
}
}
function scheduleEntry(
entry: GatewaySuspendCoordinatorEntry,
delayMs: number,
callback: () => void,
): void {
clearEntryTimer(entry);
entry.timer = setTimeout(callback, delayMs);
entry.timer.unref?.();
}
function resumeAndReopen(entry: GatewaySuspendCoordinatorEntry): boolean {
try {
entry.resumeScheduling();
} catch (err) {
entry.warn?.(`gateway scheduler recovery failed: ${String(err)}`);
enterSchedulerRecovery(entry);
return false;
}
if (COORDINATOR_STATE.current !== entry) {
return true;
}
if (!entry.reopenAdmission()) {
entry.warn?.("gateway scheduler recovery could not reopen admission");
enterSchedulerRecovery(entry);
return false;
}
clearEntryTimer(entry);
COORDINATOR_STATE.current = null;
return true;
}
function enterSchedulerRecovery(entry: GatewaySuspendCoordinatorEntry): void {
if (COORDINATOR_STATE.current !== entry) {
return;
}
if (entry.kind === "recovering") {
scheduleRecoveryRetry(entry);
return;
}
clearEntryTimer(entry);
const recovery: GatewaySchedulerRecovery = {
kind: "recovering",
owner: entry.owner,
resumeScheduling: entry.resumeScheduling,
reopenAdmission: entry.reopenAdmission,
warn: entry.warn,
};
COORDINATOR_STATE.current = recovery;
scheduleRecoveryRetry(recovery);
}
function scheduleRecoveryRetry(entry: GatewaySuspendCoordinatorEntry): void {
scheduleEntry(entry, GATEWAY_SCHEDULER_RECOVERY_RETRY_MS, () => {
if (COORDINATOR_STATE.current === entry) {
resumeAndReopen(entry);
}
});
}
function normalizeExpiredHeldSuspension(
held: HeldGatewaySuspension,
): GatewaySuspendCoordinatorEntry | null {
if (held.nowMs() < held.expiresAtMs) {
return held;
}
resumeAndReopen(held);
return COORDINATOR_STATE.current;
}
function armSchedulerRecovery(
recovery: Omit<GatewaySchedulerRecovery, "kind">,
): GatewaySchedulerRecovery {
const entry: GatewaySchedulerRecovery = { kind: "recovering", ...recovery };
scheduleRecoveryRetry(entry);
return entry;
}
// Rollback stays fail-closed: scheduler recovery must finish before admission
// reopens, otherwise an old retry can resume scheduling under a newer lease.
function resumeSchedulingBeforeReopen(params: {
owner: object;
resumeScheduling: () => void;
reopenAdmission: () => boolean;
isInvalidated: () => boolean;
warn?: (message: string) => void;
}): boolean {
if (params.isInvalidated()) {
return true;
}
try {
params.resumeScheduling();
} catch (err) {
params.warn?.(`gateway scheduler resume failed during suspension rollback: ${String(err)}`);
COORDINATOR_STATE.current = armSchedulerRecovery({
owner: params.owner,
resumeScheduling: params.resumeScheduling,
reopenAdmission: params.reopenAdmission,
warn: params.warn,
});
return false;
}
if (!params.isInvalidated()) {
params.reopenAdmission();
}
return true;
}
function armExpiry(held: Omit<HeldGatewaySuspension, "kind">): HeldGatewaySuspension {
const entry: HeldGatewaySuspension = { kind: "held", ...held };
scheduleEntry(entry, GATEWAY_SUSPEND_TTL_MS, () => {
if (COORDINATOR_STATE.current === entry) {
resumeAndReopen(entry);
}
});
return entry;
}
function renewHeldSuspension(held: HeldGatewaySuspension, nowMs: number): void {
held.expiresAtMs = nowMs + GATEWAY_SUSPEND_TTL_MS;
scheduleEntry(held, GATEWAY_SUSPEND_TTL_MS, () => {
if (COORDINATOR_STATE.current === held) {
resumeAndReopen(held);
}
});
}
/** Acquire, inspect, and either roll back immediately or hold an idle fence. */
export function prepareGatewaySuspend(params: {
requestId: string;
pauseScheduling: () => void;
resumeScheduling: () => void;
inspect?: Partial<GatewayActiveWorkInspectors>;
nowMs?: () => number;
createSuspensionId?: () => string;
warn?: (message: string) => void;
}): GatewaySuspendPrepareResult {
const nowMs = (params.nowMs ?? Date.now)();
const current = COORDINATOR_STATE.current;
if (current?.kind === "recovering") {
return schedulerRecoveryResult();
}
const existing = current ? normalizeExpiredHeldSuspension(current) : null;
if (existing?.kind === "recovering") {
return schedulerRecoveryResult();
}
if (existing) {
if (existing.requestId !== params.requestId) {
return { status: "conflict", expiresAtMs: existing.expiresAtMs };
}
existing.nowMs = params.nowMs ?? Date.now;
renewHeldSuspension(existing, nowMs);
return {
status: "ready",
suspensionId: existing.suspensionId,
expiresAtMs: existing.expiresAtMs,
activeCount: existing.snapshot.counts.totalActive,
blockers: existing.snapshot.blockers,
};
}
const owner = {};
let suspensionInvalidated = false;
const admission = tryBeginGatewaySuspendAdmission(() => {
suspensionInvalidated = true;
const activeEntry = COORDINATOR_STATE.current;
if (activeEntry?.owner !== owner) {
return;
}
clearEntryTimer(activeEntry);
COORDINATOR_STATE.current = null;
});
if (!admission) {
const snapshot = createGatewayActiveWorkSnapshot(params.inspect);
return {
status: "busy",
reason: "gateway-draining",
retryAfterMs: GATEWAY_SUSPEND_RETRY_AFTER_MS,
activeCount: snapshot.counts.totalActive,
blockers: snapshot.blockers,
};
}
let schedulingPaused = false;
let admissionCommitted = false;
try {
params.pauseScheduling();
schedulingPaused = true;
const snapshot = createGatewayActiveWorkSnapshot(params.inspect);
if (!snapshot.idle) {
const resumed = resumeSchedulingBeforeReopen({
owner,
resumeScheduling: params.resumeScheduling,
reopenAdmission: admission.rollback,
isInvalidated: () => suspensionInvalidated,
warn: params.warn,
});
schedulingPaused = false;
if (!resumed) {
return schedulerRecoveryResult();
}
return {
status: "busy",
reason: "active-work",
retryAfterMs: GATEWAY_SUSPEND_RETRY_AFTER_MS,
activeCount: snapshot.counts.totalActive,
blockers: snapshot.blockers,
};
}
if (!admission.commit()) {
throw new Error("gateway suspension admission changed during preparation");
}
admissionCommitted = true;
const suspensionId = (params.createSuspensionId ?? randomUUID)();
const expiresAtMs = nowMs + GATEWAY_SUSPEND_TTL_MS;
const held = armExpiry({
owner,
requestId: params.requestId,
suspensionId,
expiresAtMs,
snapshot,
reopenAdmission: admission.release,
resumeScheduling: params.resumeScheduling,
nowMs: params.nowMs ?? Date.now,
warn: params.warn,
});
COORDINATOR_STATE.current = held;
return {
status: "ready",
suspensionId,
expiresAtMs,
activeCount: snapshot.counts.totalActive,
blockers: snapshot.blockers,
};
} catch (err) {
if (schedulingPaused) {
const resumed = resumeSchedulingBeforeReopen({
owner,
resumeScheduling: params.resumeScheduling,
reopenAdmission: admissionCommitted ? admission.release : admission.rollback,
isInvalidated: () => suspensionInvalidated,
warn: params.warn,
});
if (!resumed) {
return schedulerRecoveryResult();
}
} else if (admissionCommitted) {
admission.release();
} else {
admission.rollback();
}
throw err;
}
}
export function getGatewaySuspendStatus(suspensionId: string): GatewaySuspendStatusResult {
const current = COORDINATOR_STATE.current;
if (current?.kind === "recovering") {
return schedulerRecoveryResult();
}
const held = current ? normalizeExpiredHeldSuspension(current) : null;
if (held?.kind === "recovering") {
return schedulerRecoveryResult();
}
if (!held) {
return { status: "running" };
}
if (held.suspensionId !== suspensionId) {
return { status: "conflict", expiresAtMs: held.expiresAtMs };
}
return { status: "ready", expiresAtMs: held.expiresAtMs };
}
export function resumeGatewaySuspend(suspensionId: string): GatewaySuspendResumeResult {
const current = COORDINATOR_STATE.current;
if (current?.kind === "recovering") {
return {
ok: false,
reason: "scheduler-resume-failed",
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
};
}
const held = current ? normalizeExpiredHeldSuspension(current) : null;
if (held?.kind === "recovering") {
return {
ok: false,
reason: "scheduler-resume-failed",
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
};
}
if (!held) {
return {
ok: true,
status: "running",
resumed: false,
};
}
if (held.suspensionId !== suspensionId) {
return { ok: false, reason: "suspension-mismatch" };
}
if (!resumeAndReopen(held)) {
return {
ok: false,
reason: "scheduler-resume-failed",
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
};
}
return {
ok: true,
status: "running",
resumed: true,
};
}
export function resetGatewaySuspendCoordinatorForTest(): void {
const current = COORDINATOR_STATE.current;
if (current) {
clearEntryTimer(current);
try {
current.resumeScheduling();
} catch (err) {
current.warn?.(`gateway scheduler resume failed during test reset: ${String(err)}`);
}
current.reopenAdmission();
COORDINATOR_STATE.current = null;
}
}
+59
View File
@@ -1,5 +1,10 @@
// Exercises heartbeat wake coalescing, retries, and skip handling.
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
getActiveGatewayRootWorkCount,
resetGatewayWorkAdmission,
tryBeginGatewaySuspendAdmission,
} from "../process/gateway-work-admission.js";
import {
HEARTBEAT_SKIP_CRON_IN_PROGRESS,
HEARTBEAT_SKIP_LANES_BUSY,
@@ -71,15 +76,69 @@ describe("heartbeat-wake", () => {
}
beforeEach(() => {
resetGatewayWorkAdmission();
resetHeartbeatWakeStateForTests();
});
afterEach(() => {
resetHeartbeatWakeStateForTests();
resetGatewayWorkAdmission();
vi.useRealTimers();
vi.restoreAllMocks();
});
it("defers a full wake while gateway suspension is prepared", async () => {
vi.useFakeTimers();
const activeRootCounts: number[] = [];
const handler = vi.fn(async () => {
activeRootCounts.push(getActiveGatewayRootWorkCount());
return { status: "ran" as const, durationMs: 1 };
});
setHeartbeatWakeHandler(handler);
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handler).not.toHaveBeenCalled();
expect(getActiveGatewayRootWorkCount()).toBe(0);
expect(suspension?.release()).toBe(true);
await vi.advanceTimersByTimeAsync(0);
expect(handler).toHaveBeenCalledOnce();
expect(activeRootCounts).toEqual([1]);
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("counts an in-flight wake until the whole handler settles", async () => {
vi.useFakeTimers();
let finishWake: (() => void) | undefined;
const wakeFinished = new Promise<void>((resolve) => {
finishWake = resolve;
});
const handler = vi.fn(async () => {
await wakeFinished;
return { status: "ran" as const, durationMs: 1 };
});
setHeartbeatWakeHandler(handler);
requestHeartbeat(wake("manual", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledOnce();
expect(getActiveGatewayRootWorkCount()).toBe(1);
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension).not.toBeNull();
expect(getActiveGatewayRootWorkCount()).toBe(1);
expect(suspension?.rollback()).toBe(true);
finishWake?.();
await vi.advanceTimersByTimeAsync(0);
expect(getActiveGatewayRootWorkCount()).toBe(0);
});
it("coalesces multiple wake requests into one run", async () => {
vi.useFakeTimers();
const handler = vi.fn().mockResolvedValue({ status: "skipped", reason: "disabled" });
+6 -1
View File
@@ -1,5 +1,6 @@
// Tracks heartbeat wake requests, busy skips, and retry timing.
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
import { resolveTimerTimeoutMs } from "../shared/number-coercion.js";
import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js";
@@ -235,7 +236,11 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") {
...(pendingWake.sessionKey ? { sessionKey: pendingWake.sessionKey } : {}),
...(pendingWake.heartbeat ? { heartbeat: pendingWake.heartbeat } : {}),
};
const res = await active(wakeOpts);
// Each wake is detached process work: admit the whole handler before
// it can mutate sessions or commitments, and keep it visible until done.
const res = await runWithGatewayIndependentRootWorkAdmission(async () =>
active(wakeOpts),
);
if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) {
// The target runtime is busy; retry this wake target soon.
queuePendingWakeReason({
+25
View File
@@ -6,6 +6,10 @@ import {
clearRuntimeConfigSnapshot,
setRuntimeConfigSnapshot,
} from "../config/config.js";
import {
isGatewayWorkAdmissionClosed,
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
import { makeNetworkInterfacesSnapshot } from "../test-helpers/network-interfaces.js";
import {
testing,
@@ -136,6 +140,26 @@ describe("infra runtime", () => {
await vi.runAllTimersAsync();
});
it("holds root admission from scheduled emission until the signal is handled", async () => {
const handler = () => {};
process.on("SIGUSR1", handler);
try {
scheduleGatewaySigusr1Restart({ delayMs: 0 });
await vi.advanceTimersByTimeAsync(0);
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
markGatewaySigusr1RestartHandled();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
root?.release();
} finally {
process.removeListener("SIGUSR1", handler);
}
});
it("backs off before an emoji that crosses the restart reason limit", () => {
const restart = scheduleGatewaySigusr1Restart({
delayMs: 0,
@@ -632,6 +656,7 @@ describe("infra runtime", () => {
expect(beforeEmit).toHaveBeenCalledTimes(1);
expect(afterEmitRejected).toHaveBeenCalledTimes(1);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
it("still emits restart when preparation fails", async () => {

Some files were not shown because too many files have changed in this diff Show More