mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(gateway): add cooperative host suspension (#103618)
* feat(gateway): add cooperative suspension preparation * style: satisfy suspension lint checks * test(gateway): reset work admission between shared suites * fix(gateway): reject upgrades during suspension * fix(gateway): preserve admitted work during suspension * test(gateway): isolate suspension and restart state * fix(gateway): close suspension false-ready gaps * refactor(protocol): slim suspension declaration graph * refactor(plugin-sdk): sever protocol registry edges * fix(gateway): preserve admitted restart follow-ups * fix(gateway): make suspension recovery fail closed * fix(protocol): keep validation formatter re-export only * test(gateway): simplify deferred fixture type * style(gateway): clarify suspension entry name * fix(gateway): retain detached work admission
This commit is contained in:
committed by
GitHub
parent
fece8c9f54
commit
1bcc4c5e70
@@ -425,6 +425,222 @@ public struct ErrorShape: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendTaskBlocker: Codable, Sendable {
|
||||
public let taskid: String
|
||||
public let status: String
|
||||
public let runtime: AnyCodable
|
||||
public let runid: String?
|
||||
public let label: String?
|
||||
public let title: String?
|
||||
|
||||
public init(
|
||||
taskid: String,
|
||||
status: String,
|
||||
runtime: AnyCodable,
|
||||
runid: String? = nil,
|
||||
label: String? = nil,
|
||||
title: String? = nil)
|
||||
{
|
||||
self.taskid = taskid
|
||||
self.status = status
|
||||
self.runtime = runtime
|
||||
self.runid = runid
|
||||
self.label = label
|
||||
self.title = title
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case taskid = "taskId"
|
||||
case status
|
||||
case runtime
|
||||
case runid = "runId"
|
||||
case label
|
||||
case title
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendBlocker: Codable, Sendable {
|
||||
public let kind: AnyCodable
|
||||
public let count: Int
|
||||
public let message: String
|
||||
public let task: GatewaySuspendTaskBlocker?
|
||||
|
||||
public init(
|
||||
kind: AnyCodable,
|
||||
count: Int,
|
||||
message: String,
|
||||
task: GatewaySuspendTaskBlocker? = nil)
|
||||
{
|
||||
self.kind = kind
|
||||
self.count = count
|
||||
self.message = message
|
||||
self.task = task
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case kind
|
||||
case count
|
||||
case message
|
||||
case task
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendPrepareParams: Codable, Sendable {
|
||||
public let requestid: String
|
||||
|
||||
public init(
|
||||
requestid: String)
|
||||
{
|
||||
self.requestid = requestid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case requestid = "requestId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendPrepareBusyResult: Codable, Sendable {
|
||||
public let status: String
|
||||
public let reason: AnyCodable
|
||||
public let retryafterms: Int
|
||||
public let activecount: Int
|
||||
public let blockers: [GatewaySuspendBlocker]
|
||||
|
||||
public init(
|
||||
status: String,
|
||||
reason: AnyCodable,
|
||||
retryafterms: Int,
|
||||
activecount: Int,
|
||||
blockers: [GatewaySuspendBlocker])
|
||||
{
|
||||
self.status = status
|
||||
self.reason = reason
|
||||
self.retryafterms = retryafterms
|
||||
self.activecount = activecount
|
||||
self.blockers = blockers
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case status
|
||||
case reason
|
||||
case retryafterms = "retryAfterMs"
|
||||
case activecount = "activeCount"
|
||||
case blockers
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendPrepareReadyResult: Codable, Sendable {
|
||||
public let status: String
|
||||
public let suspensionid: String
|
||||
public let expiresatms: Int
|
||||
public let activecount: Int
|
||||
public let blockers: [GatewaySuspendBlocker]
|
||||
|
||||
public init(
|
||||
status: String,
|
||||
suspensionid: String,
|
||||
expiresatms: Int,
|
||||
activecount: Int,
|
||||
blockers: [GatewaySuspendBlocker])
|
||||
{
|
||||
self.status = status
|
||||
self.suspensionid = suspensionid
|
||||
self.expiresatms = expiresatms
|
||||
self.activecount = activecount
|
||||
self.blockers = blockers
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case status
|
||||
case suspensionid = "suspensionId"
|
||||
case expiresatms = "expiresAtMs"
|
||||
case activecount = "activeCount"
|
||||
case blockers
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendStatusParams: Codable, Sendable {
|
||||
public let suspensionid: String
|
||||
|
||||
public init(
|
||||
suspensionid: String)
|
||||
{
|
||||
self.suspensionid = suspensionid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case suspensionid = "suspensionId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendStatusRunningResult: Codable, Sendable {
|
||||
public let status: String
|
||||
|
||||
public init(
|
||||
status: String)
|
||||
{
|
||||
self.status = status
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case status
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendStatusReadyResult: Codable, Sendable {
|
||||
public let status: String
|
||||
public let expiresatms: Int
|
||||
|
||||
public init(
|
||||
status: String,
|
||||
expiresatms: Int)
|
||||
{
|
||||
self.status = status
|
||||
self.expiresatms = expiresatms
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case status
|
||||
case expiresatms = "expiresAtMs"
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendResumeParams: Codable, Sendable {
|
||||
public let suspensionid: String
|
||||
|
||||
public init(
|
||||
suspensionid: String)
|
||||
{
|
||||
self.suspensionid = suspensionid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case suspensionid = "suspensionId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct GatewaySuspendResumeResult: Codable, Sendable {
|
||||
public let ok: Bool
|
||||
public let status: String
|
||||
public let resumed: Bool
|
||||
|
||||
public init(
|
||||
ok: Bool,
|
||||
status: String,
|
||||
resumed: Bool)
|
||||
{
|
||||
self.ok = ok
|
||||
self.status = status
|
||||
self.resumed = resumed
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case ok
|
||||
case status
|
||||
case resumed
|
||||
}
|
||||
}
|
||||
|
||||
public struct EnvironmentSummary: Codable, Sendable {
|
||||
public let id: String
|
||||
public let type: String
|
||||
@@ -9874,6 +10090,68 @@ public struct ShutdownEvent: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public enum GatewaySuspendPrepareResult: Codable, Sendable {
|
||||
case busy(GatewaySuspendPrepareBusyResult)
|
||||
case ready(GatewaySuspendPrepareReadyResult)
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case discriminator = "status"
|
||||
}
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
let discriminator = try container.decode(String.self, forKey: .discriminator)
|
||||
switch discriminator {
|
||||
case "busy": self = try .busy(GatewaySuspendPrepareBusyResult(from: decoder))
|
||||
case "ready": self = try .ready(GatewaySuspendPrepareReadyResult(from: decoder))
|
||||
default:
|
||||
throw DecodingError.dataCorruptedError(
|
||||
forKey: .discriminator,
|
||||
in: container,
|
||||
debugDescription: "Unknown GatewaySuspendPrepareResult discriminator value"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
switch self {
|
||||
case .busy(let value): try value.encode(to: encoder)
|
||||
case .ready(let value): try value.encode(to: encoder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum GatewaySuspendStatusResult: Codable, Sendable {
|
||||
case running(GatewaySuspendStatusRunningResult)
|
||||
case ready(GatewaySuspendStatusReadyResult)
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case discriminator = "status"
|
||||
}
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
let discriminator = try container.decode(String.self, forKey: .discriminator)
|
||||
switch discriminator {
|
||||
case "running": self = try .running(GatewaySuspendStatusRunningResult(from: decoder))
|
||||
case "ready": self = try .ready(GatewaySuspendStatusReadyResult(from: decoder))
|
||||
default:
|
||||
throw DecodingError.dataCorruptedError(
|
||||
forKey: .discriminator,
|
||||
in: container,
|
||||
debugDescription: "Unknown GatewaySuspendStatusResult discriminator value"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
switch self {
|
||||
case .running(let value): try value.encode(to: encoder)
|
||||
case .ready(let value): try value.encode(to: encoder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum PluginCatalogInstallAction: Codable, Sendable {
|
||||
case clawhub(PluginCatalogClawHubInstall)
|
||||
case official(PluginCatalogOfficialInstall)
|
||||
|
||||
@@ -3339,6 +3339,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- Headings:
|
||||
- H2: What is available today
|
||||
- H2: Recommended path
|
||||
- H2: Cooperative host suspension
|
||||
- H2: App code vs plugin code
|
||||
- H2: Related
|
||||
|
||||
|
||||
@@ -50,6 +50,94 @@ terminal result. For durable conversation state, use the `sessions.*` methods.
|
||||
For UI integrations, subscribe to Gateway events and render only the event
|
||||
families your app understands.
|
||||
|
||||
## Cooperative host suspension
|
||||
|
||||
Hosting controllers that freeze or snapshot a running process can use the
|
||||
host-neutral suspension handshake:
|
||||
|
||||
1. Stop admitting external ingress controlled by the host.
|
||||
2. Call `gateway.suspend.prepare` with a stable, unique `requestId`.
|
||||
3. If the response is `busy`, keep the process running and retry later.
|
||||
4. If it is `ready`, save the returned `suspensionId`, then freeze or snapshot
|
||||
the process before `expiresAtMs`.
|
||||
5. After thaw, or if suspension is abandoned, call `gateway.suspend.resume`
|
||||
with that `suspensionId` over the existing WebSocket or Admin HTTP control
|
||||
path.
|
||||
|
||||
A prepared Gateway rejects new WebSocket handshakes. A WebSocket controller
|
||||
must keep its authenticated connection open across the host operation. If that
|
||||
cannot be guaranteed, enable and use the
|
||||
[Admin HTTP RPC plugin](/plugins/admin-http-rpc) before preparing. If the
|
||||
control path is lost, wait for the two-minute lease to expire before
|
||||
reconnecting; expiry reopens admission automatically.
|
||||
|
||||
The RPC contract is:
|
||||
|
||||
- `gateway.suspend.prepare` — `operator.admin`; params
|
||||
`{ "requestId": "stable-host-operation-id" }`
|
||||
- `gateway.suspend.status` — `operator.read`; params
|
||||
`{ "suspensionId": "id-from-prepare" }`
|
||||
- `gateway.suspend.resume` — `operator.admin`; params
|
||||
`{ "suspensionId": "id-from-prepare" }`
|
||||
|
||||
IDs are trimmed, must contain a non-whitespace character, and are limited to
|
||||
128 characters. A busy prepare result has `status: "busy"`, `reason`,
|
||||
`retryAfterMs`, `activeCount`, and `blockers`. A ready result has this shape:
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"suspensionId": "2c3f...",
|
||||
"expiresAtMs": 1770000000000,
|
||||
"activeCount": 0,
|
||||
"blockers": []
|
||||
}
|
||||
```
|
||||
|
||||
Status returns `{"status":"running"}` or a ready result with `expiresAtMs`.
|
||||
Resume returns `{"ok":true,"status":"running","resumed":true}`; repeating it
|
||||
after a successful resume returns `resumed: false`.
|
||||
|
||||
A competing request ID or transient scheduler-resume failure returns retryable
|
||||
`UNAVAILABLE` with `retryAfterMs`. During scheduler recovery, prepare, status,
|
||||
and resume all return that error, the Gateway remains not-ready and
|
||||
fail-closed, and the host must not freeze or snapshot it. OpenClaw retries the
|
||||
scheduler automatically and reopens admission only after recovery succeeds. A
|
||||
mismatched resume ID returns `INVALID_REQUEST`. Prepare shares the Gateway's
|
||||
control-plane write budget of three attempts per minute; honor the returned
|
||||
retry delay. WebSocket clients are bucketed by device and IP. Admin HTTP
|
||||
controllers are bucketed by resolved client IP, so controllers behind one
|
||||
proxy can share a budget.
|
||||
|
||||
Preparation is refuse-only: OpenClaw closes new root/session/command admission,
|
||||
pauses automatic cron ticks, and inspects work synchronously. If anything is
|
||||
active, it resumes the scheduler and reopens admission before returning
|
||||
`busy`; it does not interrupt or drain that work. A ready lease lasts two
|
||||
minutes. Repeating `prepare` with the same `requestId` renews it; expiry resumes
|
||||
the scheduler before reopening admission.
|
||||
Restart emission that becomes due during a ready lease waits until the lease
|
||||
resumes; an in-flight restart makes preparation return `busy`.
|
||||
|
||||
While ready, `/healthz` remains live and `/readyz` returns `503`. Local or
|
||||
authenticated readiness responses include `gateway-draining`; unauthenticated
|
||||
remote probes receive only `{ "ready": false }`. The HTTP health probe,
|
||||
suspension methods on existing WebSocket connections, and an already-enabled
|
||||
Admin HTTP RPC route remain available. Other RPCs return retryable
|
||||
`UNAVAILABLE`. Built-in HTTP user-work routes, including OpenAI-compatible
|
||||
APIs, tool/session operations, node watches, and configured hooks, return
|
||||
`503` with `error.code: "gateway_unavailable"`.
|
||||
|
||||
This handshake does not persist incoming messages, stop third-party channel
|
||||
transports, or control the hosting platform. The host must fence its ingress
|
||||
before preparation and remains responsible for wake, snapshot/freeze, and
|
||||
stop. `activeCount` is the aggregate tracked-work count, while `blockers`
|
||||
contains the non-zero category counts and bounded task details. This is not a
|
||||
general process-quiescence barrier. Channel health, maintenance, cache refresh,
|
||||
plugin-owned HTTP routes, and plugin-owned background work can remain active.
|
||||
The hosting platform must freeze or snapshot the full process tree and its
|
||||
filesystem consistently; unregistered work cannot be proven idle by this first
|
||||
contract.
|
||||
|
||||
## App code vs plugin code
|
||||
|
||||
Use Gateway RPC when code lives outside OpenClaw:
|
||||
|
||||
@@ -348,6 +348,7 @@ methods. Treat this as feature discovery, not a full enumeration of
|
||||
- `system-event` appends a system event and can update/broadcast presence context.
|
||||
- `last-heartbeat` returns the latest persisted heartbeat event.
|
||||
- `set-heartbeats` toggles heartbeat processing on the gateway.
|
||||
- `gateway.suspend.prepare` creates a short cooperative-suspension lease only when tracked Gateway work is idle. `gateway.suspend.status` checks that lease, and `gateway.suspend.resume` releases it after thaw or an aborted host operation.
|
||||
|
||||
</Accordion>
|
||||
|
||||
|
||||
@@ -168,7 +168,7 @@ HTTP status follows the error code:
|
||||
|
||||
- discovery: `commands.list`
|
||||
Returns the HTTP RPC method names allowed by this plugin.
|
||||
- gateway: `health`, `status`, `logs.tail`, `usage.status`, `usage.cost`, `gateway.restart.request`
|
||||
- gateway: `health`, `status`, `logs.tail`, `usage.status`, `usage.cost`, `gateway.restart.request`, `gateway.suspend.prepare`, `gateway.suspend.status`, `gateway.suspend.resume`
|
||||
- config: `config.get`, `config.schema`, `config.schema.lookup`, `config.set`, `config.patch`, `config.apply`
|
||||
- channels: `channels.status`, `channels.start`, `channels.stop`, `channels.logout`
|
||||
- web: `web.login.start`, `web.login.wait`
|
||||
@@ -209,11 +209,11 @@ Shared-token WebSocket clients without a trusted device identity cannot self-dec
|
||||
|
||||
`400 INVALID_REQUEST`
|
||||
|
||||
: The request body is not valid JSON, the `method` field is missing, or the method is not in the plugin allowlist.
|
||||
: The request body is not valid JSON, the `method` field is missing, the method is not in the plugin allowlist, or a suspension resume ID does not match the active lease.
|
||||
|
||||
`503 UNAVAILABLE`
|
||||
|
||||
: The Gateway method handler is unavailable. Check Gateway logs and retry after the Gateway finishes startup.
|
||||
: The Gateway method is starting, rate-limited, suspended, or waiting on a competing suspension/resume operation. Inspect `error.details` when present and honor `error.retryAfterMs` before retrying.
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -133,6 +133,27 @@ describe("admin-http-rpc plugin handler", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["gateway.suspend.prepare", { requestId: "host-request-1" }],
|
||||
["gateway.suspend.status", { suspensionId: "suspension-1" }],
|
||||
["gateway.suspend.resume", { suspensionId: "suspension-1" }],
|
||||
] as const)("dispatches suspension method %s through Admin HTTP", async (method, params) => {
|
||||
dispatchGatewayMethod.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
payload: { status: "ok" },
|
||||
});
|
||||
|
||||
const result = await invoke({ id: "suspension", method, params });
|
||||
|
||||
expect(dispatchGatewayMethod).toHaveBeenCalledWith(method, params);
|
||||
expect(result.captured.statusCode).toBe(200);
|
||||
expect(result.json).toEqual({
|
||||
id: "suspension",
|
||||
ok: true,
|
||||
payload: { status: "ok" },
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects methods outside the admin HTTP RPC allowlist", async () => {
|
||||
const result = await invoke({ id: "bad", method: "sessions.send" });
|
||||
|
||||
|
||||
@@ -10,6 +10,9 @@ const ADMIN_HTTP_RPC_ALLOWED_METHOD_GROUPS = {
|
||||
"usage.status",
|
||||
"usage.cost",
|
||||
"gateway.restart.request",
|
||||
"gateway.suspend.prepare",
|
||||
"gateway.suspend.status",
|
||||
"gateway.suspend.resume",
|
||||
],
|
||||
discovery: ["commands.list"],
|
||||
config: [
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
// Gateway Client module implements client behavior.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type {
|
||||
ConnectParams,
|
||||
ErrorShape,
|
||||
EventFrame,
|
||||
HelloOk,
|
||||
RequestFrame,
|
||||
} from "@openclaw/gateway-protocol";
|
||||
import {
|
||||
GATEWAY_CLIENT_MODES,
|
||||
GATEWAY_CLIENT_NAMES,
|
||||
@@ -22,8 +15,13 @@ import {
|
||||
type ConnectErrorRecoveryAdvice,
|
||||
} from "@openclaw/gateway-protocol/connect-error-details";
|
||||
import {
|
||||
type ConnectParams,
|
||||
type ErrorShape,
|
||||
type EventFrame,
|
||||
type HelloOk,
|
||||
isGatewayEventFrame,
|
||||
isGatewayResponseFrame,
|
||||
type RequestFrame,
|
||||
} from "@openclaw/gateway-protocol/frame-guards";
|
||||
import { resolveGatewayStartupRetryAfterMs } from "@openclaw/gateway-protocol/startup-unavailable";
|
||||
import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION } from "@openclaw/gateway-protocol/version";
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
import type { EventFrame, ResponseFrame } from "./schema/types.js";
|
||||
import type { EventFrame, ResponseFrame } from "./schema/frames.js";
|
||||
export type {
|
||||
ConnectParams,
|
||||
ErrorShape,
|
||||
EventFrame,
|
||||
GatewayFrame,
|
||||
HelloOk,
|
||||
RequestFrame,
|
||||
ResponseFrame,
|
||||
} from "./schema/frames.js";
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
validateGatewaySuspendPrepareParams,
|
||||
validateGatewaySuspendPrepareResult,
|
||||
validateGatewaySuspendResumeResult,
|
||||
validateGatewaySuspendStatusResult,
|
||||
} from "./index.js";
|
||||
|
||||
describe("gateway suspension protocol", () => {
|
||||
it("keeps prepare params closed and bounded", () => {
|
||||
expect(validateGatewaySuspendPrepareParams({ requestId: "host-request" })).toBe(true);
|
||||
expect(validateGatewaySuspendPrepareParams({ requestId: " " })).toBe(false);
|
||||
expect(validateGatewaySuspendPrepareParams({ requestId: "host-request", extra: true })).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("validates busy and ready prepare results", () => {
|
||||
expect(
|
||||
validateGatewaySuspendPrepareResult({
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
retryAfterMs: 20_000,
|
||||
activeCount: 2,
|
||||
blockers: [
|
||||
{ kind: "queue", count: 1, message: "one queued operation" },
|
||||
{
|
||||
kind: "task",
|
||||
count: 1,
|
||||
message: "one active task",
|
||||
task: { taskId: "task-1", status: "running", runtime: "subagent" },
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
validateGatewaySuspendPrepareResult({
|
||||
status: "ready",
|
||||
suspensionId: "suspension-id",
|
||||
expiresAtMs: 123,
|
||||
activeCount: 0,
|
||||
blockers: [],
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("validates status and resume results", () => {
|
||||
expect(validateGatewaySuspendStatusResult({ status: "running" })).toBe(true);
|
||||
expect(validateGatewaySuspendStatusResult({ status: "ready", expiresAtMs: 123 })).toBe(true);
|
||||
expect(
|
||||
validateGatewaySuspendResumeResult({ ok: true, status: "running", resumed: false }),
|
||||
).toBe(true);
|
||||
expect(
|
||||
validateGatewaySuspendResumeResult({
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: false,
|
||||
warnings: [],
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps scheduler recovery on the error frame instead of success results", () => {
|
||||
const recovering = {
|
||||
status: "recovering",
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: 1_000,
|
||||
};
|
||||
|
||||
expect(validateGatewaySuspendPrepareResult(recovering)).toBe(false);
|
||||
expect(validateGatewaySuspendStatusResult(recovering)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,8 @@ export {
|
||||
type ClawHubTrustErrorDetails,
|
||||
} from "./clawhub-trust-error-details.js";
|
||||
import { Compile, type Validator as TypeBoxValidator } from "typebox/compile";
|
||||
import type { ValidationError } from "./validation-errors.js";
|
||||
export { formatValidationErrors, type ValidationError } from "./validation-errors.js";
|
||||
import {
|
||||
type AgentEvent,
|
||||
AgentEventSchema,
|
||||
@@ -185,6 +187,26 @@ import {
|
||||
UpdateStatusParamsSchema,
|
||||
type ConnectParams,
|
||||
ConnectParamsSchema,
|
||||
type GatewaySuspendBlocker,
|
||||
GatewaySuspendBlockerSchema,
|
||||
type GatewaySuspendPrepareParams,
|
||||
GatewaySuspendPrepareBusyResultSchema,
|
||||
GatewaySuspendPrepareParamsSchema,
|
||||
GatewaySuspendPrepareReadyResultSchema,
|
||||
type GatewaySuspendPrepareResult,
|
||||
GatewaySuspendPrepareResultSchema,
|
||||
type GatewaySuspendResumeParams,
|
||||
GatewaySuspendResumeParamsSchema,
|
||||
type GatewaySuspendResumeResult,
|
||||
GatewaySuspendResumeResultSchema,
|
||||
type GatewaySuspendStatusParams,
|
||||
GatewaySuspendStatusParamsSchema,
|
||||
GatewaySuspendStatusReadyResultSchema,
|
||||
type GatewaySuspendStatusResult,
|
||||
GatewaySuspendStatusResultSchema,
|
||||
GatewaySuspendStatusRunningResultSchema,
|
||||
type GatewaySuspendTaskBlocker,
|
||||
GatewaySuspendTaskBlockerSchema,
|
||||
type CronAddParams,
|
||||
CronAddParamsSchema,
|
||||
type CronAddResult,
|
||||
@@ -664,20 +686,6 @@ import {
|
||||
WorktreesBranchesResultSchema,
|
||||
} from "./schema.js";
|
||||
|
||||
/** Normalized validation error shape exposed by every protocol validator. */
|
||||
export type ValidationError = {
|
||||
/** Failed schema keyword, when the validator can report one. */
|
||||
keyword?: string;
|
||||
/** JSON-pointer path to the failing data location. */
|
||||
instancePath?: string;
|
||||
/** JSON-pointer path to the failing schema location. */
|
||||
schemaPath?: string;
|
||||
/** Validator-specific keyword parameters for richer diagnostics. */
|
||||
params?: Record<string, unknown>;
|
||||
/** Human-readable validation message. */
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Runtime validator shape shared by gateway clients and server handlers. */
|
||||
export type ProtocolValidator<T = unknown> = ((data: unknown) => data is T) & {
|
||||
/** Last validation errors, matching Ajv-style caller expectations. */
|
||||
@@ -728,6 +736,24 @@ function lazyCompile<T = unknown>(schema: unknown): ProtocolValidator<T> {
|
||||
// constants so call sites can pair validation with the wire contract directly.
|
||||
export const validateCommandsListParams = lazyCompile<CommandsListParams>(CommandsListParamsSchema);
|
||||
export const validateConnectParams = lazyCompile<ConnectParams>(ConnectParamsSchema);
|
||||
export const validateGatewaySuspendPrepareParams = lazyCompile<GatewaySuspendPrepareParams>(
|
||||
GatewaySuspendPrepareParamsSchema,
|
||||
);
|
||||
export const validateGatewaySuspendPrepareResult = lazyCompile<GatewaySuspendPrepareResult>(
|
||||
GatewaySuspendPrepareResultSchema,
|
||||
);
|
||||
export const validateGatewaySuspendStatusParams = lazyCompile<GatewaySuspendStatusParams>(
|
||||
GatewaySuspendStatusParamsSchema,
|
||||
);
|
||||
export const validateGatewaySuspendStatusResult = lazyCompile<GatewaySuspendStatusResult>(
|
||||
GatewaySuspendStatusResultSchema,
|
||||
);
|
||||
export const validateGatewaySuspendResumeParams = lazyCompile<GatewaySuspendResumeParams>(
|
||||
GatewaySuspendResumeParamsSchema,
|
||||
);
|
||||
export const validateGatewaySuspendResumeResult = lazyCompile<GatewaySuspendResumeResult>(
|
||||
GatewaySuspendResumeResultSchema,
|
||||
);
|
||||
export const validateRequestFrame = lazyCompile<RequestFrame>(RequestFrameSchema);
|
||||
export const validateResponseFrame = lazyCompile<ResponseFrame>(ResponseFrameSchema);
|
||||
export const validateEventFrame = lazyCompile<EventFrame>(EventFrameSchema);
|
||||
@@ -1199,77 +1225,22 @@ export const validateWebLoginStartParams =
|
||||
lazyCompile<WebLoginStartParams>(WebLoginStartParamsSchema);
|
||||
export const validateWebLoginWaitParams = lazyCompile<WebLoginWaitParams>(WebLoginWaitParamsSchema);
|
||||
|
||||
function firstStringParam(value: unknown): string | undefined {
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.find(
|
||||
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Convert validator errors into compact operator-facing failure text. */
|
||||
export function formatValidationErrors(errors: ValidationError[] | null | undefined) {
|
||||
if (!errors?.length) {
|
||||
return "unknown validation error";
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
|
||||
for (const err of errors) {
|
||||
const keyword = typeof err?.keyword === "string" ? err.keyword : "";
|
||||
const instancePath = typeof err?.instancePath === "string" ? err.instancePath : "";
|
||||
|
||||
if (keyword === "additionalProperties") {
|
||||
const additionalProperty =
|
||||
firstStringParam(err?.params?.additionalProperty) ??
|
||||
firstStringParam(err?.params?.additionalProperties);
|
||||
if (additionalProperty) {
|
||||
const where = instancePath ? `at ${instancePath}` : "at root";
|
||||
parts.push(`${where}: unexpected property '${additionalProperty}'`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (keyword === "required") {
|
||||
const missingProperty =
|
||||
firstStringParam(err?.params?.missingProperty) ??
|
||||
firstStringParam(err?.params?.requiredProperties);
|
||||
if (missingProperty) {
|
||||
const where = instancePath ? `at ${instancePath}: ` : "";
|
||||
parts.push(`${where}must have required property '${missingProperty}'`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const failingKeyword =
|
||||
typeof err?.params?.failingKeyword === "string" ? err.params.failingKeyword : "";
|
||||
// TypeBox reports conditional required-property misses through if/then
|
||||
// keywords, which otherwise hide the actionable missing-property context.
|
||||
const message =
|
||||
keyword === "then" || (keyword === "if" && failingKeyword === "then")
|
||||
? "must have required conditional properties"
|
||||
: typeof err?.message === "string" && err.message.trim()
|
||||
? err.message
|
||||
: "validation error";
|
||||
const where = instancePath ? `at ${instancePath}: ` : "";
|
||||
parts.push(`${where}${message}`);
|
||||
}
|
||||
|
||||
// De-dupe while preserving order.
|
||||
const unique = uniqueStrings(parts.filter((part) => part.trim()));
|
||||
if (!unique.length) {
|
||||
return "unknown validation error";
|
||||
}
|
||||
return unique.join("; ");
|
||||
}
|
||||
|
||||
// Schema exports stay explicit to make additions/removals reviewable as public
|
||||
// protocol surface changes.
|
||||
export {
|
||||
ConnectParamsSchema,
|
||||
GatewaySuspendTaskBlockerSchema,
|
||||
GatewaySuspendBlockerSchema,
|
||||
GatewaySuspendPrepareBusyResultSchema,
|
||||
GatewaySuspendPrepareParamsSchema,
|
||||
GatewaySuspendPrepareReadyResultSchema,
|
||||
GatewaySuspendPrepareResultSchema,
|
||||
GatewaySuspendStatusReadyResultSchema,
|
||||
GatewaySuspendStatusRunningResultSchema,
|
||||
GatewaySuspendStatusParamsSchema,
|
||||
GatewaySuspendStatusResultSchema,
|
||||
GatewaySuspendResumeParamsSchema,
|
||||
GatewaySuspendResumeResultSchema,
|
||||
GATEWAY_SERVER_CAPS,
|
||||
HelloOkSchema,
|
||||
RequestFrameSchema,
|
||||
@@ -1577,6 +1548,14 @@ export {
|
||||
export type {
|
||||
GatewayFrame,
|
||||
ConnectParams,
|
||||
GatewaySuspendTaskBlocker,
|
||||
GatewaySuspendBlocker,
|
||||
GatewaySuspendPrepareParams,
|
||||
GatewaySuspendPrepareResult,
|
||||
GatewaySuspendStatusParams,
|
||||
GatewaySuspendStatusResult,
|
||||
GatewaySuspendResumeParams,
|
||||
GatewaySuspendResumeResult,
|
||||
HelloOk,
|
||||
RequestFrame,
|
||||
ResponseFrame,
|
||||
@@ -1869,9 +1848,6 @@ export type {
|
||||
SessionsGroupsDeleteParams,
|
||||
SessionsGroupsMutationResult,
|
||||
};
|
||||
function uniqueStrings(values: string[]): string[] {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
// The protocol package cannot import core session types. This local structural
|
||||
// result mirrors the wire contract and keeps the package independent of src/.
|
||||
|
||||
@@ -20,6 +20,7 @@ export * from "./schema/environments.js";
|
||||
export * from "./schema/exec-approvals.js";
|
||||
export * from "./schema/devices.js";
|
||||
export * from "./schema/frames.js";
|
||||
export * from "./schema/gateway-suspend.js";
|
||||
export * from "./schema/logs-chat.js";
|
||||
export * from "./schema/nodes.js";
|
||||
export * from "./schema/protocol-schemas.js";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Gateway Protocol schema module defines protocol validation shapes.
|
||||
import type { ErrorShape } from "./types.js";
|
||||
import type { ErrorShape } from "./frames.js";
|
||||
|
||||
/** Gateway JSON-RPC style error codes shared by clients and server handlers. */
|
||||
export const ErrorCodes = {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Gateway Protocol schema module defines protocol validation shapes.
|
||||
import type { Static } from "typebox";
|
||||
import { Type } from "typebox";
|
||||
import { GatewayClientIdSchema, GatewayClientModeSchema, NonEmptyString } from "./primitives.js";
|
||||
import { SnapshotSchema, StateVersionSchema } from "./snapshot.js";
|
||||
@@ -213,3 +214,13 @@ export const GatewayFrameSchema = Type.Union(
|
||||
[RequestFrameSchema, ResponseFrameSchema, EventFrameSchema],
|
||||
{ discriminator: "type" },
|
||||
);
|
||||
|
||||
// Frame types are owner-local because they cross the public client/plugin SDK.
|
||||
// Keeping them off the aggregate registry avoids retaining every RPC schema.
|
||||
export type ConnectParams = Static<typeof ConnectParamsSchema>;
|
||||
export type HelloOk = Static<typeof HelloOkSchema>;
|
||||
export type ErrorShape = Static<typeof ErrorShapeSchema>;
|
||||
export type RequestFrame = Static<typeof RequestFrameSchema>;
|
||||
export type ResponseFrame = Static<typeof ResponseFrameSchema>;
|
||||
export type EventFrame = Static<typeof EventFrameSchema>;
|
||||
export type GatewayFrame = Static<typeof GatewayFrameSchema>;
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Gateway Protocol schemas for cooperative host suspension.
|
||||
import { Type } from "typebox";
|
||||
|
||||
const SuspensionTokenSchema = Type.String({ minLength: 1, maxLength: 128, pattern: "\\S" });
|
||||
const CountSchema = Type.Integer({ minimum: 0 });
|
||||
|
||||
export const GatewaySuspendTaskBlockerSchema = Type.Object(
|
||||
{
|
||||
taskId: Type.String(),
|
||||
status: Type.Literal("running"),
|
||||
runtime: Type.Union([
|
||||
Type.Literal("subagent"),
|
||||
Type.Literal("acp"),
|
||||
Type.Literal("cli"),
|
||||
Type.Literal("cron"),
|
||||
]),
|
||||
runId: Type.Optional(Type.String()),
|
||||
label: Type.Optional(Type.String()),
|
||||
title: Type.Optional(Type.String()),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendBlockerSchema = Type.Object(
|
||||
{
|
||||
kind: Type.Union([
|
||||
Type.Literal("queue"),
|
||||
Type.Literal("reply"),
|
||||
Type.Literal("embedded-run"),
|
||||
Type.Literal("cron-run"),
|
||||
Type.Literal("task"),
|
||||
Type.Literal("root-request"),
|
||||
Type.Literal("session-admission"),
|
||||
Type.Literal("session-mutation"),
|
||||
Type.Literal("chat-run"),
|
||||
Type.Literal("queued-turn"),
|
||||
Type.Literal("terminal-persistence"),
|
||||
Type.Literal("terminal-session"),
|
||||
]),
|
||||
count: CountSchema,
|
||||
message: Type.String(),
|
||||
task: Type.Optional(GatewaySuspendTaskBlockerSchema),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendPrepareParamsSchema = Type.Object(
|
||||
{ requestId: SuspensionTokenSchema },
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendPrepareBusyResultSchema = Type.Object(
|
||||
{
|
||||
status: Type.Literal("busy"),
|
||||
reason: Type.Union([Type.Literal("active-work"), Type.Literal("gateway-draining")]),
|
||||
retryAfterMs: CountSchema,
|
||||
activeCount: CountSchema,
|
||||
blockers: Type.Array(GatewaySuspendBlockerSchema),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendPrepareReadyResultSchema = Type.Object(
|
||||
{
|
||||
status: Type.Literal("ready"),
|
||||
suspensionId: SuspensionTokenSchema,
|
||||
expiresAtMs: CountSchema,
|
||||
activeCount: CountSchema,
|
||||
blockers: Type.Array(GatewaySuspendBlockerSchema),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendPrepareResultSchema = Type.Union([
|
||||
GatewaySuspendPrepareBusyResultSchema,
|
||||
GatewaySuspendPrepareReadyResultSchema,
|
||||
]);
|
||||
|
||||
export const GatewaySuspendStatusParamsSchema = Type.Object(
|
||||
{ suspensionId: SuspensionTokenSchema },
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendStatusRunningResultSchema = Type.Object(
|
||||
{ status: Type.Literal("running") },
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendStatusReadyResultSchema = Type.Object(
|
||||
{ status: Type.Literal("ready"), expiresAtMs: CountSchema },
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const GatewaySuspendStatusResultSchema = Type.Union([
|
||||
GatewaySuspendStatusRunningResultSchema,
|
||||
GatewaySuspendStatusReadyResultSchema,
|
||||
]);
|
||||
|
||||
export const GatewaySuspendResumeParamsSchema = GatewaySuspendStatusParamsSchema;
|
||||
|
||||
export const GatewaySuspendResumeResultSchema = Type.Object(
|
||||
{
|
||||
ok: Type.Literal(true),
|
||||
status: Type.Literal("running"),
|
||||
resumed: Type.Boolean(),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
@@ -221,6 +221,20 @@ import {
|
||||
ShutdownEventSchema,
|
||||
TickEventSchema,
|
||||
} from "./frames.js";
|
||||
import {
|
||||
GatewaySuspendBlockerSchema,
|
||||
GatewaySuspendPrepareBusyResultSchema,
|
||||
GatewaySuspendPrepareParamsSchema,
|
||||
GatewaySuspendPrepareReadyResultSchema,
|
||||
GatewaySuspendPrepareResultSchema,
|
||||
GatewaySuspendResumeParamsSchema,
|
||||
GatewaySuspendResumeResultSchema,
|
||||
GatewaySuspendStatusReadyResultSchema,
|
||||
GatewaySuspendStatusRunningResultSchema,
|
||||
GatewaySuspendStatusParamsSchema,
|
||||
GatewaySuspendStatusResultSchema,
|
||||
GatewaySuspendTaskBlockerSchema,
|
||||
} from "./gateway-suspend.js";
|
||||
import {
|
||||
ChatAbortedEventSchema,
|
||||
ChatAbortParamsSchema,
|
||||
@@ -420,6 +434,18 @@ export const ProtocolSchemas = {
|
||||
StateVersion: StateVersionSchema,
|
||||
Snapshot: SnapshotSchema,
|
||||
ErrorShape: ErrorShapeSchema,
|
||||
GatewaySuspendTaskBlocker: GatewaySuspendTaskBlockerSchema,
|
||||
GatewaySuspendBlocker: GatewaySuspendBlockerSchema,
|
||||
GatewaySuspendPrepareParams: GatewaySuspendPrepareParamsSchema,
|
||||
GatewaySuspendPrepareBusyResult: GatewaySuspendPrepareBusyResultSchema,
|
||||
GatewaySuspendPrepareReadyResult: GatewaySuspendPrepareReadyResultSchema,
|
||||
GatewaySuspendPrepareResult: GatewaySuspendPrepareResultSchema,
|
||||
GatewaySuspendStatusParams: GatewaySuspendStatusParamsSchema,
|
||||
GatewaySuspendStatusRunningResult: GatewaySuspendStatusRunningResultSchema,
|
||||
GatewaySuspendStatusReadyResult: GatewaySuspendStatusReadyResultSchema,
|
||||
GatewaySuspendStatusResult: GatewaySuspendStatusResultSchema,
|
||||
GatewaySuspendResumeParams: GatewaySuspendResumeParamsSchema,
|
||||
GatewaySuspendResumeResult: GatewaySuspendResumeResultSchema,
|
||||
|
||||
// Environment and agent-facing control RPC payloads.
|
||||
EnvironmentStatus: EnvironmentStatusSchema,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Gateway Protocol schema module defines protocol validation shapes.
|
||||
import type { Static } from "typebox";
|
||||
import { Type } from "typebox";
|
||||
import { PluginJsonValueSchema } from "./plugins.js";
|
||||
import { NonEmptyString, SessionLabelString } from "./primitives.js";
|
||||
@@ -406,6 +407,7 @@ export const SessionsPatchParamsSchema = Type.Object(
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
export type SessionsPatchParams = Static<typeof SessionsPatchParamsSchema>;
|
||||
|
||||
/** Updates or clears one plugin namespace value on a session record. */
|
||||
export const SessionsPluginPatchParamsSchema = Type.Object(
|
||||
|
||||
@@ -1,28 +1,30 @@
|
||||
/**
|
||||
* Static TypeScript types derived from the canonical gateway protocol schemas.
|
||||
*
|
||||
* Keep aliases wired through `ProtocolSchemas` so validators, runtime schemas,
|
||||
* and exported compile-time types cannot drift apart.
|
||||
* Owner-local schema modules export hot public types directly. The remaining
|
||||
* aliases stay wired through `ProtocolSchemas` so validators, runtime schemas,
|
||||
* and compile-time types cannot drift apart.
|
||||
*/
|
||||
import type { Static } from "typebox";
|
||||
import { ProtocolSchemas } from "./protocol-schemas.js";
|
||||
import type { ProtocolSchemas } from "./protocol-schemas.js";
|
||||
|
||||
/** Stable schema names registered in the protocol schema registry. */
|
||||
type ProtocolSchemaName = keyof typeof ProtocolSchemas;
|
||||
/** Inferred TypeScript type for a named TypeBox protocol schema. */
|
||||
type SchemaType<TName extends ProtocolSchemaName> = Static<(typeof ProtocolSchemas)[TName]>;
|
||||
|
||||
/** Connection handshake, envelope, snapshot, and shared error wire types. */
|
||||
export type ConnectParams = SchemaType<"ConnectParams">;
|
||||
export type HelloOk = SchemaType<"HelloOk">;
|
||||
export type RequestFrame = SchemaType<"RequestFrame">;
|
||||
export type ResponseFrame = SchemaType<"ResponseFrame">;
|
||||
export type EventFrame = SchemaType<"EventFrame">;
|
||||
export type GatewayFrame = SchemaType<"GatewayFrame">;
|
||||
/** Snapshot and shared state wire types. */
|
||||
export type Snapshot = SchemaType<"Snapshot">;
|
||||
export type PresenceEntry = SchemaType<"PresenceEntry">;
|
||||
export type ErrorShape = SchemaType<"ErrorShape">;
|
||||
export type StateVersion = SchemaType<"StateVersion">;
|
||||
export type GatewaySuspendTaskBlocker = SchemaType<"GatewaySuspendTaskBlocker">;
|
||||
export type GatewaySuspendBlocker = SchemaType<"GatewaySuspendBlocker">;
|
||||
export type GatewaySuspendPrepareParams = SchemaType<"GatewaySuspendPrepareParams">;
|
||||
export type GatewaySuspendPrepareResult = SchemaType<"GatewaySuspendPrepareResult">;
|
||||
export type GatewaySuspendStatusParams = SchemaType<"GatewaySuspendStatusParams">;
|
||||
export type GatewaySuspendStatusResult = SchemaType<"GatewaySuspendStatusResult">;
|
||||
export type GatewaySuspendResumeParams = SchemaType<"GatewaySuspendResumeParams">;
|
||||
export type GatewaySuspendResumeResult = SchemaType<"GatewaySuspendResumeResult">;
|
||||
|
||||
/** Environment status RPC payloads used by CLI and Control UI surfaces. */
|
||||
export type EnvironmentStatus = SchemaType<"EnvironmentStatus">;
|
||||
@@ -113,7 +115,6 @@ export type SessionsSendParams = SchemaType<"SessionsSendParams">;
|
||||
export type SessionsMessagesSubscribeParams = SchemaType<"SessionsMessagesSubscribeParams">;
|
||||
export type SessionsMessagesUnsubscribeParams = SchemaType<"SessionsMessagesUnsubscribeParams">;
|
||||
export type SessionsAbortParams = SchemaType<"SessionsAbortParams">;
|
||||
export type SessionsPatchParams = SchemaType<"SessionsPatchParams">;
|
||||
export type SessionsPluginPatchParams = SchemaType<"SessionsPluginPatchParams">;
|
||||
export type SessionsPluginPatchResult = SchemaType<"SessionsPluginPatchResult">;
|
||||
export type SessionsResetParams = SchemaType<"SessionsResetParams">;
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/** Normalized validation error shape exposed by every protocol validator. */
|
||||
export type ValidationError = {
|
||||
/** Failed schema keyword, when the validator can report one. */
|
||||
keyword?: string;
|
||||
/** JSON-pointer path to the failing data location. */
|
||||
instancePath?: string;
|
||||
/** JSON-pointer path to the failing schema location. */
|
||||
schemaPath?: string;
|
||||
/** Validator-specific keyword parameters for richer diagnostics. */
|
||||
params?: Record<string, unknown>;
|
||||
/** Human-readable validation message. */
|
||||
message?: string;
|
||||
};
|
||||
|
||||
function firstStringParam(value: unknown): string | undefined {
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
return value;
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.find(
|
||||
(entry): entry is string => typeof entry === "string" && entry.trim().length > 0,
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Convert validator errors into compact operator-facing failure text. */
|
||||
export function formatValidationErrors(errors: ValidationError[] | null | undefined) {
|
||||
if (!errors?.length) {
|
||||
return "unknown validation error";
|
||||
}
|
||||
|
||||
const parts: string[] = [];
|
||||
|
||||
for (const err of errors) {
|
||||
const keyword = typeof err?.keyword === "string" ? err.keyword : "";
|
||||
const instancePath = typeof err?.instancePath === "string" ? err.instancePath : "";
|
||||
|
||||
if (keyword === "additionalProperties") {
|
||||
const additionalProperty =
|
||||
firstStringParam(err?.params?.additionalProperty) ??
|
||||
firstStringParam(err?.params?.additionalProperties);
|
||||
if (additionalProperty) {
|
||||
const where = instancePath ? `at ${instancePath}` : "at root";
|
||||
parts.push(`${where}: unexpected property '${additionalProperty}'`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (keyword === "required") {
|
||||
const missingProperty =
|
||||
firstStringParam(err?.params?.missingProperty) ??
|
||||
firstStringParam(err?.params?.requiredProperties);
|
||||
if (missingProperty) {
|
||||
const where = instancePath ? `at ${instancePath}: ` : "";
|
||||
parts.push(`${where}must have required property '${missingProperty}'`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const failingKeyword =
|
||||
typeof err?.params?.failingKeyword === "string" ? err.params.failingKeyword : "";
|
||||
// TypeBox reports conditional required-property misses through if/then
|
||||
// keywords, which otherwise hide the actionable missing-property context.
|
||||
const message =
|
||||
keyword === "then" || (keyword === "if" && failingKeyword === "then")
|
||||
? "must have required conditional properties"
|
||||
: typeof err?.message === "string" && err.message.trim()
|
||||
? err.message
|
||||
: "validation error";
|
||||
const where = instancePath ? `at ${instancePath}: ` : "";
|
||||
parts.push(`${where}${message}`);
|
||||
}
|
||||
|
||||
const unique = [...new Set(parts.filter((part) => part.trim()))];
|
||||
return unique.length > 0 ? unique.join("; ") : "unknown validation error";
|
||||
}
|
||||
@@ -49,6 +49,7 @@ const exportSet = new Set(exportedNames);
|
||||
|
||||
const requiredRuntimeShimEntries = ["compat.js", "root-alias.cjs"];
|
||||
const forbiddenPublicDeclarationSpecifiers = ["@openclaw/llm-core"];
|
||||
const FORBIDDEN_PUBLIC_PROTOCOL_REGISTRY_RE = /\bdeclare\s+const\s+ProtocolSchemas(?:\$\d+)?\b/u;
|
||||
const RELATIVE_DECLARATION_SPECIFIER_RE = /\b(?:from|import)\s*(?:\(\s*)?["']([^"']+)["']/gu;
|
||||
const requiredSubpathExports = {
|
||||
"secret-input-runtime": [
|
||||
@@ -140,6 +141,12 @@ while (declarationQueue.length > 0) {
|
||||
}
|
||||
declarationPaths.add(dtsPath);
|
||||
const dtsContent = readFileSync(dtsPath, "utf8");
|
||||
if (FORBIDDEN_PUBLIC_PROTOCOL_REGISTRY_RE.test(dtsContent)) {
|
||||
console.error(
|
||||
`FORBIDDEN PUBLIC DTS REGISTRY: ${relative(resolve(scriptDir, ".."), dtsPath)} retains ProtocolSchemas`,
|
||||
);
|
||||
missing += 1;
|
||||
}
|
||||
for (const match of dtsContent.matchAll(RELATIVE_DECLARATION_SPECIFIER_RE)) {
|
||||
const specifier = match[1];
|
||||
if (!specifier?.startsWith(".")) {
|
||||
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
registerAgentRunContext,
|
||||
resetAgentRunContextForTest,
|
||||
} from "../infra/agent-events.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
INTERNAL_RUNTIME_CONTEXT_BEGIN,
|
||||
INTERNAL_RUNTIME_CONTEXT_END,
|
||||
@@ -24,6 +29,7 @@ import {
|
||||
markStartupOrphanedMainSessionsForRecovery,
|
||||
recoverStartupOrphanedMainSessions,
|
||||
recoverRestartAbortedMainSessions,
|
||||
scheduleRestartAbortedMainSessionRecovery,
|
||||
} from "./main-session-restart-recovery.js";
|
||||
import type { SessionLockInspection } from "./session-write-lock.js";
|
||||
|
||||
@@ -36,10 +42,12 @@ let tmpDir: string;
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
resetAgentRunContextForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-main-restart-recovery-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
@@ -1282,6 +1290,54 @@ describe("main-session-restart-recovery", () => {
|
||||
expect(customStore["agent:main:main"]?.abortedLastRun).toBe(false);
|
||||
});
|
||||
|
||||
it("admits each scheduled recovery attempt as independent root work", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeStore(sessionsDir, {
|
||||
"agent:main:main": {
|
||||
sessionId: "main-session",
|
||||
updatedAt: Date.now() - 10_000,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
pendingFinalDelivery: true,
|
||||
pendingFinalDeliveryText: "interrupted response",
|
||||
},
|
||||
});
|
||||
|
||||
const suspensionRef: {
|
||||
current: ReturnType<typeof tryBeginGatewaySuspendAdmission>;
|
||||
} = { current: null };
|
||||
vi.mocked(callGateway)
|
||||
.mockImplementationOnce(async () => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
suspensionRef.current = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspensionRef.current?.commit()).toBe(true);
|
||||
throw new Error("retry after suspension");
|
||||
})
|
||||
.mockImplementationOnce(async () => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
return { runId: "run-resumed" };
|
||||
});
|
||||
|
||||
scheduleRestartAbortedMainSessionRecovery({
|
||||
delayMs: 0,
|
||||
maxRetries: 2,
|
||||
stateDir: tmpDir,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(callGateway).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
expect(suspensionRef.current?.release()).toBe(true);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(callGateway).toHaveBeenCalledTimes(2);
|
||||
const store = readSessionStoreForTest(path.join(sessionsDir, "sessions.json"));
|
||||
expect(store["agent:main:main"]?.abortedLastRun).toBe(false);
|
||||
});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("fails marked sessions whose transcript tail cannot be resumed", async () => {
|
||||
const sessionsDir = await makeSessionsDir();
|
||||
await writeStore(sessionsDir, {
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
listAgentRunsForSession,
|
||||
} from "../infra/agent-events.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { CommandLane } from "../process/lanes.js";
|
||||
import {
|
||||
isAcpSessionKey,
|
||||
@@ -956,12 +957,17 @@ export function scheduleRestartAbortedMainSessionRecovery(
|
||||
const startupRecoveryCutoffMs = Date.now();
|
||||
|
||||
const runRecoveryAttempt = (attempt: number, delay: number) => {
|
||||
void recoverStartupOrphanedMainSessions({
|
||||
cfg: params.cfg,
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys,
|
||||
updatedBeforeMs: startupRecoveryCutoffMs,
|
||||
})
|
||||
// Delayed retries outlive startup; each attempt must independently block
|
||||
// host suspension while it reads and rewrites recovery session state.
|
||||
void runWithGatewayIndependentRootWorkAdmission(
|
||||
async () =>
|
||||
await recoverStartupOrphanedMainSessions({
|
||||
cfg: params.cfg,
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys,
|
||||
updatedBeforeMs: startupRecoveryCutoffMs,
|
||||
}),
|
||||
)
|
||||
.then((result) => {
|
||||
if (result.failed > 0 && attempt < maxRetries) {
|
||||
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
|
||||
|
||||
@@ -4,6 +4,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import * as sessions from "../config/sessions.js";
|
||||
import * as gateway from "../gateway/call.js";
|
||||
import * as sessionUtils from "../gateway/session-transcript-readers.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js";
|
||||
import * as announceDelivery from "./subagent-announce-delivery.js";
|
||||
import {
|
||||
@@ -130,9 +135,11 @@ describe("subagent-orphan-recovery", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
@@ -630,7 +637,17 @@ describe("subagent-orphan-recovery", () => {
|
||||
abortedLastRun: true,
|
||||
},
|
||||
});
|
||||
vi.mocked(gateway.callGateway).mockRejectedValue(new Error("service restart"));
|
||||
const admittedRootCounts: number[] = [];
|
||||
vi.mocked(gateway.callGateway).mockImplementation(async () => {
|
||||
admittedRootCounts.push(getActiveGatewayRootWorkCount());
|
||||
throw new Error("service restart");
|
||||
});
|
||||
vi.mocked(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).mockImplementation(
|
||||
async () => {
|
||||
admittedRootCounts.push(getActiveGatewayRootWorkCount());
|
||||
return 1;
|
||||
},
|
||||
);
|
||||
|
||||
const activeRuns = createActiveRuns(createTestRunRecord());
|
||||
|
||||
@@ -642,10 +659,13 @@ describe("subagent-orphan-recovery", () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await Promise.resolve();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
await vi.advanceTimersByTimeAsync(2);
|
||||
await Promise.resolve();
|
||||
|
||||
expect(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).toHaveBeenCalledOnce();
|
||||
expect(admittedRootCounts).toEqual([1, 1, 1]);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
const finalizeParams = requireRecord(
|
||||
firstCallParam(
|
||||
vi.mocked(subagentRegistrySteerRuntime.finalizeInterruptedSubagentRun).mock.calls,
|
||||
@@ -658,4 +678,29 @@ describe("subagent-orphan-recovery", () => {
|
||||
expect(finalizeParams.error).toContain("Automatic recovery failed after 2 attempts");
|
||||
expect(finalizeParams.error).toContain("service restart");
|
||||
});
|
||||
|
||||
it("waits for suspension to reopen before mutating an orphaned session", async () => {
|
||||
mockSingleAbortedSession();
|
||||
vi.mocked(gateway.callGateway).mockResolvedValue({ runId: "resumed-run" });
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
scheduleOrphanRecovery({
|
||||
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
|
||||
delayMs: 1,
|
||||
maxRetries: 0,
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(gateway.callGateway).not.toHaveBeenCalled();
|
||||
expect(sessions.updateSessionStore).not.toHaveBeenCalled();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
|
||||
expect(suspension?.release()).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(gateway.callGateway).toHaveBeenCalledOnce();
|
||||
expect(sessions.updateSessionStore).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@ import { callGateway } from "../gateway/call.js";
|
||||
import { readSessionMessagesAsync } from "../gateway/session-transcript-readers.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { truncateUtf16Safe } from "../utils.js";
|
||||
import { resolveInternalSessionEffectsTranscriptPath } from "./internal-session-effects.js";
|
||||
import {
|
||||
@@ -437,49 +438,48 @@ export function scheduleOrphanRecovery(params: {
|
||||
const resumedSessionKeys = new Set<string>();
|
||||
const attemptRecovery = (attempt: number, delay: number) => {
|
||||
setTimeout(() => {
|
||||
void recoverOrphanedSubagentSessions({
|
||||
...params,
|
||||
resumedSessionKeys,
|
||||
})
|
||||
.then((result) => {
|
||||
if (result.failed > 0 && attempt < maxRetries) {
|
||||
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
|
||||
log.info(
|
||||
`orphan recovery had ${result.failed} failure(s); retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
|
||||
);
|
||||
attemptRecovery(attempt + 1, nextDelay);
|
||||
return;
|
||||
}
|
||||
if (result.failedRuns.length === 0) {
|
||||
return;
|
||||
}
|
||||
const attempts = attempt + 1;
|
||||
void Promise.allSettled(
|
||||
result.failedRuns.map((run) =>
|
||||
finalizeInterruptedSubagentRun({
|
||||
runId: run.runId,
|
||||
childSessionKey: run.childSessionKey,
|
||||
error: buildRecoveryFailureMessage({
|
||||
attempts,
|
||||
error: run.error,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (attempt < maxRetries) {
|
||||
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
|
||||
log.warn(
|
||||
`scheduled orphan recovery failed: ${String(err)}; retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
|
||||
);
|
||||
attemptRecovery(attempt + 1, nextDelay);
|
||||
} else {
|
||||
log.warn(
|
||||
`scheduled orphan recovery failed after ${maxRetries} retries: ${String(err)}`,
|
||||
);
|
||||
}
|
||||
// Every delayed/retry scan owns a fresh root lease. Keep terminal
|
||||
// mutation in the same lease so suspension cannot become ready mid-attempt.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const result = await recoverOrphanedSubagentSessions({
|
||||
...params,
|
||||
resumedSessionKeys,
|
||||
});
|
||||
if (result.failed > 0 && attempt < maxRetries) {
|
||||
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
|
||||
log.info(
|
||||
`orphan recovery had ${result.failed} failure(s); retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
|
||||
);
|
||||
attemptRecovery(attempt + 1, nextDelay);
|
||||
return;
|
||||
}
|
||||
if (result.failedRuns.length === 0) {
|
||||
return;
|
||||
}
|
||||
const attempts = attempt + 1;
|
||||
await Promise.allSettled(
|
||||
result.failedRuns.map((run) =>
|
||||
finalizeInterruptedSubagentRun({
|
||||
runId: run.runId,
|
||||
childSessionKey: run.childSessionKey,
|
||||
error: buildRecoveryFailureMessage({
|
||||
attempts,
|
||||
error: run.error,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
);
|
||||
}).catch((err: unknown) => {
|
||||
if (attempt < maxRetries) {
|
||||
const nextDelay = delay * RETRY_BACKOFF_MULTIPLIER;
|
||||
log.warn(
|
||||
`scheduled orphan recovery failed: ${String(err)}; retrying in ${nextDelay}ms (attempt ${attempt + 1}/${maxRetries})`,
|
||||
);
|
||||
attemptRecovery(attempt + 1, nextDelay);
|
||||
} else {
|
||||
log.warn(`scheduled orphan recovery failed after ${maxRetries} retries: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
}, delay).unref?.();
|
||||
};
|
||||
|
||||
|
||||
@@ -2,6 +2,11 @@
|
||||
// detached task status, and resource retirement around child-run endings.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CallGatewayOptions } from "../gateway/call.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
|
||||
import {
|
||||
buildAnnounceIdFromChildRun,
|
||||
@@ -265,6 +270,7 @@ async function runNoReplyMirrorScenario(params: {
|
||||
|
||||
describe("subagent registry lifecycle hardening", () => {
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.clearAllMocks();
|
||||
taskExecutorMocks.completeTaskRunByRunId.mockReset();
|
||||
taskExecutorMocks.failTaskRunByRunId.mockReset();
|
||||
@@ -276,6 +282,131 @@ describe("subagent registry lifecycle hardening", () => {
|
||||
bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
it("keeps task finalization, resource retirement, and announce cleanup root-admitted", async () => {
|
||||
const entry = createRunEntry({ expectsCompletionMessage: true });
|
||||
let releaseBrowserCleanup: (() => void) | undefined;
|
||||
let releaseAnnounce: ((didAnnounce: boolean) => void) | undefined;
|
||||
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseBrowserCleanup = resolve;
|
||||
}),
|
||||
);
|
||||
const runSubagentAnnounceFlow = vi.fn(
|
||||
() =>
|
||||
new Promise<boolean>((resolve) => {
|
||||
releaseAnnounce = resolve;
|
||||
}),
|
||||
);
|
||||
const controller = createLifecycleController({ entry, runSubagentAnnounceFlow });
|
||||
|
||||
const completion = controller.completeSubagentRun({
|
||||
runId: entry.runId,
|
||||
endedAt: 4_000,
|
||||
outcome: { status: "ok" },
|
||||
reason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
triggerCleanup: true,
|
||||
});
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd,
|
||||
).toHaveBeenCalledOnce(),
|
||||
);
|
||||
expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
releaseBrowserCleanup?.();
|
||||
await vi.waitFor(() => expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce());
|
||||
await completion;
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
releaseAnnounce?.(true);
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
expect(entry.cleanupCompletedAt).toBeTypeOf("number");
|
||||
});
|
||||
|
||||
it("keeps direct delete cleanup root-admitted until the gateway call settles", async () => {
|
||||
const entry = createRunEntry({ cleanup: "delete", expectsCompletionMessage: false });
|
||||
const runs = new Map([[entry.runId, entry]]);
|
||||
let releaseDelete: (() => void) | undefined;
|
||||
gatewayMocks.callGateway.mockImplementation((opts) => {
|
||||
if (opts.method !== "sessions.delete") {
|
||||
return Promise.resolve({});
|
||||
}
|
||||
return new Promise<Record<string, unknown>>((resolve) => {
|
||||
releaseDelete = () => resolve({});
|
||||
});
|
||||
});
|
||||
const controller = createLifecycleController({ entry, runs });
|
||||
|
||||
await controller.completeSubagentRun({
|
||||
runId: entry.runId,
|
||||
endedAt: 4_000,
|
||||
outcome: { status: "ok" },
|
||||
reason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
triggerCleanup: true,
|
||||
});
|
||||
await vi.waitFor(() => expect(releaseDelete).toBeTypeOf("function"));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
releaseDelete?.();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
expect(runs.has(entry.runId)).toBe(false);
|
||||
});
|
||||
|
||||
it("retries a cleanup handoff rejected by restart drain", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const entry = createRunEntry({ expectsCompletionMessage: true });
|
||||
let releaseBrowserCleanup: (() => void) | undefined;
|
||||
browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseBrowserCleanup = resolve;
|
||||
}),
|
||||
);
|
||||
const runSubagentAnnounceFlow = vi.fn(async () => true);
|
||||
const resumeSubagentRun = vi.fn((runId: string) => {
|
||||
controller.startSubagentAnnounceCleanupFlow(runId, entry);
|
||||
});
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
resumeSubagentRun,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
const completion = controller.completeSubagentRun({
|
||||
runId: entry.runId,
|
||||
endedAt: 4_000,
|
||||
outcome: { status: "ok" },
|
||||
reason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
triggerCleanup: true,
|
||||
});
|
||||
await vi.waitFor(() => expect(releaseBrowserCleanup).toBeTypeOf("function"));
|
||||
markGatewayRestartDraining();
|
||||
releaseBrowserCleanup?.();
|
||||
await completion;
|
||||
await vi.waitFor(() =>
|
||||
expect(runtimeMocks.log).toHaveBeenCalledWith(
|
||||
expect.stringContaining("subagent cleanup admission failed"),
|
||||
),
|
||||
);
|
||||
expect(runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
expect(entry.cleanupHandled).toBe(true);
|
||||
|
||||
resetGatewayWorkAdmission();
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce());
|
||||
await vi.waitFor(() => expect(entry.cleanupCompletedAt).toBeTypeOf("number"));
|
||||
expect(resumeSubagentRun).toHaveBeenCalledWith(entry.runId);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
} finally {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not reject completion when task finalization throws", async () => {
|
||||
const persist = vi.fn();
|
||||
const persistOrThrow = vi.fn();
|
||||
@@ -1985,6 +2116,53 @@ describe("subagent registry lifecycle hardening", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a late superseded-delivery retirement root-admitted", async () => {
|
||||
const entry = createRunEntry({ expectsCompletionMessage: true, generation: 1 });
|
||||
const runs = new Map([[entry.runId, entry]]);
|
||||
let onDeliveryResult: ((delivery: SubagentAnnounceDeliveryResult) => void) | undefined;
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
onDeliveryResult = announceParams.onDeliveryResult;
|
||||
return true;
|
||||
},
|
||||
);
|
||||
let releaseRetirement = () => {};
|
||||
const retirementPending = new Promise<void>((resolve) => {
|
||||
releaseRetirement = resolve;
|
||||
});
|
||||
const retireSupersededRun = vi.fn(async () => {
|
||||
await retirementPending;
|
||||
});
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
runs,
|
||||
retireSupersededRun,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
await controller.completeSubagentRun({
|
||||
runId: entry.runId,
|
||||
endedAt: 4_000,
|
||||
outcome: { status: "ok" },
|
||||
reason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
triggerCleanup: true,
|
||||
});
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
const newer = createRunEntry({
|
||||
runId: "run-2",
|
||||
childSessionKey: entry.childSessionKey,
|
||||
generation: 2,
|
||||
});
|
||||
runs.set(newer.runId, newer);
|
||||
|
||||
onDeliveryResult?.({ delivered: false, path: "none" });
|
||||
|
||||
await vi.waitFor(() => expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
releaseRetirement();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("finalizes terminal visible-send failures without scheduling completion retry", async () => {
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
|
||||
@@ -8,6 +8,10 @@ import { isSilentReplyText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
|
||||
import type { cleanupBrowserSessionsForLifecycleEnd } from "../browser-lifecycle-cleanup.js";
|
||||
import type { callGateway as defaultCallGateway } from "../gateway/call.js";
|
||||
import { formatErrorMessage, readErrorName } from "../infra/errors.js";
|
||||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../runtime.js";
|
||||
import { emitSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js";
|
||||
import { extractTextFromChatContent } from "../shared/chat-content.js";
|
||||
@@ -215,13 +219,43 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
};
|
||||
};
|
||||
|
||||
const scheduleResumeSubagentRun = (runId: string, entry: SubagentRunRecord, delayMs: number) => {
|
||||
const scheduleResumeSubagentRun = (
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
delayMs: number,
|
||||
cleanupGeneration?: number,
|
||||
) => {
|
||||
const timer = setTimeout(() => {
|
||||
scheduledResumeTimers.delete(timer);
|
||||
if (params.runs.get(runId) !== entry) {
|
||||
return;
|
||||
}
|
||||
params.resumeSubagentRun(runId);
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (params.runs.get(runId) !== entry) {
|
||||
return;
|
||||
}
|
||||
if (cleanupGeneration !== undefined) {
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
return;
|
||||
}
|
||||
entry.cleanupHandled = false;
|
||||
params.persist();
|
||||
}
|
||||
params.resumedRuns.delete(runId);
|
||||
params.resumeSubagentRun(runId);
|
||||
}).catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
current === entry &&
|
||||
typeof current.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleResumeSubagentRun(
|
||||
runId,
|
||||
entry,
|
||||
Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS),
|
||||
cleanupGeneration,
|
||||
);
|
||||
}
|
||||
});
|
||||
}, delayMs);
|
||||
timer.unref?.();
|
||||
scheduledResumeTimers.add(timer);
|
||||
@@ -234,6 +268,49 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
scheduledResumeTimers.clear();
|
||||
};
|
||||
|
||||
const runDetachedCleanupAttempt = (args: {
|
||||
runId: string;
|
||||
entry: SubagentRunRecord;
|
||||
cleanupGeneration: number;
|
||||
run: () => Promise<void>;
|
||||
}) => {
|
||||
// Completion makes the task projection non-blocking before delivery and
|
||||
// cleanup finish. This independent lease bridges that handoff and owns the
|
||||
// full detached attempt, including its final durable registry write.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
try {
|
||||
await args.run();
|
||||
} catch (err) {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
const current = params.runs.get(args.runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.resumedRuns.delete(args.runId);
|
||||
params.persist();
|
||||
}
|
||||
}).catch((err: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
if (isGatewayRestartDraining()) {
|
||||
scheduleResumeSubagentRun(
|
||||
args.runId,
|
||||
args.entry,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
args.cleanupGeneration,
|
||||
);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const maskRunId = (runId: string): string => {
|
||||
const trimmed = runId.trim();
|
||||
if (!trimmed) {
|
||||
@@ -803,6 +880,22 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
return true;
|
||||
};
|
||||
|
||||
const retireSupersededCleanupInBackground = (
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
generation: number,
|
||||
) => {
|
||||
// Delivery callbacks are synchronous and may arrive after their announce
|
||||
// attempt returns. Give the async retirement tail its own snapshot blocker.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await retireSupersededCleanupIfNeeded(runId, entry, generation);
|
||||
}).catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent superseded cleanup retirement failed (${runId}): ${String(error)}`,
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
const isTerminalCallbackCurrent = (
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
@@ -840,20 +933,17 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
if (!beginSubagentCleanup(runId)) {
|
||||
continue;
|
||||
}
|
||||
void finalizeResumedAnnounceGiveUp({
|
||||
runDetachedCleanupAttempt({
|
||||
runId,
|
||||
entry,
|
||||
reason: "expiry",
|
||||
}).catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] Subagent expiry finalize failed during deferred retry for run ${runId}: ${String(error)}`,
|
||||
);
|
||||
const current = params.runs.get(runId);
|
||||
if (!current || current.cleanupCompletedAt) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.persist();
|
||||
cleanupGeneration: cleanupGenerations.get(entry)!,
|
||||
run: async () => {
|
||||
await finalizeResumedAnnounceGiveUp({
|
||||
runId,
|
||||
entry,
|
||||
reason: "expiry",
|
||||
});
|
||||
},
|
||||
});
|
||||
continue;
|
||||
}
|
||||
@@ -870,31 +960,46 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
preserveTranscript?: boolean;
|
||||
provisionalKill?: boolean;
|
||||
}) => {
|
||||
const runCleanupTail = (label: string, run: () => Promise<unknown>) => {
|
||||
// These best-effort tails can outlive the durable registry transition,
|
||||
// but they still mutate session-owned resources and must block snapshots.
|
||||
void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`,
|
||||
);
|
||||
});
|
||||
};
|
||||
if (!cleanupParams.preserveTranscript) {
|
||||
void removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile);
|
||||
runCleanupTail("transcript cleanup", async () => {
|
||||
await removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile);
|
||||
});
|
||||
}
|
||||
if (cleanupParams.entry.spawnMode !== "session") {
|
||||
void retireSessionMcpRuntimeForSessionKey({
|
||||
sessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "subagent-run-cleanup",
|
||||
onError: (error, sessionId) => {
|
||||
params.warn("failed to retire subagent bundle MCP runtime", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
sessionId,
|
||||
runId: maskRunId(cleanupParams.runId),
|
||||
childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey),
|
||||
});
|
||||
},
|
||||
runCleanupTail("bundle MCP cleanup", async () => {
|
||||
await retireSessionMcpRuntimeForSessionKey({
|
||||
sessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "subagent-run-cleanup",
|
||||
onError: (error, sessionId) => {
|
||||
params.warn("failed to retire subagent bundle MCP runtime", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
sessionId,
|
||||
runId: maskRunId(cleanupParams.runId),
|
||||
childSessionKey: maskSessionKey(cleanupParams.entry.childSessionKey),
|
||||
});
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
if (cleanupParams.cleanup === "delete") {
|
||||
params.clearPendingLifecycleError(cleanupParams.runId);
|
||||
if (!cleanupParams.provisionalKill) {
|
||||
void params.notifyContextEngineSubagentEnded({
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "deleted",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
runCleanupTail("context-engine cleanup", async () => {
|
||||
await params.notifyContextEngineSubagentEnded({
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "deleted",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
});
|
||||
});
|
||||
}
|
||||
params.runs.delete(cleanupParams.runId);
|
||||
@@ -903,11 +1008,13 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
return;
|
||||
}
|
||||
if (!cleanupParams.provisionalKill) {
|
||||
void params.notifyContextEngineSubagentEnded({
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "completed",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
runCleanupTail("context-engine cleanup", async () => {
|
||||
await params.notifyContextEngineSubagentEnded({
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "completed",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
});
|
||||
});
|
||||
}
|
||||
if (
|
||||
@@ -1161,20 +1268,15 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
return false;
|
||||
}
|
||||
const cleanupGeneration = cleanupGenerations.get(entry)!;
|
||||
void finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
}).catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.persist();
|
||||
runDetachedCleanupAttempt({
|
||||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
run: async () => {
|
||||
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
});
|
||||
},
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -1184,52 +1286,45 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
const cleanupGeneration = cleanupGenerations.get(entry)!;
|
||||
const skipRequesterDelivery = entry.suppressCompletionDelivery === true;
|
||||
if (entry.expectsCompletionMessage === false || skipRequesterDelivery) {
|
||||
void (async () => {
|
||||
// This driver is detached. Yield once so synchronous successor
|
||||
// registration can invalidate it before sessions.delete is submitted.
|
||||
await Promise.resolve();
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
if (cleanup === "delete") {
|
||||
// This durable boundary prevents a late yield from reviving a run
|
||||
// after deletion may already have reached the gateway.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persist();
|
||||
await deleteSubagentSessionForCleanup({
|
||||
callGateway: params.callGateway,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
spawnMode: entry.spawnMode,
|
||||
onError: (error) =>
|
||||
params.warn("sessions.delete failed during subagent cleanup", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
runId: maskRunId(runId),
|
||||
childSessionKey: maskSessionKey(entry.childSessionKey),
|
||||
}),
|
||||
runDetachedCleanupAttempt({
|
||||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
run: async () => {
|
||||
// This driver is detached. Yield once so synchronous successor
|
||||
// registration can invalidate it before sessions.delete is submitted.
|
||||
await Promise.resolve();
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
if (cleanup === "delete") {
|
||||
// This durable boundary prevents a late yield from reviving a run
|
||||
// after deletion may already have reached the gateway.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persist();
|
||||
await deleteSubagentSessionForCleanup({
|
||||
callGateway: params.callGateway,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
spawnMode: entry.spawnMode,
|
||||
onError: (error) =>
|
||||
params.warn("sessions.delete failed during subagent cleanup", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
runId: maskRunId(runId),
|
||||
childSessionKey: maskSessionKey(entry.childSessionKey),
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
skipDeliveryStatus: true,
|
||||
skipRequesterDelivery,
|
||||
});
|
||||
}
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
skipDeliveryStatus: true,
|
||||
skipRequesterDelivery,
|
||||
});
|
||||
})().catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.persist();
|
||||
},
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -1253,93 +1348,87 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
if (!didAnnounce && latestDeliveryError) {
|
||||
ensureDeliveryState(entry).lastError = latestDeliveryError;
|
||||
}
|
||||
void finalizeSubagentCleanup(
|
||||
await finalizeSubagentCleanup(
|
||||
runId,
|
||||
cleanup,
|
||||
didAnnounce || shouldCreditPriorDelivery,
|
||||
cleanupGeneration,
|
||||
).catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.persist();
|
||||
});
|
||||
);
|
||||
};
|
||||
|
||||
void params
|
||||
.runSubagentAnnounceFlow({
|
||||
childSessionKey: pendingPayload.childSessionKey,
|
||||
childRunId: pendingPayload.childRunId,
|
||||
requesterSessionKey: pendingPayload.requesterSessionKey,
|
||||
requesterOrigin,
|
||||
requesterDisplayKey: pendingPayload.requesterDisplayKey,
|
||||
task: pendingPayload.task,
|
||||
timeoutMs: params.subagentAnnounceTimeoutMs,
|
||||
cleanup,
|
||||
roundOneReply: pendingPayload.frozenResultText ?? undefined,
|
||||
fallbackReply: pendingPayload.fallbackFrozenResultText ?? undefined,
|
||||
waitForCompletion: false,
|
||||
startedAt: pendingPayload.startedAt,
|
||||
endedAt: pendingPayload.endedAt,
|
||||
label: pendingPayload.label,
|
||||
outcome: pendingPayload.outcome,
|
||||
spawnMode: pendingPayload.spawnMode,
|
||||
expectsCompletionMessage: pendingPayload.expectsCompletionMessage,
|
||||
wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true,
|
||||
onBeforeDeleteChildSession:
|
||||
cleanup === "delete"
|
||||
? () => {
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
return false;
|
||||
}
|
||||
// Announce owns delete submission; fence late yields at the
|
||||
// exact handoff instead of when cleanup merely starts.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persist();
|
||||
return true;
|
||||
const announceParams: Parameters<RunSubagentAnnounceFlow>[0] = {
|
||||
childSessionKey: pendingPayload.childSessionKey,
|
||||
childRunId: pendingPayload.childRunId,
|
||||
requesterSessionKey: pendingPayload.requesterSessionKey,
|
||||
requesterOrigin,
|
||||
requesterDisplayKey: pendingPayload.requesterDisplayKey,
|
||||
task: pendingPayload.task,
|
||||
timeoutMs: params.subagentAnnounceTimeoutMs,
|
||||
cleanup,
|
||||
roundOneReply: pendingPayload.frozenResultText ?? undefined,
|
||||
fallbackReply: pendingPayload.fallbackFrozenResultText ?? undefined,
|
||||
waitForCompletion: false,
|
||||
startedAt: pendingPayload.startedAt,
|
||||
endedAt: pendingPayload.endedAt,
|
||||
label: pendingPayload.label,
|
||||
outcome: pendingPayload.outcome,
|
||||
spawnMode: pendingPayload.spawnMode,
|
||||
expectsCompletionMessage: pendingPayload.expectsCompletionMessage,
|
||||
wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true,
|
||||
onBeforeDeleteChildSession:
|
||||
cleanup === "delete"
|
||||
? () => {
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
return false;
|
||||
}
|
||||
: undefined,
|
||||
onDeliveryResult: (delivery) => {
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
void retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
recordAnnounceDeliveryResult(entry, delivery);
|
||||
if (delivery.delivered) {
|
||||
const deliveryState = ensureDeliveryState(entry);
|
||||
if (deliveryState.lastError !== undefined) {
|
||||
deliveryState.lastError = undefined;
|
||||
// Announce owns delete submission; fence late yields at the
|
||||
// exact handoff instead of when cleanup merely starts.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persist();
|
||||
return true;
|
||||
}
|
||||
latestDeliveryError = undefined;
|
||||
return;
|
||||
}
|
||||
if (delivery.path === "none") {
|
||||
ensureDeliveryState(entry).lastDropReason = "sink_unavailable";
|
||||
}
|
||||
latestDeliveryError = formatAnnounceDeliveryError(delivery);
|
||||
if (ensureDeliveryState(entry).lastError !== latestDeliveryError) {
|
||||
ensureDeliveryState(entry).lastError = latestDeliveryError;
|
||||
: undefined,
|
||||
onDeliveryResult: (delivery) => {
|
||||
if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
retireSupersededCleanupInBackground(runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
recordAnnounceDeliveryResult(entry, delivery);
|
||||
if (delivery.delivered) {
|
||||
const deliveryState = ensureDeliveryState(entry);
|
||||
if (deliveryState.lastError !== undefined) {
|
||||
deliveryState.lastError = undefined;
|
||||
params.persist();
|
||||
}
|
||||
},
|
||||
})
|
||||
.then((didAnnounce) => {
|
||||
void finalizeAnnounceCleanup(didAnnounce);
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`,
|
||||
);
|
||||
void finalizeAnnounceCleanup(false);
|
||||
});
|
||||
latestDeliveryError = undefined;
|
||||
return;
|
||||
}
|
||||
if (delivery.path === "none") {
|
||||
ensureDeliveryState(entry).lastDropReason = "sink_unavailable";
|
||||
}
|
||||
latestDeliveryError = formatAnnounceDeliveryError(delivery);
|
||||
if (ensureDeliveryState(entry).lastError !== latestDeliveryError) {
|
||||
ensureDeliveryState(entry).lastError = latestDeliveryError;
|
||||
params.persist();
|
||||
}
|
||||
},
|
||||
};
|
||||
runDetachedCleanupAttempt({
|
||||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
run: async () => {
|
||||
let didAnnounce = false;
|
||||
try {
|
||||
didAnnounce = await params.runSubagentAnnounceFlow(announceParams);
|
||||
} catch (error) {
|
||||
defaultRuntime.log(
|
||||
`[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
await finalizeAnnounceCleanup(didAnnounce);
|
||||
},
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
@@ -1356,7 +1445,7 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
completionSnapshot?: { resultText: string | null; capturedAt: number };
|
||||
};
|
||||
|
||||
const completeSubagentRun = async (completeParams: CompleteSubagentRunParams) => {
|
||||
const completeSubagentRunAttempt = async (completeParams: CompleteSubagentRunParams) => {
|
||||
const releaseCompletionLock = await acquireTerminalCompletionLock(completeParams.runId);
|
||||
let entry: SubagentRunRecord | undefined;
|
||||
let terminalGeneration = 0;
|
||||
@@ -1843,6 +1932,16 @@ export function createSubagentRegistryLifecycleController(params: {
|
||||
startSubagentAnnounceCleanupFlow(completeParams.runId, entry);
|
||||
};
|
||||
|
||||
const completeSubagentRun = async (completeParams: CompleteSubagentRunParams) => {
|
||||
// Task finalization can make the run disappear from suspension blockers
|
||||
// before browser/MCP retirement and cleanup delivery hand off. Own this
|
||||
// entire transition as an independent root so that boundary stays atomic.
|
||||
// Callers can detach while retaining parent ALS, so nesting is intentional.
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await completeSubagentRunAttempt(completeParams);
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
clearScheduledResumeTimers,
|
||||
completeCleanupBookkeeping,
|
||||
|
||||
@@ -7,6 +7,7 @@ import { getRuntimeConfig } from "../config/config.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { callGateway } from "../gateway/call.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
type DetachedTaskFindResult,
|
||||
@@ -990,18 +991,28 @@ export function createSubagentRunManager(params: {
|
||||
finalizeKilledTask(pending.entry, pending.endedAt);
|
||||
}
|
||||
for (const entry of entriesByChildSessionKey.values()) {
|
||||
void persistSubagentSessionTiming(entry, {
|
||||
isCurrentGeneration: () => currentRunOwnsSession(entry),
|
||||
// Task finalization removes the suspension blocker before these session-owned
|
||||
// writes finish. Join them under one independent root so snapshots stay atomic.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await Promise.all([
|
||||
persistSubagentSessionTiming(entry, {
|
||||
isCurrentGeneration: () => currentRunOwnsSession(entry),
|
||||
}).catch((err: unknown) => {
|
||||
log.warn("failed to persist killed subagent session timing", {
|
||||
err,
|
||||
runId: entry.runId,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
});
|
||||
}),
|
||||
shouldDeleteAttachments(entry) ? safeRemoveAttachmentsDir(entry) : Promise.resolve(),
|
||||
]);
|
||||
}).catch((err: unknown) => {
|
||||
log.warn("failed to persist killed subagent session timing", {
|
||||
log.warn("failed to run killed subagent cleanup tail", {
|
||||
err,
|
||||
runId: entry.runId,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
});
|
||||
});
|
||||
if (shouldDeleteAttachments(entry)) {
|
||||
void safeRemoveAttachmentsDir(entry);
|
||||
}
|
||||
params.completeCleanupBookkeeping({
|
||||
runId: entry.runId,
|
||||
entry,
|
||||
|
||||
@@ -11,6 +11,11 @@ import type {
|
||||
SessionEntryPatchOptions,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import type { AgentEventPayload } from "../infra/agent-events.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
|
||||
import {
|
||||
createRunningTaskRun,
|
||||
@@ -255,6 +260,7 @@ describe("subagent registry seam flow", () => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
vi.clearAllMocks();
|
||||
vi.useFakeTimers();
|
||||
@@ -311,12 +317,172 @@ describe("subagent registry seam flow", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
mod.testing.setDepsForTest();
|
||||
mod.resetSubagentRegistryForTests({ persist: false });
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("keeps a sweeper archive mutation root-admitted until deletion settles", async () => {
|
||||
const now = Date.now();
|
||||
let releaseDelete: (() => void) | undefined;
|
||||
mocks.callGateway.mockImplementation((request: { method?: string }) => {
|
||||
if (request.method !== "sessions.delete") {
|
||||
return Promise.resolve({});
|
||||
}
|
||||
return new Promise<Record<string, unknown>>((resolve) => {
|
||||
releaseDelete = () => resolve({});
|
||||
});
|
||||
});
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-sweep-admission",
|
||||
childSessionKey: "agent:main:subagent:sweep-admission",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "archive while suspension is possible",
|
||||
cleanup: "delete",
|
||||
createdAt: now - 10_000,
|
||||
endedAt: now - 5_000,
|
||||
cleanupCompletedAt: now - 4_000,
|
||||
archiveAtMs: now - 1,
|
||||
});
|
||||
|
||||
const sweep = mod.testing.runSweeperTickForTests();
|
||||
await waitForFast(() => expect(releaseDelete).toBeTypeOf("function"));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
releaseDelete?.();
|
||||
await sweep;
|
||||
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("tracks missing-entry lifecycle result refresh until capture and persistence settle", async () => {
|
||||
const childSessionKey = "agent:main:subagent:refresh-admission";
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) =>
|
||||
request.method === "agent.wait" ? { status: "pending" } : {},
|
||||
);
|
||||
mod.registerSubagentRun({
|
||||
runId: "run-refresh-admission-old",
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "capture replacement completion",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: true,
|
||||
});
|
||||
await waitForFast(() => expect(mocks.callGateway).toHaveBeenCalled());
|
||||
const entry = mod.getSubagentRunByChildSessionKey(childSessionKey);
|
||||
expect(entry).not.toBeNull();
|
||||
Object.assign(entry ?? {}, {
|
||||
endedAt: Date.now(),
|
||||
outcome: { status: "ok" },
|
||||
});
|
||||
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
|
||||
let finishCapture: ((value: string) => void) | undefined;
|
||||
mocks.captureSubagentCompletionReply.mockImplementationOnce(
|
||||
async () =>
|
||||
await new Promise<string>((resolve) => {
|
||||
finishCapture = resolve;
|
||||
}),
|
||||
);
|
||||
mocks.persistSubagentRunsToDisk.mockClear();
|
||||
const lifecycleHandler = mocks.onAgentEvent.mock.calls.at(-1)?.[0];
|
||||
expect(lifecycleHandler).toBeTypeOf("function");
|
||||
|
||||
lifecycleHandler?.({
|
||||
runId: "run-refresh-admission-new",
|
||||
seq: 1,
|
||||
stream: "lifecycle",
|
||||
ts: Date.now(),
|
||||
sessionKey: childSessionKey,
|
||||
data: { phase: "end" },
|
||||
});
|
||||
|
||||
await waitForFast(() => expect(finishCapture).toBeTypeOf("function"));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
expect(entry?.completion?.resultText).toBeUndefined();
|
||||
|
||||
finishCapture?.("replacement final reply");
|
||||
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
expect(entry?.completion?.resultText).toBe("replacement final reply");
|
||||
expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retries a terminal completion deferred by restart drain", async () => {
|
||||
const now = Date.now();
|
||||
const runId = "run-terminal-restart-retry";
|
||||
mod.addSubagentRunForTests({
|
||||
runId,
|
||||
childSessionKey: "agent:main:subagent:terminal-restart-retry",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "deliver terminal completion after restart",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: true,
|
||||
createdAt: now - 10_000,
|
||||
startedAt: now - 9_000,
|
||||
endedAt: now - 1_000,
|
||||
endedReason: SUBAGENT_ENDED_REASON_ERROR,
|
||||
outcome: { status: "error", error: "provider interrupted" },
|
||||
});
|
||||
|
||||
markGatewayRestartDraining();
|
||||
await expect(
|
||||
mod.finalizeInterruptedSubagentRun({
|
||||
runId,
|
||||
error: "provider interrupted",
|
||||
endedAt: now - 1_000,
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
|
||||
resetGatewayWorkAdmission();
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledOnce());
|
||||
await waitForFast(() => {
|
||||
const entry = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((candidate) => candidate.runId === runId);
|
||||
expect(entry?.cleanupCompletedAt).toBeTypeOf("number");
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps killed session timing root-admitted after task finalization", async () => {
|
||||
let finishTiming: (() => void) | undefined;
|
||||
mocks.patchSessionEntry.mockImplementationOnce(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
finishTiming = resolve;
|
||||
});
|
||||
return null;
|
||||
});
|
||||
mocks.callGateway.mockImplementation(async (request: { method?: string }) =>
|
||||
request.method === "agent.wait" ? { status: "pending" } : {},
|
||||
);
|
||||
const runId = "run-kill-tail-admission";
|
||||
mod.registerSubagentRun({
|
||||
runId,
|
||||
childSessionKey: "agent:main:subagent:child",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "persist killed session state",
|
||||
cleanup: "keep",
|
||||
spawnMode: "session",
|
||||
});
|
||||
await waitForFast(() => expect(mocks.callGateway).toHaveBeenCalled());
|
||||
|
||||
expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1);
|
||||
await waitForFast(() => expect(finishTiming).toBeTypeOf("function"));
|
||||
expect(findTaskByRunIdForStatus(runId)).toMatchObject({ status: "cancelled" });
|
||||
// Bookkeeping can launch additional tracked cleanup tails; the held timing
|
||||
// write must keep at least one independent root visible until it settles.
|
||||
expect(getActiveGatewayRootWorkCount()).toBeGreaterThan(0);
|
||||
|
||||
finishTiming?.();
|
||||
await waitForFast(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("lists active and pending-delivery child sessions for maintenance preservation", () => {
|
||||
const now = Date.now();
|
||||
mod.addSubagentRunForTests({
|
||||
|
||||
+147
-35
@@ -11,6 +11,10 @@ import type { ContextEngine, SubagentEndReason } from "../context-engine/types.j
|
||||
import { callGateway } from "../gateway/call.js";
|
||||
import { getAgentRunContext, onAgentEvent } from "../infra/agent-events.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
formatAbandonedLivenessError,
|
||||
formatBlockedLivenessError,
|
||||
@@ -225,6 +229,7 @@ let restoreAttempted = false;
|
||||
const ORPHAN_RECOVERY_DEBOUNCE_MS = 1_000;
|
||||
let lastOrphanRecoveryScheduleAt = 0;
|
||||
const SUBAGENT_ANNOUNCE_TIMEOUT_MS = 120_000;
|
||||
const GATEWAY_ADMISSION_RETRY_DELAY_MS = 1_000;
|
||||
/**
|
||||
* Embedded runs can emit transient lifecycle `error` events while provider/model
|
||||
* retry is still in progress. Defer terminal error cleanup briefly so a
|
||||
@@ -379,6 +384,8 @@ export function scheduleSubagentOrphanRecovery(params?: { delayMs?: number; maxR
|
||||
lastOrphanRecoveryScheduleAt = now;
|
||||
void import("./subagent-orphan-recovery.js").then(
|
||||
({ scheduleOrphanRecovery }) => {
|
||||
// This import only installs timers. Each delayed or retrying recovery
|
||||
// attempt owns independent root admission inside the recovery module.
|
||||
scheduleOrphanRecovery({
|
||||
getActiveRuns: () => subagentRuns,
|
||||
delayMs: params?.delayMs,
|
||||
@@ -455,7 +462,10 @@ type CompleteSubagentRunParams = {
|
||||
suppressSessionEffects?: boolean;
|
||||
};
|
||||
|
||||
async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) {
|
||||
async function completeSubagentRunWithRecoveryAttempt(
|
||||
params: CompleteSubagentRunParams,
|
||||
source: string,
|
||||
) {
|
||||
try {
|
||||
await completeSubagentRun(params);
|
||||
return;
|
||||
@@ -506,6 +516,52 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams
|
||||
resumeSubagentRun(params.runId);
|
||||
}
|
||||
|
||||
function scheduleSubagentCompletionRetryAfterRestart(
|
||||
params: CompleteSubagentRunParams,
|
||||
source: string,
|
||||
expectedEntry: SubagentRunRecord,
|
||||
) {
|
||||
const expectedGeneration = expectedEntry.generation;
|
||||
const timer = setTimeout(() => {
|
||||
resumeRetryTimers.delete(timer);
|
||||
const current = subagentRuns.get(params.runId);
|
||||
if (current !== expectedEntry || current.generation !== expectedGeneration) {
|
||||
return;
|
||||
}
|
||||
void completeSubagentRunWithRecovery(params, source).catch((error: unknown) => {
|
||||
log.warn("failed to retry subagent completion after gateway restart", {
|
||||
source,
|
||||
runId: params.runId,
|
||||
error,
|
||||
});
|
||||
});
|
||||
}, GATEWAY_ADMISSION_RETRY_DELAY_MS);
|
||||
timer.unref?.();
|
||||
resumeRetryTimers.add(timer);
|
||||
}
|
||||
|
||||
async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) {
|
||||
// Each controller attempt owns its terminal transition, while this outer
|
||||
// lease closes the gap between failed attempts and fallback cleanup.
|
||||
try {
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await completeSubagentRunWithRecoveryAttempt(params, source);
|
||||
});
|
||||
} catch (error) {
|
||||
if (!isGatewayRestartDraining()) {
|
||||
throw error;
|
||||
}
|
||||
log.warn("subagent completion deferred during gateway restart", {
|
||||
source,
|
||||
runId: params.runId,
|
||||
});
|
||||
const current = subagentRuns.get(params.runId);
|
||||
if (current) {
|
||||
scheduleSubagentCompletionRetryAfterRestart(params, source, current);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function completeSubagentRunInBackground(params: CompleteSubagentRunParams, source: string) {
|
||||
void completeSubagentRunWithRecovery(params, source);
|
||||
}
|
||||
@@ -717,6 +773,61 @@ const {
|
||||
startSubagentAnnounceCleanupFlow,
|
||||
} = subagentLifecycleController;
|
||||
|
||||
function scheduleSubagentDeliveryResumeRetry(
|
||||
runId: string,
|
||||
scheduledEntry: SubagentRunRecord,
|
||||
waitMs: number,
|
||||
) {
|
||||
const timer = setTimeout(() => {
|
||||
resumeRetryTimers.delete(timer);
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (subagentRuns.get(runId) !== scheduledEntry) {
|
||||
resumedRuns.delete(runId);
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
resumeSubagentRun(runId);
|
||||
}).catch((error: unknown) => {
|
||||
log.warn("failed to resume subagent delivery retry", { runId, error });
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
subagentRuns.get(runId) === scheduledEntry &&
|
||||
typeof scheduledEntry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(
|
||||
runId,
|
||||
scheduledEntry,
|
||||
Math.max(waitMs, GATEWAY_ADMISSION_RETRY_DELAY_MS),
|
||||
);
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
});
|
||||
}, waitMs);
|
||||
timer.unref?.();
|
||||
resumeRetryTimers.add(timer);
|
||||
}
|
||||
|
||||
function finalizeResumedAnnounceGiveUpInBackground(
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
reason: "retry-limit" | "expiry",
|
||||
) {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await finalizeResumedAnnounceGiveUp({ runId, entry, reason });
|
||||
}).catch((error: unknown) => {
|
||||
log.warn("failed to finalize exhausted subagent delivery", { runId, reason, error });
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
subagentRuns.get(runId) === entry &&
|
||||
typeof entry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(runId, entry, GATEWAY_ADMISSION_RETRY_DELAY_MS);
|
||||
resumedRuns.add(runId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function resumeSubagentRun(runId: string) {
|
||||
if (!runId || resumedRuns.has(runId)) {
|
||||
return;
|
||||
@@ -738,11 +849,7 @@ function resumeSubagentRun(runId: string) {
|
||||
}
|
||||
// Skip entries that have exhausted their retry budget or expired (#18264).
|
||||
if (getDeliveryAttemptCount(entry) >= MAX_ANNOUNCE_RETRY_COUNT) {
|
||||
void finalizeResumedAnnounceGiveUp({
|
||||
runId,
|
||||
entry,
|
||||
reason: "retry-limit",
|
||||
});
|
||||
finalizeResumedAnnounceGiveUpInBackground(runId, entry, "retry-limit");
|
||||
return;
|
||||
}
|
||||
if (
|
||||
@@ -750,11 +857,7 @@ function resumeSubagentRun(runId: string) {
|
||||
typeof entry.endedAt === "number" &&
|
||||
Date.now() - entry.endedAt > ANNOUNCE_EXPIRY_MS
|
||||
) {
|
||||
void finalizeResumedAnnounceGiveUp({
|
||||
runId,
|
||||
entry,
|
||||
reason: "expiry",
|
||||
});
|
||||
finalizeResumedAnnounceGiveUpInBackground(runId, entry, "expiry");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -764,17 +867,7 @@ function resumeSubagentRun(runId: string) {
|
||||
const earliestRetryAt = (lastAttemptAt ?? 0) + delayMs;
|
||||
if (entry.expectsCompletionMessage === true && lastAttemptAt && now < earliestRetryAt) {
|
||||
const waitMs = Math.max(1, earliestRetryAt - now);
|
||||
const scheduledEntry = entry;
|
||||
const timer = setTimeout(() => {
|
||||
resumeRetryTimers.delete(timer);
|
||||
if (subagentRuns.get(runId) !== scheduledEntry) {
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
resumeSubagentRun(runId);
|
||||
}, waitMs);
|
||||
timer.unref?.();
|
||||
resumeRetryTimers.add(timer);
|
||||
scheduleSubagentDeliveryResumeRetry(runId, entry, waitMs);
|
||||
resumedRuns.add(runId);
|
||||
return;
|
||||
}
|
||||
@@ -884,12 +977,20 @@ function startSweeper() {
|
||||
|
||||
async function runSubagentSweep() {
|
||||
try {
|
||||
await sweepSubagentRuns();
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await sweepSubagentRuns();
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn(`subagent run sweep failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function runSubagentSweepCleanupTail(runId: string, label: string, run: () => Promise<unknown>) {
|
||||
void runWithGatewayIndependentRootWorkAdmission(run).catch((error: unknown) => {
|
||||
log.warn(`subagent sweep ${label} failed`, { runId, error });
|
||||
});
|
||||
}
|
||||
|
||||
function stopSweeper() {
|
||||
if (!sweeper) {
|
||||
return;
|
||||
@@ -1342,11 +1443,13 @@ async function sweepSubagentRuns() {
|
||||
now - entry.cleanupCompletedAt > SESSION_RUN_TTL_MS
|
||||
) {
|
||||
clearPendingLifecycleError(runId);
|
||||
void notifyContextEngineSubagentEnded({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
reason: "swept",
|
||||
agentDir: entry.agentDir,
|
||||
workspaceDir: entry.workspaceDir,
|
||||
runSubagentSweepCleanupTail(runId, "context-engine cleanup", async () => {
|
||||
await notifyContextEngineSubagentEnded({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
reason: "swept",
|
||||
agentDir: entry.agentDir,
|
||||
workspaceDir: entry.workspaceDir,
|
||||
});
|
||||
});
|
||||
subagentRuns.delete(runId);
|
||||
mutated = true;
|
||||
@@ -1382,11 +1485,13 @@ async function sweepSubagentRuns() {
|
||||
mutated = true;
|
||||
// Archive/purge is terminal for the run record; remove any retained attachments too.
|
||||
await safeRemoveAttachmentsDir(entry);
|
||||
void notifyContextEngineSubagentEnded({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
reason: "swept",
|
||||
agentDir: entry.agentDir,
|
||||
workspaceDir: entry.workspaceDir,
|
||||
runSubagentSweepCleanupTail(runId, "context-engine cleanup", async () => {
|
||||
await notifyContextEngineSubagentEnded({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
reason: "swept",
|
||||
agentDir: entry.agentDir,
|
||||
workspaceDir: entry.workspaceDir,
|
||||
});
|
||||
});
|
||||
}
|
||||
// Sweep orphaned pendingLifecycleError entries (absolute TTL).
|
||||
@@ -1426,7 +1531,12 @@ function ensureListener() {
|
||||
const entry = subagentRuns.get(evt.runId);
|
||||
if (!entry) {
|
||||
if (phase === "end" && typeof evt.sessionKey === "string") {
|
||||
await refreshFrozenResultFromSession(evt.sessionKey);
|
||||
const sessionKey = evt.sessionKey;
|
||||
// A replacement generation can finish after its predecessor row is
|
||||
// terminal. Keep capture + persistence inside the suspension fence.
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await refreshFrozenResultFromSession(sessionKey);
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -1573,7 +1683,9 @@ const subagentRunManager = createSubagentRunManager({
|
||||
resolveSubagentSessionStartedAt,
|
||||
notifyContextEngineSubagentEnded,
|
||||
completeCleanupBookkeeping,
|
||||
completeSubagentRun,
|
||||
completeSubagentRun: async (params) => {
|
||||
await completeSubagentRunWithRecovery(params, "subagent-wait");
|
||||
},
|
||||
resolveSubagentTask: findSubagentTaskForRun,
|
||||
});
|
||||
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
// Tests queue drain restart behavior when follow-up runs chain together.
|
||||
import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
isGatewaySubordinateWorkAdmissionClosed,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../test-utils/env.js";
|
||||
import type { FollowupRun, QueueSettings } from "./queue.js";
|
||||
import {
|
||||
clearFollowupQueue,
|
||||
clearSessionQueues,
|
||||
enqueueFollowupRun,
|
||||
FollowupRunDeferredError,
|
||||
scheduleFollowupDrain,
|
||||
@@ -18,6 +27,81 @@ import { getExistingFollowupQueue } from "./queue/state.js";
|
||||
installQueueRuntimeErrorSilencer();
|
||||
|
||||
describe("followup queue drain restart after idle window", () => {
|
||||
it("keeps a detached drain on a live root after its enqueue request returns", async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
const key = `test-detached-drain-root-${Date.now()}`;
|
||||
const settings: QueueSettings = { mode: "followup", debounceMs: 0, cap: 50 };
|
||||
const parentReleased = createDeferred<void>();
|
||||
const drained = createDeferred<void>();
|
||||
const parent = tryBeginGatewayRootWorkAdmission();
|
||||
if (!parent) {
|
||||
throw new Error("expected parent Gateway work admission");
|
||||
}
|
||||
let suspensionStarted = false;
|
||||
let subordinateAdmissionClosed: boolean | undefined;
|
||||
let activeRootCountDuringDrain: number | undefined;
|
||||
|
||||
try {
|
||||
await parent.run(async () => {
|
||||
enqueueFollowupRun(key, createRun({ prompt: "detached" }), settings);
|
||||
scheduleFollowupDrain(key, async () => {
|
||||
await parentReleased.promise;
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
suspensionStarted = suspension !== null;
|
||||
try {
|
||||
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
|
||||
activeRootCountDuringDrain = getActiveGatewayRootWorkCount();
|
||||
} finally {
|
||||
suspension?.rollback();
|
||||
drained.resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
parent.release();
|
||||
parentReleased.resolve();
|
||||
await drained.promise;
|
||||
|
||||
expect(suspensionStarted).toBe(true);
|
||||
expect(subordinateAdmissionClosed).toBe(false);
|
||||
expect(activeRootCountDuringDrain).toBe(1);
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
} finally {
|
||||
parent.release();
|
||||
parentReleased.resolve();
|
||||
clearSessionQueues([key]);
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("releases a detached drain root when its queue is cleared during debounce", async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
const env = captureEnv(["OPENCLAW_TEST_FAST"]);
|
||||
setTestEnvValue("OPENCLAW_TEST_FAST", "0");
|
||||
const key = `test-cleared-debounce-root-${Date.now()}`;
|
||||
const settings: QueueSettings = { mode: "followup", debounceMs: 60_000, cap: 50 };
|
||||
|
||||
try {
|
||||
enqueueFollowupRun(key, createRun({ prompt: "clear during debounce" }), settings);
|
||||
scheduleFollowupDrain(key, async () => {});
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
});
|
||||
|
||||
clearSessionQueues([key]);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
} finally {
|
||||
clearSessionQueues([key]);
|
||||
env.restore();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retain stale callbacks when scheduleFollowupDrain runs with an empty queue", async () => {
|
||||
const key = `test-no-stale-callback-${Date.now()}`;
|
||||
const settings: QueueSettings = { mode: "followup", debounceMs: 0, cap: 50 };
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
channelRouteCompactKey,
|
||||
channelRouteDedupeKey,
|
||||
} from "../../../plugin-sdk/channel-route.js";
|
||||
import { runWithGatewayIndependentRootWorkContinuation } from "../../../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
import {
|
||||
buildPersistedUserTurnMediaInputsFromFields,
|
||||
@@ -1129,7 +1130,9 @@ export function scheduleFollowupDrain(
|
||||
// Cache callback only when a drain actually starts. Avoid keeping stale
|
||||
// callbacks around from finalize calls where no queue work is pending.
|
||||
rememberFollowupDrainCallback(key, effectiveRunFollowup);
|
||||
void (async () => {
|
||||
// Queue drains outlive their enqueue request across debounce and retries.
|
||||
// Give the detached chain its own root so inherited request admission cannot go stale.
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
let retryDeferred = false;
|
||||
try {
|
||||
const collectState = { forceIndividualCollect: false };
|
||||
@@ -1138,7 +1141,7 @@ export function scheduleFollowupDrain(
|
||||
if (queue.items.length === 0 && queue.droppedCount === 0) {
|
||||
break;
|
||||
}
|
||||
await waitForQueueDebounce(queue);
|
||||
await waitForQueueDebounce(queue, queue.abortController.signal);
|
||||
await dropAbortedFollowups(queue.items, effectiveRunFollowup);
|
||||
if (queue.items.length === 0 && queue.droppedCount === 0) {
|
||||
break;
|
||||
@@ -1364,5 +1367,8 @@ export function scheduleFollowupDrain(
|
||||
scheduleFollowupDrain(key, effectiveRunFollowup);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}).catch((err: unknown) => {
|
||||
queue.draining = false;
|
||||
defaultRuntime.error?.(`followup queue drain admission failed for ${key}: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -5,6 +5,12 @@ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi }
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import type { HookRunner } from "../../plugins/hooks.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
import { createSuiteTempRootTracker } from "../../test-helpers/temp-dir.js";
|
||||
import { initSessionState } from "./session.js";
|
||||
|
||||
@@ -176,10 +182,12 @@ describe("session hook context wiring", () => {
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
hookRunnerMocks.hasHooks.mockReset();
|
||||
hookRunnerMocks.runSessionStart.mockReset();
|
||||
hookRunnerMocks.runSessionEnd.mockReset();
|
||||
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockClear();
|
||||
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockResolvedValue(0);
|
||||
sessionCleanupMocks.resetRegisteredAgentHarnessSessions.mockClear();
|
||||
sessionCleanupMocks.retireSessionMcpRuntime.mockClear();
|
||||
hookRunnerMocks.runSessionStart.mockResolvedValue(undefined);
|
||||
@@ -190,6 +198,7 @@ describe("session hook context wiring", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
@@ -246,6 +255,85 @@ describe("session hook context wiring", () => {
|
||||
expectFields(startContext, { sessionId: startEvent?.sessionId });
|
||||
});
|
||||
|
||||
it("keeps rollover hooks and browser cleanup root-admitted until they settle", async () => {
|
||||
const releases: Array<() => void> = [];
|
||||
const held = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
hookRunnerMocks.runSessionEnd.mockImplementationOnce(held);
|
||||
hookRunnerMocks.runSessionStart.mockImplementationOnce(held);
|
||||
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
releases.push(() => resolve(0));
|
||||
}),
|
||||
);
|
||||
const sessionKey = "agent:main:telegram:direct:held-rollover";
|
||||
const { storePath } = await createStoredSession({
|
||||
prefix: "openclaw-session-hook-held-rollover",
|
||||
sessionKey,
|
||||
sessionId: "old-held-session",
|
||||
});
|
||||
|
||||
await initSessionState({
|
||||
ctx: { Body: "/new", SessionKey: sessionKey },
|
||||
cfg: { session: { store: storePath } } as OpenClawConfig,
|
||||
commandAuthorized: true,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(3));
|
||||
await vi.waitFor(() => expect(releases).toHaveLength(3));
|
||||
for (const release of releases) {
|
||||
release();
|
||||
}
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("hands rollover hooks off after restart drain closes admission", async () => {
|
||||
const releases: Array<() => void> = [];
|
||||
const held = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
hookRunnerMocks.runSessionEnd.mockImplementationOnce(held);
|
||||
hookRunnerMocks.runSessionStart.mockImplementationOnce(held);
|
||||
sessionCleanupMocks.closeTrackedBrowserTabsForSessions.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
releases.push(() => resolve(0));
|
||||
}),
|
||||
);
|
||||
const sessionKey = "agent:main:telegram:direct:restart-handoff";
|
||||
const { storePath } = await createStoredSession({
|
||||
prefix: "openclaw-session-hook-restart-handoff",
|
||||
sessionKey,
|
||||
sessionId: "old-restart-session",
|
||||
});
|
||||
const admission = tryBeginGatewayRootWorkAdmission();
|
||||
expect(admission).not.toBeNull();
|
||||
|
||||
await admission?.run(async () => {
|
||||
markGatewayRestartDraining();
|
||||
await initSessionState({
|
||||
ctx: { Body: "/new", SessionKey: sessionKey },
|
||||
cfg: { session: { store: storePath } } as OpenClawConfig,
|
||||
commandAuthorized: true,
|
||||
});
|
||||
await vi.waitFor(() => expect(releases).toHaveLength(3));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(4);
|
||||
});
|
||||
|
||||
admission?.release();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(3);
|
||||
for (const release of releases) {
|
||||
release();
|
||||
}
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
|
||||
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("marks explicit /reset rollovers with reason reset", async () => {
|
||||
const sessionKey = "agent:main:telegram:direct:456";
|
||||
const { storePath } = await createStoredSession({
|
||||
|
||||
@@ -6,6 +6,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { loadSessionStore, type SessionEntry } from "../../config/sessions.js";
|
||||
import type { HookRunner } from "../../plugins/hooks.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
|
||||
const hookRunnerMocks = vi.hoisted(() => ({
|
||||
hasHooks: vi.fn<HookRunner["hasHooks"]>(),
|
||||
@@ -46,6 +52,7 @@ function firstSessionStartCall() {
|
||||
|
||||
describe("session-updates lifecycle hooks", () => {
|
||||
beforeEach(async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.resetModules();
|
||||
vi.doMock("../../plugins/hook-runner-global.js", () => ({
|
||||
getGlobalHookRunner: () =>
|
||||
@@ -67,6 +74,7 @@ describe("session-updates lifecycle hooks", () => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.restoreAllMocks();
|
||||
await Promise.all(
|
||||
tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })),
|
||||
@@ -109,6 +117,69 @@ describe("session-updates lifecycle hooks", () => {
|
||||
expect(startContext?.agentId).toBe("main");
|
||||
});
|
||||
|
||||
it("keeps compaction lifecycle hooks root-admitted until both settle", async () => {
|
||||
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
||||
const releases: Array<() => void> = [];
|
||||
const heldHook = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
|
||||
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
|
||||
|
||||
await incrementCompactionCount({
|
||||
cfg: { session: { store: storePath } } as OpenClawConfig,
|
||||
sessionEntry: entry,
|
||||
sessionStore,
|
||||
sessionKey,
|
||||
storePath,
|
||||
newSessionId: "s2",
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(2));
|
||||
await vi.waitFor(() => expect(releases).toHaveLength(2));
|
||||
for (const release of releases) {
|
||||
release();
|
||||
}
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("hands compaction lifecycle hooks off after restart drain closes admission", async () => {
|
||||
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
||||
const releases: Array<() => void> = [];
|
||||
const heldHook = () =>
|
||||
new Promise<void>((resolve) => {
|
||||
releases.push(resolve);
|
||||
});
|
||||
hookRunnerMocks.runSessionEnd.mockImplementationOnce(heldHook);
|
||||
hookRunnerMocks.runSessionStart.mockImplementationOnce(heldHook);
|
||||
const admission = tryBeginGatewayRootWorkAdmission();
|
||||
expect(admission).not.toBeNull();
|
||||
|
||||
await admission?.run(async () => {
|
||||
markGatewayRestartDraining();
|
||||
await incrementCompactionCount({
|
||||
cfg: { session: { store: storePath } } as OpenClawConfig,
|
||||
sessionEntry: entry,
|
||||
sessionStore,
|
||||
sessionKey,
|
||||
storePath,
|
||||
newSessionId: "s2",
|
||||
});
|
||||
await vi.waitFor(() => expect(releases).toHaveLength(2));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(3);
|
||||
});
|
||||
|
||||
admission?.release();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(2);
|
||||
for (const release of releases) {
|
||||
release();
|
||||
}
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
expect(hookRunnerMocks.runSessionEnd).toHaveBeenCalledTimes(1);
|
||||
expect(hookRunnerMocks.runSessionStart).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("recreates a complete persisted row when compaction updates a missing store row", async () => {
|
||||
const { storePath, sessionKey, sessionStore, entry } = await createFixture();
|
||||
await fs.writeFile(storePath, JSON.stringify({}, null, 2), "utf-8");
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
import { resolveStableSessionEndTranscript } from "../../gateway/session-transcript-files.fs.js";
|
||||
import { logVerbose } from "../../globals.js";
|
||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||
import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||
import { getRemoteSkillEligibility } from "../../skills/runtime/remote.js";
|
||||
import { resolveReusableWorkspaceSkillSnapshot } from "../../skills/runtime/session-snapshot.js";
|
||||
@@ -96,7 +97,9 @@ function emitCompactionSessionLifecycleHooks(params: {
|
||||
transcriptArchived: transcript.transcriptArchived,
|
||||
nextSessionId: params.nextEntry.sessionId,
|
||||
});
|
||||
void hookRunner.runSessionEnd(payload.event, payload.context).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionEnd(payload.event, payload.context);
|
||||
}).catch((err: unknown) => {
|
||||
logVerbose(`session_end hook failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
@@ -108,7 +111,9 @@ function emitCompactionSessionLifecycleHooks(params: {
|
||||
cfg: params.cfg,
|
||||
resumedFrom: params.previousEntry.sessionId,
|
||||
});
|
||||
void hookRunner.runSessionStart(payload.event, payload.context).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionStart(payload.event, payload.context);
|
||||
}).catch((err: unknown) => {
|
||||
logVerbose(`session_start hook failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ import { deliverSessionMaintenanceWarning } from "../../infra/session-maintenanc
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
|
||||
import type { PluginHookSessionEndReason } from "../../plugins/hook-types.js";
|
||||
import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gateway-work-admission.js";
|
||||
import {
|
||||
buildAgentMainSessionKey,
|
||||
isAcpSessionKey,
|
||||
@@ -1062,11 +1063,15 @@ async function initSessionStateAttemptLocked(
|
||||
// their transcript aliases main; cleanup must carry both exact keys.
|
||||
const runtimePolicySessionKey =
|
||||
resolveRuntimePolicySessionKey({ cfg, ctx: sessionCtxForState, sessionKey }) ?? sessionKey;
|
||||
void cleanupBrowserSessionsForLifecycleEnd({
|
||||
cfg,
|
||||
sessionKeys: [previousSessionEntry.sessionId, sessionKey, runtimePolicySessionKey],
|
||||
onWarn: (message) => log.warn(message),
|
||||
onError: (error) => log.warn(`browser tab cleanup failed: ${String(error)}`),
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await cleanupBrowserSessionsForLifecycleEnd({
|
||||
cfg,
|
||||
sessionKeys: [previousSessionEntry.sessionId, sessionKey, runtimePolicySessionKey],
|
||||
onWarn: (message) => log.warn(message),
|
||||
onError: (error) => log.warn(`browser tab cleanup failed: ${String(error)}`),
|
||||
});
|
||||
}).catch((error: unknown) => {
|
||||
log.warn(`browser tab cleanup admission failed: ${String(error)}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1108,7 +1113,9 @@ async function initSessionStateAttemptLocked(
|
||||
transcriptArchived: previousSessionTranscript.transcriptArchived,
|
||||
nextSessionId: effectiveSessionId,
|
||||
});
|
||||
void hookRunner.runSessionEnd(payload.event, payload.context).catch(() => {});
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionEnd(payload.event, payload.context);
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1133,7 +1140,9 @@ async function initSessionStateAttemptLocked(
|
||||
cfg,
|
||||
resumedFrom: previousSessionEntry?.sessionId,
|
||||
});
|
||||
void hookRunner.runSessionStart(payload.event, payload.context).catch(() => {});
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionStart(payload.event, payload.context);
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ export {
|
||||
markGatewaySigusr1RestartHandled,
|
||||
peekGatewaySigusr1RestartReason,
|
||||
resetGatewayRestartStateForInProcessRestart,
|
||||
rollbackGatewayRestartSignalAdmission,
|
||||
scheduleGatewaySigusr1Restart,
|
||||
} from "../../infra/restart.js";
|
||||
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
|
||||
|
||||
@@ -19,6 +19,7 @@ const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false);
|
||||
const markGatewaySigusr1RestartHandled = vi.fn();
|
||||
const peekGatewaySigusr1RestartReason = vi.fn<() => string | undefined>(() => undefined);
|
||||
const resetGatewayRestartStateForInProcessRestart = vi.fn();
|
||||
const rollbackGatewayRestartSignalAdmission = vi.fn();
|
||||
const writeGatewayRestartHandoffSync = vi.fn((_opts: unknown) => ({
|
||||
kind: "gateway-supervisor-restart-handoff" as const,
|
||||
version: 1 as const,
|
||||
@@ -129,6 +130,7 @@ vi.mock("../../infra/restart.js", () => ({
|
||||
markGatewaySigusr1RestartHandled: () => markGatewaySigusr1RestartHandled(),
|
||||
peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(),
|
||||
resetGatewayRestartStateForInProcessRestart: () => resetGatewayRestartStateForInProcessRestart(),
|
||||
rollbackGatewayRestartSignalAdmission: () => rollbackGatewayRestartSignalAdmission(),
|
||||
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
|
||||
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
|
||||
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
|
||||
@@ -492,6 +494,9 @@ describe("runGatewayLoop", () => {
|
||||
|
||||
expect(consumeGatewayRestartIntentPayloadSync).toHaveBeenCalledOnce();
|
||||
expect(markGatewayDraining).toHaveBeenCalledOnce();
|
||||
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
loadConfig.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY,
|
||||
);
|
||||
expect(waitForActiveTasks).toHaveBeenCalledWith(90_000);
|
||||
expectRestartCloseCall(closeFirst, 90_000);
|
||||
await startedSecond;
|
||||
@@ -898,6 +903,9 @@ describe("runGatewayLoop", () => {
|
||||
|
||||
await startedSecond;
|
||||
expect(start).toHaveBeenCalledTimes(2);
|
||||
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
markGatewaySigusr1RestartHandled.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
@@ -1477,6 +1485,9 @@ describe("runGatewayLoop", () => {
|
||||
|
||||
expect(consumeGatewaySigusr1RestartAuthorization).toHaveBeenCalledOnce();
|
||||
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalledOnce();
|
||||
expect(markGatewayDraining.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
markGatewaySigusr1RestartHandled.mock.invocationCallOrder[0] ?? 0,
|
||||
);
|
||||
expect(markRestartAbortedMainSessions).toHaveBeenCalledWith({
|
||||
cfg: {
|
||||
gateway: {
|
||||
@@ -2028,6 +2039,7 @@ describe("runGatewayLoop", () => {
|
||||
"SIGUSR1 handler failed: lifecycle module corrupted",
|
||||
);
|
||||
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalled();
|
||||
expect(rollbackGatewayRestartSignalAdmission).toHaveBeenCalledOnce();
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
|
||||
@@ -2063,6 +2075,7 @@ describe("runGatewayLoop", () => {
|
||||
// Restart token must be cleared so future SIGUSR1 restarts are not
|
||||
// permanently coalesced as "already in-flight".
|
||||
expect(markGatewaySigusr1RestartHandled).toHaveBeenCalled();
|
||||
expect(rollbackGatewayRestartSignalAdmission).toHaveBeenCalledOnce();
|
||||
expect(close).not.toHaveBeenCalled();
|
||||
expect(start).toHaveBeenCalledTimes(1);
|
||||
|
||||
@@ -2070,6 +2083,37 @@ describe("runGatewayLoop", () => {
|
||||
await expect(exited).resolves.toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("recloses restart admission after a failed SIGUSR1 handler rolls it back", async () => {
|
||||
vi.clearAllMocks();
|
||||
markGatewaySigusr1RestartHandled.mockImplementationOnce(() => {
|
||||
throw new Error("restart token cleanup failed");
|
||||
});
|
||||
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { close, start, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
const sigterm = captureSignal("SIGTERM");
|
||||
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => rollbackGatewayRestartSignalAdmission.mock.calls.length === 1,
|
||||
"failed SIGUSR1 handler did not roll back restart admission",
|
||||
);
|
||||
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => start.mock.calls.length === 2,
|
||||
"second SIGUSR1 did not start a restart",
|
||||
);
|
||||
|
||||
expect(close).toHaveBeenCalledTimes(1);
|
||||
expect(markGatewayDraining).toHaveBeenCalledTimes(2);
|
||||
|
||||
sigterm();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("gateway discover routing helpers", () => {
|
||||
|
||||
@@ -146,7 +146,7 @@ export async function runGatewayLoop(params: {
|
||||
let activeRestartRequest: GatewayRunSignalRequest | null = null;
|
||||
let forceActiveRestartExit: (() => void) | null = null;
|
||||
let pendingStartupForceExitTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let restartDrainingMarkPromise: Promise<void> | null = null;
|
||||
let restartDrainingMarked = false;
|
||||
let startupFailedWithoutServerHandle = false;
|
||||
const processInstanceId = randomUUID();
|
||||
const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild;
|
||||
@@ -408,17 +408,15 @@ export async function runGatewayLoop(params: {
|
||||
return DEFAULT_RESTART_DRAIN_TIMEOUT_MS;
|
||||
}
|
||||
};
|
||||
const markRestartDraining = async () => {
|
||||
if (!restartDrainingMarkPromise) {
|
||||
restartDrainingMarkPromise = (async () => {
|
||||
const { markGatewayDraining } = await loadGatewayLifecycleRuntimeModule();
|
||||
markGatewayDraining();
|
||||
})().catch((err: unknown) => {
|
||||
restartDrainingMarkPromise = null;
|
||||
throw err;
|
||||
});
|
||||
const markRestartDraining = () => {
|
||||
if (restartDrainingMarked) {
|
||||
return;
|
||||
}
|
||||
await restartDrainingMarkPromise;
|
||||
// The lifecycle module is primed before listeners are installed. Keep this
|
||||
// transition synchronous so an accepted signal cannot yield between token
|
||||
// handling and closing process-wide root admission.
|
||||
eagerLifecycleRuntime.markGatewayDraining();
|
||||
restartDrainingMarked = true;
|
||||
};
|
||||
|
||||
const runAcceptedRequest = (acceptedRequest: GatewayRunSignalRequest) => {
|
||||
@@ -572,7 +570,7 @@ export async function runGatewayLoop(params: {
|
||||
|
||||
// Reject new enqueues immediately during the drain window so
|
||||
// sessions get an explicit restart error instead of silent task loss.
|
||||
await markRestartDraining();
|
||||
markRestartDraining();
|
||||
const activeTasks = getActiveTaskCount();
|
||||
const activeRuns = getActiveEmbeddedRunCount();
|
||||
activeTasksAtDrainStart = activeTasks;
|
||||
@@ -736,8 +734,11 @@ export async function runGatewayLoop(params: {
|
||||
gatewayLog.info(`received ${signal} during shutdown; ignoring`);
|
||||
return;
|
||||
}
|
||||
shuttingDown = true;
|
||||
const isRestart = action === "restart";
|
||||
if (isRestart) {
|
||||
markRestartDraining();
|
||||
}
|
||||
shuttingDown = true;
|
||||
gatewayLog.info(`received ${signal}; ${isRestart ? "restarting" : "shutting down"}`);
|
||||
if (isRestart) {
|
||||
startGatewayRestartTrace("restart.signal.received", [
|
||||
@@ -758,9 +759,6 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
if (!server || !restartResolver) {
|
||||
pendingStartupRequest = acceptedRequest;
|
||||
void markRestartDraining().catch((err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark gateway draining for startup restart: ${String(err)}`);
|
||||
});
|
||||
armPendingStartupForceExitTimer();
|
||||
return;
|
||||
}
|
||||
@@ -803,7 +801,9 @@ export async function runGatewayLoop(params: {
|
||||
const restartIntent = consumeGatewayRestartIntentPayloadSync();
|
||||
if (restartIntent) {
|
||||
abortPendingChannelReloads();
|
||||
if (consumeGatewaySigusr1RestartAuthorization()) {
|
||||
const authorized = consumeGatewaySigusr1RestartAuthorization();
|
||||
markRestartDraining();
|
||||
if (authorized) {
|
||||
markGatewaySigusr1RestartHandled();
|
||||
}
|
||||
request("restart", "SIGUSR1", restartIntent.reason ?? "gateway.restart", restartIntent);
|
||||
@@ -836,6 +836,7 @@ export async function runGatewayLoop(params: {
|
||||
abortPendingChannelReloads();
|
||||
const sigusr1RestartIntent = consumeGatewaySigusr1RestartIntent();
|
||||
const restartReason = peekGatewaySigusr1RestartReason();
|
||||
markRestartDraining();
|
||||
markGatewaySigusr1RestartHandled();
|
||||
request(
|
||||
"restart",
|
||||
@@ -856,6 +857,14 @@ export async function runGatewayLoop(params: {
|
||||
} catch {
|
||||
// Best-effort: the eager reference itself is the recovery path.
|
||||
}
|
||||
try {
|
||||
eagerLifecycleRuntime.rollbackGatewayRestartSignalAdmission();
|
||||
// A later signal must repeat the synchronous close transition even if
|
||||
// this handler failed after marking the one-way drain.
|
||||
restartDrainingMarked = false;
|
||||
} catch {
|
||||
// Keep admission recovery independent from restart-token recovery.
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
@@ -908,7 +917,7 @@ export async function runGatewayLoop(params: {
|
||||
let isFirstStart = true;
|
||||
for (;;) {
|
||||
await onIteration();
|
||||
restartDrainingMarkPromise = null;
|
||||
restartDrainingMarked = false;
|
||||
startupStartedAt = Date.now();
|
||||
let startupFailedBeforeServerHandle = false;
|
||||
try {
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
// Every-job firing tests cover repeated schedule execution semantics.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getGatewaySuspendStatus,
|
||||
prepareGatewaySuspend,
|
||||
resetGatewaySuspendCoordinatorForTest,
|
||||
} from "../infra/gateway-suspend-coordinator.js";
|
||||
import {
|
||||
beginGatewayRestartSignalAdmission,
|
||||
isGatewayWorkAdmissionClosed,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { CronService } from "./service.js";
|
||||
import {
|
||||
createStartedCronServiceWithFinishedBarrier,
|
||||
@@ -101,6 +111,156 @@ describe("CronService interval/cron jobs fire on time", () => {
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("keeps a due timer frozen while scheduling is paused and fires it after resume", async () => {
|
||||
const store = await makeStorePath();
|
||||
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
|
||||
storePath: store.storePath,
|
||||
logger: noopLogger,
|
||||
});
|
||||
|
||||
await cron.start();
|
||||
const job = await cron.add({
|
||||
name: "suspension pause check",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 10_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "resumed-tick" },
|
||||
});
|
||||
|
||||
cron.pauseScheduling();
|
||||
await vi.advanceTimersByTimeAsync(10_005);
|
||||
expect(enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
|
||||
const finishedRun = finished.waitForOk(job.id);
|
||||
cron.resumeScheduling();
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await finishedRun;
|
||||
expectMainSystemEvent(enqueueSystemEvent, "resumed-tick", job.id);
|
||||
|
||||
cron.stop();
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("rolls a failed scheduler resume back so a retry can rearm cron", async () => {
|
||||
const store = await makeStorePath();
|
||||
const logger = createNoopLogger();
|
||||
const cron = new CronService({
|
||||
storePath: store.storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
|
||||
await cron.start();
|
||||
await cron.add({
|
||||
name: "resume retry check",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 10_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "resume" },
|
||||
});
|
||||
cron.pauseScheduling();
|
||||
logger.debug.mockImplementationOnce(() => {
|
||||
throw new Error("arm failed");
|
||||
});
|
||||
|
||||
expect(() => cron.resumeScheduling()).toThrow("arm failed");
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(() => cron.resumeScheduling()).not.toThrow();
|
||||
expect(vi.getTimerCount()).toBe(1);
|
||||
|
||||
cron.stop();
|
||||
await store.cleanup();
|
||||
});
|
||||
|
||||
it("keeps admission closed until a real cron scheduler resume retry succeeds", async () => {
|
||||
const store = await makeStorePath();
|
||||
const logger = createNoopLogger();
|
||||
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
|
||||
storePath: store.storePath,
|
||||
logger,
|
||||
});
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
|
||||
try {
|
||||
await cron.start();
|
||||
const job = await cron.add({
|
||||
name: "coordinator resume retry check",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 10_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "recovered-tick" },
|
||||
});
|
||||
logger.debug.mockImplementationOnce(() => {
|
||||
throw new Error("arm failed");
|
||||
});
|
||||
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "cron-resume-retry",
|
||||
pauseScheduling: () => cron.pauseScheduling(),
|
||||
resumeScheduling: () => cron.resumeScheduling(),
|
||||
inspect: { getQueueSize: () => 1 },
|
||||
}),
|
||||
).toMatchObject({ status: "recovering" });
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
|
||||
const finishedRun = finished.waitForOk(job.id);
|
||||
await vi.advanceTimersByTimeAsync(9_005);
|
||||
await finishedRun;
|
||||
expectMainSystemEvent(enqueueSystemEvent, "recovered-tick", job.id);
|
||||
} finally {
|
||||
cron.stop();
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
await store.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps a due timer pending when restart signal admission rolls back", async () => {
|
||||
const store = await makeStorePath();
|
||||
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
|
||||
storePath: store.storePath,
|
||||
logger: noopLogger,
|
||||
});
|
||||
resetGatewayWorkAdmission();
|
||||
|
||||
try {
|
||||
await cron.start();
|
||||
const job = await cron.add({
|
||||
name: "restart signal rollback check",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 10_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "rollback-tick" },
|
||||
});
|
||||
|
||||
const pendingSignal = beginGatewayRestartSignalAdmission();
|
||||
const finishedRun = finished.waitForOk(job.id);
|
||||
await vi.advanceTimersByTimeAsync(10_005);
|
||||
expect(enqueueSystemEvent).not.toHaveBeenCalled();
|
||||
|
||||
expect(pendingSignal.rollback()).toBe(true);
|
||||
await finishedRun;
|
||||
expectMainSystemEvent(enqueueSystemEvent, "rollback-tick", job.id);
|
||||
} finally {
|
||||
cron.stop();
|
||||
resetGatewayWorkAdmission();
|
||||
await store.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("fires a cron-expression job when the timer fires a few ms late", async () => {
|
||||
const store = await makeStorePath();
|
||||
const { cron, enqueueSystemEvent, finished } = createStartedCronServiceWithFinishedBarrier({
|
||||
|
||||
@@ -235,6 +235,8 @@ export function createMockCronStateForJobs(params: {
|
||||
durableNextRunAtMsByJobId: new Map<string, number | undefined>(),
|
||||
running: false,
|
||||
stopped: false,
|
||||
schedulingPaused: false,
|
||||
schedulerStarted: false,
|
||||
restartRecoveryPending: false,
|
||||
pendingCatchupDeferralJobIds: new Set<string>(),
|
||||
activeManualRunJobIds: new Set<string>(),
|
||||
|
||||
+56
-1
@@ -20,19 +20,74 @@ export type { CronEvent, CronServiceDeps } from "./service/state.js";
|
||||
/** Public cron service facade that owns mutable scheduler state and delegates to locked ops. */
|
||||
export class CronService implements CronServiceContract {
|
||||
private readonly state;
|
||||
private startInProgress = 0;
|
||||
private startState: { generation: number; promise: Promise<void> } | null = null;
|
||||
private lifecycleGeneration = 0;
|
||||
|
||||
constructor(deps: CronServiceDeps) {
|
||||
this.state = createCronServiceState(deps);
|
||||
}
|
||||
|
||||
async start() {
|
||||
await ops.start(this.state);
|
||||
const generation = this.lifecycleGeneration;
|
||||
const pending = this.startState;
|
||||
if (pending) {
|
||||
try {
|
||||
await pending.promise;
|
||||
} catch (err) {
|
||||
if (pending.generation === generation) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (pending.generation === generation) {
|
||||
return;
|
||||
}
|
||||
await this.start();
|
||||
return;
|
||||
}
|
||||
const promise = this.startOnce(generation);
|
||||
this.startState = { generation, promise };
|
||||
try {
|
||||
await promise;
|
||||
} finally {
|
||||
if (this.startState?.promise === promise) {
|
||||
this.startState = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async startOnce(generation: number) {
|
||||
this.startInProgress += 1;
|
||||
this.state.schedulerStarted = false;
|
||||
try {
|
||||
await ops.start(this.state);
|
||||
if (generation !== this.lifecycleGeneration) {
|
||||
ops.stop(this.state);
|
||||
return;
|
||||
}
|
||||
this.state.schedulerStarted = !this.state.stopped;
|
||||
} finally {
|
||||
this.startInProgress -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.lifecycleGeneration += 1;
|
||||
ops.stop(this.state);
|
||||
}
|
||||
|
||||
pauseScheduling() {
|
||||
ops.pauseScheduling(this.state);
|
||||
}
|
||||
|
||||
resumeScheduling() {
|
||||
ops.resumeScheduling(this.state);
|
||||
}
|
||||
|
||||
getSuspensionBlockerCount() {
|
||||
return this.startInProgress;
|
||||
}
|
||||
|
||||
async status() {
|
||||
return await ops.status(this.state);
|
||||
}
|
||||
|
||||
@@ -316,9 +316,35 @@ export async function start(state: CronServiceState) {
|
||||
/** Stops the cron service timer without mutating persisted job state. */
|
||||
export function stop(state: CronServiceState) {
|
||||
state.stopped = true;
|
||||
state.schedulerStarted = false;
|
||||
stopTimer(state);
|
||||
}
|
||||
|
||||
/** Temporarily stops automatic ticks without running startup recovery on resume. */
|
||||
export function pauseScheduling(state: CronServiceState) {
|
||||
state.schedulingPaused = true;
|
||||
stopTimer(state);
|
||||
}
|
||||
|
||||
export function resumeScheduling(state: CronServiceState) {
|
||||
if (!state.schedulingPaused) {
|
||||
return;
|
||||
}
|
||||
state.schedulingPaused = false;
|
||||
if (!state.schedulerStarted) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
armTimer(state);
|
||||
} catch (err) {
|
||||
// armTimer can install a timer before a later dependency throws. Roll the
|
||||
// whole transition back so a suspension retry cannot reopen without cron.
|
||||
state.schedulingPaused = true;
|
||||
stopTimer(state);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns cron service status after a read-only maintenance pass. */
|
||||
export async function status(state: CronServiceState) {
|
||||
return await locked(state, async () => {
|
||||
|
||||
@@ -206,6 +206,8 @@ export type CronServiceState = {
|
||||
timer: NodeJS.Timeout | null;
|
||||
running: boolean;
|
||||
stopped: boolean;
|
||||
schedulingPaused: boolean;
|
||||
schedulerStarted: boolean;
|
||||
restartRecoveryPending: boolean;
|
||||
/** Prevents maintenance reads from advancing deferred startup catch-up slots.
|
||||
* Entries are removed when the deferred job runs or becomes irrelevant. */
|
||||
@@ -234,6 +236,8 @@ export function createCronServiceState(deps: CronServiceDeps): CronServiceState
|
||||
timer: null,
|
||||
running: false,
|
||||
stopped: false,
|
||||
schedulingPaused: false,
|
||||
schedulerStarted: false,
|
||||
restartRecoveryPending: false,
|
||||
pendingCatchupDeferralJobIds: new Set<string>(),
|
||||
activeManualRunJobIds: new Set<string>(),
|
||||
|
||||
@@ -10,6 +10,10 @@ import {
|
||||
HEARTBEAT_SKIP_CRON_IN_PROGRESS,
|
||||
isRetryableHeartbeatBusySkipReason,
|
||||
} from "../../infra/heartbeat-wake.js";
|
||||
import {
|
||||
beginGatewayRootWorkAdmissionWhenOpen,
|
||||
GatewayDrainingError,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
import {
|
||||
DEFAULT_AGENT_ID,
|
||||
normalizeAgentId,
|
||||
@@ -1214,7 +1218,7 @@ export function armTimer(state: CronServiceState) {
|
||||
clearTimeout(state.timer);
|
||||
}
|
||||
state.timer = null;
|
||||
if (state.stopped) {
|
||||
if (state.stopped || state.schedulingPaused) {
|
||||
state.deps.log.debug({}, "cron: armTimer skipped - scheduler stopped");
|
||||
return;
|
||||
}
|
||||
@@ -1276,7 +1280,7 @@ export function armTimer(state: CronServiceState) {
|
||||
}
|
||||
|
||||
function armRunningRecheckTimer(state: CronServiceState) {
|
||||
if (state.stopped) {
|
||||
if (state.stopped || state.schedulingPaused) {
|
||||
return;
|
||||
}
|
||||
if (state.timer) {
|
||||
@@ -1289,9 +1293,29 @@ function armRunningRecheckTimer(state: CronServiceState) {
|
||||
}, MAX_TIMER_DELAY_MS);
|
||||
}
|
||||
|
||||
/** Handles one cron timer tick: load due jobs, reserve them, execute, persist, and re-arm. */
|
||||
/** Handles one cron timer tick under the process-wide root work admission. */
|
||||
export async function onTimer(state: CronServiceState) {
|
||||
if (state.stopped) {
|
||||
let admission;
|
||||
try {
|
||||
// A restart signal can be rejected after temporarily closing admission.
|
||||
// Wait for that decision so the consumed timer is not silently lost.
|
||||
admission = await beginGatewayRootWorkAdmissionWhenOpen();
|
||||
} catch (err) {
|
||||
if (err instanceof GatewayDrainingError) {
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
await admission.run(async () => await onAdmittedTimer(state));
|
||||
} finally {
|
||||
admission.release();
|
||||
}
|
||||
}
|
||||
|
||||
/** Loads due jobs, reserves them, executes, persists, and re-arms. */
|
||||
async function onAdmittedTimer(state: CronServiceState) {
|
||||
if (state.stopped || state.schedulingPaused) {
|
||||
return;
|
||||
}
|
||||
if (state.restartRecoveryPending) {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// abort fanout, history snapshots, and cleanup of buffered streaming state.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { isAgentRunRestartAbortReason } from "../agents/run-termination.js";
|
||||
import { onAgentEvent } from "../infra/agent-events.js";
|
||||
import {
|
||||
abortChatRunById,
|
||||
abortChatRunsForProvider,
|
||||
@@ -307,6 +308,39 @@ describe("registerChatAbortController", () => {
|
||||
});
|
||||
|
||||
describe("abortChatRunById", () => {
|
||||
it("retains terminal persistence ownership observed during abort", () => {
|
||||
const { runId, sessionKey, entry, ops } = createAbortRunFixture({});
|
||||
let terminalEvents = 0;
|
||||
const unsubscribe = onAgentEvent((event) => {
|
||||
if (event.runId === runId && event.stream === "lifecycle" && event.data.phase === "end") {
|
||||
terminalEvents += 1;
|
||||
entry.projectSessionTerminalPending = true;
|
||||
entry.projectSessionTerminalObservedAt = event.ts;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
const result = abortChatRunById(ops, { runId, sessionKey, stopReason: "user" });
|
||||
|
||||
expect(result).toEqual({ aborted: true });
|
||||
expect(entry.controller.signal.aborted).toBe(true);
|
||||
expect(entry.projectSessionActive).toBe(false);
|
||||
expect(entry.registrationCleanupRequested).toBe(true);
|
||||
expect(entry.projectSessionTerminalPending).toBe(true);
|
||||
expect(entry.projectSessionTerminalObservedAt).toEqual(expect.any(Number));
|
||||
expect(ops.chatAbortControllers.get(runId)).toBe(entry);
|
||||
|
||||
expect(abortChatRunById(ops, { runId, sessionKey, stopReason: "user" })).toEqual({
|
||||
aborted: false,
|
||||
});
|
||||
expect(terminalEvents).toBe(1);
|
||||
expect(ops.broadcast).toHaveBeenCalledOnce();
|
||||
expect(ops.removeChatRun).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
unsubscribe();
|
||||
}
|
||||
});
|
||||
|
||||
it("broadcasts aborted payload with partial message when buffered text exists", () => {
|
||||
const now = new Date("2026-01-02T03:04:05.000Z");
|
||||
const { runId, sessionKey, entry, ops } = createAbortRunFixture({
|
||||
|
||||
@@ -470,6 +470,9 @@ function resolveDefaultGlobalAgentId(ops: ChatAbortOps): string | undefined {
|
||||
}
|
||||
|
||||
export function isChatAbortControllerEntryAbortable(entry: ChatAbortControllerEntry): boolean {
|
||||
if (entry.controller.signal.aborted) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
return entry.isAbortable?.(entry) !== false;
|
||||
} catch {
|
||||
@@ -521,8 +524,13 @@ export function abortChatRunById(
|
||||
if (stopReason) {
|
||||
active.abortStopReason = stopReason;
|
||||
}
|
||||
active.projectSessionActive = false;
|
||||
// Reserve terminal ownership before abort listeners run; synchronous caller
|
||||
// cleanup must not erase the entry before Gateway observes the event below.
|
||||
active.projectSessionTerminalPending = true;
|
||||
active.projectSessionTerminalObservedAt = undefined;
|
||||
active.registrationCleanupRequested = true;
|
||||
active.controller.abort(createChatAbortSignalReason(stopReason));
|
||||
removeChatAbortControllerEntry(ops.chatAbortControllers, runId, active);
|
||||
ops.clearChatRunState(runId);
|
||||
const removed = ops.removeChatRun(runId, runId, sessionKey);
|
||||
if (active.controlUiVisible !== false) {
|
||||
@@ -551,6 +559,15 @@ export function abortChatRunById(
|
||||
endedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
// Gateway listeners synchronously stamp the terminal observation. Keep the
|
||||
// entry as suspension-visible ownership until its persistence write settles.
|
||||
if (
|
||||
ops.chatAbortControllers.get(runId) === active &&
|
||||
active.projectSessionTerminalObservedAt === undefined &&
|
||||
!active.projectSessionTerminalPersistence
|
||||
) {
|
||||
removeChatAbortControllerEntry(ops.chatAbortControllers, runId, active);
|
||||
}
|
||||
ops.agentRunSeq.delete(runId);
|
||||
if (removed?.clientRunId) {
|
||||
ops.agentRunSeq.delete(removed.clientRunId);
|
||||
|
||||
@@ -16,6 +16,11 @@ import {
|
||||
resetPluginRuntimeStateForTest,
|
||||
setActivePluginRegistry,
|
||||
} from "../plugins/runtime.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
getSkillsSnapshotVersion,
|
||||
resetSkillsRefreshStateForTest,
|
||||
@@ -723,6 +728,8 @@ function createReloaderHarness(
|
||||
promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise<boolean>;
|
||||
initialPluginInstallRecords?: Record<string, PluginInstallRecord>;
|
||||
readPluginInstallRecords?: () => Promise<Record<string, PluginInstallRecord>>;
|
||||
runTransaction?: <T>(run: () => Promise<T>) => Promise<T>;
|
||||
onRestart?: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
|
||||
} = {},
|
||||
) {
|
||||
const watcher = createWatcherMock();
|
||||
@@ -735,7 +742,9 @@ function createReloaderHarness(
|
||||
async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {},
|
||||
);
|
||||
const onHotReload = vi.fn(async (_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {});
|
||||
const onRestart = vi.fn((_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {});
|
||||
const onRestart = vi.fn(
|
||||
options.onRestart ?? ((_plan: GatewayReloadPlan, _nextConfig: OpenClawConfig) => {}),
|
||||
);
|
||||
let writeListener: ((event: ConfigWriteNotification) => void) | null = null;
|
||||
const subscribeToWrites = vi.fn((listener: (event: ConfigWriteNotification) => void) => {
|
||||
writeListener = listener;
|
||||
@@ -765,6 +774,7 @@ function createReloaderHarness(
|
||||
onNoopConfigCommit,
|
||||
onHotReload,
|
||||
onRestart,
|
||||
...(options.runTransaction ? { runTransaction: options.runTransaction } : {}),
|
||||
log,
|
||||
watchPath: "/tmp/openclaw.json",
|
||||
});
|
||||
@@ -816,10 +826,12 @@ function getOnlyPromoteSnapshotCall(promoteSnapshot: {
|
||||
|
||||
describe("startGatewayConfigReloader", () => {
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
@@ -932,6 +944,43 @@ describe("startGatewayConfigReloader", () => {
|
||||
await harness.reloader.stop();
|
||||
});
|
||||
|
||||
it("keeps restart preparation inside the accepted config root", async () => {
|
||||
let releaseRestart = () => {};
|
||||
let noteRestartStarted = () => {};
|
||||
const restartStarted = new Promise<void>((resolve) => {
|
||||
noteRestartStarted = resolve;
|
||||
});
|
||||
const restartPending = new Promise<void>((resolve) => {
|
||||
releaseRestart = resolve;
|
||||
});
|
||||
const initialConfig: OpenClawConfig = {
|
||||
gateway: { reload: { debounceMs: 0 }, terminal: { enabled: true } },
|
||||
};
|
||||
const nextConfig: OpenClawConfig = {
|
||||
gateway: { reload: { debounceMs: 0 }, terminal: { enabled: false } },
|
||||
};
|
||||
const harness = createReloaderHarness(
|
||||
async () => makeSnapshot({ config: nextConfig, hash: "restart-root" }),
|
||||
{
|
||||
initialConfig,
|
||||
runTransaction: runWithGatewayIndependentRootWorkAdmission,
|
||||
onRestart: async () => {
|
||||
noteRestartStarted();
|
||||
await restartPending;
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
harness.watcher.emit("change");
|
||||
await vi.runOnlyPendingTimersAsync();
|
||||
await restartStarted;
|
||||
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
releaseRestart();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
await harness.reloader.stop();
|
||||
});
|
||||
|
||||
it("does not notify lifecycle owners when reload mode ignores the change", async () => {
|
||||
const initialConfig: OpenClawConfig = {
|
||||
gateway: { reload: { mode: "off", debounceMs: 0 }, terminal: { enabled: true } },
|
||||
|
||||
@@ -116,6 +116,8 @@ export function startGatewayConfigReloader(opts: {
|
||||
onNoopConfigCommit: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
|
||||
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
|
||||
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
|
||||
/** Keeps one accepted config transaction inside the Gateway work fence. */
|
||||
runTransaction?: <T>(run: () => Promise<T>) => Promise<T>;
|
||||
promoteSnapshot?: (snapshot: ConfigFileSnapshot, reason: string) => Promise<boolean>;
|
||||
initialPluginInstallRecords?: PluginInstallRecords;
|
||||
readPluginInstallRecords?: () => Promise<PluginInstallRecords>;
|
||||
@@ -164,21 +166,21 @@ export function startGatewayConfigReloader(opts: {
|
||||
const schedule = () => {
|
||||
scheduleAfter(settings.debounceMs);
|
||||
};
|
||||
const queueRestart = (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => {
|
||||
const queueRestart = async (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => {
|
||||
if (restartQueued) {
|
||||
return;
|
||||
}
|
||||
restartQueued = true;
|
||||
void (async () => {
|
||||
try {
|
||||
await opts.onRestart(plan, nextConfig);
|
||||
} catch (err) {
|
||||
// Restart checks can fail (for example unresolved SecretRefs). Keep the
|
||||
// reloader alive and allow a future change to retry restart scheduling.
|
||||
restartQueued = false;
|
||||
opts.log.error(`config restart failed: ${String(err)}`);
|
||||
}
|
||||
})();
|
||||
try {
|
||||
// Restart preparation reads secrets and can mutate auth/runtime state.
|
||||
// Keep it inside the accepted config transaction instead of detaching it.
|
||||
await opts.onRestart(plan, nextConfig);
|
||||
} catch (err) {
|
||||
// Restart checks can fail (for example unresolved SecretRefs). Keep the
|
||||
// reloader alive and allow a future change to retry restart scheduling.
|
||||
restartQueued = false;
|
||||
opts.log.error(`config restart failed: ${String(err)}`);
|
||||
}
|
||||
};
|
||||
|
||||
const handleMissingSnapshot = (snapshot: ConfigFileSnapshot): boolean => {
|
||||
@@ -297,12 +299,12 @@ export function startGatewayConfigReloader(opts: {
|
||||
restartReasons: [...plan.restartReasons, followUp.reason],
|
||||
};
|
||||
await opts.onConfigChange?.(restartPlan, nextConfig);
|
||||
queueRestart(restartPlan, nextConfig);
|
||||
await queueRestart(restartPlan, nextConfig);
|
||||
return;
|
||||
}
|
||||
if (settings.mode === "restart") {
|
||||
await opts.onConfigChange?.({ ...plan, restartGateway: true }, nextConfig);
|
||||
queueRestart(plan, nextConfig);
|
||||
await queueRestart(plan, nextConfig);
|
||||
return;
|
||||
}
|
||||
if (plan.restartGateway) {
|
||||
@@ -315,7 +317,7 @@ export function startGatewayConfigReloader(opts: {
|
||||
return;
|
||||
}
|
||||
await opts.onConfigChange?.(plan, nextConfig);
|
||||
queueRestart(plan, nextConfig);
|
||||
await queueRestart(plan, nextConfig);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -335,6 +337,14 @@ export function startGatewayConfigReloader(opts: {
|
||||
}
|
||||
};
|
||||
|
||||
const runAcceptedTransaction = async (run: () => Promise<void>) => {
|
||||
if (opts.runTransaction) {
|
||||
await opts.runTransaction(run);
|
||||
return;
|
||||
}
|
||||
await run();
|
||||
};
|
||||
|
||||
const promoteAcceptedInProcessWrite = async (persistedHash: string) => {
|
||||
if (!opts.promoteSnapshot) {
|
||||
return;
|
||||
@@ -368,12 +378,14 @@ export function startGatewayConfigReloader(opts: {
|
||||
const pendingWrite = pendingInProcessConfig;
|
||||
pendingInProcessConfig = null;
|
||||
missingConfigRetries = 0;
|
||||
await applySnapshot(
|
||||
pendingWrite.config,
|
||||
pendingWrite.compareConfig,
|
||||
pendingWrite.afterWrite,
|
||||
);
|
||||
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
|
||||
await runAcceptedTransaction(async () => {
|
||||
await applySnapshot(
|
||||
pendingWrite.config,
|
||||
pendingWrite.compareConfig,
|
||||
pendingWrite.afterWrite,
|
||||
);
|
||||
await promoteAcceptedInProcessWrite(pendingWrite.persistedHash);
|
||||
});
|
||||
return;
|
||||
}
|
||||
const snapshot = await opts.readSnapshot();
|
||||
@@ -390,8 +402,10 @@ export function startGatewayConfigReloader(opts: {
|
||||
handleInvalidSnapshot(snapshot);
|
||||
return;
|
||||
}
|
||||
await applySnapshot(snapshot.config, snapshot.sourceConfig);
|
||||
await promoteAcceptedSnapshot(snapshot, "valid-config");
|
||||
await runAcceptedTransaction(async () => {
|
||||
await applySnapshot(snapshot.config, snapshot.sourceConfig);
|
||||
await promoteAcceptedSnapshot(snapshot, "valid-config");
|
||||
});
|
||||
} catch (err) {
|
||||
opts.log.error(`config reload failed: ${String(err)}`);
|
||||
} finally {
|
||||
|
||||
@@ -289,7 +289,9 @@ describe("createCronExitWatchers", () => {
|
||||
// The orphaned child is killed and the job never fires.
|
||||
expect(fake.runCancels.length).toBe(1);
|
||||
expect(fireOnExit).not.toHaveBeenCalled();
|
||||
expect(w.activeJobIds()).toEqual([]);
|
||||
expect(w.activeJobIds()).toEqual(["job-a"]);
|
||||
fake.runs[0].deferred.resolve({ exitCode: null, reason: "manual-cancel" });
|
||||
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
|
||||
});
|
||||
|
||||
it("does not arm a watcher for time-based or disabled jobs", async () => {
|
||||
@@ -325,8 +327,8 @@ describe("createCronExitWatchers", () => {
|
||||
expect(supervisor.spawn).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("cancels the watcher when the job is removed from the set", async () => {
|
||||
const { supervisor, cancelled } = makeFakeSupervisor();
|
||||
it("keeps a cancelled watcher blocking until the supervised child settles", async () => {
|
||||
const { supervisor, cancelled, runs } = makeFakeSupervisor();
|
||||
const w = createCronExitWatchers({
|
||||
getProcessSupervisor: () => supervisor as never,
|
||||
persistCompletion: vi.fn(async () => {}),
|
||||
@@ -337,7 +339,10 @@ describe("createCronExitWatchers", () => {
|
||||
await flush();
|
||||
w.reconcile([]);
|
||||
expect(cancelled).toContain("cron-exit:job-a");
|
||||
expect(w.activeJobIds()).toEqual([]);
|
||||
expect(w.activeJobIds()).toEqual(["job-a"]);
|
||||
|
||||
runs[0].deferred.resolve({ exitCode: null, reason: "manual-cancel" });
|
||||
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
|
||||
});
|
||||
|
||||
it("does not fire a job whose watcher was cancelled before exit", async () => {
|
||||
@@ -357,6 +362,35 @@ describe("createCronExitWatchers", () => {
|
||||
expect(fireOnExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains a blocker and suppresses stale fire when removed during terminal persistence", async () => {
|
||||
const { supervisor, runs } = makeFakeSupervisor();
|
||||
let releasePersist = () => {};
|
||||
const persistCompletion = vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releasePersist = resolve;
|
||||
}),
|
||||
);
|
||||
const fireOnExit = vi.fn(async () => {});
|
||||
const w = createCronExitWatchers({
|
||||
getProcessSupervisor: () => supervisor as never,
|
||||
persistCompletion,
|
||||
fireOnExit,
|
||||
logger: noopLogger,
|
||||
});
|
||||
w.reconcile([onExitJob("job-a")]);
|
||||
await flush();
|
||||
|
||||
runs[0].deferred.resolve({ exitCode: 0, reason: "exit" });
|
||||
await vi.waitFor(() => expect(persistCompletion).toHaveBeenCalledOnce());
|
||||
w.reconcile([]);
|
||||
expect(w.activeJobIds()).toEqual(["job-a"]);
|
||||
|
||||
releasePersist();
|
||||
await vi.waitFor(() => expect(w.activeJobIds()).toEqual([]));
|
||||
expect(fireOnExit).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("is one-shot: a fired job is not re-armed on a later reconcile", async () => {
|
||||
const { supervisor, runs } = makeFakeSupervisor();
|
||||
const w = createCronExitWatchers({
|
||||
|
||||
@@ -79,17 +79,32 @@ export function createCronExitWatchers(params: {
|
||||
job: OnExitCronJob;
|
||||
run: ManagedRun | undefined;
|
||||
fired: boolean;
|
||||
terminalPersisting: boolean;
|
||||
cancelled: boolean;
|
||||
lifecycleSettled: boolean;
|
||||
command: string;
|
||||
cwd: string | undefined;
|
||||
};
|
||||
const active = new Map<string, WatcherSlot>();
|
||||
// A cancelled child can keep running until the supervisor observes exit.
|
||||
// Retain those slots separately so replacement arms can own the job id while
|
||||
// suspension still sees every predecessor that is settling.
|
||||
const settlingCancelledSlots = new Set<WatcherSlot>();
|
||||
|
||||
const cancel = (jobId: string) => {
|
||||
const slot = active.get(jobId);
|
||||
if (!slot) {
|
||||
return;
|
||||
}
|
||||
active.delete(jobId);
|
||||
slot.cancelled = true;
|
||||
if (!slot.lifecycleSettled) {
|
||||
settlingCancelledSlots.add(slot);
|
||||
}
|
||||
// Terminal persistence is user-visible state. Keep the slot as a suspend
|
||||
// blocker until that write settles even when hot reload cancels the watcher.
|
||||
if (!slot.terminalPersisting) {
|
||||
active.delete(jobId);
|
||||
}
|
||||
// Cancel an already-spawned child; an in-flight spawn (run undefined) is
|
||||
// killed by the arm() ownership check once it resolves.
|
||||
slot.run?.cancel("manual-cancel");
|
||||
@@ -106,7 +121,17 @@ export function createCronExitWatchers(params: {
|
||||
const armToken: object = {};
|
||||
// Reserve the slot synchronously so a concurrent cancel/replace can observe
|
||||
// and act on this arm before the child is spawned.
|
||||
const slot: WatcherSlot = { armToken, job, run: undefined, fired: false, command, cwd };
|
||||
const slot: WatcherSlot = {
|
||||
armToken,
|
||||
job,
|
||||
run: undefined,
|
||||
fired: false,
|
||||
terminalPersisting: false,
|
||||
cancelled: false,
|
||||
lifecycleSettled: false,
|
||||
command,
|
||||
cwd,
|
||||
};
|
||||
active.set(job.id, slot);
|
||||
const owns = () => active.get(job.id) === slot && slot.armToken === armToken;
|
||||
void (async () => {
|
||||
@@ -136,8 +161,14 @@ export function createCronExitWatchers(params: {
|
||||
}
|
||||
if (!owns()) {
|
||||
// Cancelled or re-armed (changed command/cwd) while the spawn was in
|
||||
// flight — kill this now-orphaned child instead of leaking it.
|
||||
// flight — kill this now-orphaned child instead of leaking it. Wait for
|
||||
// supervisor settlement so suspension cannot snapshot a live child.
|
||||
run.cancel("manual-cancel");
|
||||
try {
|
||||
await run.wait();
|
||||
} catch {
|
||||
// The watcher was already cancelled; settlement, not outcome, matters.
|
||||
}
|
||||
return;
|
||||
}
|
||||
slot.run = run;
|
||||
@@ -165,6 +196,7 @@ export function createCronExitWatchers(params: {
|
||||
{ jobId: job.id, exitCode: exit.exitCode, reason: exit.reason },
|
||||
"cron-exit: watched command exited; firing job",
|
||||
);
|
||||
slot.terminalPersisting = true;
|
||||
// Persist the terminal one-shot state BEFORE firing. FAIL CLOSED: if the
|
||||
// store write fails we do NOT wake — waking without a persisted terminal
|
||||
// state would let a gateway restart re-arm and re-run the command.
|
||||
@@ -180,6 +212,13 @@ export function createCronExitWatchers(params: {
|
||||
);
|
||||
return;
|
||||
}
|
||||
slot.terminalPersisting = false;
|
||||
if (!owns() || slot.cancelled) {
|
||||
if (active.get(job.id) === slot) {
|
||||
active.delete(job.id);
|
||||
}
|
||||
return;
|
||||
}
|
||||
slot.fired = true;
|
||||
try {
|
||||
await params.fireOnExit(slot.job, {
|
||||
@@ -196,7 +235,13 @@ export function createCronExitWatchers(params: {
|
||||
"cron-exit: fireOnExit after exit failed",
|
||||
);
|
||||
}
|
||||
})();
|
||||
})().finally(() => {
|
||||
slot.lifecycleSettled = true;
|
||||
settlingCancelledSlots.delete(slot);
|
||||
if (slot.cancelled && active.get(job.id) === slot) {
|
||||
active.delete(job.id);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const reconcile = (jobs: CronJob[]) => {
|
||||
@@ -236,6 +281,14 @@ export function createCronExitWatchers(params: {
|
||||
reconcile,
|
||||
cancel,
|
||||
cancelAll,
|
||||
activeJobIds: () => Array.from(active.keys()),
|
||||
activeJobIds: () =>
|
||||
Array.from(
|
||||
new Set([
|
||||
...Array.from(active.entries())
|
||||
.filter(([, slot]) => !slot.fired)
|
||||
.map(([jobId]) => jobId),
|
||||
...Array.from(settlingCancelledSlots, (slot) => slot.job.id),
|
||||
]),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
import { loadManifestModelCatalog } from "../agents/model-catalog.js";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { CronServiceContract } from "../cron/service-contract.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
getPluginRuntimeGatewayRequestScope,
|
||||
@@ -12,6 +11,7 @@ import {
|
||||
import { NodeRegistry } from "./node-registry.js";
|
||||
import type { ChannelRuntimeSnapshot } from "./server-channel-runtime.types.js";
|
||||
import { createChatRunEntry, type ChatRunEntry } from "./server-chat-state.js";
|
||||
import type { GatewayCronServiceContract } from "./server-cron.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/types.js";
|
||||
|
||||
// Embedded/local agent calls need enough GatewayRequestContext to reuse server
|
||||
@@ -26,11 +26,13 @@ function cronUnavailable(): never {
|
||||
throw new Error("Cron is unavailable in local embedded agent gateway context.");
|
||||
}
|
||||
|
||||
const unavailableCron: CronServiceContract = {
|
||||
const unavailableCron: GatewayCronServiceContract = {
|
||||
start: async () => {
|
||||
cronUnavailable();
|
||||
},
|
||||
stop: () => {},
|
||||
pauseScheduling: () => {},
|
||||
resumeScheduling: () => {},
|
||||
status: async () => cronUnavailable(),
|
||||
list: async () => cronUnavailable(),
|
||||
listPage: async () => cronUnavailable(),
|
||||
|
||||
@@ -288,6 +288,15 @@ export const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
|
||||
// Session PR chips read the session's own checkout metadata, matching the
|
||||
// sessions.files.* trusted-operator read domain.
|
||||
{ name: "controlUi.sessionPullRequests", scope: "operator.read" },
|
||||
{
|
||||
name: "gateway.suspend.prepare",
|
||||
scope: "operator.admin",
|
||||
startup: true,
|
||||
controlPlaneWrite: true,
|
||||
},
|
||||
{ name: "gateway.suspend.status", scope: "operator.read" },
|
||||
// Resume is the safety escape hatch and must not sit behind write-rate limiting.
|
||||
{ name: "gateway.suspend.resume", scope: "operator.admin" },
|
||||
] as const;
|
||||
|
||||
const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap<string, CoreGatewayMethodSpec> = new Map(
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
// Covers server-local chat, cron watcher, queued-turn, and terminal blockers.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createGatewayServerActiveWorkInspectors } from "./server-active-work.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/shared-types.js";
|
||||
|
||||
vi.mock("../cron/active-jobs.js", () => ({
|
||||
getActiveCronJobCount: vi.fn(() => 2),
|
||||
}));
|
||||
|
||||
vi.mock("../tasks/cron-task-cancel.js", () => ({
|
||||
getSuspensionVisibleCronTaskRunCount: vi.fn(() => 4),
|
||||
}));
|
||||
|
||||
function controller(aborted = false): AbortController {
|
||||
const value = new AbortController();
|
||||
if (aborted) {
|
||||
value.abort();
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
describe("gateway server active work inspectors", () => {
|
||||
it("filters completed chat entries while retaining persistence and watcher blockers", () => {
|
||||
const context = {
|
||||
cron: { getSuspensionBlockerCount: () => 1 },
|
||||
chatAbortControllers: new Map([
|
||||
["active", { controller: controller() }],
|
||||
["aborted", { controller: controller(true) }],
|
||||
[
|
||||
"persisting",
|
||||
{
|
||||
controller: controller(),
|
||||
registrationCleanupRequested: true,
|
||||
controlUiVisible: true,
|
||||
projectSessionTerminalPending: true,
|
||||
},
|
||||
],
|
||||
]),
|
||||
chatQueuedTurns: new Map([
|
||||
["queued", { controller: controller() }],
|
||||
["cancelled", { controller: controller(true) }],
|
||||
]),
|
||||
terminalSessions: { size: 2 },
|
||||
} as unknown as Pick<
|
||||
GatewayRequestContext,
|
||||
"chatAbortControllers" | "chatQueuedTurns" | "cron" | "terminalSessions"
|
||||
>;
|
||||
|
||||
const inspectors = createGatewayServerActiveWorkInspectors(context);
|
||||
|
||||
expect(inspectors.getCronRuns?.()).toBe(5);
|
||||
expect(inspectors.getChatRuns?.()).toBe(1);
|
||||
expect(inspectors.getQueuedTurns?.()).toBe(1);
|
||||
expect(inspectors.getTerminalPersistence?.()).toBe(1);
|
||||
expect(inspectors.getTerminalSessions?.()).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
// Adapts server-local chat and terminal state to the shared activity inspector.
|
||||
import { getActiveCronJobCount } from "../cron/active-jobs.js";
|
||||
import type { GatewayActiveWorkInspectors } from "../infra/gateway-active-work.js";
|
||||
import { getSuspensionVisibleCronTaskRunCount } from "../tasks/cron-task-cancel.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/shared-types.js";
|
||||
|
||||
export function createGatewayServerActiveWorkInspectors(
|
||||
context: Pick<
|
||||
GatewayRequestContext,
|
||||
"chatAbortControllers" | "chatQueuedTurns" | "cron" | "terminalSessions"
|
||||
>,
|
||||
): Partial<GatewayActiveWorkInspectors> {
|
||||
return {
|
||||
getCronRuns: () =>
|
||||
Math.max(getActiveCronJobCount(), getSuspensionVisibleCronTaskRunCount()) +
|
||||
(context.cron.getSuspensionBlockerCount?.() ?? 0),
|
||||
getChatRuns: () =>
|
||||
Array.from(context.chatAbortControllers.values()).filter(
|
||||
(entry) => !entry.controller.signal.aborted && entry.registrationCleanupRequested !== true,
|
||||
).length,
|
||||
getQueuedTurns: () =>
|
||||
Array.from(context.chatQueuedTurns.values()).filter(
|
||||
(entry) => !entry.controller.signal.aborted,
|
||||
).length,
|
||||
getTerminalPersistence: () =>
|
||||
Array.from(context.chatAbortControllers.values()).filter(
|
||||
(entry) =>
|
||||
entry.controlUiVisible !== false &&
|
||||
entry.projectSessionTerminalPersisted !== true &&
|
||||
(entry.projectSessionTerminalPending === true ||
|
||||
entry.projectSessionTerminalPersistence !== undefined),
|
||||
).length,
|
||||
getTerminalSessions: () => context.terminalSessions?.size ?? 0,
|
||||
};
|
||||
}
|
||||
@@ -4,8 +4,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { CronServiceContract } from "../cron/service-contract.js";
|
||||
import type { GatewayCronState } from "./server-cron.js";
|
||||
import type { GatewayCronServiceContract, GatewayCronState } from "./server-cron.js";
|
||||
|
||||
const hoisted = vi.hoisted(() => {
|
||||
let state: unknown;
|
||||
@@ -23,6 +22,14 @@ vi.mock("./server-cron.js", () => ({
|
||||
|
||||
const { createLazyGatewayCronState } = await import("./server-cron-lazy.js");
|
||||
|
||||
function deferred() {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("createLazyGatewayCronState", () => {
|
||||
beforeEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
@@ -111,6 +118,26 @@ describe("createLazyGatewayCronState", () => {
|
||||
expect(cron["start"]).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("restarts after stop interrupts an in-flight startup", async () => {
|
||||
const finishFirstStart = deferred();
|
||||
const cron = createCronService();
|
||||
cron.start = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(async () => await finishFirstStart.promise)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
hoisted.setState(createCronState(cron));
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
const firstStart = lazy.cron.start();
|
||||
await vi.waitFor(() => expect(cron["start"]).toHaveBeenCalledOnce());
|
||||
lazy.cron.stop();
|
||||
const restarted = lazy.cron.start();
|
||||
finishFirstStart.resolve();
|
||||
await Promise.all([firstStart, restarted]);
|
||||
|
||||
expect(cron["start"]).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps synchronous wake non-blocking before the cron service is loaded", async () => {
|
||||
const cron = createCronService();
|
||||
hoisted.setState(createCronState(cron));
|
||||
@@ -134,6 +161,104 @@ describe("createLazyGatewayCronState", () => {
|
||||
expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not arm a read-loaded scheduler when suspension ends", async () => {
|
||||
const cron = createCronService();
|
||||
hoisted.setState(createCronState(cron));
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
lazy.cron.pauseScheduling();
|
||||
expect(hoisted.buildGatewayCronService).not.toHaveBeenCalled();
|
||||
|
||||
await lazy.cron.status();
|
||||
expect(cron["pauseScheduling"]).toHaveBeenCalledOnce();
|
||||
lazy.cron.resumeScheduling();
|
||||
expect(cron["resumeScheduling"]).not.toHaveBeenCalled();
|
||||
|
||||
await lazy.cron.start();
|
||||
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
|
||||
lazy.cron.pauseScheduling();
|
||||
lazy.cron.resumeScheduling();
|
||||
expect(cron["resumeScheduling"]).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("waits to start while scheduling is paused", async () => {
|
||||
const cron = createCronService();
|
||||
hoisted.setState(createCronState(cron));
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
lazy.cron.pauseScheduling();
|
||||
const startPromise = lazy.cron.start();
|
||||
await vi.waitFor(() => expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1));
|
||||
expect(cron["start"]).not.toHaveBeenCalled();
|
||||
|
||||
lazy.cron.resumeScheduling();
|
||||
await startPromise;
|
||||
expect(cron["start"]).toHaveBeenCalledOnce();
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
|
||||
});
|
||||
|
||||
it("keeps in-flight startup as a blocker until startup settles", async () => {
|
||||
const finishStart = deferred();
|
||||
const cron = createCronService();
|
||||
cron.start = vi.fn(async () => await finishStart.promise);
|
||||
hoisted.setState(createCronState(cron));
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
const startPromise = lazy.cron.start();
|
||||
await vi.waitFor(() => expect(cron["start"]).toHaveBeenCalledOnce());
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
|
||||
|
||||
lazy.cron.pauseScheduling();
|
||||
lazy.cron.resumeScheduling();
|
||||
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
|
||||
|
||||
finishStart.resolve();
|
||||
await startPromise;
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
|
||||
});
|
||||
|
||||
it("resumes a paused scheduler while exit watchers are reconciling", async () => {
|
||||
const reconcileStarted = deferred();
|
||||
const finishReconcile = deferred();
|
||||
const cron = createCronService();
|
||||
hoisted.setState({
|
||||
...createCronState(cron),
|
||||
reconcileExitWatchers: vi.fn(async () => {
|
||||
reconcileStarted.resolve();
|
||||
await finishReconcile.promise;
|
||||
}),
|
||||
});
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
const startPromise = lazy.cron.start();
|
||||
await reconcileStarted.promise;
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(1);
|
||||
|
||||
lazy.cron.pauseScheduling();
|
||||
lazy.cron.resumeScheduling();
|
||||
expect(cron["resumeScheduling"]).toHaveBeenCalledOnce();
|
||||
|
||||
finishReconcile.resolve();
|
||||
await startPromise;
|
||||
expect(lazy.cron.getSuspensionBlockerCount?.()).toBe(0);
|
||||
});
|
||||
|
||||
it("allows startup to retry after the underlying service rejects", async () => {
|
||||
const cron = createCronService();
|
||||
cron.start = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error("startup failed"))
|
||||
.mockResolvedValueOnce(undefined);
|
||||
hoisted.setState(createCronState(cron));
|
||||
const lazy = createLazyGatewayCronState(createParams());
|
||||
|
||||
await expect(lazy.cron.start()).rejects.toThrow("startup failed");
|
||||
await lazy.cron.start();
|
||||
|
||||
expect(cron["start"]).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("does not reconcile exit watchers when cron is disabled", async () => {
|
||||
const cron = createCronService();
|
||||
const reconcileExitWatchers = vi.fn(async () => {});
|
||||
@@ -161,7 +286,7 @@ function createParams(overrides: Partial<OpenClawConfig> = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function createCronState(cron: CronServiceContract): GatewayCronState {
|
||||
function createCronState(cron: GatewayCronServiceContract): GatewayCronState {
|
||||
return {
|
||||
cron,
|
||||
storePath: "/tmp/openclaw-cron.json",
|
||||
@@ -169,10 +294,12 @@ function createCronState(cron: CronServiceContract): GatewayCronState {
|
||||
} as GatewayCronState;
|
||||
}
|
||||
|
||||
function createCronService(): CronServiceContract {
|
||||
function createCronService(): GatewayCronServiceContract {
|
||||
return {
|
||||
start: vi.fn(async () => undefined),
|
||||
stop: vi.fn(),
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
status: vi.fn(async () => ({ enabled: true }) as never),
|
||||
list: vi.fn(async () => [] as never),
|
||||
listPage: vi.fn(async () => ({ items: [], total: 0 }) as never),
|
||||
|
||||
+147
-21
@@ -2,10 +2,9 @@
|
||||
// Defers scheduler startup until cron is touched by runtime or API handlers.
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { CronServiceContract } from "../cron/service-contract.js";
|
||||
import { resolveCronJobsStorePath } from "../cron/store.js";
|
||||
import { createLazyPromiseLoader } from "../shared/lazy-runtime.js";
|
||||
import type { GatewayCronState } from "./server-cron.js";
|
||||
import type { GatewayCronServiceContract, GatewayCronState } from "./server-cron.js";
|
||||
|
||||
type LazyGatewayCronParams = {
|
||||
cfg: OpenClawConfig;
|
||||
@@ -15,7 +14,12 @@ type LazyGatewayCronParams = {
|
||||
|
||||
type LoadedGatewayCronState = {
|
||||
state: GatewayCronState;
|
||||
started: boolean;
|
||||
phase: "idle" | "starting" | "started" | "stopped";
|
||||
startPromise: Promise<void> | null;
|
||||
startGeneration: number | null;
|
||||
schedulingPaused: boolean;
|
||||
underlyingStartInFlight: boolean;
|
||||
underlyingStarted: boolean;
|
||||
};
|
||||
|
||||
/** Creates a cron state proxy that imports the real cron service on first use. */
|
||||
@@ -24,13 +28,40 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
|
||||
const cronEnabled = process.env.OPENCLAW_SKIP_CRON !== "1" && params.cfg.cron?.enabled !== false;
|
||||
let loaded: LoadedGatewayCronState | null = null;
|
||||
let stopped = false;
|
||||
let lifecycleGeneration = 0;
|
||||
let schedulingPaused = false;
|
||||
const schedulingResumeWaiters = new Set<() => void>();
|
||||
const releaseSchedulingResumeWaiters = () => {
|
||||
const waiters = Array.from(schedulingResumeWaiters);
|
||||
schedulingResumeWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
const waitForSchedulingResume = async () => {
|
||||
if (!schedulingPaused) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
schedulingResumeWaiters.add(resolve);
|
||||
});
|
||||
};
|
||||
const cronStateLoader = createLazyPromiseLoader(
|
||||
() =>
|
||||
import("./server-cron.js").then(({ buildGatewayCronService }) => {
|
||||
loaded = {
|
||||
state: buildGatewayCronService(params),
|
||||
started: false,
|
||||
phase: "idle",
|
||||
startPromise: null,
|
||||
startGeneration: null,
|
||||
schedulingPaused: false,
|
||||
underlyingStartInFlight: false,
|
||||
underlyingStarted: false,
|
||||
};
|
||||
if (schedulingPaused) {
|
||||
loaded.state.cron.pauseScheduling();
|
||||
loaded.schedulingPaused = true;
|
||||
}
|
||||
return loaded;
|
||||
}),
|
||||
{ cacheRejections: true },
|
||||
@@ -45,35 +76,106 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
|
||||
return await cronStateLoader.load();
|
||||
};
|
||||
|
||||
const cron: CronServiceContract = {
|
||||
const cron: GatewayCronServiceContract = {
|
||||
async start() {
|
||||
stopped = false;
|
||||
const generation = lifecycleGeneration;
|
||||
const startCancelled = () => stopped || generation !== lifecycleGeneration;
|
||||
const resolved = await load();
|
||||
if (stopped) {
|
||||
const hasStarted = () => resolved.phase === "started";
|
||||
if (startCancelled()) {
|
||||
return;
|
||||
}
|
||||
if (resolved.started) {
|
||||
if (hasStarted()) {
|
||||
return;
|
||||
}
|
||||
resolved.started = true;
|
||||
await resolved.state.cron.start();
|
||||
// Arm on-exit watchers for jobs loaded from the store at startup (no
|
||||
// change event fires for already-persisted jobs).
|
||||
if (resolved.state.cronEnabled) {
|
||||
await resolved.state.reconcileExitWatchers?.();
|
||||
if (resolved.startPromise) {
|
||||
const pendingGeneration = resolved.startGeneration;
|
||||
try {
|
||||
await resolved.startPromise;
|
||||
} catch (err) {
|
||||
if (pendingGeneration === generation) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (startCancelled() || hasStarted()) {
|
||||
return;
|
||||
}
|
||||
if (pendingGeneration !== generation) {
|
||||
await cron.start();
|
||||
return;
|
||||
}
|
||||
}
|
||||
// If stop raced the lazy import/start path, immediately stop the loaded
|
||||
// scheduler so shutdown does not leave a background loop alive.
|
||||
if (stopped && resolved.started) {
|
||||
resolved.started = false;
|
||||
resolved.state.cron.stop();
|
||||
resolved.state.stopExitWatchers?.();
|
||||
resolved.phase = "starting";
|
||||
resolved.startGeneration = generation;
|
||||
const startPromise = (async () => {
|
||||
await waitForSchedulingResume();
|
||||
if (startCancelled()) {
|
||||
resolved.phase = "stopped";
|
||||
return;
|
||||
}
|
||||
if (resolved.schedulingPaused) {
|
||||
resolved.state.cron.resumeScheduling();
|
||||
resolved.schedulingPaused = false;
|
||||
}
|
||||
resolved.underlyingStartInFlight = true;
|
||||
try {
|
||||
await resolved.state.cron.start();
|
||||
resolved.underlyingStarted = true;
|
||||
} catch (err) {
|
||||
resolved.underlyingStarted = false;
|
||||
resolved.phase = startCancelled() ? "stopped" : "idle";
|
||||
throw err;
|
||||
} finally {
|
||||
resolved.underlyingStartInFlight = false;
|
||||
}
|
||||
if (startCancelled()) {
|
||||
resolved.phase = "stopped";
|
||||
resolved.underlyingStarted = false;
|
||||
resolved.state.cron.stop();
|
||||
resolved.state.stopExitWatchers?.();
|
||||
return;
|
||||
}
|
||||
if (schedulingPaused) {
|
||||
resolved.state.cron.pauseScheduling();
|
||||
resolved.schedulingPaused = true;
|
||||
}
|
||||
// Arm on-exit watchers for jobs loaded from the store at startup (no
|
||||
// change event fires for already-persisted jobs).
|
||||
try {
|
||||
if (resolved.state.cronEnabled) {
|
||||
await resolved.state.reconcileExitWatchers?.();
|
||||
}
|
||||
} catch (err) {
|
||||
resolved.phase = startCancelled() ? "stopped" : "started";
|
||||
throw err;
|
||||
}
|
||||
if (startCancelled()) {
|
||||
resolved.phase = "stopped";
|
||||
resolved.underlyingStarted = false;
|
||||
resolved.state.cron.stop();
|
||||
resolved.state.stopExitWatchers?.();
|
||||
return;
|
||||
}
|
||||
resolved.phase = "started";
|
||||
})();
|
||||
resolved.startPromise = startPromise;
|
||||
try {
|
||||
await startPromise;
|
||||
} finally {
|
||||
if (resolved.startPromise === startPromise) {
|
||||
resolved.startPromise = null;
|
||||
resolved.startGeneration = null;
|
||||
}
|
||||
}
|
||||
},
|
||||
stop() {
|
||||
stopped = true;
|
||||
lifecycleGeneration += 1;
|
||||
releaseSchedulingResumeWaiters();
|
||||
if (loaded) {
|
||||
loaded.started = false;
|
||||
loaded.phase = "stopped";
|
||||
loaded.underlyingStarted = false;
|
||||
loaded.state.cron.stop();
|
||||
loaded.state.stopExitWatchers?.();
|
||||
return;
|
||||
@@ -87,13 +189,37 @@ export function createLazyGatewayCronState(params: LazyGatewayCronParams): Gatew
|
||||
if (!stopped) {
|
||||
return;
|
||||
}
|
||||
resolved.started = false;
|
||||
resolved.phase = "stopped";
|
||||
resolved.underlyingStarted = false;
|
||||
resolved.state.cron.stop();
|
||||
resolved.state.stopExitWatchers?.();
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
},
|
||||
pauseScheduling() {
|
||||
schedulingPaused = true;
|
||||
if (loaded) {
|
||||
loaded.state.cron.pauseScheduling();
|
||||
loaded.schedulingPaused = true;
|
||||
}
|
||||
},
|
||||
resumeScheduling() {
|
||||
schedulingPaused = false;
|
||||
releaseSchedulingResumeWaiters();
|
||||
if (
|
||||
loaded &&
|
||||
loaded.schedulingPaused &&
|
||||
(loaded.underlyingStarted || loaded.underlyingStartInFlight)
|
||||
) {
|
||||
loaded.state.cron.resumeScheduling();
|
||||
loaded.schedulingPaused = false;
|
||||
}
|
||||
},
|
||||
getSuspensionBlockerCount() {
|
||||
const loadedBlockers = loaded?.state.cron.getSuspensionBlockerCount?.() ?? 0;
|
||||
return loaded?.phase === "starting" ? Math.max(1, loadedBlockers) : loadedBlockers;
|
||||
},
|
||||
async status() {
|
||||
return await (await load()).state.cron.status();
|
||||
},
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
// Cron notification tests protect completion-delivery warning behavior,
|
||||
// including URL redaction for invalid webhook destinations.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import { makeCronJob } from "../cron/delivery.test-helpers.js";
|
||||
import type { CronJob } from "../cron/types.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuard: vi.fn(async (_request: unknown) => ({ release: vi.fn() })),
|
||||
sendFailureNotificationAnnounce: vi.fn(),
|
||||
sendCronAnnouncePayloadStrict: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../infra/net/fetch-guard.js", () => ({
|
||||
@@ -19,10 +26,14 @@ vi.mock("../cron/delivery.js", async (importOriginal) => {
|
||||
return {
|
||||
...actual,
|
||||
sendFailureNotificationAnnounce: mocks.sendFailureNotificationAnnounce,
|
||||
sendCronAnnouncePayloadStrict: mocks.sendCronAnnouncePayloadStrict,
|
||||
};
|
||||
});
|
||||
|
||||
import { dispatchGatewayCronFinishedNotifications } from "./server-cron-notifications.js";
|
||||
import {
|
||||
dispatchGatewayCronFinishedNotifications,
|
||||
sendGatewayCronFailureAlert,
|
||||
} from "./server-cron-notifications.js";
|
||||
|
||||
function requireRecord(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object") {
|
||||
@@ -44,9 +55,184 @@ function webhookRequestBody() {
|
||||
return JSON.parse(init.body);
|
||||
}
|
||||
|
||||
function createVoidDeferred(): { promise: Promise<void>; resolve: () => void } {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function createWebhookJob(delivery: NonNullable<CronJob["delivery"]>): CronJob {
|
||||
return {
|
||||
id: "cron-notification-admission",
|
||||
name: "notification admission",
|
||||
enabled: true,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 1,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "hello" },
|
||||
delivery,
|
||||
state: {},
|
||||
};
|
||||
}
|
||||
|
||||
describe("dispatchGatewayCronFinishedNotifications", () => {
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.clearAllMocks();
|
||||
mocks.fetchWithSsrFGuard.mockImplementation(async () => ({ release: vi.fn() }));
|
||||
mocks.sendFailureNotificationAnnounce.mockResolvedValue(undefined);
|
||||
mocks.sendCronAnnouncePayloadStrict.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
it("independently admits detached completion webhook delivery", async () => {
|
||||
const deferred = createVoidDeferred();
|
||||
mocks.fetchWithSsrFGuard.mockImplementationOnce(async () => {
|
||||
await deferred.promise;
|
||||
return { release: vi.fn() };
|
||||
});
|
||||
const job = createWebhookJob({
|
||||
mode: "webhook",
|
||||
to: "https://example.invalid/cron",
|
||||
});
|
||||
const parentAdmission = tryBeginGatewayRootWorkAdmission();
|
||||
expect(parentAdmission).not.toBeNull();
|
||||
if (!parentAdmission) {
|
||||
throw new Error("expected parent Gateway work admission");
|
||||
}
|
||||
|
||||
try {
|
||||
await parentAdmission.run(async () => {
|
||||
dispatchGatewayCronFinishedNotifications({
|
||||
evt: { jobId: job.id, action: "finished", status: "ok", summary: "done" },
|
||||
job,
|
||||
deps: {} as CliDeps,
|
||||
logger: { warn: vi.fn() },
|
||||
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(2);
|
||||
});
|
||||
} finally {
|
||||
parentAdmission.release();
|
||||
}
|
||||
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
deferred.resolve();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("independently admits immediate failure alerts", async () => {
|
||||
const deferred = createVoidDeferred();
|
||||
mocks.sendCronAnnouncePayloadStrict.mockImplementationOnce(async () => {
|
||||
await deferred.promise;
|
||||
});
|
||||
const job = createWebhookJob({ mode: "announce", channel: "discord", to: "channel:ops" });
|
||||
|
||||
const delivery = sendGatewayCronFailureAlert({
|
||||
deps: {} as CliDeps,
|
||||
logger: { warn: vi.fn() },
|
||||
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
|
||||
job,
|
||||
text: "cron failed",
|
||||
channel: "discord",
|
||||
to: "channel:ops",
|
||||
mode: "announce",
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(mocks.sendCronAnnouncePayloadStrict).toHaveBeenCalledOnce());
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
deferred.resolve();
|
||||
await delivery;
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("defers detached completion delivery while suspension is prepared", async () => {
|
||||
const job = createWebhookJob({
|
||||
mode: "webhook",
|
||||
to: "https://example.invalid/cron",
|
||||
});
|
||||
const suspensionAdmission = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspensionAdmission?.commit()).toBe(true);
|
||||
|
||||
dispatchGatewayCronFinishedNotifications({
|
||||
evt: { jobId: job.id, action: "finished", status: "ok", summary: "done" },
|
||||
job,
|
||||
deps: {} as CliDeps,
|
||||
logger: { warn: vi.fn() },
|
||||
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(mocks.fetchWithSsrFGuard).not.toHaveBeenCalled();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
|
||||
expect(suspensionAdmission?.release()).toBe(true);
|
||||
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
|
||||
});
|
||||
|
||||
it("independently admits failure destination webhook delivery", async () => {
|
||||
const deferred = createVoidDeferred();
|
||||
mocks.fetchWithSsrFGuard.mockImplementationOnce(async () => {
|
||||
await deferred.promise;
|
||||
return { release: vi.fn() };
|
||||
});
|
||||
const job = createWebhookJob({
|
||||
mode: "announce",
|
||||
channel: "last",
|
||||
failureDestination: {
|
||||
mode: "webhook",
|
||||
to: "https://example.invalid/failure",
|
||||
},
|
||||
});
|
||||
|
||||
dispatchGatewayCronFinishedNotifications({
|
||||
evt: { jobId: job.id, action: "finished", status: "error", error: "boom" },
|
||||
job,
|
||||
deps: {} as CliDeps,
|
||||
logger: { warn: vi.fn() },
|
||||
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(mocks.fetchWithSsrFGuard).toHaveBeenCalledTimes(1));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
deferred.resolve();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("independently admits failure destination announce delivery", async () => {
|
||||
const deferred = createVoidDeferred();
|
||||
mocks.sendFailureNotificationAnnounce.mockImplementationOnce(() => deferred.promise);
|
||||
const job = createWebhookJob({
|
||||
mode: "announce",
|
||||
channel: "last",
|
||||
failureDestination: {
|
||||
mode: "announce",
|
||||
channel: "telegram",
|
||||
to: "-1001234567890",
|
||||
},
|
||||
});
|
||||
|
||||
dispatchGatewayCronFinishedNotifications({
|
||||
evt: { jobId: job.id, action: "finished", status: "error", error: "boom" },
|
||||
job,
|
||||
deps: {} as CliDeps,
|
||||
logger: { warn: vi.fn() },
|
||||
resolveCronAgent: () => ({ agentId: "main", cfg: {} }),
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(mocks.sendFailureNotificationAnnounce).toHaveBeenCalledTimes(1));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
deferred.resolve();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("redacts invalid completion webhook targets in warnings", () => {
|
||||
|
||||
@@ -21,6 +21,7 @@ import { normalizeHttpWebhookUrl } from "../cron/webhook-url.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
|
||||
import { SsrFBlockedError } from "../infra/net/ssrf.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
|
||||
const CRON_WEBHOOK_TIMEOUT_MS = 10_000;
|
||||
|
||||
@@ -38,6 +39,19 @@ type CronWebhookTarget = {
|
||||
source: "delivery" | "completionDestination";
|
||||
};
|
||||
|
||||
type CronFailureAlertParams = {
|
||||
deps: CliDeps;
|
||||
logger: CronLogger;
|
||||
resolveCronAgent: CronAgentResolver;
|
||||
webhookToken?: unknown;
|
||||
job: CronJob;
|
||||
text: string;
|
||||
channel: CronMessageChannel;
|
||||
to?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
accountId?: string;
|
||||
};
|
||||
|
||||
function redactWebhookUrl(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
@@ -232,19 +246,30 @@ async function postCronWebhook(params: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Sends the immediate failure alert for cron jobs that failed before normal completion delivery. */
|
||||
export async function sendGatewayCronFailureAlert(params: {
|
||||
deps: CliDeps;
|
||||
/** Detached sends outlive cron ticks; own roots block mid-delivery suspension snapshots. */
|
||||
function dispatchDetachedCronNotification(params: {
|
||||
jobId: string;
|
||||
logger: CronLogger;
|
||||
resolveCronAgent: CronAgentResolver;
|
||||
webhookToken?: unknown;
|
||||
job: CronJob;
|
||||
text: string;
|
||||
channel: CronMessageChannel;
|
||||
to?: string;
|
||||
mode?: "announce" | "webhook";
|
||||
accountId?: string;
|
||||
}): Promise<void> {
|
||||
deliver: () => Promise<void>;
|
||||
}): void {
|
||||
void runWithGatewayIndependentRootWorkAdmission(params.deliver).catch((err: unknown) => {
|
||||
params.logger.warn(
|
||||
{ jobId: params.jobId, err: formatErrorMessage(err) },
|
||||
"cron: detached notification delivery failed",
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Sends the immediate failure alert for cron jobs that failed before normal completion delivery. */
|
||||
export async function sendGatewayCronFailureAlert(params: CronFailureAlertParams): Promise<void> {
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await sendGatewayCronFailureAlertUnderAdmission(params);
|
||||
});
|
||||
}
|
||||
|
||||
async function sendGatewayCronFailureAlertUnderAdmission(
|
||||
params: CronFailureAlertParams,
|
||||
): Promise<void> {
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
|
||||
const webhookToken = normalizeOptionalString(params.webhookToken);
|
||||
|
||||
@@ -355,17 +380,20 @@ export function dispatchGatewayCronFinishedNotifications(params: {
|
||||
const payload = buildCronFinishedWebhookPayload(redactedWebhookEvent);
|
||||
// Completion notification fanout is best-effort; the cron service has
|
||||
// already recorded the run result and must not wait on slow webhooks.
|
||||
void (async () => {
|
||||
await postCronWebhook({
|
||||
webhookUrl: webhookTarget.url,
|
||||
webhookToken,
|
||||
payload,
|
||||
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
|
||||
blockedLog: "cron: webhook delivery blocked by SSRF guard",
|
||||
failedLog: "cron: webhook delivery failed",
|
||||
logger: params.logger,
|
||||
});
|
||||
})();
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: params.evt.jobId,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
postCronWebhook({
|
||||
webhookUrl: webhookTarget.url,
|
||||
webhookToken,
|
||||
payload,
|
||||
logContext: { jobId: params.evt.jobId, source: webhookTarget.source },
|
||||
blockedLog: "cron: webhook delivery blocked by SSRF guard",
|
||||
failedLog: "cron: webhook delivery failed",
|
||||
logger: params.logger,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,9 +421,10 @@ function dispatchCronFailureDestinationNotifications(params: {
|
||||
return;
|
||||
}
|
||||
|
||||
const failureDest = resolveFailureDestination(params.job, params.globalFailureDestination);
|
||||
const deliverySessionKey = resolveCronDeliverySessionKey(params.job);
|
||||
const failurePayload = buildCronFailureWebhookPayload({ evt: params.evt, job: params.job });
|
||||
const job = params.job;
|
||||
const failureDest = resolveFailureDestination(job, params.globalFailureDestination);
|
||||
const deliverySessionKey = resolveCronDeliverySessionKey(job);
|
||||
const failurePayload = buildCronFailureWebhookPayload({ evt: params.evt, job });
|
||||
|
||||
if (failureDest) {
|
||||
if (failureDest.mode === "webhook" && failureDest.to) {
|
||||
@@ -403,17 +432,20 @@ function dispatchCronFailureDestinationNotifications(params: {
|
||||
if (webhookUrl) {
|
||||
// Failure destinations mirror completion webhooks: notify in the
|
||||
// background and log failures without rewriting the cron event result.
|
||||
void (async () => {
|
||||
await postCronWebhook({
|
||||
webhookUrl,
|
||||
webhookToken: params.webhookToken,
|
||||
payload: failurePayload,
|
||||
logContext: { jobId: params.evt.jobId },
|
||||
blockedLog: "cron: failure destination webhook blocked by SSRF guard",
|
||||
failedLog: "cron: failure destination webhook failed",
|
||||
logger: params.logger,
|
||||
});
|
||||
})();
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: params.evt.jobId,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
postCronWebhook({
|
||||
webhookUrl,
|
||||
webhookToken: params.webhookToken,
|
||||
payload: failurePayload,
|
||||
logContext: { jobId: params.evt.jobId },
|
||||
blockedLog: "cron: failure destination webhook blocked by SSRF guard",
|
||||
failedLog: "cron: failure destination webhook failed",
|
||||
logger: params.logger,
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
params.logger.warn(
|
||||
{
|
||||
@@ -427,44 +459,54 @@ function dispatchCronFailureDestinationNotifications(params: {
|
||||
}
|
||||
|
||||
if (failureDest.mode === "announce") {
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
|
||||
void sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
params.job.id,
|
||||
{
|
||||
channel: failureDest.channel,
|
||||
to: failureDest.to,
|
||||
accountId: failureDest.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
// A configured failure route is already explicit; keep the cron run
|
||||
// session only for context, not for reattaching the primary topic.
|
||||
inheritSessionThread: false,
|
||||
},
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
);
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: job.id,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
job.id,
|
||||
{
|
||||
channel: failureDest.channel,
|
||||
to: failureDest.to,
|
||||
accountId: failureDest.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
// A configured failure route is already explicit; keep the cron run
|
||||
// session only for context, not for reattaching the primary topic.
|
||||
inheritSessionThread: false,
|
||||
},
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const primaryPlan = resolveCronDeliveryPlan(params.job);
|
||||
const primaryPlan = resolveCronDeliveryPlan(job);
|
||||
if (primaryPlan.mode !== "announce" || !primaryPlan.requested) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(params.job.agentId);
|
||||
void sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
params.job.id,
|
||||
{
|
||||
channel: primaryPlan.channel,
|
||||
to: primaryPlan.to,
|
||||
accountId: primaryPlan.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
},
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
);
|
||||
const { agentId, cfg: runtimeConfig } = params.resolveCronAgent(job.agentId);
|
||||
dispatchDetachedCronNotification({
|
||||
jobId: job.id,
|
||||
logger: params.logger,
|
||||
deliver: () =>
|
||||
sendFailureNotificationAnnounce(
|
||||
params.deps,
|
||||
runtimeConfig,
|
||||
agentId,
|
||||
job.id,
|
||||
{
|
||||
channel: primaryPlan.channel,
|
||||
to: primaryPlan.to,
|
||||
accountId: primaryPlan.accountId,
|
||||
sessionKey: deliverySessionKey,
|
||||
},
|
||||
`⚠️ ${failurePayload.message}`,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -6,6 +6,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { CliDeps } from "../cli/deps.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { SsrFBlockedError } from "../infra/net/ssrf.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { createDeferred } from "../test-utils/deferred.js";
|
||||
|
||||
type RunCronIsolatedAgentTurnMock = (params: {
|
||||
@@ -435,7 +439,6 @@ describe("buildGatewayCronService", () => {
|
||||
vi.setSystemTime(new Date("2026-07-10T12:00:00.000Z"));
|
||||
const cfg = createCronConfig("server-cron-hook-scheduled");
|
||||
loadConfigMock.mockReturnValue(cfg);
|
||||
|
||||
const state = buildGatewayCronService({
|
||||
cfg,
|
||||
deps: {} as CliDeps,
|
||||
@@ -486,6 +489,39 @@ describe("buildGatewayCronService", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps detached cron_changed hooks root-admitted until they settle", async () => {
|
||||
resetGatewayWorkAdmission();
|
||||
const deferred = createDeferred();
|
||||
runCronChangedMock.mockImplementationOnce(async () => await deferred.promise);
|
||||
const cfg = createCronConfig("server-cron-hook-admission");
|
||||
loadConfigMock.mockReturnValue(cfg);
|
||||
const state = buildGatewayCronService({
|
||||
cfg,
|
||||
deps: {} as CliDeps,
|
||||
broadcast: () => {},
|
||||
});
|
||||
|
||||
try {
|
||||
await state.cron.add({
|
||||
name: "held hook",
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "hello" },
|
||||
});
|
||||
await vi.waitFor(() => expect(runCronChangedMock).toHaveBeenCalledTimes(1));
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
deferred.resolve();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
} finally {
|
||||
deferred.resolve();
|
||||
state.cron.stop();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("cron_changed removed events include the deleted job snapshot", async () => {
|
||||
const cfg = createCronConfig("server-cron-hook-removed");
|
||||
loadConfigMock.mockReturnValue(cfg);
|
||||
|
||||
+76
-40
@@ -49,6 +49,7 @@ import type {
|
||||
PluginHookGatewayCronService,
|
||||
PluginHookGatewayContext,
|
||||
} from "../plugins/hook-types.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { getProcessSupervisor } from "../process/supervisor/index.js";
|
||||
import {
|
||||
normalizeAgentId,
|
||||
@@ -63,8 +64,16 @@ import {
|
||||
sendGatewayCronFailureAlert,
|
||||
} from "./server-cron-notifications.js";
|
||||
|
||||
export type GatewayCronServiceContract = CronServiceContract & {
|
||||
/** Temporarily disarm ticks without running startup recovery on resume. */
|
||||
pauseScheduling(): void;
|
||||
resumeScheduling(): void;
|
||||
/** Scheduler-owned work not represented by active cron run markers. */
|
||||
getSuspensionBlockerCount?(): number;
|
||||
};
|
||||
|
||||
export type GatewayCronState = {
|
||||
cron: CronServiceContract;
|
||||
cron: GatewayCronServiceContract;
|
||||
storePath: string;
|
||||
cronEnabled: boolean;
|
||||
reconcileExitWatchers?: () => Promise<void>;
|
||||
@@ -371,7 +380,11 @@ export function buildGatewayCronService(params: {
|
||||
config: getRuntimeConfig(),
|
||||
getCron: () => cron as PluginHookGatewayCronService,
|
||||
};
|
||||
void hookRunner.runCronChanged(evt, hookCtx).catch((err: unknown) => {
|
||||
// Hook execution is detached from the cron mutation/tick that emitted it.
|
||||
// Keep the whole plugin callback visible until its user-state effects settle.
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await hookRunner.runCronChanged(evt, hookCtx);
|
||||
}).catch((err: unknown) => {
|
||||
cronLogger.warn(
|
||||
{ err: formatErrorMessage(err), jobId: evt.jobId },
|
||||
"cron_changed hook failed",
|
||||
@@ -383,12 +396,19 @@ export function buildGatewayCronService(params: {
|
||||
const exitWatchersRef: { current: ReturnType<typeof createCronExitWatchers> | undefined } = {
|
||||
current: undefined,
|
||||
};
|
||||
let exitWatcherReconciliations = 0;
|
||||
let exitWatcherGeneration = 0;
|
||||
const reconcileExitWatchers = async () => {
|
||||
if (!exitWatchersRef.current) {
|
||||
return;
|
||||
}
|
||||
const generation = exitWatcherGeneration;
|
||||
exitWatcherReconciliations += 1;
|
||||
try {
|
||||
if (!exitWatchersRef.current) {
|
||||
return;
|
||||
}
|
||||
const result = await cron.list({ includeDisabled: true });
|
||||
if (generation !== exitWatcherGeneration) {
|
||||
return;
|
||||
}
|
||||
const jobs: CronJob[] = Array.isArray(result) ? result : (result as { jobs: CronJob[] }).jobs;
|
||||
reconcileCronExitWatchers({
|
||||
cronEnabled,
|
||||
@@ -397,6 +417,8 @@ export function buildGatewayCronService(params: {
|
||||
});
|
||||
} catch (err) {
|
||||
cronLogger.warn({ err: String(err) }, "cron-exit: reconcile failed");
|
||||
} finally {
|
||||
exitWatcherReconciliations -= 1;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -728,33 +750,35 @@ export function buildGatewayCronService(params: {
|
||||
globalFailureDestination: params.cfg.cron?.failureDestination,
|
||||
});
|
||||
|
||||
void appendCronRunLog({
|
||||
storePath,
|
||||
entry: {
|
||||
ts: Date.now(),
|
||||
jobId: evt.jobId,
|
||||
action: "finished",
|
||||
status: evt.status,
|
||||
error: evt.error,
|
||||
summary: evt.summary,
|
||||
diagnostics: evt.diagnostics,
|
||||
delivered: evt.delivered,
|
||||
deliveryStatus: evt.deliveryStatus,
|
||||
deliveryError: evt.deliveryError,
|
||||
failureNotificationDelivery: evt.failureNotificationDelivery,
|
||||
delivery: evt.delivery,
|
||||
sessionId: evt.sessionId,
|
||||
sessionKey: evt.sessionKey,
|
||||
runId: evt.runId,
|
||||
runAtMs: evt.runAtMs,
|
||||
durationMs: evt.durationMs,
|
||||
nextRunAtMs: evt.nextRunAtMs,
|
||||
triggerFired: evt.triggerFired,
|
||||
model: evt.model,
|
||||
provider: evt.provider,
|
||||
usage: evt.usage,
|
||||
},
|
||||
opts: { keepLines: runLogPrune.keepLines },
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await appendCronRunLog({
|
||||
storePath,
|
||||
entry: {
|
||||
ts: Date.now(),
|
||||
jobId: evt.jobId,
|
||||
action: "finished",
|
||||
status: evt.status,
|
||||
error: evt.error,
|
||||
summary: evt.summary,
|
||||
diagnostics: evt.diagnostics,
|
||||
delivered: evt.delivered,
|
||||
deliveryStatus: evt.deliveryStatus,
|
||||
deliveryError: evt.deliveryError,
|
||||
failureNotificationDelivery: evt.failureNotificationDelivery,
|
||||
delivery: evt.delivery,
|
||||
sessionId: evt.sessionId,
|
||||
sessionKey: evt.sessionKey,
|
||||
runId: evt.runId,
|
||||
runAtMs: evt.runAtMs,
|
||||
durationMs: evt.durationMs,
|
||||
nextRunAtMs: evt.nextRunAtMs,
|
||||
triggerFired: evt.triggerFired,
|
||||
model: evt.model,
|
||||
provider: evt.provider,
|
||||
usage: evt.usage,
|
||||
},
|
||||
opts: { keepLines: runLogPrune.keepLines },
|
||||
});
|
||||
}).catch((err: unknown) => {
|
||||
cronLogger.warn(
|
||||
{ err: String(err), storePath, jobId: evt.jobId },
|
||||
@@ -767,19 +791,31 @@ export function buildGatewayCronService(params: {
|
||||
|
||||
exitWatchersRef.current = createCronExitWatchers({
|
||||
getProcessSupervisor,
|
||||
persistCompletion: async (jobId) => {
|
||||
await cron.update(jobId, { enabled: false });
|
||||
},
|
||||
fireOnExit: (job, exit) =>
|
||||
fireOnExitJob(job, exit, {
|
||||
run: (jobId, payload) => cron.run(jobId, "force", payload ? { payload } : undefined),
|
||||
persistCompletion: async (jobId) =>
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await cron.update(jobId, { enabled: false });
|
||||
}),
|
||||
fireOnExit: (job, exit) =>
|
||||
runWithGatewayIndependentRootWorkAdmission(async () =>
|
||||
fireOnExitJob(job, exit, {
|
||||
run: (jobId, payload) => cron.run(jobId, "force", payload ? { payload } : undefined),
|
||||
}),
|
||||
),
|
||||
logger: cronLogger,
|
||||
});
|
||||
const getCronSuspensionBlockerCount = cron.getSuspensionBlockerCount.bind(cron);
|
||||
cron.getSuspensionBlockerCount = () =>
|
||||
getCronSuspensionBlockerCount() +
|
||||
exitWatcherReconciliations +
|
||||
(exitWatchersRef.current?.activeJobIds().length ?? 0);
|
||||
const stopExitWatchers = () => {
|
||||
exitWatcherGeneration += 1;
|
||||
exitWatchersRef.current?.cancelAll();
|
||||
};
|
||||
const stopCron = cron.stop.bind(cron);
|
||||
cron.stop = () => {
|
||||
stopCron();
|
||||
exitWatchersRef.current?.cancelAll();
|
||||
stopExitWatchers();
|
||||
};
|
||||
|
||||
return {
|
||||
@@ -787,6 +823,6 @@ export function buildGatewayCronService(params: {
|
||||
storePath,
|
||||
cronEnabled,
|
||||
reconcileExitWatchers,
|
||||
stopExitWatchers: () => exitWatchersRef.current?.cancelAll(),
|
||||
stopExitWatchers,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
// Server HTTP probe tests cover readiness, health, disabled compat routes, and
|
||||
// auth handling through the in-memory HTTP harness.
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
prepareGatewaySuspend,
|
||||
resetGatewaySuspendCoordinatorForTest,
|
||||
resumeGatewaySuspend,
|
||||
} from "../infra/gateway-suspend-coordinator.js";
|
||||
import { isGatewayDraining } from "../process/command-queue.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import type { ChannelManager } from "./server-channels.js";
|
||||
import {
|
||||
AUTH_TOKEN,
|
||||
AUTH_NONE,
|
||||
@@ -9,7 +21,7 @@ import {
|
||||
dispatchRequest,
|
||||
withGatewayServer,
|
||||
} from "./server-http.test-harness.js";
|
||||
import type { ReadinessChecker } from "./server/readiness.js";
|
||||
import { createReadinessChecker, type ReadinessChecker } from "./server/readiness.js";
|
||||
import { withTempConfig } from "./test-temp-config.js";
|
||||
|
||||
type GatewayServerHarness = Parameters<typeof dispatchRequest>[0];
|
||||
@@ -44,6 +56,160 @@ describe("gateway OpenAI-compatible disabled HTTP routes", () => {
|
||||
});
|
||||
|
||||
describe("gateway probe endpoints", () => {
|
||||
it("keeps liveness green while a prepared suspension lease makes readiness red", async () => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
const channelManager = {
|
||||
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
|
||||
getAutostartSuppression: () => null,
|
||||
} as unknown as ChannelManager;
|
||||
const getReadiness = createReadinessChecker({
|
||||
channelManager,
|
||||
startedAt: Date.now(),
|
||||
getGatewayDraining: isGatewayDraining,
|
||||
cacheTtlMs: 0,
|
||||
});
|
||||
|
||||
try {
|
||||
await withGatewayServer({
|
||||
prefix: "probe-suspension-lease",
|
||||
resolvedAuth: AUTH_NONE,
|
||||
overrides: { getReadiness, openAiChatCompletionsEnabled: true },
|
||||
run: async (server) => {
|
||||
const prepared = prepareGatewaySuspend({
|
||||
requestId: "request-readiness-probe",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
createSuspensionId: () => "suspension-readiness-probe",
|
||||
inspect: {
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
getRootRequests: () => 0,
|
||||
getSessionAdmissions: () => 0,
|
||||
getSessionMutations: () => 0,
|
||||
getChatRuns: () => 0,
|
||||
getQueuedTurns: () => 0,
|
||||
getTerminalPersistence: () => 0,
|
||||
getTerminalSessions: () => 0,
|
||||
},
|
||||
});
|
||||
if (prepared.status !== "ready") {
|
||||
throw new Error(`expected prepared suspension, received ${prepared.status}`);
|
||||
}
|
||||
|
||||
const health = await sendGatewayRequest(server, { path: "/healthz" });
|
||||
expect(health.res.statusCode).toBe(200);
|
||||
expect(JSON.parse(health.getBody())).toEqual({ ok: true, status: "live" });
|
||||
|
||||
const suspendedReadiness = await sendGatewayRequest(server, { path: "/readyz" });
|
||||
expect(suspendedReadiness.res.statusCode).toBe(503);
|
||||
expect(JSON.parse(suspendedReadiness.getBody())).toMatchObject({
|
||||
ready: false,
|
||||
failing: ["gateway-draining"],
|
||||
});
|
||||
|
||||
const blockedChat = await sendGatewayRequest(server, {
|
||||
path: "/v1/chat/completions",
|
||||
method: "POST",
|
||||
});
|
||||
expect(blockedChat.res.statusCode).toBe(503);
|
||||
expect(JSON.parse(blockedChat.getBody())).toMatchObject({
|
||||
error: { code: "gateway_unavailable" },
|
||||
});
|
||||
|
||||
expect(resumeGatewaySuspend(prepared.suspensionId)).toEqual({
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: true,
|
||||
});
|
||||
|
||||
const resumedReadiness = await sendGatewayRequest(server, { path: "/readyz" });
|
||||
expect(resumedReadiness.res.statusCode).toBe(200);
|
||||
expect(JSON.parse(resumedReadiness.getBody())).toMatchObject({
|
||||
ready: true,
|
||||
failing: [],
|
||||
});
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps in-flight core HTTP work visible to suspension preparation", async () => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
let releaseWatch = () => {};
|
||||
let markWatchStarted = () => {};
|
||||
const watchStarted = new Promise<void>((resolve) => {
|
||||
markWatchStarted = resolve;
|
||||
});
|
||||
const heldWatch = new Promise<void>((resolve) => {
|
||||
releaseWatch = resolve;
|
||||
});
|
||||
const handleWatchNodeRequest = vi.fn(async (_req: IncomingMessage, res: ServerResponse) => {
|
||||
markWatchStarted();
|
||||
await heldWatch;
|
||||
res.statusCode = 200;
|
||||
res.end("ok");
|
||||
return true;
|
||||
});
|
||||
|
||||
try {
|
||||
await withGatewayServer({
|
||||
prefix: "probe-http-work-admission",
|
||||
resolvedAuth: AUTH_NONE,
|
||||
overrides: { handleWatchNodeRequest },
|
||||
run: async (server) => {
|
||||
const request = createRequest({ path: "/api/nodes/watch/node-1" });
|
||||
const response = createResponse();
|
||||
const pendingRequest = dispatchRequest(server, request, response.res);
|
||||
await watchStarted;
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
const prepared = prepareGatewaySuspend({
|
||||
requestId: "request-http-work",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
inspect: {
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
getSessionAdmissions: () => 0,
|
||||
getSessionMutations: () => 0,
|
||||
getChatRuns: () => 0,
|
||||
getQueuedTurns: () => 0,
|
||||
getTerminalPersistence: () => 0,
|
||||
getTerminalSessions: () => 0,
|
||||
},
|
||||
});
|
||||
expect(prepared).toMatchObject({
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
activeCount: 1,
|
||||
});
|
||||
|
||||
releaseWatch();
|
||||
await pendingRequest;
|
||||
expect(response.res.statusCode).toBe(200);
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
releaseWatch();
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("returns detailed readiness payload for local /ready requests", async () => {
|
||||
const getReadiness: ReadinessChecker = () => ({
|
||||
ready: true,
|
||||
|
||||
+116
-47
@@ -16,6 +16,10 @@ import {
|
||||
createDiagnosticTraceContext,
|
||||
runWithDiagnosticTraceContext,
|
||||
} from "../infra/diagnostic-trace-context.js";
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import { resolveAssistantIdentity } from "./assistant-identity.js";
|
||||
import type { AuthRateLimiter } from "./auth-rate-limit.js";
|
||||
@@ -52,6 +56,7 @@ type PluginHttpRequestHandler = (
|
||||
gatewayAuthSatisfied?: boolean;
|
||||
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
|
||||
gatewayRequestOperatorScopes?: readonly string[];
|
||||
gatewayRequestClientIp?: string;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
|
||||
@@ -66,6 +71,7 @@ type PluginHttpUpgradeHandler = (
|
||||
gatewayAuthSatisfied?: boolean;
|
||||
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
|
||||
gatewayRequestOperatorScopes?: readonly string[];
|
||||
gatewayRequestClientIp?: string;
|
||||
},
|
||||
) => Promise<boolean>;
|
||||
|
||||
@@ -348,6 +354,35 @@ export async function runGatewayHttpRequestStages(
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Runs one core HTTP user-work route under the same root fence as Gateway RPCs. */
|
||||
export async function runWithGatewayHttpWorkAdmission(
|
||||
res: ServerResponse,
|
||||
run: () => Promise<boolean> | boolean,
|
||||
): Promise<boolean> {
|
||||
const admission = tryBeginGatewayRootWorkAdmission();
|
||||
if (!admission) {
|
||||
res.statusCode = 503;
|
||||
res.setHeader("Content-Type", "application/json; charset=utf-8");
|
||||
res.setHeader("Cache-Control", "no-store");
|
||||
res.setHeader("Retry-After", "1");
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
error: {
|
||||
message: "Gateway is temporarily unavailable while suspending or restarting",
|
||||
type: "service_unavailable",
|
||||
code: "gateway_unavailable",
|
||||
},
|
||||
}),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
return await admission.run(async () => await run());
|
||||
} finally {
|
||||
admission.release();
|
||||
}
|
||||
}
|
||||
|
||||
function buildPluginRequestStages(params: {
|
||||
req: IncomingMessage;
|
||||
res: ServerResponse;
|
||||
@@ -364,6 +399,11 @@ function buildPluginRequestStages(params: {
|
||||
if (!params.handlePluginRequest) {
|
||||
return [];
|
||||
}
|
||||
const requestClientIp = resolveRequestClientIp(
|
||||
params.req,
|
||||
params.trustedProxies,
|
||||
params.allowRealIpFallback,
|
||||
);
|
||||
let pluginGatewayAuthSatisfied = false;
|
||||
let pluginGatewayRequestAuth: AuthorizedGatewayHttpRequest | undefined;
|
||||
let pluginRequestOperatorScopes: string[] | undefined;
|
||||
@@ -422,6 +462,7 @@ function buildPluginRequestStages(params: {
|
||||
gatewayAuthSatisfied: pluginGatewayAuthSatisfied,
|
||||
gatewayRequestAuth: pluginGatewayRequestAuth,
|
||||
gatewayRequestOperatorScopes: pluginRequestOperatorScopes,
|
||||
gatewayRequestClientIp: requestClientIp,
|
||||
}) ?? false
|
||||
);
|
||||
},
|
||||
@@ -559,94 +600,112 @@ export function createGatewayHttpServer(opts: {
|
||||
if (opts.handleWatchNodeRequest && scopedRequestPath.startsWith("/api/nodes/watch/")) {
|
||||
requestStages.push({
|
||||
name: "watch-node",
|
||||
run: () => opts.handleWatchNodeRequest?.(req, res) ?? false,
|
||||
run: () =>
|
||||
runWithGatewayHttpWorkAdmission(
|
||||
res,
|
||||
() => opts.handleWatchNodeRequest?.(req, res) ?? false,
|
||||
),
|
||||
});
|
||||
}
|
||||
if (openAiCompatEnabled && isOpenAiModelsPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "models",
|
||||
run: async () =>
|
||||
(await getModelsHttpModule()).handleOpenAiModelsHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getModelsHttpModule()).handleOpenAiModelsHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (openAiCompatEnabled && isEmbeddingsPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "embeddings",
|
||||
run: async () =>
|
||||
(await getEmbeddingsHttpModule()).handleOpenAiEmbeddingsHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getEmbeddingsHttpModule()).handleOpenAiEmbeddingsHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (isToolsInvokePath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "tools-invoke",
|
||||
run: async () =>
|
||||
(await getToolsInvokeHttpModule()).handleToolsInvokeHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getToolsInvokeHttpModule()).handleToolsInvokeHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (isSessionKillPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "sessions-kill",
|
||||
run: async () =>
|
||||
(await getSessionKillHttpModule()).handleSessionKillHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getSessionKillHttpModule()).handleSessionKillHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (isSessionHistoryPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "sessions-history",
|
||||
run: async () =>
|
||||
(await getSessionHistoryHttpModule()).handleSessionHistoryHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
getResolvedAuth,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getSessionHistoryHttpModule()).handleSessionHistoryHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
getResolvedAuth,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (openResponsesEnabled && isOpenResponsesPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "openresponses",
|
||||
run: async () =>
|
||||
(await getOpenResponsesHttpModule()).handleOpenResponsesHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
config: openResponsesConfig,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getOpenResponsesHttpModule()).handleOpenResponsesHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
config: openResponsesConfig,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (openAiChatCompletionsEnabled && isOpenAiChatCompletionsPath(scopedRequestPath)) {
|
||||
requestStages.push({
|
||||
name: "openai",
|
||||
run: async () =>
|
||||
(await getOpenAiHttpModule()).handleOpenAiHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
config: openAiChatCompletionsConfig,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
await runWithGatewayHttpWorkAdmission(res, async () =>
|
||||
(await getOpenAiHttpModule()).handleOpenAiHttpRequest(req, res, {
|
||||
auth: resolvedAuthValue,
|
||||
config: openAiChatCompletionsConfig,
|
||||
trustedProxies,
|
||||
allowRealIpFallback,
|
||||
rateLimiter,
|
||||
}),
|
||||
),
|
||||
});
|
||||
}
|
||||
if (
|
||||
@@ -843,6 +902,7 @@ export function attachGatewayUpgradeHandler(opts: {
|
||||
const configSnapshot = getRuntimeConfig();
|
||||
const trustedProxies = configSnapshot.gateway?.trustedProxies ?? [];
|
||||
const allowRealIpFallback = configSnapshot.gateway?.allowRealIpFallback === true;
|
||||
const requestClientIp = resolveRequestClientIp(req, trustedProxies, allowRealIpFallback);
|
||||
const scopedNodeCapability = normalizePluginNodeCapabilityScopedUrl(req.url ?? "/");
|
||||
if (scopedNodeCapability.malformedScopedPath) {
|
||||
writeUpgradeAuthFailure(socket, { ok: false, reason: "unauthorized" });
|
||||
@@ -916,12 +976,21 @@ export function attachGatewayUpgradeHandler(opts: {
|
||||
gatewayAuthSatisfied: pluginGatewayAuthSatisfied,
|
||||
gatewayRequestAuth: pluginGatewayRequestAuth,
|
||||
gatewayRequestOperatorScopes: pluginGatewayRequestOperatorScopes,
|
||||
gatewayRequestClientIp: requestClientIp,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
const preauthBudgetKey = resolveRequestClientIp(req, trustedProxies, allowRealIpFallback);
|
||||
// Plugin-owned upgrade routes have already had the opportunity to claim the socket.
|
||||
// Core Gateway upgrades must stop at the HTTP boundary so a client cannot hold an
|
||||
// untracked pre-connect socket after suspension or restart admission closes.
|
||||
if (isGatewayWorkAdmissionClosed()) {
|
||||
writeUpgradeServiceUnavailable(socket, "Gateway websocket admission closed");
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
const preauthBudgetKey = requestClientIp;
|
||||
if (wss.listenerCount("connection") === 0) {
|
||||
writeUpgradeServiceUnavailable(socket, "Gateway websocket handlers unavailable");
|
||||
socket.destroy();
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
*/
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { isRetryableGatewayStartupUnavailableError } from "../../packages/gateway-protocol/src/startup-unavailable.js";
|
||||
import {
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
testing as controlPlaneRateLimitTesting,
|
||||
resolveControlPlaneRateLimitKey,
|
||||
@@ -16,12 +20,14 @@ const noWebchat = () => false;
|
||||
describe("gateway control-plane write rate limit", () => {
|
||||
beforeEach(() => {
|
||||
controlPlaneRateLimitTesting.resetControlPlaneRateLimitState();
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-02-19T00:00:00.000Z"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
resetGatewayWorkAdmission();
|
||||
controlPlaneRateLimitTesting.resetControlPlaneRateLimitState();
|
||||
});
|
||||
|
||||
@@ -145,6 +151,57 @@ describe("gateway control-plane write rate limit", () => {
|
||||
expect(handlerCalls).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it("does not consume the write budget for requests refused during suspension", async () => {
|
||||
const handlerCalls = vi.fn();
|
||||
const handler: GatewayRequestHandler = (opts) => {
|
||||
handlerCalls(opts);
|
||||
opts.respond(true, undefined, undefined);
|
||||
};
|
||||
const context = buildContext();
|
||||
const client = buildClient();
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
for (let attempt = 0; attempt < 4; attempt += 1) {
|
||||
const refused = await runRequest({ method: "config.patch", context, client, handler });
|
||||
expect(respondCall(refused)[2]).toMatchObject({
|
||||
code: "UNAVAILABLE",
|
||||
details: { reason: "gateway-suspending" },
|
||||
});
|
||||
}
|
||||
expect(suspension?.release()).toBe(true);
|
||||
|
||||
const allowed = await runRequest({ method: "config.patch", context, client, handler });
|
||||
expect(allowed).toHaveBeenCalledWith(true, undefined, undefined);
|
||||
expect(handlerCalls).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps suspension preparation rate-limited while admission is closed", async () => {
|
||||
const handlerCalls = vi.fn();
|
||||
const handler: GatewayRequestHandler = (opts) => {
|
||||
handlerCalls(opts);
|
||||
opts.respond(true, undefined, undefined);
|
||||
};
|
||||
const context = buildContext();
|
||||
const client = buildClient();
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
|
||||
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
|
||||
await runRequest({ method: "gateway.suspend.prepare", context, client, handler });
|
||||
const blocked = await runRequest({
|
||||
method: "gateway.suspend.prepare",
|
||||
context,
|
||||
client,
|
||||
handler,
|
||||
});
|
||||
|
||||
expect(handlerCalls).toHaveBeenCalledTimes(3);
|
||||
expect(respondCall(blocked)[2]).toMatchObject({ code: "UNAVAILABLE", retryable: true });
|
||||
expect(suspension?.release()).toBe(true);
|
||||
});
|
||||
|
||||
it.each(STARTUP_UNAVAILABLE_GATEWAY_METHODS)(
|
||||
"blocks startup-gated method %s before dispatch with a retryable startup error",
|
||||
async (method) => {
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
// Proves dispatcher root-work accounting and fail-closed suspension behavior.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
resetGatewaySuspendCoordinatorForTest,
|
||||
resumeGatewaySuspend,
|
||||
} from "../infra/gateway-suspend-coordinator.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
createGatewayMethodRegistry,
|
||||
createPluginGatewayMethodDescriptor,
|
||||
} from "./methods/registry.js";
|
||||
import { handleGatewayRequest } from "./server-methods.js";
|
||||
import { suspendHandlers } from "./server-methods/suspend.js";
|
||||
import type { GatewayRequestHandler } from "./server-methods/types.js";
|
||||
|
||||
function deferred() {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((done) => {
|
||||
resolve = done;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function dispatch(params: {
|
||||
method: string;
|
||||
scope: "operator.read" | "operator.write" | "operator.admin";
|
||||
handler: GatewayRequestHandler;
|
||||
requestParams?: Record<string, unknown>;
|
||||
context?: Parameters<typeof handleGatewayRequest>[0]["context"];
|
||||
}) {
|
||||
const respond = vi.fn();
|
||||
const methodRegistry = createGatewayMethodRegistry([
|
||||
createPluginGatewayMethodDescriptor({
|
||||
pluginId: "suspend-proof",
|
||||
name: params.method,
|
||||
handler: params.handler,
|
||||
scope: params.scope,
|
||||
}),
|
||||
]);
|
||||
const request = handleGatewayRequest({
|
||||
req: {
|
||||
type: "req",
|
||||
id: `request-${params.method}`,
|
||||
method: params.method,
|
||||
params: params.requestParams ?? {},
|
||||
},
|
||||
respond,
|
||||
client: {
|
||||
connId: "conn-suspend-proof",
|
||||
connect: {
|
||||
role: "operator",
|
||||
scopes: [params.scope],
|
||||
client: { id: "cli", version: "test", platform: "linux", mode: "cli" },
|
||||
minProtocol: 1,
|
||||
maxProtocol: 1,
|
||||
},
|
||||
},
|
||||
isWebchatConnect: () => false,
|
||||
context:
|
||||
params.context ??
|
||||
({ logGateway: { warn: vi.fn() } } as unknown as Parameters<
|
||||
typeof handleGatewayRequest
|
||||
>[0]["context"]),
|
||||
methodRegistry,
|
||||
});
|
||||
return { request, respond };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
describe("gateway request suspension admission", () => {
|
||||
it("keeps preparation busy while a previously admitted handler is active", async () => {
|
||||
const started = deferred();
|
||||
const finish = deferred();
|
||||
const handler = vi.fn<GatewayRequestHandler>(async ({ respond }) => {
|
||||
started.resolve();
|
||||
await finish.promise;
|
||||
respond(true, { ok: true });
|
||||
});
|
||||
const active = dispatch({
|
||||
method: "suspend-proof.run",
|
||||
scope: "operator.write",
|
||||
handler,
|
||||
});
|
||||
await started.promise;
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
expect(suspension?.rollback()).toBe(true);
|
||||
|
||||
finish.resolve();
|
||||
await active.request;
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("reports a concurrent root as busy then excludes its own prepare request", async () => {
|
||||
const started = deferred();
|
||||
const finish = deferred();
|
||||
const active = dispatch({
|
||||
method: "suspend-proof.concurrent",
|
||||
scope: "operator.write",
|
||||
handler: async ({ respond }) => {
|
||||
started.resolve();
|
||||
await finish.promise;
|
||||
respond(true, { ok: true });
|
||||
},
|
||||
});
|
||||
await started.promise;
|
||||
|
||||
const prepareHandler = suspendHandlers["gateway.suspend.prepare"];
|
||||
expect(prepareHandler).toBeTypeOf("function");
|
||||
if (!prepareHandler) {
|
||||
throw new Error("expected gateway suspension prepare handler");
|
||||
}
|
||||
const cron = {
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling: vi.fn(),
|
||||
getSuspensionBlockerCount: vi.fn(() => 0),
|
||||
};
|
||||
const context = {
|
||||
cron,
|
||||
logGateway: { warn: vi.fn() },
|
||||
chatAbortControllers: new Map(),
|
||||
chatQueuedTurns: new Map(),
|
||||
} as unknown as Parameters<typeof handleGatewayRequest>[0]["context"];
|
||||
const busy = dispatch({
|
||||
method: "gateway.suspend.prepare",
|
||||
scope: "operator.admin",
|
||||
handler: prepareHandler,
|
||||
requestParams: { requestId: "request-concurrent-root" },
|
||||
context,
|
||||
});
|
||||
await busy.request;
|
||||
|
||||
expect(busy.respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
activeCount: 1,
|
||||
blockers: expect.arrayContaining([
|
||||
expect.objectContaining({ kind: "root-request", count: 1 }),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
|
||||
finish.resolve();
|
||||
await active.request;
|
||||
const ready = dispatch({
|
||||
method: "gateway.suspend.prepare",
|
||||
scope: "operator.admin",
|
||||
handler: prepareHandler,
|
||||
requestParams: { requestId: "request-own-root-excluded" },
|
||||
context,
|
||||
});
|
||||
await ready.request;
|
||||
|
||||
expect(ready.respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
status: "ready",
|
||||
activeCount: 0,
|
||||
blockers: [],
|
||||
}),
|
||||
);
|
||||
const readyPayload = ready.respond.mock.calls[0]?.[1] as { suspensionId?: string } | undefined;
|
||||
expect(readyPayload?.suspensionId).toBeTypeOf("string");
|
||||
expect(resumeGatewaySuspend(readyPayload?.suspensionId ?? "missing")).toMatchObject({
|
||||
ok: true,
|
||||
resumed: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects new read and write handlers outside the suspension allowlist", async () => {
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
const writeHandler = vi.fn<GatewayRequestHandler>();
|
||||
const blocked = dispatch({
|
||||
method: "suspend-proof.write",
|
||||
scope: "operator.write",
|
||||
handler: writeHandler,
|
||||
});
|
||||
await blocked.request;
|
||||
expect(writeHandler).not.toHaveBeenCalled();
|
||||
expect(blocked.respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
retryable: true,
|
||||
details: expect.objectContaining({ reason: "gateway-suspending" }),
|
||||
}),
|
||||
);
|
||||
|
||||
const readHandler = vi.fn<GatewayRequestHandler>(({ respond }) => {
|
||||
respond(true, { state: "visible" });
|
||||
});
|
||||
const allowed = dispatch({
|
||||
method: "suspend-proof.read",
|
||||
scope: "operator.read",
|
||||
handler: readHandler,
|
||||
});
|
||||
await allowed.request;
|
||||
expect(readHandler).not.toHaveBeenCalled();
|
||||
expect(allowed.respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ code: "UNAVAILABLE", retryable: true }),
|
||||
);
|
||||
suspension?.release();
|
||||
});
|
||||
|
||||
it("keeps suspension status reachable while prepared", async () => {
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
const handler = vi.fn<GatewayRequestHandler>(({ respond }) => {
|
||||
respond(true, { ok: true });
|
||||
});
|
||||
|
||||
const status = dispatch({
|
||||
method: "gateway.suspend.status",
|
||||
scope: "operator.read",
|
||||
handler,
|
||||
});
|
||||
await status.request;
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(status.respond).toHaveBeenCalledWith(true, { ok: true });
|
||||
suspension?.release();
|
||||
});
|
||||
|
||||
it("rejects suspension preparation nested inside another root request", async () => {
|
||||
const root = tryBeginGatewayRootWorkAdmission();
|
||||
expect(root?.ownsRoot).toBe(true);
|
||||
const handler = vi.fn<GatewayRequestHandler>();
|
||||
|
||||
await root?.run(async () => {
|
||||
const nested = dispatch({
|
||||
method: "gateway.suspend.prepare",
|
||||
scope: "operator.admin",
|
||||
handler,
|
||||
});
|
||||
await nested.request;
|
||||
expect(nested.respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
retryable: true,
|
||||
details: expect.objectContaining({ reason: "nested-gateway-request" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
root?.release();
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
+114
-27
@@ -7,6 +7,11 @@ import {
|
||||
} from "../../packages/gateway-protocol/src/startup-unavailable.js";
|
||||
import { getPluginRegistryState } from "../plugins/runtime-state.js";
|
||||
import { withPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js";
|
||||
import {
|
||||
getGatewaySuspendAdmissionPhase,
|
||||
isGatewayRestartDraining,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { formatControlPlaneActor, resolveControlPlaneActor } from "./control-plane-audit.js";
|
||||
import { consumeControlPlaneWriteBudget } from "./control-plane-rate-limit.js";
|
||||
import {
|
||||
@@ -190,6 +195,10 @@ const loadRestartHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/restart.js"),
|
||||
(module) => module.restartHandlers,
|
||||
);
|
||||
const loadSuspendHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/suspend.js"),
|
||||
(module) => module.suspendHandlers,
|
||||
);
|
||||
const loadSendHandlers = lazyHandlerModule(
|
||||
() => import("./server-methods/send.js"),
|
||||
(module) => module.sendHandlers,
|
||||
@@ -306,6 +315,16 @@ function authorizeGatewayMethod(
|
||||
return null;
|
||||
}
|
||||
|
||||
const SUSPEND_CONTROL_METHODS = new Set([
|
||||
"gateway.suspend.prepare",
|
||||
"gateway.suspend.status",
|
||||
"gateway.suspend.resume",
|
||||
]);
|
||||
|
||||
function isGatewayMethodAllowedDuringSuspension(method: string): boolean {
|
||||
return SUSPEND_CONTROL_METHODS.has(method);
|
||||
}
|
||||
|
||||
export const coreGatewayHandlers: GatewayRequestHandlers = {
|
||||
...createLazyCoreHandlers({
|
||||
methods: ["connect"],
|
||||
@@ -665,6 +684,10 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
|
||||
methods: ["gateway.restart.request", "gateway.restart.preflight"],
|
||||
loadHandlers: loadRestartHandlers,
|
||||
}),
|
||||
...createLazyCoreHandlers({
|
||||
methods: ["gateway.suspend.prepare", "gateway.suspend.status", "gateway.suspend.resume"],
|
||||
loadHandlers: loadSuspendHandlers,
|
||||
}),
|
||||
...createLazyCoreHandlers({
|
||||
methods: ["message.action", "send", "poll"],
|
||||
loadHandlers: loadSendHandlers,
|
||||
@@ -781,33 +804,40 @@ export async function handleGatewayRequest(
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (methodRegistry.isControlPlaneWrite(req.method)) {
|
||||
const budget = consumeControlPlaneWriteBudget({ client });
|
||||
if (!budget.allowed) {
|
||||
// Control-plane writes mutate gateway-wide state; rate limit before handler lookup so
|
||||
// plugin and aux write methods share the same protection.
|
||||
const actor = resolveControlPlaneActor(client);
|
||||
context.logGateway.warn(
|
||||
`control-plane write rate-limited method=${req.method} ${formatControlPlaneActor(actor)} retryAfterMs=${budget.retryAfterMs} key=${budget.key}`,
|
||||
);
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.UNAVAILABLE,
|
||||
`rate limit exceeded for ${req.method}; retry after ${Math.ceil(budget.retryAfterMs / 1000)}s`,
|
||||
{
|
||||
retryable: true,
|
||||
retryAfterMs: budget.retryAfterMs,
|
||||
details: {
|
||||
method: req.method,
|
||||
limit: "3 per 60s",
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
return;
|
||||
const rejectRateLimitedControlPlaneWrite = (): boolean => {
|
||||
if (!methodRegistry.isControlPlaneWrite(req.method)) {
|
||||
return false;
|
||||
}
|
||||
const budget = consumeControlPlaneWriteBudget({ client });
|
||||
if (budget.allowed) {
|
||||
return false;
|
||||
}
|
||||
const actor = resolveControlPlaneActor(client);
|
||||
context.logGateway.warn(
|
||||
`control-plane write rate-limited method=${req.method} ${formatControlPlaneActor(actor)} retryAfterMs=${budget.retryAfterMs} key=${budget.key}`,
|
||||
);
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.UNAVAILABLE,
|
||||
`rate limit exceeded for ${req.method}; retry after ${Math.ceil(budget.retryAfterMs / 1000)}s`,
|
||||
{
|
||||
retryable: true,
|
||||
retryAfterMs: budget.retryAfterMs,
|
||||
details: {
|
||||
method: req.method,
|
||||
limit: "3 per 60s",
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
return true;
|
||||
};
|
||||
const isSuspendPrepare = req.method === "gateway.suspend.prepare";
|
||||
if (isSuspendPrepare && rejectRateLimitedControlPlaneWrite()) {
|
||||
// Preparation must stay protected even before it owns the root admission that it closes.
|
||||
return;
|
||||
}
|
||||
const handler = methodRegistry.getHandler(req.method) as GatewayRequestHandler | undefined;
|
||||
if (!handler) {
|
||||
@@ -818,6 +848,50 @@ export async function handleGatewayRequest(
|
||||
);
|
||||
return;
|
||||
}
|
||||
const rootWorkAdmission = tryBeginGatewayRootWorkAdmission();
|
||||
if (
|
||||
req.method === "gateway.suspend.prepare" &&
|
||||
rootWorkAdmission &&
|
||||
!rootWorkAdmission.ownsRoot
|
||||
) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.UNAVAILABLE, "gateway suspension cannot begin from a nested request", {
|
||||
retryable: true,
|
||||
retryAfterMs: 1_000,
|
||||
details: { method: req.method, reason: "nested-gateway-request" },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!rootWorkAdmission && !isGatewayMethodAllowedDuringSuspension(req.method)) {
|
||||
const restartDraining = isGatewayRestartDraining();
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.UNAVAILABLE,
|
||||
`${req.method} unavailable during gateway ${restartDraining ? "restart" : "suspension"}`,
|
||||
{
|
||||
retryable: true,
|
||||
retryAfterMs: 1_000,
|
||||
details: {
|
||||
method: req.method,
|
||||
reason: restartDraining ? "gateway-restarting" : "gateway-suspending",
|
||||
phase: getGatewaySuspendAdmissionPhase(),
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!isSuspendPrepare && rejectRateLimitedControlPlaneWrite()) {
|
||||
// A closed admission must reject first so refused writes do not exhaust the controller's
|
||||
// budget and strand it behind rate limiting after suspension resumes.
|
||||
rootWorkAdmission?.release();
|
||||
return;
|
||||
}
|
||||
const invokeHandler = () =>
|
||||
handler({
|
||||
req,
|
||||
@@ -831,5 +905,18 @@ export async function handleGatewayRequest(
|
||||
// subagent methods (e.g. context engine tools spawning sub-agents
|
||||
// during tool execution) can dispatch back into the gateway.
|
||||
// The scope also carries caller identity into plugin-owned gateway methods.
|
||||
await withPluginRuntimeGatewayRequestScope({ context, client, isWebchatConnect }, invokeHandler);
|
||||
const invokeWithRequestScope = async () =>
|
||||
await withPluginRuntimeGatewayRequestScope(
|
||||
{ context, client, isWebchatConnect },
|
||||
invokeHandler,
|
||||
);
|
||||
if (!rootWorkAdmission) {
|
||||
await invokeWithRequestScope();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await rootWorkAdmission.run(invokeWithRequestScope);
|
||||
} finally {
|
||||
rootWorkAdmission.release();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -116,6 +116,7 @@ import {
|
||||
} from "../../infra/voicewake-routing.js";
|
||||
import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
|
||||
import type { PluginHookSessionEndReason } from "../../plugins/hook-types.js";
|
||||
import { retainGatewayRootWorkAdmissionContinuation } from "../../process/gateway-work-admission.js";
|
||||
import {
|
||||
classifySessionKeyShape,
|
||||
isAcpSessionKey,
|
||||
@@ -3543,9 +3544,12 @@ export const agentHandlers: GatewayRequestHandlers = {
|
||||
if (!activeRunAbort.registered) {
|
||||
activeGatewayWorkAdmission.release();
|
||||
}
|
||||
let releaseGatewayRootContinuation: (() => void) | undefined;
|
||||
const cleanupAdmittedRun: typeof activeRunAbort.cleanup = (options) => {
|
||||
activeRunAbort.cleanup(options);
|
||||
activeGatewayWorkAdmission.release();
|
||||
releaseGatewayRootContinuation?.();
|
||||
releaseGatewayRootContinuation = undefined;
|
||||
};
|
||||
if (activeRunAbort.registered) {
|
||||
retainEmbeddedAgentRunAbortabilityForRunId(runId);
|
||||
@@ -3644,6 +3648,9 @@ export const agentHandlers: GatewayRequestHandlers = {
|
||||
// is scheduled out of this request handler so immediate agent.wait calls
|
||||
// can reach the gateway before the pre-turn runner monopolizes the loop.
|
||||
gatewayAdmissionTransferred = true;
|
||||
// Reserve the detached run before this request releases its root. Otherwise
|
||||
// its inherited ALS context becomes retired and rejects subordinate work.
|
||||
releaseGatewayRootContinuation = retainGatewayRootWorkAdmissionContinuation() ?? undefined;
|
||||
void activeGatewayWorkAdmission.run(async () => {
|
||||
await yieldAfterAgentAcceptedAck();
|
||||
|
||||
|
||||
@@ -104,6 +104,10 @@ import { deleteMediaBuffer, MEDIA_MAX_BYTES, type SavedMedia } from "../../media
|
||||
import { createChannelMessageReplyPipeline } from "../../plugin-sdk/channel-outbound.js";
|
||||
import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";
|
||||
import { isPluginOwnedSessionBindingRecord } from "../../plugins/conversation-binding.js";
|
||||
import {
|
||||
retainGatewayRootWorkAdmissionContinuation,
|
||||
runWithGatewayIndependentRootWorkContinuation,
|
||||
} from "../../process/gateway-work-admission.js";
|
||||
import { normalizeAgentId, scopeLegacySessionKeyToAgent } from "../../routing/session-key.js";
|
||||
import { normalizeInputProvenance, type InputProvenance } from "../../sessions/input-provenance.js";
|
||||
import { resolveSendPolicy } from "../../sessions/send-policy.js";
|
||||
@@ -4237,9 +4241,12 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
});
|
||||
return;
|
||||
}
|
||||
let releaseGatewayRootContinuation: (() => void) | undefined;
|
||||
const cleanupAdmittedRun: typeof activeRunAbort.cleanup = (options) => {
|
||||
activeRunAbort.cleanup(options);
|
||||
gatewayWorkAdmission?.release();
|
||||
releaseGatewayRootContinuation?.();
|
||||
releaseGatewayRootContinuation = undefined;
|
||||
};
|
||||
claimAgentRunContext(clientRunId, {
|
||||
sessionKey,
|
||||
@@ -4407,7 +4414,7 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
const chatSendAckedAtMs = chatSendTiming?.ackedAtMs ?? performance.now();
|
||||
const titleSource = stripInlineDirectiveTagsForDisplay(rawMessage).text;
|
||||
if (isDashboardSessionTitleCandidate({ sessionKey, userMessage: titleSource })) {
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
const titleEntry =
|
||||
entry?.sessionId === admittedSessionId
|
||||
? entry
|
||||
@@ -4432,7 +4439,7 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
reason: "chat.title",
|
||||
});
|
||||
}
|
||||
})().catch((err: unknown) => {
|
||||
}).catch((err: unknown) => {
|
||||
context.logGateway.warn(
|
||||
`dashboard session title generation failed: ${formatForLog(err)}`,
|
||||
);
|
||||
@@ -4790,6 +4797,9 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
}
|
||||
emitServerTiming("first-assistant-event", undefined, dispatchStartedAtMs);
|
||||
};
|
||||
// Reserve the detached dispatch before this request releases its root. Otherwise
|
||||
// its inherited ALS context becomes retired and rejects queued/session work.
|
||||
releaseGatewayRootContinuation = retainGatewayRootWorkAdmissionContinuation() ?? undefined;
|
||||
void gatewayWorkAdmission
|
||||
.run(() =>
|
||||
measureDiagnosticsTimelineSpan(
|
||||
@@ -5839,17 +5849,19 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
const dispatchError = pendingDispatchLifecycleError;
|
||||
// Reserve error projection before cleanup retires the dispatch root. Restart
|
||||
// drain may already reject fresh roots, but this accepted request must finish.
|
||||
const releaseDispatchErrorRoot = dispatchError
|
||||
? retainGatewayRootWorkAdmissionContinuation()
|
||||
: null;
|
||||
cleanupAdmittedRun();
|
||||
clearAgentRunContext(clientRunId, lifecycleGeneration);
|
||||
context.removeChatRun(clientRunId, clientRunId, sessionKey);
|
||||
if (!pendingDispatchLifecycleError) {
|
||||
if (!dispatchError) {
|
||||
return;
|
||||
}
|
||||
const persistDispatchLifecycleError = async () => {
|
||||
const dispatchError = pendingDispatchLifecycleError;
|
||||
if (!dispatchError) {
|
||||
return;
|
||||
}
|
||||
const hasActiveRun = hasTrackedActiveSessionRun({
|
||||
context,
|
||||
requestedKey: rawSessionKey,
|
||||
@@ -5888,7 +5900,13 @@ export const chatHandlers: GatewayRequestHandlers = {
|
||||
);
|
||||
}
|
||||
};
|
||||
void persistDispatchLifecycleError();
|
||||
void persistDispatchLifecycleError()
|
||||
.catch((continuationErr: unknown) => {
|
||||
context.logGateway.warn(
|
||||
`webchat session lifecycle continuation failed: ${formatForLog(continuationErr)}`,
|
||||
);
|
||||
})
|
||||
.finally(() => releaseDispatchErrorRoot?.());
|
||||
});
|
||||
} catch (err) {
|
||||
cleanupAdmittedRun({ force: true });
|
||||
|
||||
@@ -4,9 +4,11 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe
|
||||
import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
} from "../../../packages/gateway-protocol/src/schema/error-codes.js";
|
||||
import {
|
||||
formatValidationErrors,
|
||||
} from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { ValidationError } from "../../../packages/gateway-protocol/src/index.js";
|
||||
type ValidationError,
|
||||
} from "../../../packages/gateway-protocol/src/validation-errors.js";
|
||||
export { safeParseJson } from "../server-json.js";
|
||||
import { formatForLog } from "../ws-log.js";
|
||||
import type { RespondFn } from "./types.js";
|
||||
|
||||
@@ -4,12 +4,11 @@ import type {
|
||||
ConnectParams,
|
||||
ErrorShape,
|
||||
RequestFrame,
|
||||
} from "../../../packages/gateway-protocol/src/index.js";
|
||||
} from "../../../packages/gateway-protocol/src/schema/frames.js";
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import type { CliDeps } from "../../cli/deps.types.js";
|
||||
import type { HealthSummary } from "../../commands/health.types.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import type { CronServiceContract } from "../../cron/service-contract.js";
|
||||
import type { PluginApprovalRequestPayload } from "../../infra/plugin-approvals.js";
|
||||
import type { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import type { WizardSession } from "../../wizard/session.js";
|
||||
@@ -28,6 +27,7 @@ import type {
|
||||
ChatRunEntry,
|
||||
ChatRunRegistration,
|
||||
} from "../server-chat-state.js";
|
||||
import type { GatewayCronServiceContract } from "../server-cron.js";
|
||||
import type { DedupeEntry } from "../server-shared.js";
|
||||
import type { GatewayEventLoopHealth } from "../server/event-loop-health.js";
|
||||
import type { TerminalLaunchResolution } from "../terminal/launch.js";
|
||||
@@ -68,7 +68,7 @@ export type RespondFn = (
|
||||
/** Runtime services and mutable gateway state available to request handlers. */
|
||||
export type GatewayRequestContext = {
|
||||
deps: CliDeps;
|
||||
cron: CronServiceContract;
|
||||
cron: GatewayCronServiceContract;
|
||||
cronStorePath: string;
|
||||
getRuntimeConfig: () => OpenClawConfig;
|
||||
resolveTerminalLaunchPolicy: (agentId?: string) => TerminalLaunchResolution;
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
// Covers suspension RPC validation and coordinator response mapping.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { suspendHandlers } from "./suspend.js";
|
||||
|
||||
const coordinator = vi.hoisted(() => ({
|
||||
prepare: vi.fn(),
|
||||
status: vi.fn(),
|
||||
resume: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({
|
||||
prepareGatewaySuspend: coordinator.prepare,
|
||||
getGatewaySuspendStatus: coordinator.status,
|
||||
resumeGatewaySuspend: coordinator.resume,
|
||||
}));
|
||||
|
||||
vi.mock("../server-active-work.js", () => ({
|
||||
createGatewayServerActiveWorkInspectors: vi.fn(() => ({ getChatRuns: vi.fn(() => 0) })),
|
||||
}));
|
||||
|
||||
function invoke(method: keyof typeof suspendHandlers, params: unknown) {
|
||||
const respond = vi.fn();
|
||||
const pauseScheduling = vi.fn();
|
||||
const resumeScheduling = vi.fn();
|
||||
const warn = vi.fn();
|
||||
const handler = suspendHandlers[method];
|
||||
return Promise.resolve(
|
||||
handler({
|
||||
params,
|
||||
respond,
|
||||
context: {
|
||||
cron: { pauseScheduling, resumeScheduling },
|
||||
logGateway: { warn },
|
||||
chatAbortControllers: new Map(),
|
||||
chatQueuedTurns: new Map(),
|
||||
},
|
||||
} as unknown as Parameters<typeof handler>[0]),
|
||||
).then(() => ({ respond, pauseScheduling, resumeScheduling }));
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("gateway suspend handlers", () => {
|
||||
it("validates the closed prepare params shape", async () => {
|
||||
const { respond } = await invoke("gateway.suspend.prepare", {
|
||||
requestId: "request-1",
|
||||
extra: true,
|
||||
});
|
||||
|
||||
expect(coordinator.prepare).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(false, undefined, {
|
||||
code: "INVALID_REQUEST",
|
||||
message: "invalid gateway.suspend.prepare params",
|
||||
});
|
||||
});
|
||||
|
||||
it("wires prepare to scheduler pause/resume and returns busy or ready", async () => {
|
||||
coordinator.prepare.mockReturnValueOnce({
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
activeCount: 1,
|
||||
blockers: [{ kind: "queue", count: 1, message: "busy" }],
|
||||
});
|
||||
const { respond, pauseScheduling, resumeScheduling } = await invoke("gateway.suspend.prepare", {
|
||||
requestId: "request-1",
|
||||
});
|
||||
|
||||
expect(coordinator.prepare).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
requestId: "request-1",
|
||||
pauseScheduling: expect.any(Function),
|
||||
resumeScheduling: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
const options = coordinator.prepare.mock.calls[0]?.[0];
|
||||
options.pauseScheduling();
|
||||
options.resumeScheduling();
|
||||
expect(pauseScheduling).toHaveBeenCalledOnce();
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({ status: "busy", reason: "active-work" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("maps a competing prepared lease to retryable unavailable", async () => {
|
||||
coordinator.prepare.mockReturnValueOnce({ status: "conflict", expiresAtMs: Date.now() + 5000 });
|
||||
const { respond } = await invoke("gateway.suspend.prepare", { requestId: "request-2" });
|
||||
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
details: expect.objectContaining({ reason: "gateway-suspension-conflict" }),
|
||||
retryable: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("maps prepare and status recovery to the same retryable unavailable error", async () => {
|
||||
const recovering = {
|
||||
status: "recovering",
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: 1_000,
|
||||
};
|
||||
coordinator.prepare.mockReturnValueOnce(recovering);
|
||||
coordinator.status.mockReturnValueOnce(recovering);
|
||||
|
||||
const prepared = await invoke("gateway.suspend.prepare", { requestId: "request-recovery" });
|
||||
const status = await invoke("gateway.suspend.status", { suspensionId: "stale-id" });
|
||||
|
||||
for (const respond of [prepared.respond, status.respond]) {
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
message: "gateway scheduler recovery is pending",
|
||||
retryable: true,
|
||||
retryAfterMs: 1_000,
|
||||
details: { reason: "scheduler-resume-failed" },
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps resume idempotent and rejects a mismatched active lease", async () => {
|
||||
coordinator.resume.mockReturnValueOnce({ ok: false, reason: "suspension-mismatch" });
|
||||
const mismatch = await invoke("gateway.suspend.resume", {
|
||||
suspensionId: "suspension-wrong",
|
||||
});
|
||||
expect(mismatch.respond).toHaveBeenCalledWith(false, undefined, {
|
||||
code: "INVALID_REQUEST",
|
||||
message: "gateway suspension id does not match",
|
||||
});
|
||||
|
||||
coordinator.resume.mockReturnValueOnce({
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: false,
|
||||
});
|
||||
const resumed = await invoke("gateway.suspend.resume", { suspensionId: "suspension-1" });
|
||||
expect(resumed.respond).toHaveBeenCalledWith(true, {
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("returns retryable unavailable when scheduler resume needs retry", async () => {
|
||||
coordinator.resume.mockReturnValueOnce({
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: 1_000,
|
||||
});
|
||||
|
||||
const { respond } = await invoke("gateway.suspend.resume", {
|
||||
suspensionId: "suspension-1",
|
||||
});
|
||||
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: "UNAVAILABLE",
|
||||
message: "gateway scheduler recovery is pending",
|
||||
retryable: true,
|
||||
retryAfterMs: 1_000,
|
||||
details: { reason: "scheduler-resume-failed" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
// Gateway RPC handlers for cooperative, host-neutral process suspension.
|
||||
import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
validateGatewaySuspendPrepareParams,
|
||||
validateGatewaySuspendResumeParams,
|
||||
validateGatewaySuspendStatusParams,
|
||||
} from "../../../packages/gateway-protocol/src/index.js";
|
||||
import {
|
||||
getGatewaySuspendStatus,
|
||||
prepareGatewaySuspend,
|
||||
resumeGatewaySuspend,
|
||||
} from "../../infra/gateway-suspend-coordinator.js";
|
||||
import { createGatewayServerActiveWorkInspectors } from "../server-active-work.js";
|
||||
import type { GatewayRequestHandlers } from "./types.js";
|
||||
|
||||
function invalidParams(method: string) {
|
||||
return errorShape(ErrorCodes.INVALID_REQUEST, `invalid ${method} params`);
|
||||
}
|
||||
|
||||
function schedulerRecoveryError(retryAfterMs: number) {
|
||||
return errorShape(ErrorCodes.UNAVAILABLE, "gateway scheduler recovery is pending", {
|
||||
retryable: true,
|
||||
retryAfterMs,
|
||||
details: { reason: "scheduler-resume-failed" },
|
||||
});
|
||||
}
|
||||
|
||||
export const suspendHandlers: GatewayRequestHandlers = {
|
||||
"gateway.suspend.prepare": async ({ respond, params, context }) => {
|
||||
if (!validateGatewaySuspendPrepareParams(params)) {
|
||||
respond(false, undefined, invalidParams("gateway.suspend.prepare"));
|
||||
return;
|
||||
}
|
||||
const requestId = params.requestId.trim();
|
||||
const result = prepareGatewaySuspend({
|
||||
requestId,
|
||||
pauseScheduling: () => context.cron.pauseScheduling(),
|
||||
resumeScheduling: () => context.cron.resumeScheduling(),
|
||||
inspect: createGatewayServerActiveWorkInspectors(context),
|
||||
warn: (message) => context.logGateway.warn(message),
|
||||
});
|
||||
if (result.status === "conflict") {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.UNAVAILABLE, "another gateway suspension is already prepared", {
|
||||
retryable: true,
|
||||
retryAfterMs: Math.max(0, result.expiresAtMs - Date.now()),
|
||||
details: { reason: "gateway-suspension-conflict", expiresAtMs: result.expiresAtMs },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (result.status === "recovering") {
|
||||
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
|
||||
return;
|
||||
}
|
||||
respond(true, result);
|
||||
},
|
||||
"gateway.suspend.status": async ({ respond, params }) => {
|
||||
if (!validateGatewaySuspendStatusParams(params)) {
|
||||
respond(false, undefined, invalidParams("gateway.suspend.status"));
|
||||
return;
|
||||
}
|
||||
const suspensionId = params.suspensionId.trim();
|
||||
const result = getGatewaySuspendStatus(suspensionId);
|
||||
if (result.status === "conflict") {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.UNAVAILABLE, "a different gateway suspension is prepared", {
|
||||
retryable: true,
|
||||
retryAfterMs: Math.max(0, result.expiresAtMs - Date.now()),
|
||||
details: { reason: "gateway-suspension-conflict", expiresAtMs: result.expiresAtMs },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (result.status === "recovering") {
|
||||
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
|
||||
return;
|
||||
}
|
||||
respond(true, result);
|
||||
},
|
||||
"gateway.suspend.resume": async ({ respond, params }) => {
|
||||
if (!validateGatewaySuspendResumeParams(params)) {
|
||||
respond(false, undefined, invalidParams("gateway.suspend.resume"));
|
||||
return;
|
||||
}
|
||||
const suspensionId = params.suspensionId.trim();
|
||||
const result = resumeGatewaySuspend(suspensionId);
|
||||
if (!result.ok) {
|
||||
if (result.reason === "scheduler-resume-failed") {
|
||||
respond(false, undefined, schedulerRecoveryError(result.retryAfterMs));
|
||||
return;
|
||||
}
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.INVALID_REQUEST, "gateway suspension id does not match"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
respond(true, result);
|
||||
},
|
||||
};
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
GATEWAY_CLIENT_IDS,
|
||||
GATEWAY_CLIENT_MODES,
|
||||
} from "../../packages/gateway-protocol/src/client-info.js";
|
||||
import type { ErrorShape } from "../../packages/gateway-protocol/src/index.js";
|
||||
import type { ErrorShape } from "../../packages/gateway-protocol/src/schema/frames.js";
|
||||
import { PROTOCOL_VERSION } from "../../packages/gateway-protocol/src/version.js";
|
||||
import { normalizeModelRef, parseModelRef } from "../agents/model-selection.js";
|
||||
import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
|
||||
|
||||
@@ -4,11 +4,20 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ConfigWriteNotification } from "../config/config.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { consumeGatewaySigusr1RestartIntent } from "../infra/restart.js";
|
||||
import {
|
||||
consumeGatewaySigusr1RestartIntent,
|
||||
markGatewaySigusr1RestartHandled,
|
||||
testing as restartTesting,
|
||||
} from "../infra/restart.js";
|
||||
import {
|
||||
pinActivePluginChannelRegistry,
|
||||
releasePinnedPluginChannelRegistry,
|
||||
} from "../plugins/runtime.js";
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { createEmptyRuntimeWebToolsMetadata } from "../secrets/runtime-fast-path.js";
|
||||
import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot } from "../secrets/runtime.js";
|
||||
import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js";
|
||||
@@ -449,6 +458,48 @@ describe("gateway hot reload model state", () => {
|
||||
});
|
||||
|
||||
describe("gateway restart deferral preflight", () => {
|
||||
it("holds root admission across an immediate config-reload restart signal", () => {
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
const signalSpy = vi.fn();
|
||||
process.once("SIGUSR1", signalSpy);
|
||||
const { requestGatewayRestart } = createReloadHandlersForTest();
|
||||
|
||||
try {
|
||||
expect(
|
||||
requestGatewayRestart(
|
||||
{
|
||||
changedPaths: ["gateway.port"],
|
||||
restartGateway: true,
|
||||
restartReasons: ["gateway.port"],
|
||||
hotReasons: [],
|
||||
reloadHooks: false,
|
||||
restartGmailWatcher: false,
|
||||
restartCron: false,
|
||||
restartHeartbeat: false,
|
||||
restartHealthMonitor: false,
|
||||
reloadPlugins: false,
|
||||
restartChannels: new Set(),
|
||||
disposeMcpRuntimes: false,
|
||||
noopPaths: [],
|
||||
},
|
||||
{},
|
||||
),
|
||||
).toBe(true);
|
||||
|
||||
expect(signalSpy).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
|
||||
|
||||
markGatewaySigusr1RestartHandled();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", signalSpy);
|
||||
restartTesting.resetSigusr1State();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
});
|
||||
|
||||
it("defers channel hot reload until active embedded work drains", async () => {
|
||||
const previousSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS;
|
||||
const previousSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS;
|
||||
@@ -815,7 +866,6 @@ describe("gateway restart deferral preflight", () => {
|
||||
});
|
||||
|
||||
it("logs active task run ids before waiting and when forcing after timeout", async () => {
|
||||
const restartTesting = (await import("../infra/restart.js")).testing;
|
||||
restartTesting.resetSigusr1State();
|
||||
const logReload = { info: vi.fn(), warn: vi.fn() };
|
||||
const { requestGatewayRestart } = createReloadHandlersForTest(logReload);
|
||||
@@ -902,7 +952,6 @@ describe("gateway restart deferral preflight", () => {
|
||||
});
|
||||
|
||||
it("uses the default restart deferral timeout when config omits deferralTimeoutMs", async () => {
|
||||
const restartTesting = (await import("../infra/restart.js")).testing;
|
||||
restartTesting.resetSigusr1State();
|
||||
const { requestGatewayRestart } = createReloadHandlersForTest();
|
||||
hoisted.activeTaskCount.value = 1;
|
||||
|
||||
@@ -23,11 +23,12 @@ import type { HeartbeatRunner } from "../infra/heartbeat-runner.js";
|
||||
import { resetDirectoryCache } from "../infra/outbound/target-resolver.js";
|
||||
import {
|
||||
deferGatewayRestartUntilIdle,
|
||||
emitGatewayRestart,
|
||||
emitGatewayRestartWithSignalAdmission,
|
||||
resolveGatewayRestartDeferralTimeoutMs,
|
||||
setGatewaySigusr1RestartPolicy,
|
||||
} from "../infra/restart.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
clearSecretsRuntimeSnapshot,
|
||||
getActiveSecretsRuntimeSnapshot,
|
||||
@@ -713,7 +714,10 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
}
|
||||
// No active operations or pending replies, restart immediately
|
||||
params.logReload.warn(`config change requires gateway restart (${reasons})`);
|
||||
const emitted = emitGatewayRestart();
|
||||
// The managed reloader owns independent root admission until onRestart
|
||||
// returns. Extend that fence across signal delivery until the run loop
|
||||
// atomically promotes it to one-way restart drain.
|
||||
const emitted = emitGatewayRestartWithSignalAdmission();
|
||||
if (!emitted) {
|
||||
params.logReload.info("gateway restart already scheduled; skipping duplicate signal");
|
||||
}
|
||||
@@ -778,6 +782,7 @@ export function startManagedGatewayConfigReloader(
|
||||
initialConfig: params.initialConfig,
|
||||
initialCompareConfig: params.initialCompareConfig,
|
||||
initialInternalWriteHash: params.initialInternalWriteHash,
|
||||
runTransaction: runWithGatewayIndependentRootWorkAdmission,
|
||||
readSnapshot: params.readSnapshot,
|
||||
promoteSnapshot: async (snapshot, _reason) => await params.promoteSnapshot(snapshot),
|
||||
subscribeToWrites: params.subscribeToWrites,
|
||||
|
||||
@@ -158,6 +158,7 @@ const mocks = vi.hoisted(() => {
|
||||
recordInboundSessionAndDispatchReply: vi.fn(
|
||||
async (_params: RecordInboundSessionAndDispatchReplyParams) => {},
|
||||
),
|
||||
logDebug: vi.fn(),
|
||||
logInfo: vi.fn(),
|
||||
logWarn: vi.fn(),
|
||||
logError: vi.fn(),
|
||||
@@ -283,6 +284,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
|
||||
|
||||
vi.mock("../logging/subsystem.js", () => {
|
||||
const logger = {
|
||||
debug: mocks.logDebug,
|
||||
info: mocks.logInfo,
|
||||
warn: mocks.logWarn,
|
||||
error: mocks.logError,
|
||||
@@ -305,6 +307,12 @@ const {
|
||||
refreshLatestUpdateRestartSentinel,
|
||||
scheduleRestartSentinelWake,
|
||||
} = await import("./server-restart-sentinel.js");
|
||||
const {
|
||||
getActiveGatewayRootWorkCount,
|
||||
getGatewaySuspendAdmissionPhase,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} = await import("../process/gateway-work-admission.js");
|
||||
|
||||
function expectRecordFields(
|
||||
record: unknown,
|
||||
@@ -371,12 +379,15 @@ function expectContinuationDispatchFields(
|
||||
|
||||
describe("scheduleRestartSentinelWake", () => {
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
mocks.queuedSessionDelivery = null;
|
||||
mocks.readRestartSentinel.mockReset();
|
||||
mocks.readRestartSentinel.mockResolvedValue({
|
||||
version: 1,
|
||||
payload: {
|
||||
@@ -476,6 +487,47 @@ describe("scheduleRestartSentinelWake", () => {
|
||||
expect(mocks.logWarn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("defers pending update retries until suspension resumes", async () => {
|
||||
vi.useFakeTimers();
|
||||
const pendingPayload: RestartSentinelPayload = {
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
ts: 123,
|
||||
stats: {
|
||||
mode: "git",
|
||||
handoffId: "handoff-1",
|
||||
reason: "managed-service-handoff-started",
|
||||
},
|
||||
};
|
||||
let finishRetryRead: ((value: RestartSentinel | null) => void) | undefined;
|
||||
const retryRead = new Promise<RestartSentinel | null>((resolve) => {
|
||||
finishRetryRead = resolve;
|
||||
});
|
||||
mocks.readRestartSentinel
|
||||
.mockResolvedValueOnce({ version: 1, payload: pendingPayload })
|
||||
.mockImplementationOnce(async () => (await retryRead) as RestartSentinel);
|
||||
|
||||
await scheduleRestartSentinelWake({ deps: {} as never });
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(getGatewaySuspendAdmissionPhase()).toBe("prepared");
|
||||
expect(mocks.readRestartSentinel).toHaveBeenCalledTimes(1);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
|
||||
expect(suspension?.release()).toBe(true);
|
||||
await vi.waitFor(() => {
|
||||
expect(mocks.readRestartSentinel).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
finishRetryRead?.(null);
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("retries outbound delivery once and logs a warning without dropping the agent wake", async () => {
|
||||
vi.useFakeTimers();
|
||||
mocks.deliverOutboundPayloads
|
||||
|
||||
@@ -41,6 +41,7 @@ import { isPendingControlPlaneUpdateRestartSentinel } from "../infra/update-cont
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { stringifyRouteThreadId } from "../plugin-sdk/channel-route.js";
|
||||
import type { OutboundReplyPayload } from "../plugin-sdk/reply-payload.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
deliveryContextFromSession,
|
||||
mergeDeliveryContext,
|
||||
@@ -468,9 +469,11 @@ async function loadRestartSentinelStartupTask(params: {
|
||||
const attempt = params.attempt ?? 0;
|
||||
if (attempt < CONTROL_PLANE_UPDATE_PENDING_MAX_ATTEMPTS) {
|
||||
const timer = setTimeout(() => {
|
||||
void scheduleRestartSentinelWakeAttempt({
|
||||
deps: params.deps,
|
||||
attempt: attempt + 1,
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await scheduleRestartSentinelWakeAttempt({
|
||||
deps: params.deps,
|
||||
attempt: attempt + 1,
|
||||
});
|
||||
}).catch((err: unknown) => {
|
||||
log.warn(`restart sentinel pending update retry failed: ${formatErrorMessage(err)}`);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isVitestRuntimeEnv } from "../infra/env.js";
|
||||
import { startHeartbeatRunner, type HeartbeatRunner } from "../infra/heartbeat-runner.js";
|
||||
import type { PluginMetadataRegistryView } from "../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { isGatewayModelPricingEnabled } from "./model-pricing-config.js";
|
||||
import type { startGatewayMaintenanceTimers } from "./server-maintenance.js";
|
||||
import {
|
||||
@@ -30,10 +31,10 @@ export function startGatewayCronWithLogging(params: {
|
||||
afterStart?: () => Promise<void>;
|
||||
logCron: { error: (message: string) => void };
|
||||
}): void {
|
||||
void params.cron
|
||||
.start()
|
||||
.then(() => params.afterStart?.())
|
||||
.catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`));
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await params.cron.start();
|
||||
await params.afterStart?.();
|
||||
}).catch((err: unknown) => params.logCron.error(`failed to start: ${String(err)}`));
|
||||
}
|
||||
|
||||
function clearGatewayMaintenanceHandles(maintenance: GatewayMaintenanceHandles | null): void {
|
||||
@@ -100,38 +101,42 @@ export function scheduleGatewayPostReadyMaintenance(params: {
|
||||
if (params.isClosing()) {
|
||||
return;
|
||||
}
|
||||
void runGatewayPostReadyMaintenance({
|
||||
startMaintenance: async () => {
|
||||
if (params.isClosing()) {
|
||||
return null;
|
||||
}
|
||||
const maintenance = await params.startMaintenance();
|
||||
if (params.isClosing()) {
|
||||
// Maintenance can allocate intervals before shutdown is observed; clear them here
|
||||
// instead of handing live timers to a closing gateway.
|
||||
clearGatewayMaintenanceHandles(maintenance);
|
||||
return null;
|
||||
}
|
||||
return maintenance;
|
||||
},
|
||||
applyMaintenance: (maintenance) => {
|
||||
if (params.isClosing()) {
|
||||
clearGatewayMaintenanceHandles(maintenance);
|
||||
return;
|
||||
}
|
||||
params.applyMaintenance(maintenance);
|
||||
},
|
||||
shouldStartCron: () => !params.isClosing() && params.shouldStartCron(),
|
||||
markCronStartHandled: params.markCronStartHandled,
|
||||
cron: params.cron,
|
||||
logCron: params.logCron,
|
||||
log: params.log,
|
||||
recordPostReadyMemory: () => {
|
||||
if (!params.isClosing()) {
|
||||
params.recordPostReadyMemory();
|
||||
}
|
||||
},
|
||||
});
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () =>
|
||||
runGatewayPostReadyMaintenance({
|
||||
startMaintenance: async () => {
|
||||
if (params.isClosing()) {
|
||||
return null;
|
||||
}
|
||||
const maintenance = await params.startMaintenance();
|
||||
if (params.isClosing()) {
|
||||
// Maintenance can allocate intervals before shutdown is observed; clear them here
|
||||
// instead of handing live timers to a closing gateway.
|
||||
clearGatewayMaintenanceHandles(maintenance);
|
||||
return null;
|
||||
}
|
||||
return maintenance;
|
||||
},
|
||||
applyMaintenance: (maintenance) => {
|
||||
if (params.isClosing()) {
|
||||
clearGatewayMaintenanceHandles(maintenance);
|
||||
return;
|
||||
}
|
||||
params.applyMaintenance(maintenance);
|
||||
},
|
||||
shouldStartCron: () => !params.isClosing() && params.shouldStartCron(),
|
||||
markCronStartHandled: params.markCronStartHandled,
|
||||
cron: params.cron,
|
||||
logCron: params.logCron,
|
||||
log: params.log,
|
||||
recordPostReadyMemory: () => {
|
||||
if (!params.isClosing()) {
|
||||
params.recordPostReadyMemory();
|
||||
}
|
||||
},
|
||||
}),
|
||||
).catch((err: unknown) =>
|
||||
params.log.warn(`gateway post-ready maintenance deferred task failed: ${String(err)}`),
|
||||
);
|
||||
}, params.delayMs);
|
||||
timer.unref?.();
|
||||
return timer;
|
||||
@@ -143,7 +148,7 @@ function recoverPendingOutboundDeliveries(params: {
|
||||
}): void {
|
||||
// Recovery is best-effort background work; startup must continue even if outbound modules fail
|
||||
// to import or queued delivery replay fails.
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { recoverPendingDeliveries } = await import("../infra/outbound/delivery-queue.js");
|
||||
const { deliverOutboundPayloadsInternal } = await import("../infra/outbound/deliver.js");
|
||||
const logRecovery = params.log.child("delivery-recovery");
|
||||
@@ -152,7 +157,7 @@ function recoverPendingOutboundDeliveries(params: {
|
||||
log: logRecovery,
|
||||
cfg: params.cfg,
|
||||
});
|
||||
})().catch((err: unknown) => params.log.error(`Delivery recovery failed: ${String(err)}`));
|
||||
}).catch((err: unknown) => params.log.error(`Delivery recovery failed: ${String(err)}`));
|
||||
}
|
||||
|
||||
function recoverPendingSessionDeliveries(params: {
|
||||
@@ -163,7 +168,7 @@ function recoverPendingSessionDeliveries(params: {
|
||||
// Delay session continuation recovery so the gateway has time to publish ready state and
|
||||
// request routing before replaying restart-sentinel deliveries.
|
||||
const timer = setTimeout(() => {
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { recoverPendingRestartContinuationDeliveries } =
|
||||
await import("./server-restart-sentinel.js");
|
||||
const logRecovery = params.log.child("session-delivery-recovery");
|
||||
@@ -172,7 +177,7 @@ function recoverPendingSessionDeliveries(params: {
|
||||
log: logRecovery,
|
||||
maxEnqueuedAt: params.maxEnqueuedAt,
|
||||
});
|
||||
})().catch((err: unknown) =>
|
||||
}).catch((err: unknown) =>
|
||||
params.log.error(`Session delivery recovery failed: ${String(err)}`),
|
||||
);
|
||||
}, 1_250);
|
||||
@@ -191,7 +196,7 @@ function startGatewayModelPricingRefreshOnDemand(params: {
|
||||
let stopRefresh: (() => void) | undefined;
|
||||
// Import pricing refresh lazily; many gateway starts never use model-pricing metadata.
|
||||
// The stopped flag closes the race where shutdown happens before the import resolves.
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { startGatewayModelPricingRefresh } = await import("./model-pricing-cache.js");
|
||||
if (stopped) {
|
||||
return;
|
||||
@@ -204,7 +209,7 @@ function startGatewayModelPricingRefreshOnDemand(params: {
|
||||
stopRefresh();
|
||||
stopRefresh = undefined;
|
||||
}
|
||||
})().catch((err: unknown) =>
|
||||
}).catch((err: unknown) =>
|
||||
params.log.error(`Model pricing refresh failed to start: ${String(err)}`),
|
||||
);
|
||||
return () => {
|
||||
|
||||
@@ -30,7 +30,11 @@ import {
|
||||
createToolEventRecipientRegistry,
|
||||
} from "./server-chat-state.js";
|
||||
import { MAX_PREAUTH_PAYLOAD_BYTES } from "./server-constants.js";
|
||||
import { attachGatewayUpgradeHandler, createGatewayHttpServer } from "./server-http.js";
|
||||
import {
|
||||
attachGatewayUpgradeHandler,
|
||||
createGatewayHttpServer,
|
||||
runWithGatewayHttpWorkAdmission,
|
||||
} from "./server-http.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/types.js";
|
||||
import type { DedupeEntry } from "./server-shared.js";
|
||||
import type { HookClientIpConfig, HooksRequestHandler } from "./server/hooks-request-handler.js";
|
||||
@@ -153,20 +157,22 @@ export async function createGatewayRuntimeState(params: {
|
||||
if (url.pathname !== basePath && !url.pathname.startsWith(`${basePath}/`)) {
|
||||
return false;
|
||||
}
|
||||
if (!loadedHooksRequestHandler) {
|
||||
// Hooks are cold for most gateway starts; create the handler only after a request
|
||||
// matches the configured base path so startup avoids importing hook runtime code.
|
||||
const { createGatewayHooksRequestHandler } = await import("./server/hooks.js");
|
||||
loadedHooksRequestHandler = createGatewayHooksRequestHandler({
|
||||
deps: params.deps,
|
||||
getHooksConfig: params.hooksConfig,
|
||||
getClientIpConfig: params.getHookClientIpConfig,
|
||||
bindHost: params.bindHost,
|
||||
port: params.port,
|
||||
logHooks: params.logHooks,
|
||||
});
|
||||
}
|
||||
return await loadedHooksRequestHandler(req, res);
|
||||
return await runWithGatewayHttpWorkAdmission(res, async () => {
|
||||
if (!loadedHooksRequestHandler) {
|
||||
// Hooks are cold for most gateway starts; create the handler only after a request
|
||||
// matches the configured base path so startup avoids importing hook runtime code.
|
||||
const { createGatewayHooksRequestHandler } = await import("./server/hooks.js");
|
||||
loadedHooksRequestHandler = createGatewayHooksRequestHandler({
|
||||
deps: params.deps,
|
||||
getHooksConfig: params.hooksConfig,
|
||||
getClientIpConfig: params.getHookClientIpConfig,
|
||||
bindHost: params.bindHost,
|
||||
port: params.port,
|
||||
logHooks: params.logHooks,
|
||||
});
|
||||
}
|
||||
return await loadedHooksRequestHandler(req, res);
|
||||
});
|
||||
};
|
||||
|
||||
let loadedPluginRequestHandler: GatewayPluginRequestHandler | null = null;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Gateway shared request-state types.
|
||||
// Defines cached dedupe entries for idempotent Gateway method calls.
|
||||
import type { ErrorShape } from "../../packages/gateway-protocol/src/index.js";
|
||||
import type { ErrorShape } from "../../packages/gateway-protocol/src/schema/frames.js";
|
||||
|
||||
export const PENDING_CHAT_SEND_DEDUPE_PREFIX = "pending-chat:";
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@ import type {
|
||||
PluginHookGatewayStartEvent,
|
||||
} from "../plugins/hook-types.js";
|
||||
import type { PluginServicesHandle } from "../plugins/services.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
|
||||
@@ -40,7 +44,8 @@ const hoisted = vi.hoisted(() => {
|
||||
skipped: 0,
|
||||
}));
|
||||
const scheduleRestartAbortedMainSessionRecovery = vi.fn();
|
||||
const scheduleRestartSentinelWake = vi.fn();
|
||||
const scheduleRestartSentinelWake =
|
||||
vi.fn<typeof import("./server-restart-sentinel.js").scheduleRestartSentinelWake>();
|
||||
const refreshLatestUpdateRestartSentinel = vi.fn<
|
||||
typeof import("./server-restart-sentinel.js").refreshLatestUpdateRestartSentinel
|
||||
>(async () => null);
|
||||
@@ -301,6 +306,7 @@ function firstGatewayStartCall(
|
||||
|
||||
describe("startGatewayPostAttachRuntime", () => {
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
vi.stubEnv("OPENCLAW_SKIP_CHANNELS", "0");
|
||||
vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", "0");
|
||||
@@ -364,6 +370,7 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllEnvs();
|
||||
@@ -371,6 +378,10 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
|
||||
it("re-enables startup-gated methods after post-attach sidecars start", async () => {
|
||||
const unavailableGatewayMethods = new Set<string>(["chat.history", "models.list"]);
|
||||
const methodsAtRecoveryRegistration: string[][] = [];
|
||||
hoisted.scheduleRestartAbortedMainSessionRecovery.mockImplementationOnce(() => {
|
||||
methodsAtRecoveryRegistration.push([...unavailableGatewayMethods]);
|
||||
});
|
||||
const onSidecarsReady = vi.fn();
|
||||
const log = { info: vi.fn(), warn: vi.fn() };
|
||||
|
||||
@@ -399,6 +410,7 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
expect(hoisted.scheduleRestartAbortedMainSessionRecovery).toHaveBeenCalledWith({
|
||||
cfg: { hooks: { internal: { enabled: false } } },
|
||||
});
|
||||
expect(methodsAtRecoveryRegistration).toStrictEqual([["chat.history", "models.list"]]);
|
||||
expect(hoisted.startGatewayMemoryBackend).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
@@ -430,6 +442,29 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
expect(events).toEqual(["sidecars", "returned", "sentinel"]);
|
||||
});
|
||||
|
||||
it("keeps delayed restart sentinel recovery admitted until wake work completes", async () => {
|
||||
vi.useFakeTimers();
|
||||
let finishWake: (() => void) | undefined;
|
||||
const wake = new Promise<void>((resolve) => {
|
||||
finishWake = resolve;
|
||||
});
|
||||
hoisted.scheduleRestartSentinelWake.mockReturnValueOnce(wake);
|
||||
|
||||
testing.scheduleRestartSentinelWakeAfterReady({
|
||||
deps: {} as never,
|
||||
log: { warn: vi.fn() },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(750);
|
||||
|
||||
expect(hoisted.scheduleRestartSentinelWake).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
|
||||
finishWake?.();
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it("starts sidecars while startup logging is still pending", async () => {
|
||||
const events: string[] = [];
|
||||
let finishStartupLog: (() => void) | undefined;
|
||||
@@ -2050,6 +2085,13 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
it("dispatches registered gateway startup internal hooks without configured hook packs", async () => {
|
||||
vi.useFakeTimers();
|
||||
hoisted.hasInternalHookListeners.mockReturnValue(true);
|
||||
let releaseHook = () => {};
|
||||
hoisted.triggerInternalHook.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
releaseHook = resolve;
|
||||
}),
|
||||
);
|
||||
const cfg = {} as never;
|
||||
const deps = {} as never;
|
||||
|
||||
@@ -2088,7 +2130,11 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
},
|
||||
);
|
||||
expect(hoisted.triggerInternalHook).toHaveBeenCalledWith(hoisted.startupHookEvent);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
releaseHook();
|
||||
await vi.waitFor(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
} finally {
|
||||
releaseHook();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@ import type { loadOpenClawPlugins } from "../plugins/loader.js";
|
||||
import { getPluginModuleLoaderStats } from "../plugins/plugin-module-loader-cache.js";
|
||||
import type { PluginRegistry } from "../plugins/registry.js";
|
||||
import type { PluginServicesHandle } from "../plugins/services.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import {
|
||||
GATEWAY_EVENT_UPDATE_AVAILABLE,
|
||||
@@ -157,13 +158,12 @@ function scheduleGatewayMemoryBackend(params: {
|
||||
return;
|
||||
}
|
||||
const start = () => {
|
||||
void import("./server-startup-memory.js")
|
||||
.then(({ startGatewayMemoryBackend }) =>
|
||||
startGatewayMemoryBackend({ cfg: params.cfg, log: params.log }),
|
||||
)
|
||||
.catch((err: unknown) => {
|
||||
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
|
||||
});
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { startGatewayMemoryBackend } = await import("./server-startup-memory.js");
|
||||
await startGatewayMemoryBackend({ cfg: params.cfg, log: params.log });
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`qmd memory startup initialization failed: ${String(err)}`);
|
||||
});
|
||||
};
|
||||
if (params.policy.mode === "immediate") {
|
||||
setImmediate(start);
|
||||
@@ -181,12 +181,10 @@ function schedulePostAttachUpdateSentinelRefresh(params: {
|
||||
>;
|
||||
}): void {
|
||||
const handle = setImmediate(() => {
|
||||
void measureStartup(params.startupTrace, "post-attach.update-sentinel", async () => {
|
||||
try {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await measureStartup(params.startupTrace, "post-attach.update-sentinel", async () => {
|
||||
await params.refreshLatestUpdateRestartSentinel();
|
||||
} catch (err) {
|
||||
params.log.warn(`restart sentinel refresh failed: ${String(err)}`);
|
||||
}
|
||||
});
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`restart sentinel refresh failed: ${String(err)}`);
|
||||
});
|
||||
@@ -210,39 +208,41 @@ function scheduleProviderAuthStatePrewarm(params: {
|
||||
let pendingRewarmReason: string | undefined;
|
||||
const isStopped = () => stopped;
|
||||
const delayMs = params.delayMs ?? PROVIDER_AUTH_PREWARM_START_DELAY_MS;
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const [{ setAuthProfileFailureHook }, { clearCurrentProviderAuthState }] = await Promise.all([
|
||||
import("../agents/auth-profiles/failure-hook.js"),
|
||||
import("../agents/model-provider-auth-state.js"),
|
||||
]);
|
||||
const loadProviderAuthWarmModule = () => import("../agents/model-provider-auth.js");
|
||||
const runRewarm = async (reason: string) => {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
const cfg = params.getConfig();
|
||||
rewarmInFlight = true;
|
||||
try {
|
||||
const { warmCurrentProviderAuthStateOffMainThread } = await loadProviderAuthWarmModule();
|
||||
const metrics = await measureProviderAuthWarm(() =>
|
||||
warmCurrentProviderAuthStateOffMainThread(cfg, { isCancelled: isStopped }),
|
||||
);
|
||||
await runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
params.log.info(
|
||||
`provider auth state re-warmed (${reason}) ${formatProviderAuthWarmMetrics(metrics)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
params.log.warn(`provider auth state rewarm failed: ${String(err)}`);
|
||||
} finally {
|
||||
rewarmInFlight = false;
|
||||
const nextReason = pendingRewarmReason;
|
||||
pendingRewarmReason = undefined;
|
||||
if (nextReason && !isStopped()) {
|
||||
scheduleAuthMapRewarm(nextReason);
|
||||
const cfg = params.getConfig();
|
||||
rewarmInFlight = true;
|
||||
try {
|
||||
const { warmCurrentProviderAuthStateOffMainThread } = await loadProviderAuthWarmModule();
|
||||
const metrics = await measureProviderAuthWarm(() =>
|
||||
warmCurrentProviderAuthStateOffMainThread(cfg, { isCancelled: isStopped }),
|
||||
);
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
params.log.info(
|
||||
`provider auth state re-warmed (${reason}) ${formatProviderAuthWarmMetrics(metrics)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
params.log.warn(`provider auth state rewarm failed: ${String(err)}`);
|
||||
} finally {
|
||||
rewarmInFlight = false;
|
||||
const nextReason = pendingRewarmReason;
|
||||
pendingRewarmReason = undefined;
|
||||
if (nextReason && !isStopped()) {
|
||||
scheduleAuthMapRewarm(nextReason);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
const scheduleAuthMapRewarm = (reason: string) => {
|
||||
// Collapse repeated auth-profile failures into one rewarm turn while a
|
||||
@@ -279,7 +279,7 @@ function scheduleProviderAuthStatePrewarm(params: {
|
||||
}
|
||||
startupTimer = setTimeout(
|
||||
() => {
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
@@ -294,14 +294,14 @@ function scheduleProviderAuthStatePrewarm(params: {
|
||||
params.log.info(
|
||||
`provider auth state pre-warmed ${formatProviderAuthWarmMetrics(metrics)}`,
|
||||
);
|
||||
})().catch((err: unknown) => {
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`provider auth state pre-warm failed: ${String(err)}`);
|
||||
});
|
||||
},
|
||||
Math.max(0, delayMs),
|
||||
);
|
||||
startupTimer.unref?.();
|
||||
})().catch((err: unknown) => {
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`provider auth state pre-warm setup failed: ${String(err)}`);
|
||||
});
|
||||
return {
|
||||
@@ -335,26 +335,28 @@ function scheduleAgentRuntimePluginPrewarm(params: {
|
||||
timer = setTimeout(
|
||||
() => {
|
||||
timer = undefined;
|
||||
void measureStartup(params.startupTrace, "post-ready.agent-runtime-plugins", async () => {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
const started = performance.now();
|
||||
const { ensureRuntimePluginsLoaded } = await import("../agents/runtime-plugins.js");
|
||||
const cfg = params.getConfig();
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
ensureRuntimePluginsLoaded({
|
||||
config: cfg,
|
||||
workspaceDir: params.workspaceDir,
|
||||
allowGatewaySubagentBinding: true,
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await measureStartup(params.startupTrace, "post-ready.agent-runtime-plugins", async () => {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
const started = performance.now();
|
||||
const { ensureRuntimePluginsLoaded } = await import("../agents/runtime-plugins.js");
|
||||
const cfg = params.getConfig();
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
ensureRuntimePluginsLoaded({
|
||||
config: cfg,
|
||||
workspaceDir: params.workspaceDir,
|
||||
allowGatewaySubagentBinding: true,
|
||||
});
|
||||
if (!isStopped()) {
|
||||
params.log.info(
|
||||
`agent runtime plugins pre-warmed in ${(performance.now() - started).toFixed(0)}ms`,
|
||||
);
|
||||
}
|
||||
});
|
||||
if (!isStopped()) {
|
||||
params.log.info(
|
||||
`agent runtime plugins pre-warmed in ${(performance.now() - started).toFixed(0)}ms`,
|
||||
);
|
||||
}
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`agent runtime plugin pre-warm failed: ${String(err)}`);
|
||||
});
|
||||
@@ -387,9 +389,11 @@ function schedulePostReadySidecarTask(params: {
|
||||
if (isStopped()) {
|
||||
return;
|
||||
}
|
||||
void measureStartup(params.startupTrace, params.name, () =>
|
||||
params.run(isStopped, abortController.signal),
|
||||
).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await measureStartup(params.startupTrace, params.name, () =>
|
||||
params.run(isStopped, abortController.signal),
|
||||
);
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`${params.name} failed after gateway ready: ${String(err)}`);
|
||||
});
|
||||
});
|
||||
@@ -406,6 +410,20 @@ function schedulePostReadySidecarTask(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function scheduleRestartSentinelWakeAfterReady(params: {
|
||||
deps: CliDeps;
|
||||
log: { warn: (msg: string) => void };
|
||||
}): void {
|
||||
setTimeout(() => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { scheduleRestartSentinelWake } = await loadGatewayRestartSentinelModule();
|
||||
await scheduleRestartSentinelWake({ deps: params.deps });
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`);
|
||||
});
|
||||
}, 750);
|
||||
}
|
||||
|
||||
type CleanStaleLockFiles = typeof import("../agents/session-write-lock.js").cleanStaleLockFiles;
|
||||
type MarkRestartAbortedMainSessionsFromLocks =
|
||||
typeof import("../agents/main-session-restart-recovery.js").markRestartAbortedMainSessionsFromLocks;
|
||||
@@ -779,19 +797,22 @@ export async function startGatewaySidecars(params: {
|
||||
// Run startup hooks after sidecar startup has yielded once so gateway bind
|
||||
// and channel startup are not delayed by hook handlers.
|
||||
setTimeout(() => {
|
||||
void loadInternalHooksModule().then(({ createInternalHookEvent, triggerInternalHook }) => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const { createInternalHookEvent, triggerInternalHook } = await loadInternalHooksModule();
|
||||
const hookEvent = createInternalHookEvent("gateway", "startup", "gateway:startup", {
|
||||
cfg: params.cfg,
|
||||
deps: params.deps,
|
||||
workspaceDir: params.defaultWorkspaceDir,
|
||||
});
|
||||
void triggerInternalHook(hookEvent);
|
||||
await triggerInternalHook(hookEvent);
|
||||
}).catch((err: unknown) => {
|
||||
params.logHooks.warn(`gateway startup hook failed: ${String(err)}`);
|
||||
});
|
||||
}, 250);
|
||||
}
|
||||
|
||||
if (params.cfg.acp?.enabled) {
|
||||
void (async () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const ready = await measureStartup(params.startupTrace, "sidecars.acp.runtime-ready", () =>
|
||||
waitForAcpRuntimeBackendReady({ backendId: params.cfg.acp?.backend }),
|
||||
);
|
||||
@@ -815,7 +836,7 @@ export async function startGatewaySidecars(params: {
|
||||
`acp startup identity reconcile (renderer=${ACP_SESSION_IDENTITY_RENDERER_VERSION}): checked=${result.checked} resolved=${result.resolved} failed=${result.failed}`,
|
||||
);
|
||||
});
|
||||
})().catch((err: unknown) => {
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`acp startup identity reconcile failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
@@ -864,15 +885,7 @@ export async function startGatewaySidecars(params: {
|
||||
if (!(await hasRestartSentinelFast())) {
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
void loadGatewayRestartSentinelModule()
|
||||
.then(({ scheduleRestartSentinelWake }) =>
|
||||
scheduleRestartSentinelWake({ deps: params.deps }),
|
||||
)
|
||||
.catch((err: unknown) => {
|
||||
params.log.warn(`restart sentinel wake failed to schedule: ${String(err)}`);
|
||||
});
|
||||
}, 750);
|
||||
scheduleRestartSentinelWakeAfterReady({ deps: params.deps, log: params.log });
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1025,16 +1038,19 @@ function createDeferredGatewayUpdateCheck(params: {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
void measureStartup(params.startupTrace, "post-attach.update-check", () =>
|
||||
params.runtimeDeps.scheduleGatewayUpdateCheck({
|
||||
cfg: params.cfg,
|
||||
log: params.log,
|
||||
isNixMode: params.isNixMode,
|
||||
onUpdateAvailableChange: (updateAvailable) => {
|
||||
const payload: GatewayUpdateAvailableEventPayload = { updateAvailable };
|
||||
params.broadcast(GATEWAY_EVENT_UPDATE_AVAILABLE, payload, { dropIfSlow: true });
|
||||
},
|
||||
}),
|
||||
void runWithGatewayIndependentRootWorkAdmission(
|
||||
async () =>
|
||||
await measureStartup(params.startupTrace, "post-attach.update-check", () =>
|
||||
params.runtimeDeps.scheduleGatewayUpdateCheck({
|
||||
cfg: params.cfg,
|
||||
log: params.log,
|
||||
isNixMode: params.isNixMode,
|
||||
onUpdateAvailableChange: (updateAvailable) => {
|
||||
const payload: GatewayUpdateAvailableEventPayload = { updateAvailable };
|
||||
params.broadcast(GATEWAY_EVENT_UPDATE_AVAILABLE, payload, { dropIfSlow: true });
|
||||
},
|
||||
}),
|
||||
),
|
||||
)
|
||||
.then((nextStop) => {
|
||||
if (stopped) {
|
||||
@@ -1259,9 +1275,6 @@ export async function startGatewayPostAttachRuntime(
|
||||
loaderStatsAfter.sourceTransformFallbacks - loaderStatsBefore.sourceTransformFallbacks,
|
||||
],
|
||||
]);
|
||||
for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) {
|
||||
params.unavailableGatewayMethods.delete(method);
|
||||
}
|
||||
try {
|
||||
const { scheduleRestartAbortedMainSessionRecovery } =
|
||||
await loadMainSessionRestartRecoveryModule();
|
||||
@@ -1269,6 +1282,11 @@ export async function startGatewayPostAttachRuntime(
|
||||
} catch (err) {
|
||||
params.log.warn(`main-session restart recovery failed to schedule: ${String(err)}`);
|
||||
}
|
||||
// Capture the orphan-recovery cutoff before new startup-gated agent
|
||||
// work can create sessions that the recovery scan must leave alone.
|
||||
for (const method of STARTUP_UNAVAILABLE_GATEWAY_METHODS) {
|
||||
params.unavailableGatewayMethods.delete(method);
|
||||
}
|
||||
if (!pluginServicesReported) {
|
||||
reportPluginServices(result.pluginServices);
|
||||
}
|
||||
@@ -1343,19 +1361,21 @@ export async function startGatewayPostAttachRuntime(
|
||||
const hookRunner = await runtimeDeps.getGlobalHookRunner();
|
||||
if (hookRunner?.hasHooks("gateway_start")) {
|
||||
const { withPluginHttpRouteRegistry } = await import("../plugins/http-registry.js");
|
||||
void withPluginHttpRouteRegistry(sidecarsResult.pluginRegistry, () =>
|
||||
hookRunner.runGatewayStart(
|
||||
{ port: params.port },
|
||||
{
|
||||
port: params.port,
|
||||
config: params.gatewayPluginConfigAtStart,
|
||||
workspaceDir: params.defaultWorkspaceDir,
|
||||
getCron: () =>
|
||||
params.getCronService?.() ??
|
||||
(params.deps.cron as PluginHookGatewayCronService | undefined),
|
||||
},
|
||||
),
|
||||
).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await withPluginHttpRouteRegistry(sidecarsResult.pluginRegistry, () =>
|
||||
hookRunner.runGatewayStart(
|
||||
{ port: params.port },
|
||||
{
|
||||
port: params.port,
|
||||
config: params.gatewayPluginConfigAtStart,
|
||||
workspaceDir: params.defaultWorkspaceDir,
|
||||
getCron: () =>
|
||||
params.getCronService?.() ??
|
||||
(params.deps.cron as PluginHookGatewayCronService | undefined),
|
||||
},
|
||||
),
|
||||
);
|
||||
}).catch((err: unknown) => {
|
||||
params.log.warn(`gateway_start hook failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
@@ -1398,6 +1418,7 @@ export const testing = {
|
||||
cleanupStaleSessionLocks,
|
||||
scheduleProviderAuthStatePrewarm,
|
||||
schedulePrimaryModelPrewarm,
|
||||
scheduleRestartSentinelWakeAfterReady,
|
||||
shouldSkipStartupModelPrewarm,
|
||||
stopPostReadySidecarsAfterCloseStarted,
|
||||
};
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import type { ChannelPlugin } from "../channels/plugins/types.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
isGatewaySubordinateWorkAdmissionClosed,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
createChannelTestPluginBase,
|
||||
createDirectOutboundTestAdapter,
|
||||
@@ -215,6 +220,42 @@ describe("gateway server agent", () => {
|
||||
testState.allowFrom = undefined;
|
||||
});
|
||||
|
||||
test("keeps accepted detached agent work on its retained request root", async () => {
|
||||
await setTestSessionStore({
|
||||
entries: {
|
||||
main: {
|
||||
sessionId: "sess-agent-detached-root",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
},
|
||||
});
|
||||
let subordinateAdmissionClosed: boolean | undefined;
|
||||
vi.mocked(agentCommand).mockImplementationOnce(async () => {
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension).not.toBeNull();
|
||||
try {
|
||||
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
|
||||
} finally {
|
||||
suspension?.rollback();
|
||||
}
|
||||
});
|
||||
|
||||
const res = await rpcReq(gatewaySuite.ws, "agent", {
|
||||
message: "prove detached root transfer",
|
||||
sessionKey: "main",
|
||||
idempotencyKey: "idem-agent-detached-root",
|
||||
});
|
||||
|
||||
expect(res.ok).toBe(true);
|
||||
expect(res.payload?.status).toBe("accepted");
|
||||
await vi.waitFor(() => {
|
||||
expect(subordinateAdmissionClosed).toBe(false);
|
||||
});
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
test("agent marks implicit delivery when lastTo is stale", async () => {
|
||||
testState.allowFrom = ["+436769770569"];
|
||||
await setTestSessionStore({
|
||||
|
||||
@@ -6,9 +6,18 @@ import path from "node:path";
|
||||
import { beforeEach, describe, expect, test, vi } from "vitest";
|
||||
import { WebSocket } from "ws";
|
||||
import { emitAgentEvent, registerAgentRunContext } from "../infra/agent-events.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
isGatewaySubordinateWorkAdmissionClosed,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { extractFirstTextBlock } from "../shared/chat-message-content.js";
|
||||
import { createDeferred } from "../test-utils/deferred.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js";
|
||||
import * as sessionLifecycleState from "./session-lifecycle-state.js";
|
||||
import {
|
||||
connectOk,
|
||||
dispatchInboundMessageMock,
|
||||
@@ -201,6 +210,64 @@ describe("gateway server chat", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("keeps started chat dispatch on its retained request root", async () => {
|
||||
await withMainSessionStore(async () => {
|
||||
let subordinateAdmissionClosed: boolean | undefined;
|
||||
dispatchInboundMessageMock.mockImplementationOnce(async (...args: unknown[]) => {
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension).not.toBeNull();
|
||||
try {
|
||||
subordinateAdmissionClosed = isGatewaySubordinateWorkAdmissionClosed();
|
||||
} finally {
|
||||
suspension?.rollback();
|
||||
}
|
||||
const [params] = args as [
|
||||
{
|
||||
dispatcher: {
|
||||
sendFinalReply: (payload: { text: string }) => boolean;
|
||||
markComplete: () => void;
|
||||
waitForIdle: () => Promise<void>;
|
||||
getQueuedCounts: () => { final: number; block: number; tool: number };
|
||||
};
|
||||
},
|
||||
];
|
||||
params.dispatcher.sendFinalReply({ text: "detached root stayed live" });
|
||||
params.dispatcher.markComplete();
|
||||
await params.dispatcher.waitForIdle();
|
||||
return {
|
||||
queuedFinal: true,
|
||||
counts: params.dispatcher.getQueuedCounts(),
|
||||
};
|
||||
});
|
||||
const finalPromise = onceMessage(
|
||||
ws,
|
||||
(message) =>
|
||||
message.type === "event" &&
|
||||
message.event === "chat" &&
|
||||
message.payload?.state === "final" &&
|
||||
message.payload?.runId === "idem-chat-detached-root",
|
||||
8_000,
|
||||
);
|
||||
|
||||
const res = await rpcReq(ws, "chat.send", {
|
||||
sessionKey: "main",
|
||||
message: "prove detached root transfer",
|
||||
idempotencyKey: "idem-chat-detached-root",
|
||||
});
|
||||
|
||||
expect(res.ok).toBe(true);
|
||||
expect(res.payload?.status).toBe("started");
|
||||
await vi.waitFor(() => {
|
||||
expect(subordinateAdmissionClosed).toBe(false);
|
||||
});
|
||||
await finalPromise;
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
const waitForAgentRunOk = async (runId: string, timeoutMs = 1_000) => {
|
||||
const res = await rpcReq(ws, "agent.wait", {
|
||||
runId,
|
||||
@@ -714,7 +781,7 @@ describe("gateway server chat", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("marks a running webchat session failed when dispatch rejects before a reply", async () => {
|
||||
test("marks a running webchat session failed when restart drain overlaps dispatch rejection", async () => {
|
||||
await withMainSessionStore(async (dir) => {
|
||||
await writeSessionStore({
|
||||
entries: {
|
||||
@@ -729,34 +796,72 @@ describe("gateway server chat", () => {
|
||||
});
|
||||
const subscribeRes = await rpcReq(ws, "sessions.subscribe", {});
|
||||
expect(subscribeRes.ok).toBe(true);
|
||||
dispatchInboundMessageMock.mockRejectedValueOnce(new Error("provider rejected request"));
|
||||
|
||||
const errorPromise = onceMessage(
|
||||
ws,
|
||||
(o) =>
|
||||
o.type === "event" &&
|
||||
o.event === "chat" &&
|
||||
o.payload?.state === "error" &&
|
||||
o.payload?.runId === "idem-dispatch-error-1",
|
||||
8_000,
|
||||
);
|
||||
const sessionChangedPromise = onceMessage(
|
||||
ws,
|
||||
(o) =>
|
||||
o.type === "event" &&
|
||||
o.event === "sessions.changed" &&
|
||||
o.payload?.reason === "chat.dispatch-error" &&
|
||||
o.payload?.sessionKey === "agent:main:main",
|
||||
8_000,
|
||||
);
|
||||
const res = await rpcReq(ws, "chat.send", {
|
||||
sessionKey: "main",
|
||||
message: "run: pwd",
|
||||
idempotencyKey: "idem-dispatch-error-1",
|
||||
});
|
||||
expect(res.ok).toBe(true);
|
||||
await errorPromise;
|
||||
const sessionChanged = await sessionChangedPromise;
|
||||
const rejectDispatch = createDeferred<void>();
|
||||
const releasePersistence = createDeferred<void>();
|
||||
let dispatchStarted = false;
|
||||
let persistenceEntered = false;
|
||||
const persistLifecycleEvent = sessionLifecycleState.persistGatewaySessionLifecycleEvent;
|
||||
const persistSpy = vi
|
||||
.spyOn(sessionLifecycleState, "persistGatewaySessionLifecycleEvent")
|
||||
.mockImplementation(async (params) => {
|
||||
persistenceEntered = true;
|
||||
await releasePersistence.promise;
|
||||
await persistLifecycleEvent(params);
|
||||
});
|
||||
const sessionChanged = await (async () => {
|
||||
try {
|
||||
dispatchInboundMessageMock.mockImplementationOnce(async () => {
|
||||
dispatchStarted = true;
|
||||
await rejectDispatch.promise;
|
||||
throw new Error("provider rejected request");
|
||||
});
|
||||
const errorPromise = onceMessage(
|
||||
ws,
|
||||
(o) =>
|
||||
o.type === "event" &&
|
||||
o.event === "chat" &&
|
||||
o.payload?.state === "error" &&
|
||||
o.payload?.runId === "idem-dispatch-error-1",
|
||||
8_000,
|
||||
);
|
||||
const sessionChangedPromise = onceMessage(
|
||||
ws,
|
||||
(o) =>
|
||||
o.type === "event" &&
|
||||
o.event === "sessions.changed" &&
|
||||
o.payload?.reason === "chat.dispatch-error" &&
|
||||
o.payload?.sessionKey === "agent:main:main",
|
||||
8_000,
|
||||
);
|
||||
const res = await rpcReq(ws, "chat.send", {
|
||||
sessionKey: "main",
|
||||
message: "run: pwd",
|
||||
idempotencyKey: "idem-dispatch-error-1",
|
||||
});
|
||||
expect(res.ok).toBe(true);
|
||||
await vi.waitFor(() => {
|
||||
expect(dispatchStarted).toBe(true);
|
||||
});
|
||||
markGatewayRestartDraining();
|
||||
rejectDispatch.resolve();
|
||||
await errorPromise;
|
||||
await vi.waitFor(() => {
|
||||
expect(persistenceEntered).toBe(true);
|
||||
});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
releasePersistence.resolve();
|
||||
const changed = await sessionChangedPromise;
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
return changed;
|
||||
} finally {
|
||||
rejectDispatch.resolve();
|
||||
releasePersistence.resolve();
|
||||
persistSpy.mockRestore();
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
})();
|
||||
expectRecordFields(sessionChanged.payload, {
|
||||
sessionId: "sess-main",
|
||||
status: "failed",
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
resetDiagnosticEventsForTest,
|
||||
type DiagnosticEventPayload,
|
||||
} from "../infra/diagnostic-events.js";
|
||||
import { tryBeginGatewaySuspendAdmission } from "../process/gateway-work-admission.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
import type { ResolvedGatewayAuth } from "./auth.js";
|
||||
import { MAX_PREAUTH_PAYLOAD_BYTES } from "./server-constants.js";
|
||||
@@ -144,6 +145,22 @@ describe("gateway pre-auth hardening", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects core websocket upgrades while suspension admission is closed", async () => {
|
||||
const harness = await createGatewaySuiteHarness();
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
try {
|
||||
await expect(requestUpgradeRejection(harness.port)).resolves.toEqual({
|
||||
status: 503,
|
||||
body: "Gateway websocket admission closed",
|
||||
});
|
||||
} finally {
|
||||
suspension?.release();
|
||||
await harness.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("closes idle unauthenticated sockets after the handshake timeout", async () => {
|
||||
setEnvForTest("OPENCLAW_TEST_HANDSHAKE_TIMEOUT_MS", "200");
|
||||
|
||||
|
||||
@@ -445,6 +445,9 @@ describe("gateway update.run", () => {
|
||||
await vi.waitFor(() => {
|
||||
expect(updateMock).toHaveBeenCalledOnce();
|
||||
}, FAST_WAIT_OPTS);
|
||||
await vi.waitFor(() => {
|
||||
expect(sigusr1).toHaveBeenCalled();
|
||||
}, FAST_WAIT_OPTS);
|
||||
} finally {
|
||||
process.off("SIGUSR1", sigusr1);
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
setActivePluginRegistry,
|
||||
} from "../../plugins/runtime.js";
|
||||
import { getPluginRuntimeGatewayRequestScope } from "../../plugins/runtime/gateway-request-scope.js";
|
||||
import { resolveControlPlaneRateLimitKey } from "../control-plane-rate-limit.js";
|
||||
import { ExecApprovalManager } from "../exec-approval-manager.js";
|
||||
import type { AuthorizedGatewayHttpRequest } from "../http-utils.js";
|
||||
import { authorizeOperatorScopesForMethod, CLI_DEFAULT_OPERATOR_SCOPES } from "../method-scopes.js";
|
||||
@@ -248,6 +249,60 @@ describe("plugin HTTP route runtime scopes", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("threads resolved HTTP client IP into the shared control-plane rate identity", async () => {
|
||||
const observed: Array<{ clientIp?: string; connId?: string; rateLimitKey: string }> = [];
|
||||
const handler = createPluginRequestHandler({
|
||||
routes: [
|
||||
createRoute({
|
||||
path: SECURE_HOOK_PATH,
|
||||
auth: "gateway",
|
||||
gatewayRuntimeScopeSurface: "trusted-operator",
|
||||
gatewayMethodDispatchAllowed: true,
|
||||
handler: async () => {
|
||||
const client = getPluginRuntimeGatewayRequestScope()?.client;
|
||||
observed.push({
|
||||
...(client?.clientIp ? { clientIp: client.clientIp } : {}),
|
||||
...(client?.connId ? { connId: client.connId } : {}),
|
||||
rateLimitKey: resolveControlPlaneRateLimitKey(client ?? null),
|
||||
});
|
||||
return true;
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
for (const clientIp of ["203.0.113.10", "203.0.113.10", "203.0.113.11"]) {
|
||||
const { handled, res } = await dispatchPluginRequest(handler, {
|
||||
path: SECURE_HOOK_PATH,
|
||||
authContext: {
|
||||
gatewayAuthSatisfied: true,
|
||||
gatewayRequestAuth: { authMethod: "token", trustDeclaredOperatorScopes: false },
|
||||
gatewayRequestClientIp: clientIp,
|
||||
},
|
||||
});
|
||||
expect(handled).toBe(true);
|
||||
expect(res.statusCode).toBe(200);
|
||||
}
|
||||
|
||||
expect(observed).toEqual([
|
||||
{
|
||||
clientIp: "203.0.113.10",
|
||||
connId: "plugin-http:203.0.113.10",
|
||||
rateLimitKey: "unknown-device|203.0.113.10",
|
||||
},
|
||||
{
|
||||
clientIp: "203.0.113.10",
|
||||
connId: "plugin-http:203.0.113.10",
|
||||
rateLimitKey: "unknown-device|203.0.113.10",
|
||||
},
|
||||
{
|
||||
clientIp: "203.0.113.11",
|
||||
connId: "plugin-http:203.0.113.11",
|
||||
rateLimitKey: "unknown-device|203.0.113.11",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses server-local routes and gateway context when the active registry belongs to another gateway", async () => {
|
||||
const serverAContext = { label: "server-a" } as unknown as GatewayRequestContext;
|
||||
const serverBContext = { label: "server-b" } as unknown as GatewayRequestContext;
|
||||
|
||||
@@ -46,8 +46,11 @@ function resolvePluginRoutePathContextForRequest(
|
||||
|
||||
function createPluginRouteRuntimeClient(
|
||||
scopes: readonly string[],
|
||||
clientIp: string | undefined,
|
||||
): GatewayRequestOptions["client"] {
|
||||
return {
|
||||
connId: `plugin-http:${clientIp ?? "unknown"}`,
|
||||
...(clientIp ? { clientIp } : {}),
|
||||
connect: {
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
maxProtocol: PROTOCOL_VERSION,
|
||||
@@ -71,6 +74,7 @@ function writeUpgradeUnauthorized(socket: Duplex) {
|
||||
type PluginRouteRuntimeDispatchContext = {
|
||||
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
|
||||
gatewayRequestOperatorScopes?: readonly string[];
|
||||
gatewayRequestClientIp?: string;
|
||||
};
|
||||
|
||||
function getMissingPluginRouteRuntimeContext(
|
||||
@@ -92,6 +96,7 @@ function createPluginRouteRuntimeScope(params: {
|
||||
gatewayRequestContext?: GatewayRequestContext;
|
||||
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
|
||||
gatewayRequestOperatorScopes?: readonly string[];
|
||||
gatewayRequestClientIp?: string;
|
||||
}): PluginRouteRuntimeScope {
|
||||
const runtimeScopes =
|
||||
params.route.auth !== "gateway"
|
||||
@@ -103,7 +108,10 @@ function createPluginRouteRuntimeScope(params: {
|
||||
"trusted-operator",
|
||||
)
|
||||
: params.gatewayRequestOperatorScopes!;
|
||||
const runtimeClient = createPluginRouteRuntimeClient(runtimeScopes);
|
||||
const runtimeClient = createPluginRouteRuntimeClient(
|
||||
runtimeScopes,
|
||||
params.gatewayRequestClientIp,
|
||||
);
|
||||
return {
|
||||
...(params.gatewayRequestContext ? { context: params.gatewayRequestContext } : {}),
|
||||
client: runtimeClient,
|
||||
@@ -120,6 +128,7 @@ export type PluginRouteDispatchContext = {
|
||||
gatewayAuthSatisfied?: boolean;
|
||||
gatewayRequestAuth?: AuthorizedGatewayHttpRequest;
|
||||
gatewayRequestOperatorScopes?: readonly string[];
|
||||
gatewayRequestClientIp?: string;
|
||||
};
|
||||
|
||||
export type PluginHttpRequestHandler = (
|
||||
@@ -189,6 +198,7 @@ export function createGatewayPluginRequestHandler(params: {
|
||||
gatewayRequestContext,
|
||||
gatewayRequestAuth,
|
||||
gatewayRequestOperatorScopes,
|
||||
gatewayRequestClientIp: dispatchContext?.gatewayRequestClientIp,
|
||||
}),
|
||||
async () => route.handler(req, res),
|
||||
);
|
||||
@@ -265,6 +275,7 @@ export function createGatewayPluginUpgradeHandler(params: {
|
||||
gatewayRequestContext,
|
||||
gatewayRequestAuth,
|
||||
gatewayRequestOperatorScopes,
|
||||
gatewayRequestClientIp: dispatchContext?.gatewayRequestClientIp,
|
||||
}),
|
||||
async () => route.handleUpgrade?.(req, socket, head),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// WebSocket connect suspension tests cover root admission before handshake mutations.
|
||||
import type { IncomingMessage } from "node:http";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { WebSocket } from "ws";
|
||||
import { PROTOCOL_VERSION } from "../../../../packages/gateway-protocol/src/index.js";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import type { GatewayRequestContext } from "../../server-methods/types.js";
|
||||
|
||||
const { incrementPresenceVersionMock, loadConfigMock, upsertPresenceMock } = vi.hoisted(() => ({
|
||||
incrementPresenceVersionMock: vi.fn(() => 2),
|
||||
loadConfigMock: vi.fn(() => ({ gateway: { auth: { mode: "none" } } })),
|
||||
upsertPresenceMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../../config/config.js", () => ({
|
||||
getRuntimeConfig: loadConfigMock,
|
||||
loadConfig: loadConfigMock,
|
||||
}));
|
||||
vi.mock("../../../config/io.js", () => ({
|
||||
getRuntimeConfig: loadConfigMock,
|
||||
}));
|
||||
vi.mock("../../../infra/system-presence.js", () => ({
|
||||
upsertPresence: upsertPresenceMock,
|
||||
}));
|
||||
vi.mock("../health-state.js", () => ({
|
||||
buildGatewaySnapshot: vi.fn(() => ({
|
||||
presence: [],
|
||||
health: {},
|
||||
stateVersion: { presence: 1, health: 1 },
|
||||
uptimeMs: 1,
|
||||
sessionDefaults: {
|
||||
defaultAgentId: "main",
|
||||
mainKey: "main",
|
||||
mainSessionKey: "main",
|
||||
scope: "per-sender",
|
||||
},
|
||||
})),
|
||||
getHealthCache: vi.fn(() => null),
|
||||
getHealthVersion: vi.fn(() => 1),
|
||||
incrementPresenceVersion: incrementPresenceVersionMock,
|
||||
}));
|
||||
|
||||
import { attachGatewayWsMessageHandler } from "./message-handler.js";
|
||||
|
||||
function createLogger() {
|
||||
return {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
function attachHarness(params: { deferSocketSend?: boolean } = {}) {
|
||||
let onMessage: ((data: string) => void) | undefined;
|
||||
let finishSocketSend: (() => void) | undefined;
|
||||
let client: unknown = null;
|
||||
const socketSend = vi.fn((_payload: string, callback?: (error?: Error) => void) => {
|
||||
if (params.deferSocketSend) {
|
||||
finishSocketSend = () => callback?.();
|
||||
return;
|
||||
}
|
||||
callback?.();
|
||||
});
|
||||
const socket = {
|
||||
_receiver: {},
|
||||
send: socketSend,
|
||||
on: vi.fn((event: string, handler: (data: string) => void) => {
|
||||
if (event === "message") {
|
||||
onMessage = handler;
|
||||
}
|
||||
return socket;
|
||||
}),
|
||||
} as unknown as WebSocket;
|
||||
const close = vi.fn();
|
||||
const setClient = vi.fn((next: unknown) => {
|
||||
client = next;
|
||||
return true;
|
||||
});
|
||||
|
||||
attachGatewayWsMessageHandler({
|
||||
socket,
|
||||
upgradeReq: {
|
||||
headers: { host: "127.0.0.1:19001" },
|
||||
socket: { localAddress: "127.0.0.1", remoteAddress: "127.0.0.1" },
|
||||
} as unknown as IncomingMessage,
|
||||
connId: "suspension-connect",
|
||||
remoteAddr: "127.0.0.1",
|
||||
localAddr: "127.0.0.1",
|
||||
requestHost: "127.0.0.1:19001",
|
||||
connectNonce: "suspension-connect-nonce",
|
||||
getResolvedAuth: () => ({ mode: "none", allowTailscale: false }),
|
||||
gatewayMethods: [],
|
||||
events: [],
|
||||
extraHandlers: {},
|
||||
buildRequestContext: () => ({}) as GatewayRequestContext,
|
||||
refreshHealthSnapshot: vi.fn(async () => ({}) as never),
|
||||
send: vi.fn(),
|
||||
close,
|
||||
isClosed: vi.fn(() => false),
|
||||
clearHandshakeTimer: vi.fn(),
|
||||
getClient: () => client as never,
|
||||
setClient: setClient as never,
|
||||
setHandshakeState: vi.fn(),
|
||||
advanceHandshakePhase: vi.fn(),
|
||||
setCloseCause: vi.fn(),
|
||||
setLastFrameMeta: vi.fn(),
|
||||
originCheckMetrics: { hostHeaderFallbackAccepted: 0 },
|
||||
logGateway: createLogger() as never,
|
||||
logHealth: createLogger() as never,
|
||||
logWsControl: createLogger() as never,
|
||||
});
|
||||
if (!onMessage) {
|
||||
throw new Error("expected websocket message handler");
|
||||
}
|
||||
|
||||
return {
|
||||
close,
|
||||
finishSocketSend: () => finishSocketSend?.(),
|
||||
get client() {
|
||||
return client;
|
||||
},
|
||||
sendConnect: () =>
|
||||
onMessage?.(
|
||||
JSON.stringify({
|
||||
type: "req",
|
||||
id: "connect-1",
|
||||
method: "connect",
|
||||
params: {
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
maxProtocol: PROTOCOL_VERSION,
|
||||
client: {
|
||||
id: "gateway-client",
|
||||
version: "dev",
|
||||
platform: "test",
|
||||
mode: "backend",
|
||||
},
|
||||
role: "operator",
|
||||
scopes: [],
|
||||
caps: [],
|
||||
},
|
||||
}),
|
||||
),
|
||||
setClient,
|
||||
socketSend,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(resetGatewayWorkAdmission);
|
||||
|
||||
describe("WebSocket connect suspension admission", () => {
|
||||
it.each(["preparing", "prepared"] as const)(
|
||||
"rejects a validated connect while suspension is %s before session mutations",
|
||||
async (phase) => {
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension).not.toBeNull();
|
||||
if (phase === "prepared") {
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
}
|
||||
const harness = attachHarness();
|
||||
|
||||
harness.sendConnect();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(harness.socketSend).toHaveBeenCalledOnce();
|
||||
});
|
||||
const response = JSON.parse(harness.socketSend.mock.calls[0]?.[0] ?? "{}") as {
|
||||
error?: {
|
||||
code?: string;
|
||||
retryable?: boolean;
|
||||
retryAfterMs?: number;
|
||||
details?: Record<string, unknown>;
|
||||
};
|
||||
};
|
||||
expect(response.error).toMatchObject({
|
||||
code: "UNAVAILABLE",
|
||||
retryable: true,
|
||||
retryAfterMs: 1_000,
|
||||
details: {
|
||||
method: "connect",
|
||||
reason: "gateway-suspending",
|
||||
phase,
|
||||
},
|
||||
});
|
||||
expect(harness.client).toBeNull();
|
||||
expect(harness.setClient).not.toHaveBeenCalled();
|
||||
expect(upsertPresenceMock).not.toHaveBeenCalled();
|
||||
expect(incrementPresenceVersionMock).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => {
|
||||
expect(harness.close).toHaveBeenCalledWith(1013, "gateway suspension in progress");
|
||||
});
|
||||
|
||||
if (phase === "prepared") {
|
||||
suspension?.release();
|
||||
} else {
|
||||
suspension?.rollback();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps an accepted handshake visible as root work until hello is sent", async () => {
|
||||
const harness = attachHarness({ deferSocketSend: true });
|
||||
|
||||
harness.sendConnect();
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(harness.socketSend).toHaveBeenCalledOnce();
|
||||
});
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension).not.toBeNull();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
expect(suspension?.rollback()).toBe(true);
|
||||
|
||||
harness.finishSocketSend();
|
||||
await vi.waitFor(() => {
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -92,6 +92,12 @@ import { loadVoiceWakeConfig } from "../../../infra/voicewake.js";
|
||||
import { rawDataToString } from "../../../infra/ws.js";
|
||||
import { logRejectedLargePayload } from "../../../logging/diagnostic-payload.js";
|
||||
import type { createSubsystemLogger } from "../../../logging/subsystem.js";
|
||||
import {
|
||||
getGatewaySuspendAdmissionPhase,
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import {
|
||||
BOOTSTRAP_HANDOFF_OPERATOR_SCOPES,
|
||||
isPairingSetupBootstrapProfile,
|
||||
@@ -188,6 +194,8 @@ import { isUnauthorizedRoleError, UnauthorizedFloodGuard } from "./unauthorized-
|
||||
type SubsystemLogger = ReturnType<typeof createSubsystemLogger>;
|
||||
|
||||
const DEVICE_SIGNATURE_SKEW_MS = 2 * 60 * 1000;
|
||||
const GATEWAY_WORK_ADMISSION_RETRY_AFTER_MS = 1_000;
|
||||
const GATEWAY_WORK_ADMISSION_CLOSE_CODE = 1013;
|
||||
const DEVICE_CREDENTIAL_INVALIDATING_METHODS = new Set([
|
||||
"device.pair.remove",
|
||||
"device.token.rotate",
|
||||
@@ -662,6 +670,11 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
|
||||
close(4001, `client invalidated: ${reason}`);
|
||||
return true;
|
||||
};
|
||||
const runDetachedConnectWork = (run: () => Promise<void>, onError: (error: unknown) => void) => {
|
||||
// Connect-triggered mutations outlive hello-ok. Give each tail its own
|
||||
// root lease so suspension cannot report ready while one is still active.
|
||||
void runWithGatewayIndependentRootWorkAdmission(run).catch(onError);
|
||||
};
|
||||
|
||||
const handleMessage = async (data: RawData) => {
|
||||
if (isClosed()) {
|
||||
@@ -2179,10 +2192,16 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
|
||||
nodeIdsForPairing.add(instanceIdLocal);
|
||||
}
|
||||
for (const nodeId of nodeIdsForPairing) {
|
||||
void updatePairedNodeMetadata(nodeId, {
|
||||
lastConnectedAtMs: nodeSession.connectedAtMs,
|
||||
}).catch((err: unknown) =>
|
||||
logGateway.warn(`failed to record last connect for ${nodeId}: ${formatForLog(err)}`),
|
||||
runDetachedConnectWork(
|
||||
async () => {
|
||||
await updatePairedNodeMetadata(nodeId, {
|
||||
lastConnectedAtMs: nodeSession.connectedAtMs,
|
||||
});
|
||||
},
|
||||
(err) =>
|
||||
logGateway.warn(
|
||||
`failed to record last connect for ${nodeId}: ${formatForLog(err)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
recordRemoteNodeInfo({
|
||||
@@ -2194,42 +2213,48 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
|
||||
commands: nodeSession.commands,
|
||||
remoteIp: nodeSession.remoteIp,
|
||||
});
|
||||
void refreshRemoteNodeBins({
|
||||
nodeId: nodeSession.nodeId,
|
||||
platform: nodeSession.platform,
|
||||
deviceFamily: nodeSession.deviceFamily,
|
||||
commands: nodeSession.commands,
|
||||
cfg: getRuntimeConfig(),
|
||||
// The node socket is registered before macOS app command handlers finish warming.
|
||||
// Delay only the connect-time probe; later skill refreshes use the live session.
|
||||
readinessDelayMs: 5_000,
|
||||
}).catch((err: unknown) =>
|
||||
logGateway.warn(
|
||||
`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
|
||||
),
|
||||
runDetachedConnectWork(
|
||||
async () => {
|
||||
await refreshRemoteNodeBins({
|
||||
nodeId: nodeSession.nodeId,
|
||||
platform: nodeSession.platform,
|
||||
deviceFamily: nodeSession.deviceFamily,
|
||||
commands: nodeSession.commands,
|
||||
cfg: getRuntimeConfig(),
|
||||
// The node socket is registered before macOS app command handlers finish warming.
|
||||
// Delay only the connect-time probe; later skill refreshes use the live session.
|
||||
readinessDelayMs: 5_000,
|
||||
});
|
||||
},
|
||||
(err) =>
|
||||
logGateway.warn(
|
||||
`remote bin probe failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
|
||||
),
|
||||
);
|
||||
void loadVoiceWakeConfig()
|
||||
.then((cfg) => {
|
||||
runDetachedConnectWork(
|
||||
async () => {
|
||||
const cfg = await loadVoiceWakeConfig();
|
||||
context.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.changed", {
|
||||
triggers: cfg.triggers,
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) =>
|
||||
},
|
||||
(err) =>
|
||||
logGateway.warn(
|
||||
`voicewake snapshot failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
|
||||
),
|
||||
);
|
||||
void loadVoiceWakeRoutingConfig()
|
||||
.then((routing) => {
|
||||
);
|
||||
runDetachedConnectWork(
|
||||
async () => {
|
||||
const routing = await loadVoiceWakeRoutingConfig();
|
||||
context.nodeRegistry.sendEvent(nodeSession.nodeId, "voicewake.routing.changed", {
|
||||
config: routing,
|
||||
});
|
||||
})
|
||||
.catch((err: unknown) =>
|
||||
},
|
||||
(err) =>
|
||||
logGateway.warn(
|
||||
`voicewake routing snapshot failed for ${nodeSession.nodeId}: ${formatForLog(err)}`,
|
||||
),
|
||||
);
|
||||
);
|
||||
}
|
||||
|
||||
const snapshot = buildGatewaySnapshot({
|
||||
@@ -2510,8 +2535,80 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
|
||||
}
|
||||
};
|
||||
|
||||
const rejectConnectForClosedAdmission = async (data: RawData): Promise<boolean> => {
|
||||
if (isClosed() || getRawDataByteLength(data) > MAX_PREAUTH_PAYLOAD_BYTES) {
|
||||
return false;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawDataToString(data));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
!validateRequestFrame(parsed) ||
|
||||
parsed.method !== "connect" ||
|
||||
!validateConnectParams(parsed.params)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const restartDraining = isGatewayRestartDraining();
|
||||
const reason = restartDraining ? "gateway-restarting" : "gateway-suspending";
|
||||
const operation = restartDraining ? "restart" : "suspension";
|
||||
const phase = getGatewaySuspendAdmissionPhase();
|
||||
setLastFrameMeta({ type: "req", method: "connect", id: parsed.id });
|
||||
setHandshakeState("failed");
|
||||
setCloseCause(reason, {
|
||||
method: "connect",
|
||||
phase,
|
||||
});
|
||||
await sendFrame({
|
||||
type: "res",
|
||||
id: parsed.id,
|
||||
ok: false,
|
||||
error: errorShape(ErrorCodes.UNAVAILABLE, `connect unavailable during gateway ${operation}`, {
|
||||
retryable: true,
|
||||
retryAfterMs: GATEWAY_WORK_ADMISSION_RETRY_AFTER_MS,
|
||||
details: {
|
||||
method: "connect",
|
||||
reason,
|
||||
phase,
|
||||
},
|
||||
}),
|
||||
}).catch(() => {});
|
||||
queueMicrotask(() =>
|
||||
close(GATEWAY_WORK_ADMISSION_CLOSE_CODE, `gateway ${operation} in progress`),
|
||||
);
|
||||
return true;
|
||||
};
|
||||
|
||||
const handleIncomingMessage = async (data: RawData) => {
|
||||
if (getClient()) {
|
||||
await handleMessage(data);
|
||||
return;
|
||||
}
|
||||
const admission = tryBeginGatewayRootWorkAdmission();
|
||||
if (!admission) {
|
||||
if (await rejectConnectForClosedAdmission(data)) {
|
||||
return;
|
||||
}
|
||||
// Malformed pre-auth frames still use the established validation and
|
||||
// close path; only a validated connect can cross into mutable work.
|
||||
await handleMessage(data);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await admission.run(() => handleMessage(data));
|
||||
} finally {
|
||||
admission.release();
|
||||
}
|
||||
};
|
||||
|
||||
socket.on("message", (data) => {
|
||||
void runWithDiagnosticTraceContext(createDiagnosticTraceContext(), () => handleMessage(data));
|
||||
void runWithDiagnosticTraceContext(createDiagnosticTraceContext(), () =>
|
||||
handleIncomingMessage(data),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Gateway WebSocket client types describe authenticated client state retained by the server.
|
||||
import type { WebSocket } from "ws";
|
||||
import type { ConnectParams } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { ConnectParams } from "../../../packages/gateway-protocol/src/schema/frames.js";
|
||||
import type { AgentRuntimeIdentity } from "../agent-runtime-identity-token.js";
|
||||
import type { PluginNodeCapabilityClient } from "../plugin-node-capability.js";
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ import { getSessionBindingService } from "../infra/outbound/session-binding-serv
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import { runPluginHostCleanup } from "../plugins/host-hook-cleanup.js";
|
||||
import { getActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import { runWithGatewayIndependentRootWorkContinuation } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
isSubagentSessionKey,
|
||||
normalizeAgentId,
|
||||
@@ -188,7 +189,9 @@ export function emitGatewaySessionEndPluginHook(params: {
|
||||
nextSessionId: params.nextSessionId,
|
||||
nextSessionKey: params.nextSessionKey,
|
||||
});
|
||||
void hookRunner.runSessionEnd(payload.event, payload.context).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionEnd(payload.event, payload.context);
|
||||
}).catch((err: unknown) => {
|
||||
logVerbose(`session_end hook failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
@@ -231,7 +234,9 @@ export function emitGatewaySessionStartPluginHook(params: {
|
||||
cfg: params.cfg,
|
||||
resumedFrom: params.resumedFrom,
|
||||
});
|
||||
void hookRunner.runSessionStart(payload.event, payload.context).catch((err: unknown) => {
|
||||
void runWithGatewayIndependentRootWorkContinuation(async () => {
|
||||
await hookRunner.runSessionStart(payload.event, payload.context);
|
||||
}).catch((err: unknown) => {
|
||||
logVerbose(`session_start hook failed: ${String(err)}`);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -26,10 +26,13 @@ import {
|
||||
getPairedDevice,
|
||||
requestDevicePairing,
|
||||
} from "../infra/device-pairing.js";
|
||||
import { resetGatewaySuspendCoordinatorForTest } from "../infra/gateway-suspend-coordinator.js";
|
||||
import { __testing as restartTesting } from "../infra/restart.js";
|
||||
import { drainSystemEvents, peekSystemEvents } from "../infra/system-events.js";
|
||||
import { rawDataToString } from "../infra/ws.js";
|
||||
import { resetLogger, setLoggerOverride } from "../logging.js";
|
||||
import { clearGatewaySubagentRuntime } from "../plugins/runtime/gateway-bindings.js";
|
||||
import { resetGatewayWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
DEFAULT_AGENT_ID,
|
||||
normalizeMainKey,
|
||||
@@ -250,9 +253,22 @@ function applyGatewaySkipEnv() {
|
||||
: "openclaw-test-no-bundled-extensions";
|
||||
}
|
||||
|
||||
function resetGatewayLifecycleTestState(options: { preserveRuntimeBindings: boolean }): void {
|
||||
// Resume a held scheduler before hard admission reset invalidates and forgets its
|
||||
// lease. Then cancel restart timers and retire their module-local signal lease.
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
if (options.preserveRuntimeBindings) {
|
||||
restartTesting.resetSigusr1TransientState();
|
||||
} else {
|
||||
restartTesting.resetSigusr1State();
|
||||
}
|
||||
resetGatewayWorkAdmission();
|
||||
}
|
||||
|
||||
async function resetGatewayTestState(options: { uniqueConfigRoot: boolean }) {
|
||||
// Some tests intentionally use fake timers; ensure they don't leak into gateway suites.
|
||||
vi.useRealTimers();
|
||||
resetGatewayLifecycleTestState({ preserveRuntimeBindings: false });
|
||||
setLoggerOverride({ level: "silent", consoleLevel: "silent" });
|
||||
if (!tempHome) {
|
||||
throw new Error("resetGatewayTestState called before temp home was initialized");
|
||||
@@ -369,6 +385,7 @@ async function resetGatewayTestState(options: { uniqueConfigRoot: boolean }) {
|
||||
|
||||
async function cleanupGatewayTestHome(options: { restoreEnv: boolean }) {
|
||||
vi.useRealTimers();
|
||||
resetGatewayLifecycleTestState({ preserveRuntimeBindings: activeSuiteGatewayServerCount > 0 });
|
||||
clearGatewaySubagentRuntime();
|
||||
resetLogger();
|
||||
resetTaskRegistryForTests({ persist: false });
|
||||
@@ -395,6 +412,7 @@ async function cleanupGatewayTestHome(options: { restoreEnv: boolean }) {
|
||||
|
||||
async function resetGatewayTestRuntimeOnly() {
|
||||
vi.useRealTimers();
|
||||
resetGatewayLifecycleTestState({ preserveRuntimeBindings: true });
|
||||
setLoggerOverride({ level: "silent", consoleLevel: "silent" });
|
||||
applyGatewaySkipEnv();
|
||||
delete process.env.OPENCLAW_GATEWAY_TOKEN;
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
* like command processing, session lifecycle, etc.
|
||||
*/
|
||||
|
||||
import type { SessionsPatchParams } from "../../packages/gateway-protocol/src/schema.js";
|
||||
import type { SessionsPatchParams } from "../../packages/gateway-protocol/src/schema/sessions.js";
|
||||
import type { WorkspaceBootstrapFile } from "../agents/workspace.js";
|
||||
import type { CliDeps } from "../cli/outbound-send-deps.js";
|
||||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Runs the gateway-backed runtime that delivers native approval events.
|
||||
import { readConnectErrorDetailCode } from "../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import type { EventFrame } from "../../packages/gateway-protocol/src/index.js";
|
||||
import type { EventFrame } from "../../packages/gateway-protocol/src/schema/frames.js";
|
||||
import { startGatewayClientWhenEventLoopReady } from "../gateway/client-start-readiness.js";
|
||||
import type { GatewayClient, GatewayReconnectPausedInfo } from "../gateway/client.js";
|
||||
import { isApprovalMethod } from "../gateway/method-scopes.js";
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// Collects process activity shared by restart and host-suspension decisions.
|
||||
import { getActiveEmbeddedRunCount } from "../agents/embedded-agent-runner/run-state.js";
|
||||
import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js";
|
||||
import { getActiveCronJobCount } from "../cron/active-jobs.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import { getActiveGatewayRootWorkCount } from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
getActiveSessionLifecycleMutationCount,
|
||||
getActiveSessionWorkAdmissionCount,
|
||||
} from "../sessions/session-lifecycle-admission.js";
|
||||
import { getSuspensionVisibleCronTaskRunCount } from "../tasks/cron-task-cancel.js";
|
||||
import { getInspectableActiveTaskRestartBlockers } from "../tasks/task-registry.maintenance.js";
|
||||
import {
|
||||
type ActiveTaskRestartBlocker,
|
||||
formatActiveTaskRestartBlocker,
|
||||
} from "../tasks/task-restart-blocker.js";
|
||||
|
||||
export type GatewayActiveWorkCounts = {
|
||||
queueSize: number;
|
||||
pendingReplies: number;
|
||||
embeddedRuns: number;
|
||||
cronRuns: number;
|
||||
activeTasks: number;
|
||||
rootRequests: number;
|
||||
sessionAdmissions: number;
|
||||
sessionMutations: number;
|
||||
chatRuns: number;
|
||||
queuedTurns: number;
|
||||
terminalPersistence: number;
|
||||
terminalSessions: number;
|
||||
/** Compatibility aggregate. Categories can overlap; use individual counts for diagnostics. */
|
||||
totalActive: number;
|
||||
};
|
||||
|
||||
export type GatewayActiveWorkBlocker = {
|
||||
kind:
|
||||
| "queue"
|
||||
| "reply"
|
||||
| "embedded-run"
|
||||
| "cron-run"
|
||||
| "task"
|
||||
| "root-request"
|
||||
| "session-admission"
|
||||
| "session-mutation"
|
||||
| "chat-run"
|
||||
| "queued-turn"
|
||||
| "terminal-persistence"
|
||||
| "terminal-session";
|
||||
count: number;
|
||||
message: string;
|
||||
task?: ActiveTaskRestartBlocker;
|
||||
};
|
||||
|
||||
export type GatewayActiveWorkSnapshot = {
|
||||
idle: boolean;
|
||||
counts: GatewayActiveWorkCounts;
|
||||
blockers: GatewayActiveWorkBlocker[];
|
||||
};
|
||||
|
||||
export type GatewayActiveWorkInspectors = {
|
||||
getQueueSize: () => number;
|
||||
getPendingReplies: () => number;
|
||||
getEmbeddedRuns: () => number;
|
||||
getCronRuns: () => number;
|
||||
getActiveTasks: () => number;
|
||||
getTaskBlockers: () => ActiveTaskRestartBlocker[];
|
||||
getRootRequests: () => number;
|
||||
getSessionAdmissions: () => number;
|
||||
getSessionMutations: () => number;
|
||||
getChatRuns: () => number;
|
||||
getQueuedTurns: () => number;
|
||||
getTerminalPersistence: () => number;
|
||||
getTerminalSessions: () => number;
|
||||
};
|
||||
|
||||
const defaultInspectors: GatewayActiveWorkInspectors = {
|
||||
getQueueSize: getTotalQueueSize,
|
||||
getPendingReplies: getTotalPendingReplies,
|
||||
getEmbeddedRuns: getActiveEmbeddedRunCount,
|
||||
getCronRuns: () => Math.max(getActiveCronJobCount(), getSuspensionVisibleCronTaskRunCount()),
|
||||
getActiveTasks: () => getInspectableActiveTaskRestartBlockers().length,
|
||||
getTaskBlockers: getInspectableActiveTaskRestartBlockers,
|
||||
getRootRequests: () => getActiveGatewayRootWorkCount({ excludeCurrent: true }),
|
||||
getSessionAdmissions: getActiveSessionWorkAdmissionCount,
|
||||
getSessionMutations: getActiveSessionLifecycleMutationCount,
|
||||
getChatRuns: () => 0,
|
||||
getQueuedTurns: () => 0,
|
||||
getTerminalPersistence: () => 0,
|
||||
getTerminalSessions: () => 0,
|
||||
};
|
||||
|
||||
function normalizeCount(value: number): number {
|
||||
return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
||||
}
|
||||
|
||||
export function createGatewayActiveWorkSnapshot(
|
||||
inspectors: Partial<GatewayActiveWorkInspectors> = {},
|
||||
): GatewayActiveWorkSnapshot {
|
||||
const resolved = { ...defaultInspectors, ...inspectors };
|
||||
const counts: GatewayActiveWorkCounts = {
|
||||
queueSize: normalizeCount(resolved.getQueueSize()),
|
||||
pendingReplies: normalizeCount(resolved.getPendingReplies()),
|
||||
embeddedRuns: normalizeCount(resolved.getEmbeddedRuns()),
|
||||
cronRuns: normalizeCount(resolved.getCronRuns()),
|
||||
activeTasks: normalizeCount(resolved.getActiveTasks()),
|
||||
rootRequests: normalizeCount(resolved.getRootRequests()),
|
||||
sessionAdmissions: normalizeCount(resolved.getSessionAdmissions()),
|
||||
sessionMutations: normalizeCount(resolved.getSessionMutations()),
|
||||
chatRuns: normalizeCount(resolved.getChatRuns()),
|
||||
queuedTurns: normalizeCount(resolved.getQueuedTurns()),
|
||||
terminalPersistence: normalizeCount(resolved.getTerminalPersistence()),
|
||||
terminalSessions: normalizeCount(resolved.getTerminalSessions()),
|
||||
totalActive: 0,
|
||||
};
|
||||
counts.totalActive = Object.entries(counts).reduce(
|
||||
(total, [key, count]) => (key === "totalActive" ? total : total + count),
|
||||
0,
|
||||
);
|
||||
|
||||
const blockers: GatewayActiveWorkBlocker[] = [];
|
||||
const add = (count: number, kind: GatewayActiveWorkBlocker["kind"], message: string) => {
|
||||
if (count > 0) {
|
||||
blockers.push({ kind, count, message });
|
||||
}
|
||||
};
|
||||
add(counts.queueSize, "queue", `${counts.queueSize} queued or active operation(s)`);
|
||||
add(
|
||||
counts.pendingReplies,
|
||||
"reply",
|
||||
`${counts.pendingReplies} pending reply delivery operation(s)`,
|
||||
);
|
||||
add(counts.embeddedRuns, "embedded-run", `${counts.embeddedRuns} active embedded run(s)`);
|
||||
add(counts.cronRuns, "cron-run", `${counts.cronRuns} active cron run(s)`);
|
||||
add(counts.rootRequests, "root-request", `${counts.rootRequests} active gateway request(s)`);
|
||||
add(
|
||||
counts.sessionAdmissions,
|
||||
"session-admission",
|
||||
`${counts.sessionAdmissions} admitted session turn(s)`,
|
||||
);
|
||||
add(
|
||||
counts.sessionMutations,
|
||||
"session-mutation",
|
||||
`${counts.sessionMutations} active session lifecycle mutation(s)`,
|
||||
);
|
||||
add(counts.chatRuns, "chat-run", `${counts.chatRuns} active chat run(s)`);
|
||||
add(counts.queuedTurns, "queued-turn", `${counts.queuedTurns} queued chat turn(s)`);
|
||||
add(
|
||||
counts.terminalPersistence,
|
||||
"terminal-persistence",
|
||||
`${counts.terminalPersistence} pending terminal session write(s)`,
|
||||
);
|
||||
add(
|
||||
counts.terminalSessions,
|
||||
"terminal-session",
|
||||
`${counts.terminalSessions} open terminal session(s)`,
|
||||
);
|
||||
|
||||
if (counts.activeTasks > 0) {
|
||||
const taskBlockers = resolved.getTaskBlockers();
|
||||
if (taskBlockers.length === 0) {
|
||||
blockers.push({
|
||||
kind: "task",
|
||||
count: counts.activeTasks,
|
||||
message: `${counts.activeTasks} active background task run(s)`,
|
||||
});
|
||||
} else {
|
||||
const shownTaskBlockers = taskBlockers.slice(0, 8);
|
||||
for (const task of shownTaskBlockers) {
|
||||
blockers.push({
|
||||
kind: "task",
|
||||
count: 1,
|
||||
message: formatActiveTaskRestartBlocker(task),
|
||||
task,
|
||||
});
|
||||
}
|
||||
const omitted = counts.activeTasks - shownTaskBlockers.length;
|
||||
if (omitted > 0) {
|
||||
blockers.push({
|
||||
kind: "task",
|
||||
count: omitted,
|
||||
message: `${omitted} additional active background task run(s)`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { idle: counts.totalActive === 0, counts, blockers };
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// Covers atomic refuse-only suspension preparation, renewal, and release.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
markGatewayRestartDraining,
|
||||
resetGatewayWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import type { GatewayActiveWorkInspectors } from "./gateway-active-work.js";
|
||||
import {
|
||||
GATEWAY_SUSPEND_TTL_MS,
|
||||
getGatewaySuspendStatus,
|
||||
prepareGatewaySuspend,
|
||||
resetGatewaySuspendCoordinatorForTest,
|
||||
resumeGatewaySuspend,
|
||||
} from "./gateway-suspend-coordinator.js";
|
||||
|
||||
function inspectors(
|
||||
overrides: Partial<GatewayActiveWorkInspectors> = {},
|
||||
): GatewayActiveWorkInspectors {
|
||||
return {
|
||||
getQueueSize: () => 0,
|
||||
getPendingReplies: () => 0,
|
||||
getEmbeddedRuns: () => 0,
|
||||
getCronRuns: () => 0,
|
||||
getActiveTasks: () => 0,
|
||||
getTaskBlockers: () => [],
|
||||
getRootRequests: () => 0,
|
||||
getSessionAdmissions: () => 0,
|
||||
getSessionMutations: () => 0,
|
||||
getChatRuns: () => 0,
|
||||
getQueuedTurns: () => 0,
|
||||
getTerminalPersistence: () => 0,
|
||||
getTerminalSessions: () => 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
});
|
||||
|
||||
describe("gateway suspend coordinator", () => {
|
||||
it("test reset resumes a held scheduler before admission is cleared", () => {
|
||||
const resumeScheduling = vi.fn(() => {
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
});
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-test-reset",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
}),
|
||||
).toMatchObject({ status: "ready" });
|
||||
|
||||
resetGatewaySuspendCoordinatorForTest();
|
||||
resetGatewayWorkAdmission();
|
||||
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
});
|
||||
|
||||
it("reopens admission in the same turn when active work refuses preparation", () => {
|
||||
const events: string[] = [];
|
||||
const result = prepareGatewaySuspend({
|
||||
requestId: "request-busy",
|
||||
pauseScheduling: () => events.push("pause"),
|
||||
resumeScheduling: () => events.push("resume"),
|
||||
inspect: inspectors({
|
||||
getQueueSize: () => {
|
||||
events.push("inspect");
|
||||
return 1;
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.status).toBe("busy");
|
||||
expect(events).toEqual(["pause", "inspect", "resume"]);
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps admission closed until a failed busy rollback resumes scheduling", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error("timer unavailable");
|
||||
})
|
||||
.mockImplementationOnce(() => {});
|
||||
const first = prepareGatewaySuspend({
|
||||
requestId: "request-busy-resume-retry",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors({ getQueueSize: () => 1 }),
|
||||
});
|
||||
|
||||
expect(first).toEqual({
|
||||
status: "recovering",
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: 1_000,
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(getGatewaySuspendStatus("stale-id")).toEqual(first);
|
||||
expect(resumeGatewaySuspend("stale-id")).toEqual({
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: 1_000,
|
||||
});
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-before-scheduler-resume",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
}),
|
||||
).toEqual(first);
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(resumeScheduling).toHaveBeenCalledTimes(2);
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
|
||||
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-after-scheduler-resume",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
createSuspensionId: () => "suspension-after-scheduler-resume",
|
||||
}),
|
||||
).toMatchObject({
|
||||
status: "ready",
|
||||
suspensionId: "suspension-after-scheduler-resume",
|
||||
});
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(resumeScheduling).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels scheduler recovery when restart supersedes suspension", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi.fn(() => {
|
||||
throw new Error("timer unavailable");
|
||||
});
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-recovery-restart",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors({ getQueueSize: () => 1 }),
|
||||
}),
|
||||
).toMatchObject({ status: "recovering" });
|
||||
|
||||
markGatewayRestartDraining();
|
||||
vi.advanceTimersByTime(1_000);
|
||||
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(getGatewaySuspendStatus("stale-id")).toEqual({ status: "running" });
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("owns recovery when inspection fails before admission commits", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error("timer unavailable");
|
||||
})
|
||||
.mockImplementationOnce(() => {});
|
||||
const result = prepareGatewaySuspend({
|
||||
requestId: "request-inspection-failure",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors({
|
||||
getQueueSize: () => {
|
||||
throw new Error("inspection failed");
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ status: "recovering" });
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(resumeScheduling).toHaveBeenCalledTimes(2);
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("renews one ready lease and resumes only with the matching id", () => {
|
||||
const resumeScheduling = vi.fn();
|
||||
const first = prepareGatewaySuspend({
|
||||
requestId: "request-ready",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
nowMs: () => 1_000,
|
||||
createSuspensionId: () => "suspension-1",
|
||||
});
|
||||
expect(first).toMatchObject({
|
||||
status: "ready",
|
||||
suspensionId: "suspension-1",
|
||||
expiresAtMs: 1_000 + GATEWAY_SUSPEND_TTL_MS,
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(getGatewaySuspendStatus("suspension-1")).toEqual({
|
||||
status: "ready",
|
||||
expiresAtMs: 1_000 + GATEWAY_SUSPEND_TTL_MS,
|
||||
});
|
||||
|
||||
const renewed = prepareGatewaySuspend({
|
||||
requestId: "request-ready",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors({ getQueueSize: () => 99 }),
|
||||
nowMs: () => 2_000,
|
||||
});
|
||||
expect(renewed).toMatchObject({
|
||||
status: "ready",
|
||||
suspensionId: "suspension-1",
|
||||
expiresAtMs: 2_000 + GATEWAY_SUSPEND_TTL_MS,
|
||||
});
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-other",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
}).status,
|
||||
).toBe("conflict");
|
||||
|
||||
expect(resumeGatewaySuspend("wrong-id")).toEqual({
|
||||
ok: false,
|
||||
reason: "suspension-mismatch",
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(resumeGatewaySuspend("suspension-1")).toEqual({
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: true,
|
||||
});
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
expect(resumeGatewaySuspend("suspension-1")).toEqual({
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("lets restart supersede a suspension without reopening its scheduler", () => {
|
||||
const resumeScheduling = vi.fn();
|
||||
const result = prepareGatewaySuspend({
|
||||
requestId: "request-restart",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
createSuspensionId: () => "suspension-restart",
|
||||
});
|
||||
expect(result.status).toBe("ready");
|
||||
|
||||
markGatewayRestartDraining();
|
||||
|
||||
expect(getGatewaySuspendStatus("suspension-restart")).toEqual({ status: "running" });
|
||||
expect(resumeScheduling).not.toHaveBeenCalled();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
});
|
||||
|
||||
it("exposes scheduler recovery after a ready lease cannot resume", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error("timer unavailable");
|
||||
})
|
||||
.mockImplementationOnce(() => {});
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-resume-retry",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
createSuspensionId: () => "suspension-resume-retry",
|
||||
});
|
||||
|
||||
expect(resumeGatewaySuspend("suspension-resume-retry")).toMatchObject({
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(getGatewaySuspendStatus("suspension-resume-retry")).toMatchObject({
|
||||
status: "recovering",
|
||||
});
|
||||
expect(
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-resume-retry",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
}),
|
||||
).toMatchObject({ status: "recovering" });
|
||||
expect(resumeGatewaySuspend("suspension-resume-retry")).toMatchObject({
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
});
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(resumeScheduling).toHaveBeenCalledTimes(2);
|
||||
expect(getGatewaySuspendStatus("suspension-resume-retry")).toEqual({ status: "running" });
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("auto-resumes an abandoned ready lease at expiry", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi.fn();
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-expiry",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
createSuspensionId: () => "suspension-expiry",
|
||||
});
|
||||
|
||||
vi.advanceTimersByTime(GATEWAY_SUSPEND_TTL_MS);
|
||||
|
||||
expect(getGatewaySuspendStatus("suspension-expiry")).toEqual({ status: "running" });
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("enters recovery when lease expiry cannot resume the scheduler", () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const resumeScheduling = vi
|
||||
.fn()
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error("timer unavailable");
|
||||
})
|
||||
.mockImplementationOnce(() => {});
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-expiry-recovery",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
createSuspensionId: () => "suspension-expiry-recovery",
|
||||
});
|
||||
|
||||
vi.advanceTimersByTime(GATEWAY_SUSPEND_TTL_MS);
|
||||
expect(getGatewaySuspendStatus("suspension-expiry-recovery")).toMatchObject({
|
||||
status: "recovering",
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(resumeScheduling).toHaveBeenCalledTimes(2);
|
||||
expect(getGatewaySuspendStatus("suspension-expiry-recovery")).toEqual({
|
||||
status: "running",
|
||||
});
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("expires synchronously when timer delivery is delayed", () => {
|
||||
let nowMs = 10_000;
|
||||
const resumeScheduling = vi.fn();
|
||||
prepareGatewaySuspend({
|
||||
requestId: "request-delayed-expiry",
|
||||
pauseScheduling: vi.fn(),
|
||||
resumeScheduling,
|
||||
inspect: inspectors(),
|
||||
nowMs: () => nowMs,
|
||||
createSuspensionId: () => "suspension-delayed-expiry",
|
||||
});
|
||||
|
||||
nowMs += GATEWAY_SUSPEND_TTL_MS;
|
||||
|
||||
expect(getGatewaySuspendStatus("suspension-delayed-expiry")).toEqual({ status: "running" });
|
||||
expect(resumeScheduling).toHaveBeenCalledOnce();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,415 @@
|
||||
// Coordinates an atomic, refuse-only host suspension preparation lease.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type {
|
||||
GatewaySuspendPrepareResult as GatewaySuspendPrepareWireResult,
|
||||
GatewaySuspendResumeResult as GatewaySuspendResumeWireResult,
|
||||
GatewaySuspendStatusResult as GatewaySuspendStatusWireResult,
|
||||
} from "../../packages/gateway-protocol/src/index.js";
|
||||
import { tryBeginGatewaySuspendAdmission } from "../process/gateway-work-admission.js";
|
||||
import { resolveGlobalSingleton } from "../shared/global-singleton.js";
|
||||
import {
|
||||
createGatewayActiveWorkSnapshot,
|
||||
type GatewayActiveWorkInspectors,
|
||||
type GatewayActiveWorkSnapshot,
|
||||
} from "./gateway-active-work.js";
|
||||
|
||||
export const GATEWAY_SUSPEND_TTL_MS = 2 * 60_000;
|
||||
export const GATEWAY_SUSPEND_RETRY_AFTER_MS = 20_000;
|
||||
const GATEWAY_SCHEDULER_RECOVERY_RETRY_MS = 1_000;
|
||||
|
||||
type GatewaySchedulerRecoveryResult = {
|
||||
status: "recovering";
|
||||
reason: "scheduler-resume-failed";
|
||||
retryAfterMs: number;
|
||||
};
|
||||
|
||||
export type GatewaySuspendPrepareResult =
|
||||
| GatewaySuspendPrepareWireResult
|
||||
| { status: "conflict"; expiresAtMs: number }
|
||||
| GatewaySchedulerRecoveryResult;
|
||||
|
||||
export type GatewaySuspendStatusResult =
|
||||
| GatewaySuspendStatusWireResult
|
||||
| { status: "conflict"; expiresAtMs: number }
|
||||
| GatewaySchedulerRecoveryResult;
|
||||
|
||||
export type GatewaySuspendResumeResult =
|
||||
| GatewaySuspendResumeWireResult
|
||||
| { ok: false; reason: "suspension-mismatch" }
|
||||
| { ok: false; reason: "scheduler-resume-failed"; retryAfterMs: number };
|
||||
|
||||
type GatewaySuspendCoordinatorEntryBase = {
|
||||
owner: object;
|
||||
resumeScheduling: () => void;
|
||||
reopenAdmission: () => boolean;
|
||||
warn?: (message: string) => void;
|
||||
timer?: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
type HeldGatewaySuspension = GatewaySuspendCoordinatorEntryBase & {
|
||||
kind: "held";
|
||||
requestId: string;
|
||||
suspensionId: string;
|
||||
expiresAtMs: number;
|
||||
snapshot: GatewayActiveWorkSnapshot;
|
||||
nowMs: () => number;
|
||||
};
|
||||
|
||||
type GatewaySchedulerRecovery = GatewaySuspendCoordinatorEntryBase & {
|
||||
kind: "recovering";
|
||||
};
|
||||
|
||||
type GatewaySuspendCoordinatorEntry = HeldGatewaySuspension | GatewaySchedulerRecovery;
|
||||
|
||||
type GatewaySuspendCoordinatorState = {
|
||||
current: GatewaySuspendCoordinatorEntry | null;
|
||||
};
|
||||
|
||||
const COORDINATOR_STATE = resolveGlobalSingleton(
|
||||
Symbol.for("openclaw.gatewaySuspendCoordinatorState"),
|
||||
(): GatewaySuspendCoordinatorState => ({
|
||||
current: null,
|
||||
}),
|
||||
);
|
||||
|
||||
function schedulerRecoveryResult(): GatewaySchedulerRecoveryResult {
|
||||
return {
|
||||
status: "recovering",
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
|
||||
};
|
||||
}
|
||||
|
||||
function clearEntryTimer(entry: GatewaySuspendCoordinatorEntry): void {
|
||||
if (entry.timer) {
|
||||
clearTimeout(entry.timer);
|
||||
entry.timer = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleEntry(
|
||||
entry: GatewaySuspendCoordinatorEntry,
|
||||
delayMs: number,
|
||||
callback: () => void,
|
||||
): void {
|
||||
clearEntryTimer(entry);
|
||||
entry.timer = setTimeout(callback, delayMs);
|
||||
entry.timer.unref?.();
|
||||
}
|
||||
|
||||
function resumeAndReopen(entry: GatewaySuspendCoordinatorEntry): boolean {
|
||||
try {
|
||||
entry.resumeScheduling();
|
||||
} catch (err) {
|
||||
entry.warn?.(`gateway scheduler recovery failed: ${String(err)}`);
|
||||
enterSchedulerRecovery(entry);
|
||||
return false;
|
||||
}
|
||||
if (COORDINATOR_STATE.current !== entry) {
|
||||
return true;
|
||||
}
|
||||
if (!entry.reopenAdmission()) {
|
||||
entry.warn?.("gateway scheduler recovery could not reopen admission");
|
||||
enterSchedulerRecovery(entry);
|
||||
return false;
|
||||
}
|
||||
clearEntryTimer(entry);
|
||||
COORDINATOR_STATE.current = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
function enterSchedulerRecovery(entry: GatewaySuspendCoordinatorEntry): void {
|
||||
if (COORDINATOR_STATE.current !== entry) {
|
||||
return;
|
||||
}
|
||||
if (entry.kind === "recovering") {
|
||||
scheduleRecoveryRetry(entry);
|
||||
return;
|
||||
}
|
||||
clearEntryTimer(entry);
|
||||
const recovery: GatewaySchedulerRecovery = {
|
||||
kind: "recovering",
|
||||
owner: entry.owner,
|
||||
resumeScheduling: entry.resumeScheduling,
|
||||
reopenAdmission: entry.reopenAdmission,
|
||||
warn: entry.warn,
|
||||
};
|
||||
COORDINATOR_STATE.current = recovery;
|
||||
scheduleRecoveryRetry(recovery);
|
||||
}
|
||||
|
||||
function scheduleRecoveryRetry(entry: GatewaySuspendCoordinatorEntry): void {
|
||||
scheduleEntry(entry, GATEWAY_SCHEDULER_RECOVERY_RETRY_MS, () => {
|
||||
if (COORDINATOR_STATE.current === entry) {
|
||||
resumeAndReopen(entry);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeExpiredHeldSuspension(
|
||||
held: HeldGatewaySuspension,
|
||||
): GatewaySuspendCoordinatorEntry | null {
|
||||
if (held.nowMs() < held.expiresAtMs) {
|
||||
return held;
|
||||
}
|
||||
resumeAndReopen(held);
|
||||
return COORDINATOR_STATE.current;
|
||||
}
|
||||
|
||||
function armSchedulerRecovery(
|
||||
recovery: Omit<GatewaySchedulerRecovery, "kind">,
|
||||
): GatewaySchedulerRecovery {
|
||||
const entry: GatewaySchedulerRecovery = { kind: "recovering", ...recovery };
|
||||
scheduleRecoveryRetry(entry);
|
||||
return entry;
|
||||
}
|
||||
|
||||
// Rollback stays fail-closed: scheduler recovery must finish before admission
|
||||
// reopens, otherwise an old retry can resume scheduling under a newer lease.
|
||||
function resumeSchedulingBeforeReopen(params: {
|
||||
owner: object;
|
||||
resumeScheduling: () => void;
|
||||
reopenAdmission: () => boolean;
|
||||
isInvalidated: () => boolean;
|
||||
warn?: (message: string) => void;
|
||||
}): boolean {
|
||||
if (params.isInvalidated()) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
params.resumeScheduling();
|
||||
} catch (err) {
|
||||
params.warn?.(`gateway scheduler resume failed during suspension rollback: ${String(err)}`);
|
||||
COORDINATOR_STATE.current = armSchedulerRecovery({
|
||||
owner: params.owner,
|
||||
resumeScheduling: params.resumeScheduling,
|
||||
reopenAdmission: params.reopenAdmission,
|
||||
warn: params.warn,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
if (!params.isInvalidated()) {
|
||||
params.reopenAdmission();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function armExpiry(held: Omit<HeldGatewaySuspension, "kind">): HeldGatewaySuspension {
|
||||
const entry: HeldGatewaySuspension = { kind: "held", ...held };
|
||||
scheduleEntry(entry, GATEWAY_SUSPEND_TTL_MS, () => {
|
||||
if (COORDINATOR_STATE.current === entry) {
|
||||
resumeAndReopen(entry);
|
||||
}
|
||||
});
|
||||
return entry;
|
||||
}
|
||||
|
||||
function renewHeldSuspension(held: HeldGatewaySuspension, nowMs: number): void {
|
||||
held.expiresAtMs = nowMs + GATEWAY_SUSPEND_TTL_MS;
|
||||
scheduleEntry(held, GATEWAY_SUSPEND_TTL_MS, () => {
|
||||
if (COORDINATOR_STATE.current === held) {
|
||||
resumeAndReopen(held);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Acquire, inspect, and either roll back immediately or hold an idle fence. */
|
||||
export function prepareGatewaySuspend(params: {
|
||||
requestId: string;
|
||||
pauseScheduling: () => void;
|
||||
resumeScheduling: () => void;
|
||||
inspect?: Partial<GatewayActiveWorkInspectors>;
|
||||
nowMs?: () => number;
|
||||
createSuspensionId?: () => string;
|
||||
warn?: (message: string) => void;
|
||||
}): GatewaySuspendPrepareResult {
|
||||
const nowMs = (params.nowMs ?? Date.now)();
|
||||
const current = COORDINATOR_STATE.current;
|
||||
if (current?.kind === "recovering") {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
const existing = current ? normalizeExpiredHeldSuspension(current) : null;
|
||||
if (existing?.kind === "recovering") {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
if (existing) {
|
||||
if (existing.requestId !== params.requestId) {
|
||||
return { status: "conflict", expiresAtMs: existing.expiresAtMs };
|
||||
}
|
||||
existing.nowMs = params.nowMs ?? Date.now;
|
||||
renewHeldSuspension(existing, nowMs);
|
||||
return {
|
||||
status: "ready",
|
||||
suspensionId: existing.suspensionId,
|
||||
expiresAtMs: existing.expiresAtMs,
|
||||
activeCount: existing.snapshot.counts.totalActive,
|
||||
blockers: existing.snapshot.blockers,
|
||||
};
|
||||
}
|
||||
|
||||
const owner = {};
|
||||
let suspensionInvalidated = false;
|
||||
const admission = tryBeginGatewaySuspendAdmission(() => {
|
||||
suspensionInvalidated = true;
|
||||
const activeEntry = COORDINATOR_STATE.current;
|
||||
if (activeEntry?.owner !== owner) {
|
||||
return;
|
||||
}
|
||||
clearEntryTimer(activeEntry);
|
||||
COORDINATOR_STATE.current = null;
|
||||
});
|
||||
if (!admission) {
|
||||
const snapshot = createGatewayActiveWorkSnapshot(params.inspect);
|
||||
return {
|
||||
status: "busy",
|
||||
reason: "gateway-draining",
|
||||
retryAfterMs: GATEWAY_SUSPEND_RETRY_AFTER_MS,
|
||||
activeCount: snapshot.counts.totalActive,
|
||||
blockers: snapshot.blockers,
|
||||
};
|
||||
}
|
||||
|
||||
let schedulingPaused = false;
|
||||
let admissionCommitted = false;
|
||||
try {
|
||||
params.pauseScheduling();
|
||||
schedulingPaused = true;
|
||||
const snapshot = createGatewayActiveWorkSnapshot(params.inspect);
|
||||
if (!snapshot.idle) {
|
||||
const resumed = resumeSchedulingBeforeReopen({
|
||||
owner,
|
||||
resumeScheduling: params.resumeScheduling,
|
||||
reopenAdmission: admission.rollback,
|
||||
isInvalidated: () => suspensionInvalidated,
|
||||
warn: params.warn,
|
||||
});
|
||||
schedulingPaused = false;
|
||||
if (!resumed) {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
return {
|
||||
status: "busy",
|
||||
reason: "active-work",
|
||||
retryAfterMs: GATEWAY_SUSPEND_RETRY_AFTER_MS,
|
||||
activeCount: snapshot.counts.totalActive,
|
||||
blockers: snapshot.blockers,
|
||||
};
|
||||
}
|
||||
if (!admission.commit()) {
|
||||
throw new Error("gateway suspension admission changed during preparation");
|
||||
}
|
||||
admissionCommitted = true;
|
||||
const suspensionId = (params.createSuspensionId ?? randomUUID)();
|
||||
const expiresAtMs = nowMs + GATEWAY_SUSPEND_TTL_MS;
|
||||
const held = armExpiry({
|
||||
owner,
|
||||
requestId: params.requestId,
|
||||
suspensionId,
|
||||
expiresAtMs,
|
||||
snapshot,
|
||||
reopenAdmission: admission.release,
|
||||
resumeScheduling: params.resumeScheduling,
|
||||
nowMs: params.nowMs ?? Date.now,
|
||||
warn: params.warn,
|
||||
});
|
||||
COORDINATOR_STATE.current = held;
|
||||
return {
|
||||
status: "ready",
|
||||
suspensionId,
|
||||
expiresAtMs,
|
||||
activeCount: snapshot.counts.totalActive,
|
||||
blockers: snapshot.blockers,
|
||||
};
|
||||
} catch (err) {
|
||||
if (schedulingPaused) {
|
||||
const resumed = resumeSchedulingBeforeReopen({
|
||||
owner,
|
||||
resumeScheduling: params.resumeScheduling,
|
||||
reopenAdmission: admissionCommitted ? admission.release : admission.rollback,
|
||||
isInvalidated: () => suspensionInvalidated,
|
||||
warn: params.warn,
|
||||
});
|
||||
if (!resumed) {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
} else if (admissionCommitted) {
|
||||
admission.release();
|
||||
} else {
|
||||
admission.rollback();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
export function getGatewaySuspendStatus(suspensionId: string): GatewaySuspendStatusResult {
|
||||
const current = COORDINATOR_STATE.current;
|
||||
if (current?.kind === "recovering") {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
const held = current ? normalizeExpiredHeldSuspension(current) : null;
|
||||
if (held?.kind === "recovering") {
|
||||
return schedulerRecoveryResult();
|
||||
}
|
||||
if (!held) {
|
||||
return { status: "running" };
|
||||
}
|
||||
if (held.suspensionId !== suspensionId) {
|
||||
return { status: "conflict", expiresAtMs: held.expiresAtMs };
|
||||
}
|
||||
return { status: "ready", expiresAtMs: held.expiresAtMs };
|
||||
}
|
||||
|
||||
export function resumeGatewaySuspend(suspensionId: string): GatewaySuspendResumeResult {
|
||||
const current = COORDINATOR_STATE.current;
|
||||
if (current?.kind === "recovering") {
|
||||
return {
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
|
||||
};
|
||||
}
|
||||
const held = current ? normalizeExpiredHeldSuspension(current) : null;
|
||||
if (held?.kind === "recovering") {
|
||||
return {
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
|
||||
};
|
||||
}
|
||||
if (!held) {
|
||||
return {
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: false,
|
||||
};
|
||||
}
|
||||
if (held.suspensionId !== suspensionId) {
|
||||
return { ok: false, reason: "suspension-mismatch" };
|
||||
}
|
||||
if (!resumeAndReopen(held)) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: "scheduler-resume-failed",
|
||||
retryAfterMs: GATEWAY_SCHEDULER_RECOVERY_RETRY_MS,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
status: "running",
|
||||
resumed: true,
|
||||
};
|
||||
}
|
||||
|
||||
export function resetGatewaySuspendCoordinatorForTest(): void {
|
||||
const current = COORDINATOR_STATE.current;
|
||||
if (current) {
|
||||
clearEntryTimer(current);
|
||||
try {
|
||||
current.resumeScheduling();
|
||||
} catch (err) {
|
||||
current.warn?.(`gateway scheduler resume failed during test reset: ${String(err)}`);
|
||||
}
|
||||
current.reopenAdmission();
|
||||
COORDINATOR_STATE.current = null;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
// Exercises heartbeat wake coalescing, retries, and skip handling.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getActiveGatewayRootWorkCount,
|
||||
resetGatewayWorkAdmission,
|
||||
tryBeginGatewaySuspendAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import {
|
||||
HEARTBEAT_SKIP_CRON_IN_PROGRESS,
|
||||
HEARTBEAT_SKIP_LANES_BUSY,
|
||||
@@ -71,15 +76,69 @@ describe("heartbeat-wake", () => {
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
resetGatewayWorkAdmission();
|
||||
resetHeartbeatWakeStateForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetHeartbeatWakeStateForTests();
|
||||
resetGatewayWorkAdmission();
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("defers a full wake while gateway suspension is prepared", async () => {
|
||||
vi.useFakeTimers();
|
||||
const activeRootCounts: number[] = [];
|
||||
const handler = vi.fn(async () => {
|
||||
activeRootCounts.push(getActiveGatewayRootWorkCount());
|
||||
return { status: "ran" as const, durationMs: 1 };
|
||||
});
|
||||
setHeartbeatWakeHandler(handler);
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension?.commit()).toBe(true);
|
||||
|
||||
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
|
||||
expect(suspension?.release()).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(activeRootCounts).toEqual([1]);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("counts an in-flight wake until the whole handler settles", async () => {
|
||||
vi.useFakeTimers();
|
||||
let finishWake: (() => void) | undefined;
|
||||
const wakeFinished = new Promise<void>((resolve) => {
|
||||
finishWake = resolve;
|
||||
});
|
||||
const handler = vi.fn(async () => {
|
||||
await wakeFinished;
|
||||
return { status: "ran" as const, durationMs: 1 };
|
||||
});
|
||||
setHeartbeatWakeHandler(handler);
|
||||
|
||||
requestHeartbeat(wake("manual", { coalesceMs: 0 }));
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
|
||||
expect(handler).toHaveBeenCalledOnce();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
const suspension = tryBeginGatewaySuspendAdmission(() => {});
|
||||
expect(suspension).not.toBeNull();
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
expect(suspension?.rollback()).toBe(true);
|
||||
|
||||
finishWake?.();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("coalesces multiple wake requests into one run", async () => {
|
||||
vi.useFakeTimers();
|
||||
const handler = vi.fn().mockResolvedValue({ status: "skipped", reason: "disabled" });
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Tracks heartbeat wake requests, busy skips, and retry timing.
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../process/gateway-work-admission.js";
|
||||
import { resolveTimerTimeoutMs } from "../shared/number-coercion.js";
|
||||
import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js";
|
||||
|
||||
@@ -235,7 +236,11 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") {
|
||||
...(pendingWake.sessionKey ? { sessionKey: pendingWake.sessionKey } : {}),
|
||||
...(pendingWake.heartbeat ? { heartbeat: pendingWake.heartbeat } : {}),
|
||||
};
|
||||
const res = await active(wakeOpts);
|
||||
// Each wake is detached process work: admit the whole handler before
|
||||
// it can mutate sessions or commitments, and keep it visible until done.
|
||||
const res = await runWithGatewayIndependentRootWorkAdmission(async () =>
|
||||
active(wakeOpts),
|
||||
);
|
||||
if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) {
|
||||
// The target runtime is busy; retry this wake target soon.
|
||||
queuePendingWakeReason({
|
||||
|
||||
@@ -6,6 +6,10 @@ import {
|
||||
clearRuntimeConfigSnapshot,
|
||||
setRuntimeConfigSnapshot,
|
||||
} from "../config/config.js";
|
||||
import {
|
||||
isGatewayWorkAdmissionClosed,
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
import { makeNetworkInterfacesSnapshot } from "../test-helpers/network-interfaces.js";
|
||||
import {
|
||||
testing,
|
||||
@@ -136,6 +140,26 @@ describe("infra runtime", () => {
|
||||
await vi.runAllTimersAsync();
|
||||
});
|
||||
|
||||
it("holds root admission from scheduled emission until the signal is handled", async () => {
|
||||
const handler = () => {};
|
||||
process.on("SIGUSR1", handler);
|
||||
try {
|
||||
scheduleGatewaySigusr1Restart({ delayMs: 0 });
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
|
||||
|
||||
markGatewaySigusr1RestartHandled();
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
const root = tryBeginGatewayRootWorkAdmission();
|
||||
expect(root).not.toBeNull();
|
||||
root?.release();
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", handler);
|
||||
}
|
||||
});
|
||||
|
||||
it("backs off before an emoji that crosses the restart reason limit", () => {
|
||||
const restart = scheduleGatewaySigusr1Restart({
|
||||
delayMs: 0,
|
||||
@@ -632,6 +656,7 @@ describe("infra runtime", () => {
|
||||
|
||||
expect(beforeEmit).toHaveBeenCalledTimes(1);
|
||||
expect(afterEmitRejected).toHaveBeenCalledTimes(1);
|
||||
expect(isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
});
|
||||
|
||||
it("still emits restart when preparation fails", async () => {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user