mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-26 12:26:38 -06:00
18b79d99ab
* fix(gateway): bound busy channel health by real run age The channel health policy treats a channel as healthy-busy even while disconnected, bounded only by a 25 minute stale ceiling measured from lastRunActivityAt. The run-state heartbeat refreshes lastRunActivityAt every 60 seconds for as long as any run is active, so a run that hangs forever (for example a send blocking on a dead socket after the transport already reported connected:false) keeps that timestamp fresh and the stuck ceiling is never reached. The account is then reported healthy forever by the health monitor, readiness probe, and health CLI, and no restart ever fires. createRunStateMachine now tracks each in-flight run's start time keyed by an opaque run handle and publishes the oldest still-active run's start as activeRunStartedAt. The health policy busy override keys its ceiling off the real run age, so a run stuck longer than the threshold reports stuck and the monitor can restart it. Because the reported start is the oldest active run and advances to the next-oldest as runs complete, a channel churning through many short overlapping runs (activeRuns above 1 across concurrent queue keys) stays healthy; only a genuinely hung run breaches the ceiling. Short and active runs stay healthy and the existing lastRunActivityAt fallback is preserved for snapshots without a start time. * fix(channels): retain run-state callback compatibility Keep the released zero-argument onRunEnd callback source-compatible while allowing internal queue callers to pass a run handle for exact concurrent-run accounting. The compatibility path closes the oldest active run, preserving existing lifecycle behavior for consumers that do not use handles. * fix(channels): keep anonymous runs out of age tracking The zero-argument lifecycle callbacks cannot identify which concurrent run completed, so they must not update the identity-sensitive run start used by channel health. Keep their busy count separately and reserve exact start tracking for the shared queue's handle-aware lifecycle path. * fix(channels): keep tracked runs internal Keep the public run-state lifecycle callbacks unchanged. The channel queue now owns opaque run identity and augments its status updates with the oldest active queue run, so implementation details do not expand the SDK surface. * fix(channels): type queue run start status Keep activeRunStartedAt in the internal status patch type so the queue can publish its private tracked-run age through the existing status sink. * fix(channels): wrap isActive to satisfy unbound-method lint * fix(gateway): gate busy run-age ceiling on disconnected transport
206 lines
6.2 KiB
TypeScript
206 lines
6.2 KiB
TypeScript
// Channel lifecycle core contracts define account lifecycle snapshots and sync hooks.
|
|
import type { ChannelAccountSnapshot } from "../channels/plugins/types.core.js";
|
|
import { createRunStateMachine, type RunStateStatusSink } from "../channels/run-state-machine.js";
|
|
import { KeyedAsyncQueue } from "./keyed-async-queue.js";
|
|
|
|
type CloseAwareServer = {
|
|
once: (event: "close", listener: () => void) => unknown;
|
|
};
|
|
|
|
type PassiveAccountLifecycleParams<Handle> = {
|
|
abortSignal?: AbortSignal;
|
|
start: () => Promise<Handle>;
|
|
stop?: (handle: Handle) => void | Promise<void>;
|
|
onStop?: () => void | Promise<void>;
|
|
};
|
|
|
|
/** Runtime context passed to queued channel work. */
|
|
export type ChannelRunQueueTaskContext = {
|
|
/** Signal tied to the channel/account lifecycle that owns the queued work. */
|
|
lifecycleSignal?: AbortSignal;
|
|
};
|
|
|
|
/** Per-key async queue used by channel plugins to serialize account or thread work. */
|
|
export type ChannelRunQueue = {
|
|
/** Enqueue work under a serialization key such as account id, thread id, or chat id. */
|
|
enqueue: (key: string, task: (context: ChannelRunQueueTaskContext) => Promise<void>) => void;
|
|
/** Stop accepting meaningful work and mark the lifecycle as inactive. */
|
|
deactivate: () => void;
|
|
};
|
|
|
|
/** Hooks used to wire channel queue state into runtime status and error reporting. */
|
|
export type ChannelRunQueueParams = {
|
|
/** Receives busy/idle lifecycle snapshots from the shared run-state machine. */
|
|
setStatus?: RunStateStatusSink;
|
|
/** Lifecycle signal propagated to queued tasks. */
|
|
abortSignal?: AbortSignal;
|
|
/** Best-effort sink for task failures after enqueueing. */
|
|
onError?: (error: unknown) => void;
|
|
};
|
|
|
|
/** Bind a fixed account id into a status writer so lifecycle code can emit partial snapshots. */
|
|
export function createAccountStatusSink(params: {
|
|
accountId: string;
|
|
setStatus: (next: ChannelAccountSnapshot) => void;
|
|
}): (patch: Omit<ChannelAccountSnapshot, "accountId">) => void {
|
|
return (patch) => {
|
|
params.setStatus({ accountId: params.accountId, ...patch });
|
|
};
|
|
}
|
|
|
|
function createTrackedRunState(params: ChannelRunQueueParams) {
|
|
const runStarts = new Map<symbol, number>();
|
|
const oldestRunStart = () => Math.min(...runStarts.values());
|
|
const runState = createRunStateMachine({
|
|
setStatus: (patch) => {
|
|
params.setStatus?.({
|
|
...patch,
|
|
activeRunStartedAt: runStarts.size > 0 ? oldestRunStart() : null,
|
|
});
|
|
},
|
|
abortSignal: params.abortSignal,
|
|
});
|
|
|
|
return {
|
|
isActive: () => runState.isActive(),
|
|
deactivate: runState.deactivate,
|
|
onRunStart() {
|
|
const handle = Symbol();
|
|
runStarts.set(handle, Date.now());
|
|
runState.onRunStart();
|
|
return handle;
|
|
},
|
|
onRunEnd(handle: symbol) {
|
|
runStarts.delete(handle);
|
|
runState.onRunEnd();
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Serialize channel work per key while keeping lifecycle/busy accounting out of
|
|
* channel-specific message handlers. The queue does not impose run timeouts;
|
|
* callers should rely on session/tool/runtime lifecycle for long-running work.
|
|
*/
|
|
export function createChannelRunQueue(params: ChannelRunQueueParams): ChannelRunQueue {
|
|
const queue = new KeyedAsyncQueue();
|
|
const runState = createTrackedRunState(params);
|
|
const reportError = (error: unknown) => {
|
|
try {
|
|
params.onError?.(error);
|
|
} catch {
|
|
// Keep queue error handling best-effort; callers should not create a
|
|
// secondary unhandled rejection from their reporting hook.
|
|
}
|
|
};
|
|
|
|
return {
|
|
enqueue(key, task) {
|
|
void queue
|
|
.enqueue(key, async () => {
|
|
if (!runState.isActive()) {
|
|
return;
|
|
}
|
|
const runHandle = runState.onRunStart();
|
|
try {
|
|
// Deactivation can happen while this key waited behind older work.
|
|
if (!runState.isActive()) {
|
|
return;
|
|
}
|
|
await task({ lifecycleSignal: params.abortSignal });
|
|
} finally {
|
|
runState.onRunEnd(runHandle);
|
|
}
|
|
})
|
|
.catch(reportError);
|
|
},
|
|
deactivate: runState.deactivate,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Return a promise that resolves when the signal is aborted.
|
|
*
|
|
* If no signal is provided, the promise stays pending forever. When provided,
|
|
* `onAbort` runs once before the promise resolves.
|
|
*/
|
|
export function waitUntilAbort(
|
|
signal?: AbortSignal,
|
|
onAbort?: () => void | Promise<void>,
|
|
): Promise<void> {
|
|
return new Promise<void>((resolve, reject) => {
|
|
const complete = () => {
|
|
Promise.resolve(onAbort?.()).then(() => resolve(), reject);
|
|
};
|
|
if (!signal) {
|
|
return;
|
|
}
|
|
if (signal.aborted) {
|
|
complete();
|
|
return;
|
|
}
|
|
signal.addEventListener("abort", complete, { once: true });
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Keep a passive account task alive until abort, then run optional cleanup.
|
|
*/
|
|
export async function runPassiveAccountLifecycle<Handle>(
|
|
params: PassiveAccountLifecycleParams<Handle>,
|
|
): Promise<void> {
|
|
const handle = await params.start();
|
|
|
|
try {
|
|
await waitUntilAbort(params.abortSignal);
|
|
} finally {
|
|
await params.stop?.(handle);
|
|
await params.onStop?.();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Keep a channel/provider task pending until the HTTP server closes.
|
|
*
|
|
* When an abort signal is provided, `onAbort` is invoked once and should
|
|
* trigger server shutdown. The returned promise resolves only after `close`.
|
|
*/
|
|
export async function keepHttpServerTaskAlive(params: {
|
|
server: CloseAwareServer;
|
|
abortSignal?: AbortSignal;
|
|
onAbort?: () => void | Promise<void>;
|
|
}): Promise<void> {
|
|
const { server, abortSignal, onAbort } = params;
|
|
let abortTask: Promise<void> = Promise.resolve();
|
|
let abortTriggered = false;
|
|
|
|
const triggerAbort = () => {
|
|
if (abortTriggered) {
|
|
return;
|
|
}
|
|
abortTriggered = true;
|
|
abortTask = Promise.resolve(onAbort?.()).then(() => undefined);
|
|
};
|
|
|
|
const onAbortSignal = () => {
|
|
triggerAbort();
|
|
};
|
|
|
|
if (abortSignal) {
|
|
if (abortSignal.aborted) {
|
|
triggerAbort();
|
|
} else {
|
|
abortSignal.addEventListener("abort", onAbortSignal, { once: true });
|
|
}
|
|
}
|
|
|
|
await new Promise<void>((resolve) => {
|
|
server.once("close", () => resolve());
|
|
});
|
|
|
|
if (abortSignal) {
|
|
abortSignal.removeEventListener("abort", onAbortSignal);
|
|
}
|
|
await abortTask;
|
|
}
|