mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
improve(mattermost): restart only the changed account on config reload (#99312)
* improve(gateway): restart only the changed account on channel config reload
When a config change is scoped entirely to one channel account
(channels.{kind}.accounts.{accountId}[.*]), restart just that account
instead of the whole channel. Wholesale channel restarts disconnect every
account on the channel; on gateways running many accounts, each
account-scoped config write (adding an account, changing one account's
settings) briefly dropped every other account's connection.
The reload plan gains a restartChannelAccounts bucket populated only when
all of a channel's changed paths are account-scoped; any channel-global
path falls back to the existing wholesale restart, and a channel scheduled
for wholesale restart drops its per-account entries so each (channel,
account) pair restarts at most once. The executor runs per-account
restarts through the existing stopChannel/startChannel accountId parameter.
* fix(gateway): preserve account reload admission
* fix(gateway): scope surgical reloads to isolated plugins
* test(gateway): avoid preactivating reload snapshot
* docs(channels): define account reload isolation contract
* fix(gateway): preflight scoped account reloads
* fix(gateway): re-drain live scoped reload targets
* docs: refresh generated docs map
* fix(mattermost): align scoped reload with durable ingress
---------
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -7385,6 +7385,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
|
||||
- H2: Message adapter
|
||||
- H3: Inbound ingress (experimental)
|
||||
- H3: Durable ingress and replay dedupe
|
||||
- H4: Account-scoped restart contract
|
||||
- H3: Typing indicators
|
||||
- H3: Media source params
|
||||
- H3: Native payload shaping
|
||||
|
||||
@@ -116,6 +116,32 @@ guard when adopting the drain and size `completedTtlMs`/`completedMaxEntries`
|
||||
to cover the old guard window instead. Non-dedupe protections (age fences,
|
||||
outbound echo caches) are unrelated to this rule and stay.
|
||||
|
||||
#### Account-scoped restart contract
|
||||
|
||||
Channel config changes restart the whole channel by default. A multi-account
|
||||
channel may set `reload.accountScopedRestart: true` only when configuration
|
||||
resolution reads channel-wide shared fields plus the selected account, never a
|
||||
sibling account, and the Gateway can stop and start one `(channel, accountId)`
|
||||
runtime without replacing sibling runtimes.
|
||||
|
||||
The scoped path applies only to changes under
|
||||
`channels.<channel>.accounts.<non-default-id>.*`. Changes to shared channel
|
||||
fields, `accounts.default`, removed or unresolvable accounts, and mixed changes
|
||||
that can affect inheritance are promoted to a whole-channel restart. Plugins
|
||||
that do not opt in always use the whole-channel path.
|
||||
|
||||
For channels using the durable ingress drain, the account monitor's stop path
|
||||
must first settle all accepted transport admissions, then dispose and await its
|
||||
drain. Starting the account opens the same account-keyed queue, whose initial
|
||||
drain recovers undispatched durable rows. Do not add a second reload-specific
|
||||
replay pass; queue recovery is the canonical restart path.
|
||||
|
||||
Treat this flag as a capability claim, not a performance preference. Contract
|
||||
tests should prove that adding and editing one named account leaves a sibling's
|
||||
resolved config unchanged, stopping one account settles only that account's
|
||||
monitor and drain, and a fresh monitor recovers that account's rows exactly
|
||||
once. If any guarantee cannot be proved, omit the flag.
|
||||
|
||||
### Typing indicators
|
||||
|
||||
If your channel supports typing indicators outside inbound replies, expose
|
||||
|
||||
@@ -24,7 +24,14 @@ export const mattermostSetupPlugin: ChannelPlugin<ResolvedMattermostAccount> = {
|
||||
media: true,
|
||||
nativeCommands: true,
|
||||
},
|
||||
reload: { configPrefixes: ["channels.mattermost"] },
|
||||
reload: {
|
||||
configPrefixes: ["channels.mattermost"],
|
||||
/**
|
||||
* accounts.default is promoted; named resolution merges only channel-wide fields
|
||||
* plus the selected account. Runtime monitor, debounce, and ingress use accountId.
|
||||
*/
|
||||
accountScopedRestart: true,
|
||||
},
|
||||
configSchema: MattermostChannelConfigSchema,
|
||||
config: {
|
||||
...mattermostConfigAdapter,
|
||||
|
||||
@@ -165,6 +165,64 @@ describe("mattermostPlugin", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("opts into account-scoped config restarts", () => {
|
||||
expect(mattermostPlugin.reload).toMatchObject({ accountScopedRestart: true });
|
||||
});
|
||||
|
||||
it("keeps sibling resolution stable across named-account additions and edits", () => {
|
||||
const before: OpenClawConfig = {
|
||||
channels: {
|
||||
mattermost: {
|
||||
replyToMode: "first",
|
||||
accounts: {
|
||||
beta: {
|
||||
baseUrl: "https://beta.example.com",
|
||||
chatmode: "onmessage",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const afterAdd: OpenClawConfig = {
|
||||
channels: {
|
||||
mattermost: {
|
||||
replyToMode: "first",
|
||||
accounts: {
|
||||
alpha: {
|
||||
baseUrl: "https://alpha.example.com",
|
||||
chatmode: "oncall",
|
||||
},
|
||||
beta: {
|
||||
baseUrl: "https://beta.example.com",
|
||||
chatmode: "onmessage",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const afterEdit: OpenClawConfig = {
|
||||
channels: {
|
||||
mattermost: {
|
||||
replyToMode: "first",
|
||||
accounts: {
|
||||
alpha: {
|
||||
baseUrl: "https://alpha-new.example.com",
|
||||
chatmode: "onchar",
|
||||
},
|
||||
beta: {
|
||||
baseUrl: "https://beta.example.com",
|
||||
chatmode: "onmessage",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const expectedBeta = mattermostPlugin.config.resolveAccount(before, "beta");
|
||||
expect(mattermostPlugin.config.resolveAccount(afterAdd, "beta")).toEqual(expectedBeta);
|
||||
expect(mattermostPlugin.config.resolveAccount(afterEdit, "beta")).toEqual(expectedBeta);
|
||||
});
|
||||
|
||||
describe("messaging", () => {
|
||||
it("keeps @username targets", () => {
|
||||
const normalize = requireMattermostNormalizeTarget();
|
||||
|
||||
@@ -798,7 +798,14 @@ export const mattermostPlugin: ChannelPlugin<ResolvedMattermostAccount> = create
|
||||
streaming: {
|
||||
blockStreamingCoalesceDefaults: { minChars: 1500, idleMs: 1000 },
|
||||
},
|
||||
reload: { configPrefixes: ["channels.mattermost"] },
|
||||
reload: {
|
||||
configPrefixes: ["channels.mattermost"],
|
||||
/**
|
||||
* accounts.default is promoted; named resolution merges only channel-wide fields
|
||||
* plus the selected account. Monitor debounce and durable ingress use accountId.
|
||||
*/
|
||||
accountScopedRestart: true,
|
||||
},
|
||||
configSchema: MattermostChannelConfigSchema,
|
||||
config: {
|
||||
...mattermostConfigAdapter,
|
||||
|
||||
@@ -38,9 +38,13 @@ function postedEvent(params?: {
|
||||
});
|
||||
}
|
||||
|
||||
function startMonitor(queue: MattermostIngressQueue, dispatch: MattermostIngressDispatch) {
|
||||
function startMonitor(
|
||||
queue: MattermostIngressQueue,
|
||||
dispatch: MattermostIngressDispatch,
|
||||
accountId = "default",
|
||||
) {
|
||||
return createMattermostIngressMonitor({
|
||||
accountId: "default",
|
||||
accountId,
|
||||
queue,
|
||||
dispatch,
|
||||
runtime: { error: vi.fn(), log: vi.fn() },
|
||||
@@ -49,22 +53,37 @@ function startMonitor(queue: MattermostIngressQueue, dispatch: MattermostIngress
|
||||
});
|
||||
}
|
||||
|
||||
async function withQueue<T>(fn: (queue: MattermostIngressQueue) => Promise<T>): Promise<T> {
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mattermost-ingress-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
const queue = createChannelIngressQueueForTests<MattermostIngressPayload>({
|
||||
function createQueue(stateDir: string, accountId: string): MattermostIngressQueue {
|
||||
return createChannelIngressQueueForTests<MattermostIngressPayload>({
|
||||
channelId: "mattermost",
|
||||
accountId: "default",
|
||||
accountId,
|
||||
stateDir,
|
||||
});
|
||||
}
|
||||
|
||||
async function withStateDir<T>(fn: (stateDir: string) => Promise<T>): Promise<T> {
|
||||
const created = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mattermost-ingress-"));
|
||||
const stateDir = await fs.realpath(created);
|
||||
try {
|
||||
return await fn(queue);
|
||||
return await fn(stateDir);
|
||||
} finally {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(stateDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function withQueue<T>(fn: (queue: MattermostIngressQueue) => Promise<T>): Promise<T> {
|
||||
return await withStateDir(async (stateDir) => await fn(createQueue(stateDir, "default")));
|
||||
}
|
||||
|
||||
function createDeferred(): { promise: Promise<void>; resolve: () => void } {
|
||||
let resolvePromise = () => {};
|
||||
const promise = new Promise<void>((resolve) => {
|
||||
resolvePromise = resolve;
|
||||
});
|
||||
return { promise, resolve: resolvePromise };
|
||||
}
|
||||
|
||||
function testLifecycle() {
|
||||
const calls = {
|
||||
adopted: vi.fn(async () => {}),
|
||||
@@ -131,6 +150,88 @@ describe("Mattermost durable ingress", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("settles only the restarted account and recovers its durable row exactly once", async () => {
|
||||
await withStateDir(async (stateDir) => {
|
||||
const queueA = createQueue(stateDir, "account-a");
|
||||
const queueB = createQueue(stateDir, "account-b");
|
||||
const dispatchA = vi.fn<MattermostIngressDispatch>(async (_post, _payload, lifecycle) => {
|
||||
lifecycle.onDeferred();
|
||||
return { kind: "deferred" } as const;
|
||||
});
|
||||
const dispatchBBeforeRestart = vi.fn<MattermostIngressDispatch>(async () => undefined);
|
||||
const monitorA = startMonitor(queueA, dispatchA, "account-a");
|
||||
const monitorBBeforeRestart = startMonitor(queueB, dispatchBBeforeRestart, "account-b");
|
||||
const admissionStored = createDeferred();
|
||||
const releaseAdmission = createDeferred();
|
||||
const enqueueB = queueB.enqueue.bind(queueB);
|
||||
queueB.enqueue = async (...args) => {
|
||||
const result = await enqueueB(...args);
|
||||
admissionStored.resolve();
|
||||
await releaseAdmission.promise;
|
||||
return result;
|
||||
};
|
||||
let admittingB: Promise<void> | undefined;
|
||||
let stoppingB: Promise<void> | undefined;
|
||||
|
||||
try {
|
||||
await Promise.all([monitorA.waitForIdle(), monitorBBeforeRestart.waitForIdle()]);
|
||||
await monitorA.receive(postedEvent({ postId: "post-account-a" }));
|
||||
await monitorA.waitForIdle();
|
||||
const claimsABefore = await queueA.listClaims();
|
||||
expect(claimsABefore).toHaveLength(1);
|
||||
expect(dispatchA).toHaveBeenCalledTimes(1);
|
||||
|
||||
admittingB = monitorBBeforeRestart.receive(postedEvent({ postId: "post-account-b" }));
|
||||
await admissionStored.promise;
|
||||
let stopSettled = false;
|
||||
stoppingB = monitorBBeforeRestart.stop().then(() => {
|
||||
stopSettled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
// stop() must wait for the serialized append admission. Account A's
|
||||
// drain claim and debounce handoff stay owned by its live monitor.
|
||||
expect(stopSettled).toBe(false);
|
||||
expect(await queueA.listClaims()).toEqual(claimsABefore);
|
||||
expect(dispatchA).toHaveBeenCalledTimes(1);
|
||||
|
||||
releaseAdmission.resolve();
|
||||
await Promise.all([admittingB, stoppingB]);
|
||||
expect(dispatchBBeforeRestart).not.toHaveBeenCalled();
|
||||
expect(await queueB.listPending({ limit: "all" })).toEqual([
|
||||
expect.objectContaining({ id: "post-account-b" }),
|
||||
]);
|
||||
expect(await queueA.listClaims()).toEqual(claimsABefore);
|
||||
|
||||
const dispatchBAfterRestart = vi.fn<MattermostIngressDispatch>(
|
||||
async (_post, _payload, lifecycle) => {
|
||||
await lifecycle.onAdopted();
|
||||
},
|
||||
);
|
||||
const monitorBAfterRestart = startMonitor(queueB, dispatchBAfterRestart, "account-b");
|
||||
try {
|
||||
await monitorBAfterRestart.waitForIdle();
|
||||
expect(dispatchBAfterRestart).toHaveBeenCalledTimes(1);
|
||||
expect(dispatchBAfterRestart.mock.calls[0]?.[0].id).toBe("post-account-b");
|
||||
|
||||
await monitorBAfterRestart.receive(postedEvent({ postId: "post-account-b" }));
|
||||
await monitorBAfterRestart.waitForIdle();
|
||||
expect(dispatchBAfterRestart).toHaveBeenCalledTimes(1);
|
||||
expect(await queueA.listClaims()).toEqual(claimsABefore);
|
||||
expect(dispatchA).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
await monitorBAfterRestart.stop();
|
||||
}
|
||||
} finally {
|
||||
releaseAdmission.resolve();
|
||||
await Promise.allSettled(
|
||||
[admittingB, stoppingB].filter((task): task is Promise<void> => task !== undefined),
|
||||
);
|
||||
await Promise.allSettled([monitorA.stop(), monitorBBeforeRestart.stop()]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("retains completion so a duplicate post id cannot dispatch twice", async () => {
|
||||
await withQueue(async (queue) => {
|
||||
const dispatch = vi.fn(async (_post, _payload, lifecycle) => {
|
||||
|
||||
@@ -65,7 +65,17 @@ export type ChannelPlugin<ResolvedAccount = any, Probe = unknown, Audit = unknow
|
||||
debounceMs?: number;
|
||||
};
|
||||
};
|
||||
reload?: { configPrefixes: string[]; noopPrefixes?: string[] };
|
||||
reload?: {
|
||||
configPrefixes: string[];
|
||||
noopPrefixes?: string[];
|
||||
/**
|
||||
* Opt into restarting only the changed non-default named account.
|
||||
* Set only when sibling account resolution and lifecycle state are isolated and
|
||||
* account stop fully settles owned work. Shared, default, removed, or unresolved
|
||||
* account changes still restart the whole channel.
|
||||
*/
|
||||
accountScopedRestart?: boolean;
|
||||
};
|
||||
setupWizard?: ChannelPluginSetupWizard;
|
||||
config: ChannelConfigAdapter<ResolvedAccount>;
|
||||
configSchema?: ChannelConfigSchema;
|
||||
|
||||
@@ -1051,7 +1051,7 @@ function configApplyHintForPaths(paths: string[], afterConfig: OpenClawConfig):
|
||||
if (paths.some(isPluginEntryConfigPath)) {
|
||||
return RESTART_HINT;
|
||||
}
|
||||
const plan = buildGatewayReloadPlan(paths);
|
||||
const plan = buildGatewayReloadPlan(paths, { candidateConfig: afterConfig });
|
||||
if (plan.restartGateway) {
|
||||
return RESTART_HINT;
|
||||
}
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
// Gateway config reload planner.
|
||||
// Maps changed config paths to hot-reload actions, no-ops, or full restarts.
|
||||
import { type ChannelId, listChannelPlugins } from "../channels/plugins/index.js";
|
||||
import {
|
||||
type ChannelId,
|
||||
type ChannelPlugin,
|
||||
listChannelPlugins,
|
||||
} from "../channels/plugins/index.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
getActivePluginChannelRegistryVersion,
|
||||
getActivePluginHttpRouteRegistry,
|
||||
getActivePluginHttpRouteRegistryVersion,
|
||||
} from "../plugins/runtime.js";
|
||||
import { DEFAULT_ACCOUNT_ID } from "../routing/account-id.js";
|
||||
import { isPlainObject } from "../utils.js";
|
||||
|
||||
export type ChannelKind = ChannelId;
|
||||
@@ -23,6 +29,8 @@ export type GatewayReloadPlan = {
|
||||
reloadPlugins: boolean;
|
||||
restartChannels: Set<ChannelKind>;
|
||||
disposeMcpRuntimes: boolean;
|
||||
/** Account targets; absent means no targeted restarts for hand-built plans. */
|
||||
restartChannelAccounts?: Map<ChannelKind, Set<string>>;
|
||||
noopPaths: string[];
|
||||
};
|
||||
|
||||
@@ -30,6 +38,7 @@ type ReloadRule = {
|
||||
prefix: string;
|
||||
kind: "restart" | "hot" | "none";
|
||||
actions?: ReloadAction[];
|
||||
accountScopedPlugin?: ChannelPlugin;
|
||||
};
|
||||
|
||||
type ConfigReloadMetadata = {
|
||||
@@ -44,11 +53,14 @@ type ReloadAction =
|
||||
| "restart-health-monitor"
|
||||
| "reload-plugins"
|
||||
| "dispose-mcp-runtimes"
|
||||
| `restart-channel-account:${ChannelId}`
|
||||
| `restart-channel:${ChannelId}`;
|
||||
|
||||
type GatewayReloadPlanOptions = {
|
||||
noopPaths?: Iterable<string>;
|
||||
forceChangedPaths?: Iterable<string>;
|
||||
/** Candidate config used to reject removed, unknown, or unresolvable account targets. */
|
||||
candidateConfig?: OpenClawConfig;
|
||||
};
|
||||
|
||||
const PLUGIN_INSTALL_TIMESTAMP_KEYS = ["installedAt", "resolvedAt"] as const;
|
||||
@@ -179,15 +191,22 @@ function listReloadRules(): ReloadRule[] {
|
||||
return cachedReloadRules;
|
||||
}
|
||||
// Channel docking: plugins contribute hot reload/no-op prefixes here.
|
||||
const channelReloadRules: ReloadRule[] = listChannelPlugins().flatMap((plugin) =>
|
||||
(plugin.reload?.configPrefixes ?? [])
|
||||
.map(
|
||||
(prefix): ReloadRule => ({
|
||||
const channelReloadRules: ReloadRule[] = listChannelPlugins().flatMap((plugin) => {
|
||||
const restartAction = plugin.reload?.accountScopedRestart
|
||||
? (`restart-channel-account:${plugin.id}` as ReloadAction)
|
||||
: (`restart-channel:${plugin.id}` as ReloadAction);
|
||||
return (plugin.reload?.configPrefixes ?? [])
|
||||
.map((prefix): ReloadRule => {
|
||||
const rule: ReloadRule = {
|
||||
prefix,
|
||||
kind: "hot",
|
||||
actions: [`restart-channel:${plugin.id}` as ReloadAction],
|
||||
}),
|
||||
)
|
||||
actions: [restartAction],
|
||||
};
|
||||
if (plugin.reload?.accountScopedRestart) {
|
||||
rule.accountScopedPlugin = plugin;
|
||||
}
|
||||
return rule;
|
||||
})
|
||||
.concat(
|
||||
(plugin.reload?.noopPrefixes ?? []).map(
|
||||
(prefix): ReloadRule => ({
|
||||
@@ -195,8 +214,8 @@ function listReloadRules(): ReloadRule[] {
|
||||
kind: "none",
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
);
|
||||
});
|
||||
const channelPluginStateRules: ReloadRule[] = listChannelPlugins().flatMap((plugin) => [
|
||||
{
|
||||
prefix: `plugins.entries.${plugin.id}`,
|
||||
@@ -338,12 +357,53 @@ export function listPluginInstallWholeRecordPaths(
|
||||
);
|
||||
}
|
||||
|
||||
function extractAccountIdFromPath(channel: ChannelId, path: string): string | null {
|
||||
const prefix = `channels.${channel}.accounts.`;
|
||||
if (!path.startsWith(prefix)) {
|
||||
return null;
|
||||
}
|
||||
const rest = path.slice(prefix.length);
|
||||
if (rest.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const dotIdx = rest.indexOf(".");
|
||||
const id = dotIdx === -1 ? rest : rest.slice(0, dotIdx);
|
||||
if (id.length === 0) {
|
||||
return null;
|
||||
}
|
||||
// Default config is the inheritance base, so it can change every account.
|
||||
if (id === DEFAULT_ACCOUNT_ID) {
|
||||
return null;
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
function isResolvableChannelAccount(params: {
|
||||
plugin: ChannelPlugin | undefined;
|
||||
accountId: string;
|
||||
config: OpenClawConfig;
|
||||
}): boolean {
|
||||
if (!params.plugin) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
if (!params.plugin.config.listAccountIds(params.config).includes(params.accountId)) {
|
||||
return false;
|
||||
}
|
||||
params.plugin.config.resolveAccount(params.config, params.accountId);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildGatewayReloadPlan(
|
||||
changedPaths: string[],
|
||||
options: GatewayReloadPlanOptions = {},
|
||||
): GatewayReloadPlan {
|
||||
const noopPaths = new Set(options.noopPaths);
|
||||
const forceChangedPaths = new Set(options.forceChangedPaths);
|
||||
const restartChannelAccounts = new Map<ChannelKind, Set<string>>();
|
||||
const plan: GatewayReloadPlan = {
|
||||
changedPaths,
|
||||
restartGateway: false,
|
||||
@@ -357,10 +417,41 @@ export function buildGatewayReloadPlan(
|
||||
reloadPlugins: false,
|
||||
restartChannels: new Set(),
|
||||
disposeMcpRuntimes: false,
|
||||
restartChannelAccounts,
|
||||
noopPaths: [],
|
||||
};
|
||||
|
||||
const applyAction = (action: ReloadAction) => {
|
||||
const applyAction = (
|
||||
action: ReloadAction,
|
||||
originatingPath: string,
|
||||
accountScopedPlugin?: ChannelPlugin,
|
||||
) => {
|
||||
if (action.startsWith("restart-channel-account:")) {
|
||||
const channel = action.slice("restart-channel-account:".length) as ChannelId;
|
||||
const accountId = extractAccountIdFromPath(channel, originatingPath);
|
||||
if (accountId !== null) {
|
||||
if (
|
||||
options.candidateConfig &&
|
||||
!isResolvableChannelAccount({
|
||||
plugin: accountScopedPlugin,
|
||||
accountId,
|
||||
config: options.candidateConfig,
|
||||
})
|
||||
) {
|
||||
plan.restartChannels.add(channel);
|
||||
return;
|
||||
}
|
||||
let set = restartChannelAccounts.get(channel);
|
||||
if (!set) {
|
||||
set = new Set<string>();
|
||||
restartChannelAccounts.set(channel, set);
|
||||
}
|
||||
set.add(accountId);
|
||||
return;
|
||||
}
|
||||
plan.restartChannels.add(channel);
|
||||
return;
|
||||
}
|
||||
if (action.startsWith("restart-channel:")) {
|
||||
const channel = action.slice("restart-channel:".length) as ChannelId;
|
||||
plan.restartChannels.add(channel);
|
||||
@@ -418,10 +509,15 @@ export function buildGatewayReloadPlan(
|
||||
}
|
||||
plan.hotReasons.push(path);
|
||||
for (const action of rule.actions ?? []) {
|
||||
applyAction(action);
|
||||
applyAction(action, path, rule.accountScopedPlugin);
|
||||
}
|
||||
}
|
||||
|
||||
// A wholesale restart covers its account targets and must run only once.
|
||||
for (const channel of plan.restartChannels) {
|
||||
restartChannelAccounts.delete(channel);
|
||||
}
|
||||
|
||||
if (plan.restartGmailWatcher) {
|
||||
plan.reloadHooks = true;
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ export function reloadPlanNeedsRecovery(plan: GatewayReloadPlan): boolean {
|
||||
plan.restartGmailWatcher ||
|
||||
plan.reloadPlugins ||
|
||||
plan.restartChannels.size > 0 ||
|
||||
(plan.restartChannelAccounts?.size ?? 0) > 0 ||
|
||||
shouldRefreshContextWindowCache(plan)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { PluginInstallRecord } from "../config/types.plugins.js";
|
||||
import {
|
||||
pinActivePluginChannelRegistry,
|
||||
pinActivePluginHttpRouteRegistry,
|
||||
releasePinnedPluginChannelRegistry,
|
||||
resetPluginRuntimeStateForTest,
|
||||
setActivePluginRegistry,
|
||||
} from "../plugins/runtime.js";
|
||||
@@ -28,7 +29,11 @@ import {
|
||||
} from "../skills/runtime/refresh-state.js";
|
||||
import { createTestRegistry } from "../test-utils/channel-plugins.js";
|
||||
import { diffConfigPaths, diffGatewayReloadPaths } from "./config-diff.js";
|
||||
import { buildGatewayReloadPlan, resolveConfigReloadMetadata } from "./config-reload-plan.js";
|
||||
import {
|
||||
buildGatewayReloadPlan,
|
||||
type ChannelKind,
|
||||
resolveConfigReloadMetadata,
|
||||
} from "./config-reload-plan.js";
|
||||
import { resolveGatewayReloadSettings } from "./config-reload-settings.js";
|
||||
import {
|
||||
type GatewayConfigReloadTransactionOwnership,
|
||||
@@ -179,9 +184,26 @@ describe("buildGatewayReloadPlan", () => {
|
||||
noopPrefixes: ["channels.whatsapp"],
|
||||
},
|
||||
};
|
||||
const mattermostPlugin: ChannelPlugin = {
|
||||
id: "mattermost",
|
||||
meta: {
|
||||
id: "mattermost",
|
||||
label: "Mattermost",
|
||||
selectionLabel: "Mattermost",
|
||||
docsPath: "/channels/mattermost",
|
||||
blurb: "test",
|
||||
},
|
||||
capabilities: { chatTypes: ["direct"] },
|
||||
config: {
|
||||
listAccountIds: (cfg) => Object.keys(cfg.channels?.mattermost?.accounts ?? {}),
|
||||
resolveAccount: () => ({}),
|
||||
},
|
||||
reload: { configPrefixes: ["channels.mattermost"], accountScopedRestart: true },
|
||||
};
|
||||
const registry = createTestRegistry([
|
||||
{ pluginId: "telegram", plugin: telegramPlugin, source: "test" },
|
||||
{ pluginId: "whatsapp", plugin: whatsappPlugin, source: "test" },
|
||||
{ pluginId: "mattermost", plugin: mattermostPlugin, source: "test" },
|
||||
]);
|
||||
registry.reloads = [
|
||||
{
|
||||
@@ -373,6 +395,75 @@ describe("buildGatewayReloadPlan", () => {
|
||||
expect(plan.restartChannels).toEqual(new Set(["telegram"]));
|
||||
});
|
||||
|
||||
const mattermostAccountConfig = {
|
||||
channels: {
|
||||
mattermost: {
|
||||
accounts: {
|
||||
alpha: { enabled: true },
|
||||
beta: { enabled: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "targets changed named accounts",
|
||||
paths: [
|
||||
"channels.mattermost.accounts.alpha.enabled",
|
||||
"channels.mattermost.accounts.beta.commands",
|
||||
],
|
||||
expectedChannels: new Set<ChannelKind>(),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>([
|
||||
["mattermost", new Set(["alpha", "beta"])],
|
||||
]),
|
||||
},
|
||||
{
|
||||
label: "promotes accounts.default changes",
|
||||
paths: ["channels.mattermost.accounts.default.commands"],
|
||||
expectedChannels: new Set<ChannelKind>(["mattermost"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
{
|
||||
label: "promotes channel-global changes",
|
||||
paths: ["channels.mattermost.botToken"],
|
||||
expectedChannels: new Set<ChannelKind>(["mattermost"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
{
|
||||
label: "promotes unlisted account changes",
|
||||
paths: ["channels.mattermost.accounts.removed.enabled"],
|
||||
expectedChannels: new Set<ChannelKind>(["mattermost"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
{
|
||||
label: "lets an unlisted account replace earlier scoped targets",
|
||||
paths: [
|
||||
"channels.mattermost.accounts.alpha.enabled",
|
||||
"channels.mattermost.accounts.removed.enabled",
|
||||
],
|
||||
expectedChannels: new Set<ChannelKind>(["mattermost"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
{
|
||||
label: "lets a mixed global change replace scoped targets",
|
||||
paths: ["channels.mattermost.accounts.alpha.enabled", "channels.mattermost.botToken"],
|
||||
expectedChannels: new Set<ChannelKind>(["mattermost"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
{
|
||||
label: "keeps non-opted-in channels wholesale",
|
||||
paths: ["channels.telegram.accounts.alpha.enabled"],
|
||||
expectedChannels: new Set<ChannelKind>(["telegram"]),
|
||||
expectedAccounts: new Map<ChannelKind, Set<string>>(),
|
||||
},
|
||||
])("$label", ({ paths, expectedChannels, expectedAccounts }) => {
|
||||
const plan = buildGatewayReloadPlan(paths, { candidateConfig: mattermostAccountConfig });
|
||||
|
||||
expect(plan.restartChannels).toEqual(expectedChannels);
|
||||
expect(plan.restartChannelAccounts).toEqual(expectedAccounts);
|
||||
});
|
||||
|
||||
it("restarts every channel whose config prefix matches", () => {
|
||||
const plan = buildGatewayReloadPlan(["web.enabled", "channels.telegram.botToken"]);
|
||||
|
||||
@@ -1661,6 +1752,53 @@ describe("startGatewayConfigReloader", () => {
|
||||
await harness.reloader.stop();
|
||||
});
|
||||
|
||||
it("runs account-scoped channel changes through hot reload", async () => {
|
||||
const channelRegistry = createTestRegistry([
|
||||
{
|
||||
pluginId: "mattermost",
|
||||
plugin: {
|
||||
id: "mattermost",
|
||||
meta: {
|
||||
id: "mattermost",
|
||||
label: "Mattermost",
|
||||
selectionLabel: "Mattermost",
|
||||
docsPath: "/channels/mattermost",
|
||||
blurb: "test",
|
||||
},
|
||||
capabilities: { chatTypes: ["direct"] },
|
||||
config: { listAccountIds: () => ["default", "alpha"], resolveAccount: () => ({}) },
|
||||
reload: { configPrefixes: ["channels.mattermost"], accountScopedRestart: true },
|
||||
} satisfies ChannelPlugin,
|
||||
source: "test",
|
||||
},
|
||||
]);
|
||||
const initialConfig = {
|
||||
gateway: { reload: { debounceMs: 0 } },
|
||||
channels: { mattermost: { accounts: { alpha: { enabled: false } } } },
|
||||
} as OpenClawConfig;
|
||||
const nextConfig = {
|
||||
gateway: { reload: { debounceMs: 0 } },
|
||||
channels: { mattermost: { accounts: { alpha: { enabled: true } } } },
|
||||
} as OpenClawConfig;
|
||||
const harness = createReloaderHarness(
|
||||
vi.fn(async () => makeSnapshot({ config: nextConfig, hash: "account-reload" })),
|
||||
{ initialConfig },
|
||||
);
|
||||
|
||||
pinActivePluginChannelRegistry(channelRegistry);
|
||||
try {
|
||||
harness.watcher.emit("change");
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
const [plan] = getOnlyHotReloadCall(harness);
|
||||
expect(plan.restartChannelAccounts).toEqual(new Map([["mattermost", new Set(["alpha"])]]));
|
||||
expect(harness.onNoopConfigCommit).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
releasePinnedPluginChannelRegistry(channelRegistry);
|
||||
await harness.reloader.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("plans one immutable runtime override snapshot per candidate", async () => {
|
||||
const initialConfig: OpenClawConfig = {
|
||||
gateway: { reload: { debounceMs: 0 } },
|
||||
|
||||
@@ -82,7 +82,8 @@ function isNoopReloadPlan(plan: GatewayReloadPlan): boolean {
|
||||
!plan.restartHealthMonitor &&
|
||||
!plan.reloadPlugins &&
|
||||
!plan.disposeMcpRuntimes &&
|
||||
plan.restartChannels.size === 0
|
||||
plan.restartChannels.size === 0 &&
|
||||
(plan.restartChannelAccounts?.size ?? 0) === 0
|
||||
);
|
||||
}
|
||||
|
||||
@@ -530,6 +531,7 @@ export function startGatewayConfigReloader(opts: {
|
||||
const plan = buildGatewayReloadPlan(changedPaths, {
|
||||
noopPaths: pluginInstallTimestampNoopPaths,
|
||||
forceChangedPaths: pluginInstallWholeRecordPaths,
|
||||
candidateConfig: nextConfig,
|
||||
});
|
||||
if (nextSettings.mode === "off") {
|
||||
opts.log.info("config reload disabled (gateway.reload.mode=off)");
|
||||
|
||||
@@ -47,6 +47,7 @@ function createReloadPlan(overrides?: Partial<GatewayReloadPlan>): GatewayReload
|
||||
restartHealthMonitor: overrides?.restartHealthMonitor ?? false,
|
||||
reloadPlugins: overrides?.reloadPlugins ?? false,
|
||||
restartChannels: overrides?.restartChannels ?? new Set(),
|
||||
restartChannelAccounts: overrides?.restartChannelAccounts,
|
||||
disposeMcpRuntimes: overrides?.disposeMcpRuntimes ?? false,
|
||||
noopPaths: overrides?.noopPaths ?? [],
|
||||
};
|
||||
@@ -304,6 +305,31 @@ describe("gateway aux handlers", () => {
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it("restarts the whole channel when a secret change is scoped to one account", async () => {
|
||||
// secrets.reload has no per-account restart path — account-scoped plan
|
||||
// entries must still produce a channel restart so rotated credentials
|
||||
// are applied.
|
||||
const buildReloadPlan = () =>
|
||||
createReloadPlan({
|
||||
restartChannels: new Set(),
|
||||
restartChannelAccounts: new Map([["slack", new Set(["ops"])]]),
|
||||
});
|
||||
activateSnapshot(slackConfig("old-slack-secret"));
|
||||
const prepared = createSnapshot(slackConfig("new-slack-secret"));
|
||||
const activateRuntimeSecrets = vi.fn().mockResolvedValue(prepared);
|
||||
const { reload, respond, startChannel, stopChannel } =
|
||||
createSecretsReloadHarnessWithChannelMocks({
|
||||
activateRuntimeSecrets,
|
||||
buildReloadPlan,
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]);
|
||||
expect(startChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]);
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it("coalesces concurrent secrets.reload calls so channels are not restarted twice", async () => {
|
||||
const buildReloadPlan = buildRestartChannelsPlan("slack");
|
||||
activateSnapshot(slackConfig("old-slack-secret"));
|
||||
|
||||
@@ -361,8 +361,15 @@ export function createGatewayAuxHandlers(params: {
|
||||
expectedGeneration: nextSharedGatewaySessionGeneration,
|
||||
});
|
||||
}
|
||||
if (plan.restartChannels.size > 0) {
|
||||
const restartChannels = [...plan.restartChannels];
|
||||
// Account-scoped changes restart their whole channel here:
|
||||
// secrets.reload has no per-account restart path, and a missed
|
||||
// restart would leave rotated credentials unapplied.
|
||||
const channelsToRestart = new Set<ChannelKind>([
|
||||
...plan.restartChannels,
|
||||
...(plan.restartChannelAccounts?.keys() ?? []),
|
||||
]);
|
||||
if (channelsToRestart.size > 0) {
|
||||
const restartChannels = [...channelsToRestart];
|
||||
if (
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) ||
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS)
|
||||
|
||||
@@ -141,7 +141,8 @@ function isNoopConfigReloadPlan(plan: ReturnType<typeof buildGatewayReloadPlan>)
|
||||
!plan.restartHealthMonitor &&
|
||||
!plan.reloadPlugins &&
|
||||
!plan.disposeMcpRuntimes &&
|
||||
plan.restartChannels.size === 0
|
||||
plan.restartChannels.size === 0 &&
|
||||
(plan.restartChannelAccounts?.size ?? 0) === 0
|
||||
);
|
||||
}
|
||||
|
||||
@@ -150,7 +151,7 @@ function resolveConfigRestartRequirement(params: {
|
||||
nextConfig: OpenClawConfig;
|
||||
}): { requiresRestart: boolean; scheduleDirectRestart: boolean } {
|
||||
const reloadSettings = resolveGatewayReloadSettings(params.nextConfig);
|
||||
const plan = buildGatewayReloadPlan(params.changedPaths);
|
||||
const plan = buildGatewayReloadPlan(params.changedPaths, { candidateConfig: params.nextConfig });
|
||||
if (isNoopConfigReloadPlan(plan)) {
|
||||
return { requiresRestart: false, scheduleDirectRestart: false };
|
||||
}
|
||||
|
||||
@@ -332,12 +332,15 @@ function createDeferredVoid() {
|
||||
function createReloadHandlersForTest(
|
||||
logReload = { info: vi.fn(), warn: vi.fn() },
|
||||
channels?: {
|
||||
start: (channel: ChannelKind) => Promise<void>;
|
||||
stop: (channel: ChannelKind) => Promise<void>;
|
||||
start: ReloadHandlerParams["startChannel"];
|
||||
stop: ReloadHandlerParams["stopChannel"];
|
||||
},
|
||||
reloadPlugins?: Parameters<typeof createGatewayReloadHandlers>[0]["reloadPlugins"],
|
||||
stopPostReadySidecars = vi.fn(),
|
||||
recovery: boolean | NonNullable<ReloadHandlerParams["requestRecoveryRestart"]> = true,
|
||||
options?: {
|
||||
getChannelAutostartSuppression?: ReloadHandlerParams["getChannelAutostartSuppression"];
|
||||
},
|
||||
) {
|
||||
const cron = { start: vi.fn(async () => {}), stop: vi.fn() };
|
||||
const stopExitWatchers = vi.fn();
|
||||
@@ -362,6 +365,7 @@ function createReloadHandlersForTest(
|
||||
});
|
||||
const cronReconciliation = createTestCronReconciliation();
|
||||
const logCron = { error: vi.fn() };
|
||||
const logChannels = { info: vi.fn(), error: vi.fn() };
|
||||
const handlers = createGatewayReloadHandlers({
|
||||
deps: {} as never,
|
||||
broadcast: vi.fn(),
|
||||
@@ -369,6 +373,7 @@ function createReloadHandlersForTest(
|
||||
setState,
|
||||
startChannel: channels?.start ?? vi.fn(async () => {}),
|
||||
stopChannel: channels?.stop ?? vi.fn(async () => {}),
|
||||
getChannelAutostartSuppression: options?.getChannelAutostartSuppression,
|
||||
stopPostReadySidecars,
|
||||
reloadPlugins:
|
||||
reloadPlugins ??
|
||||
@@ -379,7 +384,7 @@ function createReloadHandlersForTest(
|
||||
}),
|
||||
),
|
||||
logHooks: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
logChannels: { info: vi.fn(), error: vi.fn() },
|
||||
logChannels,
|
||||
logCron,
|
||||
logReload,
|
||||
cronReconciliation,
|
||||
@@ -397,6 +402,7 @@ function createReloadHandlersForTest(
|
||||
cron,
|
||||
cronReconciliation,
|
||||
heartbeatRunner,
|
||||
logChannels,
|
||||
logCron,
|
||||
setState,
|
||||
stopExitWatchers,
|
||||
@@ -3114,6 +3120,366 @@ describe("gateway channel hot reload handlers", () => {
|
||||
}
|
||||
}
|
||||
|
||||
function createAccountReloadPlan(
|
||||
accountIds: string[],
|
||||
overrides: Partial<GatewayReloadPlan> = {},
|
||||
): GatewayReloadPlan {
|
||||
return {
|
||||
...createChannelReloadPlan([]),
|
||||
changedPaths: accountIds.map((accountId) => `channels.discord.accounts.${accountId}`),
|
||||
restartChannelAccounts: new Map([["discord", new Set(accountIds)]]),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function withDiscordAccountResolver(
|
||||
listAccountIds: () => string[],
|
||||
run: () => Promise<void>,
|
||||
resolveAccount: (cfg: OpenClawConfig, accountId?: string | null) => unknown = () => ({}),
|
||||
) {
|
||||
const registry = createTestRegistry([
|
||||
{
|
||||
pluginId: "discord",
|
||||
plugin: {
|
||||
...createChannelTestPluginBase({
|
||||
id: "discord",
|
||||
config: { listAccountIds, resolveAccount },
|
||||
}),
|
||||
},
|
||||
source: "test",
|
||||
},
|
||||
]);
|
||||
pinActivePluginChannelRegistry(registry);
|
||||
try {
|
||||
await run();
|
||||
} finally {
|
||||
releasePinnedPluginChannelRegistry(registry);
|
||||
}
|
||||
}
|
||||
|
||||
async function withDiscordAccounts(accountIds: string[], run: () => Promise<void>) {
|
||||
await withDiscordAccountResolver(() => accountIds, run);
|
||||
}
|
||||
|
||||
it("restarts only the changed account", async () => {
|
||||
const events: string[] = [];
|
||||
const startRootCounts: number[] = [];
|
||||
const accountStopSettled = createDeferredVoid();
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
await accountStopSettled.promise;
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
startRootCounts.push(getActiveGatewayRootWorkCount({ excludeCurrent: true }));
|
||||
}),
|
||||
};
|
||||
const { applyHotReload } = createReloadHandlersForTest(undefined, channels);
|
||||
const root = tryBeginGatewayRootWorkAdmission();
|
||||
expect(root).not.toBeNull();
|
||||
let reload: Promise<void> | undefined;
|
||||
|
||||
try {
|
||||
await root?.run(async () => {
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha", "beta"], async () => {
|
||||
reload = applyHotReload(createAccountReloadPlan(["alpha"]), {});
|
||||
await waitForFast(() => expect(events).toEqual(["stop:discord:alpha"]));
|
||||
expect(channels.start).not.toHaveBeenCalled();
|
||||
|
||||
accountStopSettled.resolve();
|
||||
await reload;
|
||||
});
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
accountStopSettled.resolve();
|
||||
await reload?.catch(() => {});
|
||||
root?.release();
|
||||
}
|
||||
|
||||
expect(events).toEqual(["stop:discord:alpha", "start:discord:alpha"]);
|
||||
expect(startRootCounts).toEqual([1]);
|
||||
expect(channels.stop).toHaveBeenCalledOnce();
|
||||
expect(channels.start).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("continues targeted restarts after an account failure", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
if (accountId === "alpha") {
|
||||
throw new Error("stop failed");
|
||||
}
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const }));
|
||||
const { applyHotReload } = createReloadHandlersForTest(
|
||||
undefined,
|
||||
channels,
|
||||
undefined,
|
||||
undefined,
|
||||
requestRecoveryRestart,
|
||||
);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha", "beta"], async () => {
|
||||
await applyHotReload(createAccountReloadPlan(["alpha", "beta"]), {});
|
||||
});
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:alpha", "stop:discord:beta", "start:discord:beta"]);
|
||||
expect(requestRecoveryRestart).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("promotes unlisted accounts to a wholesale restart", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const { applyHotReload } = createReloadHandlersForTest(undefined, channels);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha"], async () => {
|
||||
await applyHotReload(createAccountReloadPlan(["removed-account"]), {});
|
||||
});
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:undefined", "start:discord:undefined"]);
|
||||
});
|
||||
|
||||
it("promotes unresolvable accounts to a wholesale restart before stopping any account", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const { applyHotReload, logChannels } = createReloadHandlersForTest(undefined, channels);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccountResolver(
|
||||
() => ["default", "alpha", "beta"],
|
||||
async () => {
|
||||
await applyHotReload(createAccountReloadPlan(["alpha", "beta"]), {});
|
||||
},
|
||||
(_cfg, accountId) => {
|
||||
if (accountId === "beta") {
|
||||
throw new Error("account resolution failed");
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:undefined", "start:discord:undefined"]);
|
||||
expect(logChannels.info).toHaveBeenCalledWith(
|
||||
"promoting discord account reload to whole-channel restart after account resolution failed: account resolution failed",
|
||||
);
|
||||
});
|
||||
|
||||
it("requests recovery when account enumeration fails after config commit", async () => {
|
||||
const channels = {
|
||||
stop: vi.fn(async () => {}),
|
||||
start: vi.fn(async () => {}),
|
||||
};
|
||||
const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const }));
|
||||
const { applyHotReload } = createReloadHandlersForTest(
|
||||
undefined,
|
||||
channels,
|
||||
undefined,
|
||||
undefined,
|
||||
requestRecoveryRestart,
|
||||
);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccountResolver(
|
||||
() => {
|
||||
throw new Error("account enumeration failed");
|
||||
},
|
||||
async () => {
|
||||
await applyHotReload(createAccountReloadPlan(["alpha"]), {});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
expect(channels.stop).not.toHaveBeenCalled();
|
||||
expect(channels.start).not.toHaveBeenCalled();
|
||||
expect(requestRecoveryRestart).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("skips per-account restarts for channels already queued for wholesale restart", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const { applyHotReload } = createReloadHandlersForTest(undefined, channels);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha"], async () => {
|
||||
await applyHotReload(
|
||||
createAccountReloadPlan(["alpha"], { restartChannels: new Set(["discord"]) }),
|
||||
{},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:undefined", "start:discord:undefined"]);
|
||||
});
|
||||
|
||||
it("aggregates targeted and wholesale stop failures into one suppressed recovery request", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
throw new Error("stop failed");
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const requestRecoveryRestart = vi.fn(() => ({ status: "emitted" as const }));
|
||||
const { applyHotReload } = createReloadHandlersForTest(
|
||||
undefined,
|
||||
channels,
|
||||
undefined,
|
||||
undefined,
|
||||
requestRecoveryRestart,
|
||||
{
|
||||
getChannelAutostartSuppression: () => ({
|
||||
reason: "crash-loop-breaker",
|
||||
message: "safe mode",
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha"], async () => {
|
||||
await applyHotReload(
|
||||
createAccountReloadPlan(["alpha"], {
|
||||
restartChannels: new Set<ChannelKind>(["telegram"]),
|
||||
}),
|
||||
{},
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:alpha", "stop:telegram:undefined"]);
|
||||
expect(requestRecoveryRestart).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("stops account targets without restarting them while autostart is suppressed", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const { applyHotReload } = createReloadHandlersForTest(
|
||||
undefined,
|
||||
channels,
|
||||
undefined,
|
||||
undefined,
|
||||
true,
|
||||
{
|
||||
getChannelAutostartSuppression: () => ({
|
||||
reason: "crash-loop-breaker",
|
||||
message: "safe mode",
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha"], async () => {
|
||||
await applyHotReload(createAccountReloadPlan(["alpha"]), {});
|
||||
});
|
||||
});
|
||||
|
||||
expect(events).toEqual(["stop:discord:alpha"]);
|
||||
});
|
||||
|
||||
it("rechecks agent work admitted after plugin reload leaves the channel running", async () => {
|
||||
const events: string[] = [];
|
||||
const channels = {
|
||||
stop: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`stop:${channel}:${accountId}`);
|
||||
}),
|
||||
start: vi.fn(async (channel: ChannelKind, accountId?: string) => {
|
||||
events.push(`start:${channel}:${accountId}`);
|
||||
}),
|
||||
};
|
||||
const reloadPlugins = vi.fn(async (params): Promise<GatewayPluginReloadResult> => {
|
||||
await params.beforeReplace(new Set());
|
||||
hoisted.activeEmbeddedRunCount.value = 1;
|
||||
return {
|
||||
restartChannels: new Set(),
|
||||
activeChannels: new Set(["discord"]),
|
||||
};
|
||||
});
|
||||
const logReload = { info: vi.fn(), warn: vi.fn() };
|
||||
const { applyHotReload } = createReloadHandlersForTest(logReload, channels, reloadPlugins);
|
||||
vi.useFakeTimers();
|
||||
let reload: Promise<void> | undefined;
|
||||
|
||||
try {
|
||||
await withChannelReloadsEnabled(async () => {
|
||||
await withDiscordAccounts(["default", "alpha"], async () => {
|
||||
reload = applyHotReload(createAccountReloadPlan(["alpha"], { reloadPlugins: true }), {});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(events).toEqual([]);
|
||||
expect(logReload.warn).toHaveBeenCalledWith(expect.stringContaining("(discord)"));
|
||||
|
||||
hoisted.activeEmbeddedRunCount.value = 0;
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await reload;
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
hoisted.activeEmbeddedRunCount.value = 0;
|
||||
await vi.advanceTimersByTimeAsync(500).catch(() => {});
|
||||
vi.useRealTimers();
|
||||
await reload?.catch(() => {});
|
||||
}
|
||||
|
||||
expect(events).toEqual(["stop:discord:alpha", "start:discord:alpha"]);
|
||||
expect(reloadPlugins).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("requires a recovery owner for targeted account reloads", async () => {
|
||||
const { applyHotReload } = createReloadHandlersForTest(
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
);
|
||||
|
||||
await expect(applyHotReload(createAccountReloadPlan(["alpha"]), {})).rejects.toThrow(
|
||||
"config reload requires a managed gateway restart owner for irreversible hot reload",
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses channel restarts while crash-loop safe mode suppresses autostart", async () => {
|
||||
const logChannels = { info: vi.fn(), error: vi.fn() };
|
||||
const channels = {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
warmCurrentProviderAuthStateOffMainThread,
|
||||
} from "../agents/model-provider-auth.js";
|
||||
import { getTotalPendingReplies } from "../auto-reply/reply/dispatcher-registry.js";
|
||||
import { getChannelPlugin } from "../channels/plugins/index.js";
|
||||
import type { CliDeps } from "../cli/deps.types.js";
|
||||
import { isRestartEnabled } from "../config/commands.flags.js";
|
||||
import { getConfigValueAtPath } from "../config/config-paths.js";
|
||||
@@ -562,6 +563,8 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
resetDirectoryCache();
|
||||
|
||||
const channelsToRestart = new Set(plan.restartChannels);
|
||||
const restartChannelAccounts =
|
||||
plan.restartChannelAccounts ?? new Map<ChannelKind, Set<string>>();
|
||||
const channelsStoppedBeforePluginReload = new Set<ChannelKind>();
|
||||
let activePluginChannelsAfterReload: ReadonlySet<ChannelKind> | null = null;
|
||||
let pluginReloadAborted = false;
|
||||
@@ -578,6 +581,8 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
const shouldSkipChannelRestart =
|
||||
isTruthyEnvValue(candidateEnv.OPENCLAW_SKIP_CHANNELS) ||
|
||||
isTruthyEnvValue(candidateEnv.OPENCLAW_SKIP_PROVIDERS);
|
||||
const channelReloadTargets = () =>
|
||||
new Set<ChannelKind>([...channelsToRestart, ...restartChannelAccounts.keys()]);
|
||||
const getChannelAutostartSuppression = () => params.getChannelAutostartSuppression?.() ?? null;
|
||||
const logSuppressedChannelRestart = (
|
||||
channels: ReadonlySet<ChannelKind>,
|
||||
@@ -757,16 +762,11 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
for (const channel of channels) {
|
||||
channelsToRestart.add(channel);
|
||||
}
|
||||
if (channelsToRestart.size === 0 || shouldSkipChannelRestart) {
|
||||
const targets = channelReloadTargets();
|
||||
if (targets.size === 0 || shouldSkipChannelRestart) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
await waitForActiveWorkBeforeChannelReload(
|
||||
channelsToRestart,
|
||||
nextConfig,
|
||||
isTransactionCurrent,
|
||||
)
|
||||
) {
|
||||
if (await waitForActiveWorkBeforeChannelReload(targets, nextConfig, isTransactionCurrent)) {
|
||||
params.logChannels.info(
|
||||
"channel reload before plugin replace cancelled by config supersession or restart",
|
||||
);
|
||||
@@ -870,9 +870,15 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
}
|
||||
}
|
||||
|
||||
if (!plan.reloadPlugins && channelsToRestart.size > 0 && !shouldSkipChannelRestart) {
|
||||
const channelTargets = channelReloadTargets();
|
||||
const hasLiveChannelTargets = [...channelTargets].some(
|
||||
(channel) => !channelsStoppedBeforePluginReload.has(channel),
|
||||
);
|
||||
// Plugin replacement can admit new agent work while an account monitor stays live.
|
||||
// Recheck that work here; durable ingress replay remains owned by the fresh monitor drain.
|
||||
if (!pluginReloadAborted && hasLiveChannelTargets && !shouldSkipChannelRestart) {
|
||||
pluginReloadAborted = await waitForActiveWorkBeforeChannelReload(
|
||||
channelsToRestart,
|
||||
channelTargets,
|
||||
nextConfig,
|
||||
isTransactionCurrent,
|
||||
);
|
||||
@@ -946,7 +952,48 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
}
|
||||
}
|
||||
|
||||
if (channelsToRestart.size > 0) {
|
||||
// Suppressed and normal reloads share fallback selection so stale account
|
||||
// ids always reach the wholesale path that evicts their old runtime.
|
||||
const collectChannelAccountTargets = (): Array<[ChannelKind, string]> => {
|
||||
const targets: Array<[ChannelKind, string]> = [];
|
||||
for (const [channel, accountIds] of restartChannelAccounts) {
|
||||
if (
|
||||
channelsToRestart.has(channel) ||
|
||||
(plan.reloadPlugins && activePluginChannelsAfterReload?.has(channel) === false)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const plugin = getChannelPlugin(channel);
|
||||
let listedAccountIds: Set<string>;
|
||||
try {
|
||||
listedAccountIds = new Set(plugin?.config.listAccountIds(nextConfig) ?? []);
|
||||
} catch (err) {
|
||||
scheduleRecoveryRestart(`channel account enumeration (${channel})`, err);
|
||||
continue;
|
||||
}
|
||||
if ([...accountIds].some((accountId) => !listedAccountIds.has(accountId))) {
|
||||
channelsToRestart.add(channel);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
for (const accountId of accountIds) {
|
||||
plugin?.config.resolveAccount(nextConfig, accountId);
|
||||
}
|
||||
} catch (err) {
|
||||
params.logChannels.info(
|
||||
`promoting ${channel} account reload to whole-channel restart after account resolution failed: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
channelsToRestart.add(channel);
|
||||
continue;
|
||||
}
|
||||
for (const accountId of accountIds) {
|
||||
targets.push([channel, accountId]);
|
||||
}
|
||||
}
|
||||
return targets;
|
||||
};
|
||||
|
||||
if (channelsToRestart.size > 0 || restartChannelAccounts.size > 0) {
|
||||
if (shouldSkipChannelRestart) {
|
||||
params.logChannels.info(
|
||||
"skipping channel reload (OPENCLAW_SKIP_CHANNELS=1 or OPENCLAW_SKIP_PROVIDERS=1)",
|
||||
@@ -956,6 +1003,21 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
if (cancelledByRestart) {
|
||||
params.logChannels.info("channel restart cancelled by in-process restart");
|
||||
} else {
|
||||
const accountStops = collectChannelAccountTargets();
|
||||
const accountStopFailures: string[] = [];
|
||||
for (const [channel, accountId] of accountStops) {
|
||||
try {
|
||||
params.logChannels.info(
|
||||
`stopping ${channel} account ${accountId} before suppressed hot reload`,
|
||||
);
|
||||
await params.stopChannel(channel, accountId, { manual: false });
|
||||
} catch (err) {
|
||||
accountStopFailures.push(`${channel}[${accountId}]`);
|
||||
params.logChannels.error(
|
||||
`failed to stop ${channel} account ${accountId} during suppressed hot reload: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const stopFailures = await collectChannelOperationFailures({
|
||||
channels: channelsToRestart,
|
||||
run: async (channel) => {
|
||||
@@ -976,16 +1038,36 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
);
|
||||
},
|
||||
});
|
||||
if (stopFailures.length > 0) {
|
||||
scheduleRecoveryRestart(`channel stop (${stopFailures.join(", ")})`);
|
||||
const allStopFailures = [...accountStopFailures, ...stopFailures];
|
||||
if (allStopFailures.length > 0) {
|
||||
scheduleRecoveryRestart(`channel stop (${allStopFailures.join(", ")})`);
|
||||
}
|
||||
logSuppressedChannelRestart(channelsToRestart, "channel restart during hot reload");
|
||||
logSuppressedChannelRestart(channelReloadTargets(), "channel restart during hot reload");
|
||||
}
|
||||
} else {
|
||||
const cancelledByRestart = pluginReloadAborted;
|
||||
if (cancelledByRestart) {
|
||||
params.logChannels.info("channel restart cancelled by in-process restart");
|
||||
} else {
|
||||
const accountRestarts = collectChannelAccountTargets();
|
||||
const accountRestartFailures: string[] = [];
|
||||
for (const [channel, accountId] of accountRestarts) {
|
||||
try {
|
||||
params.logChannels.info(`restarting ${channel} account ${accountId}`);
|
||||
await params.stopChannel(channel, accountId, { manual: false });
|
||||
if (isLifecycleReloadAborted()) {
|
||||
continue;
|
||||
}
|
||||
await runOutsideGatewayRootWorkAdmission(() =>
|
||||
params.startChannel(channel, accountId),
|
||||
);
|
||||
} catch (err) {
|
||||
accountRestartFailures.push(`${channel}[${accountId}]`);
|
||||
params.logChannels.error(
|
||||
`failed to restart ${channel} account ${accountId} during hot reload: ${formatErrorMessage(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const restartChannel = async (name: ChannelKind) => {
|
||||
if (plan.reloadPlugins && activePluginChannelsAfterReload?.has(name) === false) {
|
||||
return;
|
||||
@@ -1008,8 +1090,9 @@ export function createGatewayReloadHandlers(params: GatewayReloadHandlerParams)
|
||||
);
|
||||
},
|
||||
});
|
||||
if (restartFailures.length > 0) {
|
||||
scheduleRecoveryRestart(`channel restart (${restartFailures.join(", ")})`);
|
||||
const allRestartFailures = [...accountRestartFailures, ...restartFailures];
|
||||
if (allRestartFailures.length > 0) {
|
||||
scheduleRecoveryRestart(`channel restart (${allRestartFailures.join(", ")})`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user