mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 20:05:46 -06:00
fix(gateway): recover credential-file accounts after secrets reload (#126999)
* fix(gateway): recover credential-file accounts on secrets reload Preserve independently discovered credential-file degradation across runtime snapshot refreshes, and re-inspect only the affected account when secrets are reloaded. Healthy sibling accounts remain running while status and doctor retain exact-owner diagnostics until recovery or teardown. * test(gateway): prove credential-file reload recovery * test(gateway): assert redacted reload error code
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
9feb1db00d
commit
2ecd342fbb
@@ -2,7 +2,11 @@
|
||||
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { SESSION_TOTAL_TOKENS_VERSION } from "../config/sessions/types.js";
|
||||
import { setActiveDegradedPlugins } from "../plugins/runtime-degraded-state.js";
|
||||
import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js";
|
||||
import {
|
||||
clearActiveCredentialDegradedOwner,
|
||||
setActiveCredentialDegradedOwner,
|
||||
setActiveDegradedSecretOwners,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import type { TaskAuditFinding } from "../tasks/task-registry.audit.js";
|
||||
import type { TaskRecord, TaskRegistrySummary } from "../tasks/task-registry.types.js";
|
||||
import { normalizeSessionDeliveryState } from "../utils/delivery-context.shared.js";
|
||||
@@ -219,6 +223,7 @@ describe("getStatusSummary", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
setActiveDegradedPlugins([]);
|
||||
clearActiveCredentialDegradedOwner("account", "telegram:work");
|
||||
setActiveDegradedSecretOwners([]);
|
||||
statusSummaryMocks.taskRegistrySummary = {
|
||||
total: 0,
|
||||
@@ -406,28 +411,44 @@ describe("getStatusSummary", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports degraded SecretRef owners without exposing ref identifiers", async () => {
|
||||
it("reports stale snapshot and cold credential owners without exposing ref identifiers", async () => {
|
||||
setActiveDegradedSecretOwners([
|
||||
{
|
||||
ownerKind: "account",
|
||||
ownerId: "discord:ops",
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
state: "unavailable",
|
||||
degradationState: "cold",
|
||||
paths: ["channels.discord.accounts.ops.token"],
|
||||
degradationState: "stale",
|
||||
paths: ["models.providers.openai.apiKey"],
|
||||
refKeys: ["env:default:PRIVATE_REF_ID"],
|
||||
reason: "provider SecretRef is unresolved (env:default:PRIVATE_REF_ID)",
|
||||
},
|
||||
]);
|
||||
setActiveCredentialDegradedOwner({
|
||||
ownerKind: "account",
|
||||
ownerId: "telegram:work",
|
||||
state: "unavailable",
|
||||
paths: ["channels.telegram.accounts.work.tokenFile"],
|
||||
refKeys: [],
|
||||
reason: "credential failure includes PRIVATE_REF_ID",
|
||||
});
|
||||
|
||||
const summary = await getStatusSummary();
|
||||
|
||||
expect(summary.degradedSecretOwners).toEqual([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
state: "unavailable",
|
||||
degradationState: "stale",
|
||||
paths: ["models.providers.openai.apiKey"],
|
||||
reason: "secret resolution failed",
|
||||
},
|
||||
{
|
||||
ownerKind: "account",
|
||||
ownerId: "discord:ops",
|
||||
ownerId: "telegram:work",
|
||||
state: "unavailable",
|
||||
degradationState: "cold",
|
||||
paths: ["channels.discord.accounts.ops.token"],
|
||||
paths: ["channels.telegram.accounts.work.tokenFile"],
|
||||
reason: "secret resolution failed",
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -33,6 +33,10 @@ import {
|
||||
claimAgentRunDelegatedAuthority,
|
||||
releaseAgentRunDelegatedAuthority,
|
||||
} from "../infra/agent-run-registry.js";
|
||||
import {
|
||||
setActiveCredentialDegradedOwner,
|
||||
type DegradedSecretOwner,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
@@ -47,6 +51,10 @@ import {
|
||||
import { createAgentRuntimeApprovalAuthorityValidator } from "./agent-runtime-identity-token.js";
|
||||
import type { GatewayReloadPlan } from "./config-reload.js";
|
||||
import { createGatewayAuxHandlers } from "./server-aux-handlers.js";
|
||||
import {
|
||||
registerGatewaySecretCredentialReloadCases,
|
||||
type CredentialReloadHarnessOptions,
|
||||
} from "./server-secrets-reload.test-support.js";
|
||||
import { enforceSharedGatewaySessionGenerationForConfigWrite } from "./server-shared-auth-generation.js";
|
||||
import { createWorkerSessionPlacementStore } from "./worker-environments/placement-store.js";
|
||||
|
||||
@@ -185,7 +193,8 @@ async function invokeSecretStoreSet(params: {
|
||||
|
||||
type RespondCall = [boolean, unknown, { message?: string } | undefined];
|
||||
type GatewayAuxHandlerParams = Parameters<typeof createGatewayAuxHandlers>[0];
|
||||
type ChannelName = Parameters<GatewayAuxHandlerParams["startChannel"]>[0];
|
||||
type GatewayChannelManager = GatewayAuxHandlerParams["channelManager"];
|
||||
type ChannelName = Parameters<GatewayChannelManager["startChannel"]>[0];
|
||||
|
||||
function firstRespondCall(respond: ReturnType<typeof vi.fn>): RespondCall {
|
||||
const call = respond.mock.calls[0];
|
||||
@@ -208,8 +217,10 @@ type SecretsReloadHarnessParams = {
|
||||
sharedGatewaySessionGenerationState?: GatewayAuxHandlerParams["sharedGatewaySessionGenerationState"];
|
||||
resolveSharedGatewaySessionGenerationForConfig?: GatewayAuxHandlerParams["resolveSharedGatewaySessionGenerationForConfig"];
|
||||
clients?: GatewayAuxHandlerParams["clients"];
|
||||
startChannel?: GatewayAuxHandlerParams["startChannel"];
|
||||
stopChannel?: GatewayAuxHandlerParams["stopChannel"];
|
||||
startChannel?: GatewayChannelManager["startChannel"];
|
||||
stopChannel?: GatewayChannelManager["stopChannel"];
|
||||
isManuallyStopped?: (channel: ChannelName, accountId: string) => boolean;
|
||||
resolveRuntimeAccountId?: (channel: ChannelName, accountId: string) => string | undefined;
|
||||
getChannelAutostartSuppression?: GatewayAuxHandlerParams["getChannelAutostartSuppression"];
|
||||
logChannelsInfo?: GatewayAuxHandlerParams["logChannels"]["info"];
|
||||
respond?: ReturnType<typeof vi.fn>;
|
||||
@@ -232,8 +243,13 @@ function createSecretsReloadHarness(params: SecretsReloadHarnessParams) {
|
||||
resolveSharedGatewaySessionGenerationForConfig:
|
||||
params.resolveSharedGatewaySessionGenerationForConfig ?? (() => undefined),
|
||||
clients: params.clients ?? [],
|
||||
startChannel: params.startChannel ?? (async () => {}),
|
||||
stopChannel: params.stopChannel ?? (async () => {}),
|
||||
channelManager: {
|
||||
startChannel: params.startChannel ?? (async () => {}),
|
||||
stopChannel: params.stopChannel ?? (async () => {}),
|
||||
isManuallyStopped: params.isManuallyStopped ?? (() => false),
|
||||
resolveRuntimeAccountId:
|
||||
params.resolveRuntimeAccountId ?? ((_channel, accountId) => accountId),
|
||||
},
|
||||
getChannelAutostartSuppression: params.getChannelAutostartSuppression,
|
||||
logChannels: { info: params.logChannelsInfo ?? vi.fn() },
|
||||
onApprovalLifecycle: params.onApprovalLifecycle,
|
||||
@@ -267,6 +283,42 @@ function createSecretsReloadHarnessWithChannelMocks(
|
||||
};
|
||||
}
|
||||
|
||||
function createCredentialReloadHarness(options: CredentialReloadHarnessOptions = {}) {
|
||||
const ownerAccountId = options.ownerAccountId ?? "ops";
|
||||
const owner: DegradedSecretOwner = {
|
||||
ownerKind: "account",
|
||||
ownerId: `slack:${ownerAccountId}`,
|
||||
state: "unavailable",
|
||||
paths: ["env.SERVICE_ACCOUNT_FILE"],
|
||||
refKeys: [],
|
||||
reason: "credential file is unavailable",
|
||||
};
|
||||
const config = slackConfig("unchanged-secret");
|
||||
activateSnapshot(config);
|
||||
setActiveCredentialDegradedOwner(owner);
|
||||
const startChannel = vi.fn().mockImplementation(async () => {
|
||||
if (options.createFailure) {
|
||||
throw options.createFailure(owner);
|
||||
}
|
||||
});
|
||||
const stopChannel = vi.fn().mockResolvedValue(undefined);
|
||||
const isManuallyStopped = vi.fn(() => options.manualStop ?? false);
|
||||
return {
|
||||
...createSecretsReloadHarness({
|
||||
activateRuntimeSecrets: mockResolvedSecrets(config),
|
||||
buildReloadPlan: () => createReloadPlan(),
|
||||
startChannel,
|
||||
stopChannel,
|
||||
isManuallyStopped,
|
||||
resolveRuntimeAccountId: () => options.runtimeAccountId ?? ownerAccountId,
|
||||
}),
|
||||
owner,
|
||||
startChannel,
|
||||
stopChannel,
|
||||
isManuallyStopped,
|
||||
};
|
||||
}
|
||||
|
||||
// Other gateway test helpers (e.g. test-helpers.mocks.ts, test-helpers.server.ts)
|
||||
// set OPENCLAW_SKIP_CHANNELS / OPENCLAW_SKIP_PROVIDERS at module load. When a
|
||||
// shared vitest worker imports those helpers before this file's tests run,
|
||||
@@ -551,18 +603,13 @@ describe("gateway aux handlers", () => {
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it("restarts the whole channel when a secret change is scoped to one account", async () => {
|
||||
// secrets.reload has no per-account restart path — account-scoped plan
|
||||
// entries must still produce a channel restart so rotated credentials
|
||||
// are applied.
|
||||
it("restarts only the changed account when a secret change is account-scoped", async () => {
|
||||
const buildReloadPlan = () =>
|
||||
createReloadPlan({
|
||||
restartChannels: new Set(),
|
||||
restartChannelAccounts: new Map([["slack", new Set(["ops"])]]),
|
||||
});
|
||||
activateSnapshot(slackConfig("old-slack-secret"));
|
||||
const prepared = createSnapshot(slackConfig("new-slack-secret"));
|
||||
const activateRuntimeSecrets = vi.fn().mockResolvedValue(prepared);
|
||||
const activateRuntimeSecrets = mockResolvedSecrets(slackConfig("new-slack-secret"));
|
||||
const { reload, respond, startChannel, stopChannel } =
|
||||
createSecretsReloadHarnessWithChannelMocks({
|
||||
activateRuntimeSecrets,
|
||||
@@ -571,11 +618,30 @@ describe("gateway aux handlers", () => {
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]);
|
||||
expect(startChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]);
|
||||
expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]);
|
||||
expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]);
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
registerGatewaySecretCredentialReloadCases(createCredentialReloadHarness);
|
||||
|
||||
it("does not restart account targets already covered by a whole-channel target", async () => {
|
||||
activateSnapshot(slackConfig("old-secret"));
|
||||
const { reload, startChannel, stopChannel } = createSecretsReloadHarnessWithChannelMocks({
|
||||
activateRuntimeSecrets: mockResolvedSecrets(slackConfig("new-secret")),
|
||||
buildReloadPlan: () =>
|
||||
createReloadPlan({
|
||||
restartChannels: new Set(["slack"]),
|
||||
restartChannelAccounts: new Map([["slack", new Set(["ops"])]]),
|
||||
}),
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel.mock.calls).toEqual([["slack"]]);
|
||||
expect(startChannel.mock.calls).toEqual([["slack"]]);
|
||||
});
|
||||
|
||||
it("coalesces concurrent secrets.reload calls so channels are not restarted twice", async () => {
|
||||
const buildReloadPlan = buildRestartChannelsPlan("slack");
|
||||
activateSnapshot(slackConfig("old-slack-secret"));
|
||||
@@ -731,9 +797,15 @@ describe("gateway aux handlers", () => {
|
||||
expect(firstRespondCall(respond)[0]).toBe(true);
|
||||
});
|
||||
|
||||
it("rolls back stopped channels when a later restart fails", async () => {
|
||||
it("rolls back only exact stopped accounts when a later account restart fails", async () => {
|
||||
const authAgentDir = "/tmp/openclaw-secrets-reload-concurrent-oauth";
|
||||
const buildReloadPlan = buildRestartChannelsPlan("slack", "zalo");
|
||||
const buildReloadPlan = () =>
|
||||
createReloadPlan({
|
||||
restartChannelAccounts: new Map([
|
||||
["slack", new Set(["ops"])],
|
||||
["zalo", new Set(["alerts"])],
|
||||
]),
|
||||
});
|
||||
activateSnapshot(slackZaloConfig("old-slack-secret", "old-zalo-secret"));
|
||||
const activateRuntimeSecrets = mockResolvedSecrets(
|
||||
slackZaloConfig("new-slack-secret", "new-zalo-secret"),
|
||||
@@ -779,21 +851,30 @@ describe("gateway aux handlers", () => {
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel.mock.calls).toEqual([["slack"], ["zalo"], ["slack"]]);
|
||||
expect(startChannel.mock.calls).toEqual([["slack"], ["zalo"], ["slack"], ["zalo"]]);
|
||||
expect(stopChannel.mock.calls).toEqual([
|
||||
["slack", "ops", { manual: false }],
|
||||
["zalo", "alerts", { manual: false }],
|
||||
["slack", "ops", { manual: false }],
|
||||
]);
|
||||
expect(startChannel.mock.calls).toEqual([
|
||||
["slack", "ops", { preserveManualStop: true }],
|
||||
["zalo", "alerts", { preserveManualStop: true }],
|
||||
["slack", "ops", { preserveManualStop: true }],
|
||||
["zalo", "alerts", { preserveManualStop: true }],
|
||||
]);
|
||||
expect(
|
||||
logChannelsInfo.mock.calls.some(([msg]) =>
|
||||
String(msg).startsWith("failed to restart zalo channel after secrets reload"),
|
||||
String(msg).startsWith("failed to restart zalo account alerts after secrets reload"),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
logChannelsInfo.mock.calls.some(([msg]) =>
|
||||
String(msg).startsWith("rolling back slack channel after secrets reload failure"),
|
||||
String(msg).startsWith("rolling back slack account ops after secrets reload failure"),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
logChannelsInfo.mock.calls.some(([msg]) =>
|
||||
String(msg).startsWith("rolling back zalo channel after secrets reload failure"),
|
||||
String(msg).startsWith("rolling back zalo account alerts after secrets reload failure"),
|
||||
),
|
||||
).toBe(true);
|
||||
// The handler surfaces the partial-failure so the caller can retry/alert
|
||||
@@ -815,19 +896,13 @@ describe("gateway aux handlers", () => {
|
||||
).toMatchObject({ access: "access-new", refresh: "refresh-new" });
|
||||
});
|
||||
|
||||
it("does not roll back over a snapshot published after secrets.reload activation", async () => {
|
||||
const buildReloadPlan = buildRestartChannelsPlan("slack");
|
||||
it("fences account-scoped rollback when a newer snapshot and generation supersede reload", async () => {
|
||||
const buildReloadPlan = () =>
|
||||
createReloadPlan({ restartChannelAccounts: new Map([["slack", new Set(["ops"])]]) });
|
||||
activateSnapshot(slackConfig("old-slack-secret"));
|
||||
const prepared = createSnapshot(slackConfig("reload-secret"));
|
||||
const concurrent = createSnapshot(slackConfig("concurrent-secret"));
|
||||
const activateRuntimeSecrets = vi.fn(
|
||||
async (
|
||||
_config: OpenClawConfig,
|
||||
_activationParams: Parameters<GatewayAuxHandlerParams["activateRuntimeSecrets"]>[1],
|
||||
) => {
|
||||
return prepared;
|
||||
},
|
||||
);
|
||||
const activateRuntimeSecrets = vi.fn(async () => prepared);
|
||||
const sharedGatewaySessionGenerationState = {
|
||||
current: "gen-old" as string | undefined,
|
||||
required: "gen-old" as string | undefined | null,
|
||||
@@ -841,19 +916,24 @@ describe("gateway aux handlers", () => {
|
||||
})
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
const stopChannel = vi.fn().mockResolvedValue(undefined);
|
||||
const { reload, respond } = createSecretsReloadHarness({
|
||||
activateRuntimeSecrets,
|
||||
buildReloadPlan,
|
||||
sharedGatewaySessionGenerationState,
|
||||
resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload",
|
||||
startChannel,
|
||||
stopChannel: vi.fn().mockResolvedValue(undefined),
|
||||
stopChannel,
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(firstRespondCall(respond)[0]).toBe(false);
|
||||
expect(startChannel).toHaveBeenCalledTimes(2);
|
||||
expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]);
|
||||
expect(startChannel.mock.calls).toEqual([
|
||||
["slack", "ops", { preserveManualStop: true }],
|
||||
["slack", "ops", { preserveManualStop: true }],
|
||||
]);
|
||||
expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(slackConfig("concurrent-secret"));
|
||||
expect(sharedGatewaySessionGenerationState).toEqual({
|
||||
current: "gen-concurrent",
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
// Gateway auxiliary method handlers.
|
||||
// Wires reload, secrets, exec approval, and plugin approval RPC handlers.
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
type AgentRunDelegatedAuthority,
|
||||
registerAgentRunDelegatedAuthorityClosedHandler,
|
||||
} from "../infra/agent-run-registry.js";
|
||||
import type { ChannelApprovalKind } from "../infra/approval-types.js";
|
||||
import { isTruthyEnvValue } from "../infra/env.js";
|
||||
import { createExecApprovalForwarder } from "../infra/exec-approval-forwarder.js";
|
||||
import {
|
||||
type ExecApprovalDecision,
|
||||
@@ -24,21 +22,10 @@ import {
|
||||
resolveCommandSecretsFromActiveRuntimeSnapshot,
|
||||
type CommandSecretAssignment,
|
||||
} from "../secrets/runtime-command-secrets.js";
|
||||
import {
|
||||
getActiveSecretsRuntimeSnapshotState,
|
||||
getActiveSecretsRuntimeSnapshotRevisionState,
|
||||
type PreparedSecretsRuntimeSnapshot,
|
||||
} from "../secrets/runtime-state.js";
|
||||
import { createLazyPromise } from "../shared/lazy-runtime.js";
|
||||
import type { AgentRuntimeDelegatedAuthority } from "./agent-runtime-identity-token.js";
|
||||
import { resolveApprovalSessionAudienceWithFallback } from "./approval-session-audience.js";
|
||||
import type { ChatAbortControllerEntry } from "./chat-abort.js";
|
||||
import { diffConfigPaths } from "./config-diff.js";
|
||||
import {
|
||||
buildGatewayReloadPlan,
|
||||
type ChannelKind,
|
||||
type GatewayReloadPlan,
|
||||
} from "./config-reload-plan.js";
|
||||
import {
|
||||
createExecApprovalIosPushDelivery,
|
||||
createPluginApprovalIosPushDelivery,
|
||||
@@ -53,7 +40,6 @@ import {
|
||||
pruneTerminalOperatorApprovals,
|
||||
} from "./operator-approval-store.js";
|
||||
import { QuestionManager } from "./question-manager.js";
|
||||
import type { ChannelAutostartSuppression } from "./server-channels.js";
|
||||
import { publishAppliedApprovalResolution } from "./server-methods/approval-publication.js";
|
||||
import {
|
||||
cancelAgentRuntimeBoundApprovals,
|
||||
@@ -62,17 +48,9 @@ import {
|
||||
} from "./server-methods/approval-run-cancellation.js";
|
||||
import type { GatewayRequestContext } from "./server-methods/types.js";
|
||||
import {
|
||||
captureSharedGatewaySessionGenerationOwnership,
|
||||
claimSharedGatewaySessionGenerationIfOwned,
|
||||
disconnectStaleSharedGatewayAuthClients,
|
||||
finalizeOwnedSharedGatewaySessionGeneration,
|
||||
isSharedGatewaySessionGenerationOwnershipCurrent,
|
||||
replaceOwnedSharedGatewaySessionGenerationState,
|
||||
type SharedGatewayAuthClient,
|
||||
type SharedGatewaySessionGenerationOwnership,
|
||||
type SharedGatewaySessionGenerationState,
|
||||
} from "./server-shared-auth-generation.js";
|
||||
import type { ActivateRuntimeSecrets } from "./server-startup-config.js";
|
||||
createGatewaySecretsReloader,
|
||||
type GatewaySecretsReloaderParams,
|
||||
} from "./server-secrets-reload.js";
|
||||
import type { WorkerSessionTurnClaim } from "./worker-environments/placement-record.js";
|
||||
|
||||
type GatewayAuxHandlerLogger = {
|
||||
@@ -81,63 +59,19 @@ type GatewayAuxHandlerLogger = {
|
||||
debug?: (message: string) => void;
|
||||
};
|
||||
|
||||
type ReloadSecretsResult = {
|
||||
warningCount: number;
|
||||
};
|
||||
|
||||
async function activateSecretsRuntimeSnapshotIfCurrent(
|
||||
snapshot: PreparedSecretsRuntimeSnapshot,
|
||||
expectedRevision: number,
|
||||
options?: {
|
||||
canActivate?: () => boolean;
|
||||
onActivated?: () => void;
|
||||
},
|
||||
): Promise<number | null> {
|
||||
const runtime = await import("../secrets/runtime.js");
|
||||
if (options?.canActivate && !options.canActivate()) {
|
||||
return null;
|
||||
}
|
||||
if (!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)) {
|
||||
return null;
|
||||
}
|
||||
options?.onActivated?.();
|
||||
return runtime.getActiveSecretsRuntimeSnapshotRevision();
|
||||
}
|
||||
|
||||
async function restoreSecretsRuntimeSnapshotIfCurrent(
|
||||
snapshot: PreparedSecretsRuntimeSnapshot,
|
||||
expectedRevision: number,
|
||||
ownedSnapshot: PreparedSecretsRuntimeSnapshot,
|
||||
options?: { onActivated?: () => void },
|
||||
): Promise<number | null> {
|
||||
const runtime = await import("../secrets/runtime.js");
|
||||
if (!runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) {
|
||||
return null;
|
||||
}
|
||||
options?.onActivated?.();
|
||||
return runtime.getActiveSecretsRuntimeSnapshotRevision();
|
||||
}
|
||||
|
||||
/** Create auxiliary gateway handlers that are not part of the core descriptor set. */
|
||||
export function createGatewayAuxHandlers(params: {
|
||||
log: GatewayAuxHandlerLogger;
|
||||
activateRuntimeSecrets: ActivateRuntimeSecrets;
|
||||
buildReloadPlan?: (changedPaths: string[]) => GatewayReloadPlan;
|
||||
sharedGatewaySessionGenerationState: SharedGatewaySessionGenerationState;
|
||||
resolveSharedGatewaySessionGenerationForConfig: (config: OpenClawConfig) => string | undefined;
|
||||
clients: Iterable<SharedGatewayAuthClient>;
|
||||
startChannel: (name: ChannelKind) => Promise<void>;
|
||||
stopChannel: (name: ChannelKind) => Promise<void>;
|
||||
getChannelAutostartSuppression?: () => ChannelAutostartSuppression | null;
|
||||
logChannels: { info: (msg: string) => void };
|
||||
onApprovalLifecycle?: (event: OperatorApprovalLifecycleEvent) => void;
|
||||
onAgentRunAuthorityClosed?: (authority: AgentRunDelegatedAuthority) => void;
|
||||
validateAgentRuntimeDelegatedAuthority?: (authority: AgentRuntimeDelegatedAuthority) => boolean;
|
||||
chatAbortControllers?: Map<string, ChatAbortControllerEntry>;
|
||||
registerWorkerTurnClaimClosedHandler?: (
|
||||
handler: (claim: WorkerSessionTurnClaim) => void,
|
||||
) => () => void;
|
||||
}) {
|
||||
export function createGatewayAuxHandlers(
|
||||
params: GatewaySecretsReloaderParams & {
|
||||
log: GatewayAuxHandlerLogger;
|
||||
onApprovalLifecycle?: (event: OperatorApprovalLifecycleEvent) => void;
|
||||
onAgentRunAuthorityClosed?: (authority: AgentRunDelegatedAuthority) => void;
|
||||
validateAgentRuntimeDelegatedAuthority?: (authority: AgentRuntimeDelegatedAuthority) => boolean;
|
||||
chatAbortControllers?: Map<string, ChatAbortControllerEntry>;
|
||||
registerWorkerTurnClaimClosedHandler?: (
|
||||
handler: (claim: WorkerSessionTurnClaim) => void,
|
||||
) => () => void;
|
||||
},
|
||||
) {
|
||||
// Both approval kinds share one durable first-answer-wins registry and
|
||||
// Gateway-lifetime epoch while retaining separate in-process waiter maps.
|
||||
// A newly constructed Gateway cannot resume the prior lifetime's waiters.
|
||||
@@ -198,7 +132,6 @@ export function createGatewayAuxHandlers(params: {
|
||||
),
|
||||
{ cacheRejections: true },
|
||||
);
|
||||
const buildReloadPlan = params.buildReloadPlan ?? buildGatewayReloadPlan;
|
||||
const pluginApprovalManager = createApprovalManager<PluginApprovalRequestPayload>(
|
||||
"plugin",
|
||||
resolveCanonicalPluginApprovalRequestAllowedDecisions,
|
||||
@@ -344,305 +277,11 @@ export function createGatewayAuxHandlers(params: {
|
||||
),
|
||||
{ cacheRejections: true },
|
||||
);
|
||||
// Serialize the entire `secrets.reload` path (activation + channel restart)
|
||||
// so concurrent callers cannot overlap the stop/start loop and so the
|
||||
// "before" snapshot used for the reload-plan diff is always the snapshot
|
||||
// replaced by this call's activation, not one captured by a prior caller.
|
||||
let reloadInFlight: Promise<ReloadSecretsResult> | null = null;
|
||||
const runExclusiveReload = (
|
||||
fn: () => Promise<ReloadSecretsResult>,
|
||||
options: { joinInFlight?: boolean } = {},
|
||||
): Promise<ReloadSecretsResult> => {
|
||||
if (reloadInFlight) {
|
||||
if (options.joinInFlight !== false) {
|
||||
return reloadInFlight;
|
||||
}
|
||||
const precedingReload = reloadInFlight;
|
||||
return precedingReload.catch(() => undefined).then(() => runExclusiveReload(fn, options));
|
||||
}
|
||||
const run = (async () => {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
reloadInFlight = null;
|
||||
}
|
||||
})();
|
||||
reloadInFlight = run;
|
||||
return run;
|
||||
};
|
||||
const loadSecretsHandlers = createLazyPromise(
|
||||
() =>
|
||||
import("./server-methods/secrets.js").then(({ createSecretsHandlers }) =>
|
||||
createSecretsHandlers({
|
||||
reloadSecrets: (reloadOptions) =>
|
||||
runExclusiveReload(async () => {
|
||||
let transaction:
|
||||
| {
|
||||
previousSnapshot: PreparedSecretsRuntimeSnapshot;
|
||||
previousSharedGatewaySessionGeneration: string | undefined;
|
||||
previousSharedGatewaySessionGenerationRequired: string | undefined | null;
|
||||
prepared: PreparedSecretsRuntimeSnapshot;
|
||||
plan: GatewayReloadPlan;
|
||||
nextSharedGatewaySessionGeneration: string | undefined;
|
||||
sharedGatewaySessionGenerationChanged: boolean;
|
||||
generationOwnership: SharedGatewaySessionGenerationOwnership;
|
||||
publishedSnapshotRevision: number;
|
||||
}
|
||||
| undefined;
|
||||
const stoppedChannels: ChannelKind[] = [];
|
||||
const restartedChannels = new Set<ChannelKind>();
|
||||
try {
|
||||
for (;;) {
|
||||
const previousSnapshot = getActiveSecretsRuntimeSnapshotState();
|
||||
if (!previousSnapshot) {
|
||||
throw new Error("Secrets runtime snapshot is not active.");
|
||||
}
|
||||
const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState();
|
||||
const previousGenerationOwnership =
|
||||
captureSharedGatewaySessionGenerationOwnership(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
);
|
||||
// Snapshot both generation fields with the candidate revision.
|
||||
// A stale preparation retries all three owners together.
|
||||
const previousSharedGatewaySessionGeneration =
|
||||
previousGenerationOwnership.generation;
|
||||
const previousSharedGatewaySessionGenerationRequired =
|
||||
params.sharedGatewaySessionGenerationState.required;
|
||||
const prepared = await params.activateRuntimeSecrets(
|
||||
previousSnapshot.sourceConfig,
|
||||
{
|
||||
reason: "reload",
|
||||
activate: false,
|
||||
publishFailureAsDegraded: true,
|
||||
forceColdRefKeys: reloadOptions?.forceColdRefKeys,
|
||||
canPublishFailureAsDegraded: () =>
|
||||
getActiveSecretsRuntimeSnapshotRevisionState() === previousSnapshotRevision,
|
||||
},
|
||||
);
|
||||
const plan = buildReloadPlan(
|
||||
diffConfigPaths(previousSnapshot.config, prepared.config),
|
||||
);
|
||||
const nextSharedGatewaySessionGeneration =
|
||||
params.resolveSharedGatewaySessionGenerationForConfig(prepared.config);
|
||||
let publishedSnapshotRevision: number | null = null;
|
||||
let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null;
|
||||
const activateIfCurrent =
|
||||
params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent;
|
||||
if (activateIfCurrent) {
|
||||
const activated = await activateIfCurrent(
|
||||
prepared,
|
||||
previousSnapshotRevision,
|
||||
{
|
||||
reason: "reload",
|
||||
activate: true,
|
||||
},
|
||||
async () => {
|
||||
publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState();
|
||||
generationOwnership = claimSharedGatewaySessionGenerationIfOwned(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousGenerationOwnership,
|
||||
nextSharedGatewaySessionGeneration,
|
||||
);
|
||||
},
|
||||
() =>
|
||||
isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousGenerationOwnership,
|
||||
),
|
||||
);
|
||||
if (!activated) {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
publishedSnapshotRevision = await activateSecretsRuntimeSnapshotIfCurrent(
|
||||
prepared,
|
||||
previousSnapshotRevision,
|
||||
{
|
||||
canActivate: () =>
|
||||
isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousGenerationOwnership,
|
||||
),
|
||||
onActivated: () => {
|
||||
generationOwnership = claimSharedGatewaySessionGenerationIfOwned(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousGenerationOwnership,
|
||||
nextSharedGatewaySessionGeneration,
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
if (publishedSnapshotRevision === null) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (publishedSnapshotRevision === null || generationOwnership === null) {
|
||||
throw new Error("Secrets runtime activation did not publish ownership.");
|
||||
}
|
||||
transaction = {
|
||||
previousSnapshot,
|
||||
previousSharedGatewaySessionGeneration,
|
||||
previousSharedGatewaySessionGenerationRequired,
|
||||
prepared,
|
||||
plan,
|
||||
nextSharedGatewaySessionGeneration,
|
||||
sharedGatewaySessionGenerationChanged:
|
||||
previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration,
|
||||
generationOwnership,
|
||||
publishedSnapshotRevision,
|
||||
};
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
break;
|
||||
}
|
||||
const {
|
||||
prepared,
|
||||
plan,
|
||||
generationOwnership,
|
||||
nextSharedGatewaySessionGeneration,
|
||||
sharedGatewaySessionGenerationChanged,
|
||||
} = transaction;
|
||||
if (sharedGatewaySessionGenerationChanged) {
|
||||
disconnectStaleSharedGatewayAuthClients({
|
||||
clients: params.clients,
|
||||
expectedGeneration: nextSharedGatewaySessionGeneration,
|
||||
});
|
||||
}
|
||||
// Account-scoped changes restart their whole channel here:
|
||||
// secrets.reload has no per-account restart path, and a missed
|
||||
// restart would leave rotated credentials unapplied.
|
||||
const channelsToRestart = new Set<ChannelKind>([
|
||||
...plan.restartChannels,
|
||||
...(plan.restartChannelAccounts?.keys() ?? []),
|
||||
]);
|
||||
if (channelsToRestart.size > 0) {
|
||||
const restartChannels = [...channelsToRestart];
|
||||
if (
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) ||
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS)
|
||||
) {
|
||||
throw new Error(
|
||||
`secrets.reload requires restarting channels: ${restartChannels.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (params.getChannelAutostartSuppression?.()) {
|
||||
throw new Error(
|
||||
`secrets.reload requires restarting channels but channel autostart is suppressed by crash-loop breaker: ${restartChannels.join(", ")}`,
|
||||
);
|
||||
}
|
||||
const restartFailures: ChannelKind[] = [];
|
||||
for (const channel of restartChannels) {
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
params.logChannels.info(`restarting ${channel} channel after secrets reload`);
|
||||
// Track for rollback before awaiting stopChannel: if stopChannel
|
||||
// throws after partially stopping the channel, still attempt recovery.
|
||||
stoppedChannels.push(channel);
|
||||
try {
|
||||
await params.stopChannel(channel);
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
await params.startChannel(channel);
|
||||
restartedChannels.add(channel);
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
} catch {
|
||||
params.logChannels.info(
|
||||
`failed to restart ${channel} channel after secrets reload`,
|
||||
);
|
||||
restartFailures.push(channel);
|
||||
}
|
||||
}
|
||||
if (restartFailures.length > 0) {
|
||||
throw new Error(
|
||||
`failed to restart channels after secrets reload: ${restartFailures.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (
|
||||
!finalizeOwnedSharedGatewaySessionGeneration(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
return { warningCount: prepared.warnings.length };
|
||||
} catch (err) {
|
||||
let generationRestored = false;
|
||||
if (transaction) {
|
||||
const failedTransaction = transaction;
|
||||
await restoreSecretsRuntimeSnapshotIfCurrent(
|
||||
failedTransaction.previousSnapshot,
|
||||
failedTransaction.publishedSnapshotRevision,
|
||||
failedTransaction.prepared,
|
||||
{
|
||||
onActivated: () => {
|
||||
generationRestored = replaceOwnedSharedGatewaySessionGenerationState(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
failedTransaction.generationOwnership,
|
||||
{
|
||||
current: failedTransaction.previousSharedGatewaySessionGeneration,
|
||||
required:
|
||||
failedTransaction.previousSharedGatewaySessionGenerationRequired,
|
||||
},
|
||||
);
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
if (generationRestored && transaction) {
|
||||
if (transaction.sharedGatewaySessionGenerationChanged) {
|
||||
disconnectStaleSharedGatewayAuthClients({
|
||||
clients: params.clients,
|
||||
expectedGeneration: transaction.previousSharedGatewaySessionGeneration,
|
||||
});
|
||||
}
|
||||
}
|
||||
// Generation ownership fences state rollback, not liveness.
|
||||
// Restart stopped channels against whichever runtime is current now.
|
||||
for (const channel of stoppedChannels) {
|
||||
params.logChannels.info(
|
||||
`rolling back ${channel} channel after secrets reload failure`,
|
||||
);
|
||||
try {
|
||||
if (restartedChannels.has(channel)) {
|
||||
await params.stopChannel(channel);
|
||||
}
|
||||
await params.startChannel(channel);
|
||||
} catch {
|
||||
params.logChannels.info(
|
||||
`failed to roll back ${channel} channel after secrets reload`,
|
||||
);
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}, reloadOptions),
|
||||
reloadSecrets: createGatewaySecretsReloader(params),
|
||||
log: params.log,
|
||||
resolveSecrets: async ({
|
||||
allowedPaths,
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
/**
|
||||
* Server channel lifecycle tests.
|
||||
*/
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createDeferred } from "../../test/helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { ChannelIngressUnavailableError } from "../channels/message/ingress-unavailable.js";
|
||||
import type {
|
||||
ChannelAccountLinkState,
|
||||
@@ -15,6 +18,7 @@ import type {
|
||||
} from "../channels/plugins/types.public.js";
|
||||
import { formatGatewayChannelsStatusLines } from "../commands/channels/status.runtime.js";
|
||||
import type { GatewayNativeApprovalRuntime } from "../infra/approval-gateway-runtime.types.js";
|
||||
import { tryReadSecretFileSync } from "../infra/secret-file.js";
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
type SubsystemLogger,
|
||||
@@ -34,6 +38,7 @@ import {
|
||||
import { DEFAULT_ACCOUNT_ID } from "../routing/session-key.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import {
|
||||
clearActiveCredentialDegradedOwner,
|
||||
listActiveDegradedSecretOwners,
|
||||
setActiveDegradedSecretOwners,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
@@ -102,6 +107,7 @@ const CHANNEL_APPROVAL_GATEWAY_RUNTIME_CONTEXT_CAPABILITY = "approval.gateway";
|
||||
type ApprovalGatewayRequestRuntime = Pick<GatewayNativeApprovalRuntime, "request">;
|
||||
|
||||
const createdManagers: Array<{ manager: ChannelManager; channelIds: ChannelId[] }> = [];
|
||||
const channelTempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function healthOf(account: ChannelAccountSnapshot | undefined) {
|
||||
return evaluateChannelHealth(account ?? {}, {
|
||||
@@ -315,6 +321,9 @@ describe("server-channels auto restart", () => {
|
||||
hoisted.sleepWithAbort.mockClear();
|
||||
hoisted.startChannelApprovalHandlerBootstrap.mockReset();
|
||||
hoisted.startChannelApprovalHandlerBootstrap.mockResolvedValue(async () => {});
|
||||
for (const owner of listActiveDegradedSecretOwners()) {
|
||||
clearActiveCredentialDegradedOwner(owner.ownerKind, owner.ownerId);
|
||||
}
|
||||
setActiveDegradedSecretOwners([]);
|
||||
});
|
||||
|
||||
@@ -330,6 +339,9 @@ describe("server-channels auto restart", () => {
|
||||
vi.clearAllTimers();
|
||||
vi.useRealTimers();
|
||||
resetGatewayWorkAdmission();
|
||||
for (const owner of listActiveDegradedSecretOwners()) {
|
||||
clearActiveCredentialDegradedOwner(owner.ownerKind, owner.ownerId);
|
||||
}
|
||||
setActiveDegradedSecretOwners([]);
|
||||
setActivePluginRegistry(previousRegistry ?? createEmptyPluginRegistry());
|
||||
});
|
||||
@@ -2724,7 +2736,8 @@ describe("server-channels auto restart", () => {
|
||||
});
|
||||
|
||||
it("keeps one file-credential account cold and recovers it without restarting siblings", async () => {
|
||||
let broken = true;
|
||||
const credentialPath = path.join(channelTempDirs.make("openclaw-channel-credential-"), "token");
|
||||
const credentialConfigPath = "channels.telegram.accounts.broken.tokenFile";
|
||||
const startAccount = vi.fn(
|
||||
async ({ abortSignal }: ChannelGatewayContext<TestAccount>) =>
|
||||
await new Promise<void>((resolve) => {
|
||||
@@ -2733,56 +2746,70 @@ describe("server-channels auto restart", () => {
|
||||
);
|
||||
installTestRegistry(
|
||||
createTestPlugin({
|
||||
id: "discord",
|
||||
id: "telegram",
|
||||
listAccountIds: () => ["broken", "healthy"],
|
||||
resolveAccount: (_cfg, accountId) => ({
|
||||
enabled: true,
|
||||
configured: true,
|
||||
...(accountId === "broken" && broken
|
||||
? {
|
||||
credentialDiagnostics: [
|
||||
resolveAccount: (_cfg, accountId) => {
|
||||
const credential =
|
||||
accountId === "broken"
|
||||
? tryReadSecretFileSync(
|
||||
credentialPath,
|
||||
"Telegram bot token",
|
||||
{},
|
||||
{
|
||||
code: "CREDENTIAL_FILE_UNAVAILABLE" as const,
|
||||
path: "channels.discord.accounts.broken.tokenFile",
|
||||
reason: "not-found",
|
||||
configPath: credentialConfigPath,
|
||||
},
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
)
|
||||
: { status: "available" as const, value: "healthy-token" };
|
||||
return {
|
||||
enabled: true,
|
||||
configured: true,
|
||||
...(credential.status === "configured_unavailable"
|
||||
? { credentialDiagnostics: [credential.diagnostic] }
|
||||
: {}),
|
||||
};
|
||||
},
|
||||
startAccount,
|
||||
}),
|
||||
);
|
||||
const manager = createManager({ channelIds: ["discord"] });
|
||||
const manager = createManager({ channelIds: ["telegram"] });
|
||||
|
||||
await expect(manager.startChannels()).resolves.toBeUndefined();
|
||||
|
||||
expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual(["healthy"]);
|
||||
expect(manager.getRuntimeSnapshot().channelAccounts.discord?.broken).toMatchObject({
|
||||
expect(manager.getRuntimeSnapshot().channelAccounts.telegram?.broken).toMatchObject({
|
||||
configured: true,
|
||||
running: false,
|
||||
lastError:
|
||||
"Secret owner account:discord:broken is configured but unavailable (credential file is unavailable).",
|
||||
"Secret owner account:telegram:broken is configured but unavailable (credential file is unavailable).",
|
||||
});
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ownerId: "discord:broken",
|
||||
paths: ["channels.discord.accounts.broken.tokenFile"],
|
||||
ownerId: "telegram:broken",
|
||||
paths: [credentialConfigPath],
|
||||
refKeys: [],
|
||||
}),
|
||||
);
|
||||
|
||||
broken = false;
|
||||
await manager.startChannel("discord", "broken");
|
||||
await expect(manager.startChannel("telegram", "broken")).rejects.toMatchObject({
|
||||
code: "SECRET_SURFACE_UNAVAILABLE",
|
||||
ownerId: "telegram:broken",
|
||||
});
|
||||
expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual(["healthy"]);
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(
|
||||
expect.objectContaining({ ownerId: "telegram:broken" }),
|
||||
);
|
||||
|
||||
fs.writeFileSync(credentialPath, "repaired-token", { mode: 0o600 });
|
||||
await manager.startChannel("telegram", "broken");
|
||||
|
||||
expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([
|
||||
"healthy",
|
||||
"broken",
|
||||
]);
|
||||
expect(listActiveDegradedSecretOwners()).not.toContainEqual(
|
||||
expect.objectContaining({ ownerId: "discord:broken" }),
|
||||
expect.objectContaining({ ownerId: "telegram:broken" }),
|
||||
);
|
||||
await manager.stopChannel("discord");
|
||||
await manager.stopChannel("telegram");
|
||||
});
|
||||
|
||||
it("uses fallback logger and runtime when a channel is missing startup wiring", async () => {
|
||||
@@ -3023,6 +3050,76 @@ describe("server-channels auto restart", () => {
|
||||
await manager.stopChannel("discord");
|
||||
});
|
||||
|
||||
it("retires only the credential owner for an evicted channel account", async () => {
|
||||
let accountIds = ["removed", "retained"];
|
||||
installTestRegistry(
|
||||
createTestPlugin({
|
||||
listAccountIds: () => accountIds,
|
||||
startAccount: async () => {},
|
||||
resolveAccount: (_cfg, accountId) => ({
|
||||
enabled: true,
|
||||
configured: true,
|
||||
credentialDiagnostics: [
|
||||
{
|
||||
code: "CREDENTIAL_FILE_UNAVAILABLE" as const,
|
||||
path: `channels.discord.accounts.${accountId}.tokenFile`,
|
||||
reason: "not-found",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const manager = createManager();
|
||||
|
||||
await manager.startChannels();
|
||||
expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([
|
||||
"discord:removed",
|
||||
"discord:retained",
|
||||
]);
|
||||
|
||||
accountIds = ["retained"];
|
||||
await expect(manager.startChannel("discord")).rejects.toMatchObject({
|
||||
ownerId: "discord:retained",
|
||||
});
|
||||
|
||||
expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([
|
||||
"discord:retained",
|
||||
]);
|
||||
});
|
||||
|
||||
it("resolves only an unambiguous authoritative runtime account for a normalized owner", async () => {
|
||||
let accountIds = ["Ops Team"];
|
||||
installTestRegistry(
|
||||
createTestPlugin({
|
||||
id: "line",
|
||||
listAccountIds: () => accountIds,
|
||||
startAccount: async () => {},
|
||||
resolveAccount: (_cfg, accountId) => ({
|
||||
enabled: true,
|
||||
configured: true,
|
||||
credentialDiagnostics: [
|
||||
{
|
||||
code: "CREDENTIAL_FILE_UNAVAILABLE" as const,
|
||||
path: `channels.line.accounts.${accountId}.channelAccessTokenFile`,
|
||||
reason: "not-found",
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const manager = createManager({ channelIds: ["line"] });
|
||||
|
||||
await manager.startChannels();
|
||||
|
||||
expect(manager.resolveRuntimeAccountId("line", "ops-team")).toBe("Ops Team");
|
||||
expect(manager.resolveRuntimeAccountId("line", "missing")).toBeUndefined();
|
||||
|
||||
accountIds = ["Ops Team", "ops-team"];
|
||||
await manager.startChannels();
|
||||
|
||||
expect(manager.resolveRuntimeAccountId("line", "ops-team")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("reuses plugin account resolution for health monitor overrides", () => {
|
||||
installTestRegistry(
|
||||
createTestPlugin({
|
||||
|
||||
@@ -283,7 +283,9 @@ export type ChannelManager = {
|
||||
};
|
||||
|
||||
// Channel docking: lifecycle hooks (`plugin.gateway`) flow through this manager.
|
||||
export function createChannelManager(opts: ChannelManagerOptions): ChannelManager {
|
||||
export function createChannelManager(opts: ChannelManagerOptions): ChannelManager & {
|
||||
resolveRuntimeAccountId: (channelId: ChannelId, accountId: string) => string | undefined;
|
||||
} {
|
||||
const {
|
||||
getRuntimeConfig,
|
||||
channelLogs,
|
||||
@@ -490,6 +492,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
|
||||
continue;
|
||||
}
|
||||
store.runtimes.delete(id);
|
||||
clearActiveCredentialDegradedOwner("account", restartKey(channelId, normalizeAccountId(id)));
|
||||
store.pluginCommandCatalogOwners.delete(id);
|
||||
restarts.delete(restartKey(channelId, id));
|
||||
manuallyStopped.delete(restartKey(channelId, id));
|
||||
@@ -1429,6 +1432,12 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
|
||||
ambientAutostartSuppressedChannelIds.has(channelId),
|
||||
markChannelLoggedOut,
|
||||
isManuallyStopped: isManuallyStoppedFlag,
|
||||
resolveRuntimeAccountId: (channelId, accountId) => {
|
||||
const matches = [...(channelStores.get(channelId)?.runtimes.keys() ?? [])].filter(
|
||||
(id) => normalizeAccountId(id) === accountId,
|
||||
);
|
||||
return matches.length === 1 ? matches[0] : undefined;
|
||||
},
|
||||
isAutoRestartScheduled,
|
||||
resetRestartAttempts,
|
||||
isHealthMonitorEnabled,
|
||||
|
||||
@@ -127,8 +127,6 @@ export async function startGatewayCoreRuntime(input: {
|
||||
readinessEventLoopHealth,
|
||||
workerDispatchAuthority,
|
||||
clients,
|
||||
startChannel,
|
||||
stopChannel,
|
||||
sharedGatewaySessionGenerationState,
|
||||
resolveSharedGatewaySessionGenerationForConfig,
|
||||
sessionMessageSubscribers,
|
||||
@@ -339,8 +337,7 @@ export async function startGatewayCoreRuntime(input: {
|
||||
sharedGatewaySessionGenerationState,
|
||||
resolveSharedGatewaySessionGenerationForConfig,
|
||||
clients,
|
||||
startChannel,
|
||||
stopChannel,
|
||||
channelManager,
|
||||
getChannelAutostartSuppression: channelManager.getAutostartSuppression,
|
||||
logChannels,
|
||||
registerWorkerTurnClaimClosedHandler: workerEnvironmentStartup?.placementStore
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
// Additional credential-owner lifecycle cases registered in the existing auxiliary-handler suite.
|
||||
import { expect, it, vi } from "vitest";
|
||||
import {
|
||||
listActiveDegradedSecretOwners,
|
||||
SecretSurfaceUnavailableError,
|
||||
type DegradedSecretOwner,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
|
||||
export type CredentialReloadHarnessOptions = {
|
||||
ownerAccountId?: string;
|
||||
runtimeAccountId?: string;
|
||||
manualStop?: boolean;
|
||||
createFailure?: (owner: DegradedSecretOwner) => Error;
|
||||
};
|
||||
|
||||
type CredentialReloadHarness = {
|
||||
owner: DegradedSecretOwner;
|
||||
reload: () => Promise<void>;
|
||||
respond: ReturnType<typeof vi.fn>;
|
||||
startChannel: ReturnType<typeof vi.fn>;
|
||||
stopChannel: ReturnType<typeof vi.fn>;
|
||||
isManuallyStopped: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
|
||||
/** Registers focused account recovery cases against the existing RPC owner and shared fixture. */
|
||||
export function registerGatewaySecretCredentialReloadCases(
|
||||
createHarness: (options?: CredentialReloadHarnessOptions) => CredentialReloadHarness,
|
||||
): void {
|
||||
it("reinspects a previously degraded account without stopping healthy siblings when config is unchanged", async () => {
|
||||
const { reload, respond, startChannel, stopChannel } = createHarness();
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel).not.toHaveBeenCalled();
|
||||
expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]);
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it("accepts only the same trusted credential owner remaining degraded after reinspection", async () => {
|
||||
const { owner, reload, respond, startChannel, stopChannel } = createHarness({
|
||||
createFailure: (degradedOwner) => new SecretSurfaceUnavailableError(degradedOwner),
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel).not.toHaveBeenCalled();
|
||||
expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]);
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(expect.objectContaining(owner));
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it.each(["forged", "different-owner"] as const)(
|
||||
"rejects a %s unavailable error instead of treating account recovery as degraded success",
|
||||
async (failureKind) => {
|
||||
const { owner, reload, respond, stopChannel } = createHarness({
|
||||
createFailure: (degradedOwner) =>
|
||||
failureKind === "different-owner"
|
||||
? new SecretSurfaceUnavailableError({ ...degradedOwner, ownerId: "slack:other" })
|
||||
: Object.setPrototypeOf(
|
||||
Object.assign(new Error("forged unavailable error"), {
|
||||
name: "SecretSurfaceUnavailableError",
|
||||
code: "SECRET_SURFACE_UNAVAILABLE",
|
||||
ownerKind: "account",
|
||||
ownerId: degradedOwner.ownerId,
|
||||
}),
|
||||
SecretSurfaceUnavailableError.prototype,
|
||||
),
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(respond.mock.calls[0]?.[0]).toBe(false);
|
||||
expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]);
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(expect.objectContaining(owner));
|
||||
},
|
||||
);
|
||||
|
||||
it("does not reinspect a degraded account that an operator manually stopped", async () => {
|
||||
const { reload, respond, startChannel, stopChannel } = createHarness({ manualStop: true });
|
||||
|
||||
await reload();
|
||||
|
||||
expect(stopChannel).not.toHaveBeenCalled();
|
||||
expect(startChannel).not.toHaveBeenCalled();
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(
|
||||
expect.objectContaining({ ownerId: "slack:ops" }),
|
||||
);
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 });
|
||||
});
|
||||
|
||||
it("recovers the authoritative raw account identity instead of its normalized owner suffix", async () => {
|
||||
const { reload, startChannel, stopChannel, isManuallyStopped } = createHarness({
|
||||
ownerAccountId: "ops-team",
|
||||
runtimeAccountId: "Ops Team",
|
||||
});
|
||||
|
||||
await reload();
|
||||
|
||||
expect(isManuallyStopped).toHaveBeenCalledWith("slack", "Ops Team");
|
||||
expect(stopChannel).not.toHaveBeenCalled();
|
||||
expect(startChannel.mock.calls).toEqual([["slack", "Ops Team", { preserveManualStop: true }]]);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
// Owns serialized secrets snapshot replacement and exact channel-account lifecycle recovery.
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isTruthyEnvValue } from "../infra/env.js";
|
||||
import {
|
||||
isTrustedSecretSurfaceUnavailableError,
|
||||
listActiveCredentialDegradedOwners,
|
||||
type DegradedSecretOwner,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import {
|
||||
getActiveSecretsRuntimeSnapshotRevisionState,
|
||||
getActiveSecretsRuntimeSnapshotState,
|
||||
type PreparedSecretsRuntimeSnapshot,
|
||||
} from "../secrets/runtime-state.js";
|
||||
import { diffConfigPaths } from "./config-diff.js";
|
||||
import {
|
||||
buildGatewayReloadPlan,
|
||||
type ChannelKind,
|
||||
type GatewayReloadPlan,
|
||||
} from "./config-reload-plan.js";
|
||||
import type { ChannelAutostartSuppression, createChannelManager } from "./server-channels.js";
|
||||
import {
|
||||
captureSharedGatewaySessionGenerationOwnership,
|
||||
claimSharedGatewaySessionGenerationIfOwned,
|
||||
disconnectStaleSharedGatewayAuthClients,
|
||||
finalizeOwnedSharedGatewaySessionGeneration,
|
||||
isSharedGatewaySessionGenerationOwnershipCurrent,
|
||||
replaceOwnedSharedGatewaySessionGenerationState,
|
||||
type SharedGatewayAuthClient,
|
||||
type SharedGatewaySessionGenerationOwnership,
|
||||
type SharedGatewaySessionGenerationState,
|
||||
} from "./server-shared-auth-generation.js";
|
||||
import type { ActivateRuntimeSecrets } from "./server-startup-config.js";
|
||||
|
||||
type ReloadSecretsResult = { warningCount: number };
|
||||
type ReloadSecretsOptions = { forceColdRefKeys?: ReadonlySet<string>; joinInFlight?: boolean };
|
||||
type ReloadChannelTarget = {
|
||||
channel: ChannelKind;
|
||||
accountId?: string;
|
||||
credentialOwnerId?: string;
|
||||
inspectOnly?: boolean;
|
||||
};
|
||||
|
||||
export type GatewaySecretsReloaderParams = {
|
||||
activateRuntimeSecrets: ActivateRuntimeSecrets;
|
||||
buildReloadPlan?: (changedPaths: string[]) => GatewayReloadPlan;
|
||||
sharedGatewaySessionGenerationState: SharedGatewaySessionGenerationState;
|
||||
resolveSharedGatewaySessionGenerationForConfig: (config: OpenClawConfig) => string | undefined;
|
||||
clients: Iterable<SharedGatewayAuthClient>;
|
||||
channelManager: Pick<
|
||||
ReturnType<typeof createChannelManager>,
|
||||
"startChannel" | "stopChannel" | "isManuallyStopped" | "resolveRuntimeAccountId"
|
||||
>;
|
||||
getChannelAutostartSuppression?: () => ChannelAutostartSuppression | null;
|
||||
logChannels: { info: (message: string) => void };
|
||||
};
|
||||
|
||||
async function activateSnapshotIfCurrent(
|
||||
snapshot: PreparedSecretsRuntimeSnapshot,
|
||||
expectedRevision: number,
|
||||
options: { canActivate: () => boolean; onActivated: () => void },
|
||||
): Promise<number | null> {
|
||||
const runtime = await import("../secrets/runtime.js");
|
||||
if (
|
||||
!options.canActivate() ||
|
||||
!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
options.onActivated();
|
||||
return runtime.getActiveSecretsRuntimeSnapshotRevision();
|
||||
}
|
||||
|
||||
async function restoreSnapshotIfCurrent(
|
||||
snapshot: PreparedSecretsRuntimeSnapshot,
|
||||
expectedRevision: number,
|
||||
ownedSnapshot: PreparedSecretsRuntimeSnapshot,
|
||||
onActivated: () => void,
|
||||
): Promise<void> {
|
||||
const runtime = await import("../secrets/runtime.js");
|
||||
if (runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) {
|
||||
onActivated();
|
||||
}
|
||||
}
|
||||
|
||||
/** Keeps snapshot CAS, generation ownership, and exact account recovery in one transaction. */
|
||||
export function createGatewaySecretsReloader(params: GatewaySecretsReloaderParams) {
|
||||
const buildReloadPlan = params.buildReloadPlan ?? buildGatewayReloadPlan;
|
||||
const manager = params.channelManager;
|
||||
let reloadInFlight: Promise<ReloadSecretsResult> | null = null;
|
||||
const runExclusiveReload = (
|
||||
fn: () => Promise<ReloadSecretsResult>,
|
||||
options: ReloadSecretsOptions = {},
|
||||
): Promise<ReloadSecretsResult> => {
|
||||
if (reloadInFlight) {
|
||||
return options.joinInFlight === false
|
||||
? reloadInFlight.catch(() => undefined).then(() => runExclusiveReload(fn, options))
|
||||
: reloadInFlight;
|
||||
}
|
||||
const run = (async () => {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
reloadInFlight = null;
|
||||
}
|
||||
})();
|
||||
reloadInFlight = run;
|
||||
return run;
|
||||
};
|
||||
|
||||
return (reloadOptions?: ReloadSecretsOptions) =>
|
||||
runExclusiveReload(async () => {
|
||||
let transaction:
|
||||
| {
|
||||
previousSnapshot: PreparedSecretsRuntimeSnapshot;
|
||||
previousGeneration: string | undefined;
|
||||
previousRequiredGeneration: string | undefined | null;
|
||||
prepared: PreparedSecretsRuntimeSnapshot;
|
||||
plan: GatewayReloadPlan;
|
||||
credentialOwners: DegradedSecretOwner[];
|
||||
nextGeneration: string | undefined;
|
||||
generationChanged: boolean;
|
||||
generationOwnership: SharedGatewaySessionGenerationOwnership;
|
||||
publishedSnapshotRevision: number;
|
||||
}
|
||||
| undefined;
|
||||
const touchedTargets: Array<{ target: ReloadChannelTarget; restarted: boolean }> = [];
|
||||
const startTarget = ({ channel, accountId }: ReloadChannelTarget) =>
|
||||
accountId
|
||||
? manager.startChannel(channel, accountId, { preserveManualStop: true })
|
||||
: manager.startChannel(channel);
|
||||
const stopTarget = ({ channel, accountId }: ReloadChannelTarget) =>
|
||||
accountId
|
||||
? manager.stopChannel(channel, accountId, { manual: false })
|
||||
: manager.stopChannel(channel);
|
||||
|
||||
try {
|
||||
for (;;) {
|
||||
const previousSnapshot = getActiveSecretsRuntimeSnapshotState();
|
||||
if (!previousSnapshot) {
|
||||
throw new Error("Secrets runtime snapshot is not active.");
|
||||
}
|
||||
const previousRevision = getActiveSecretsRuntimeSnapshotRevisionState();
|
||||
const previousOwnership = captureSharedGatewaySessionGenerationOwnership(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
);
|
||||
const previousGeneration = previousOwnership.generation;
|
||||
const previousRequiredGeneration = params.sharedGatewaySessionGenerationState.required;
|
||||
const prepared = await params.activateRuntimeSecrets(previousSnapshot.sourceConfig, {
|
||||
reason: "reload",
|
||||
activate: false,
|
||||
publishFailureAsDegraded: true,
|
||||
forceColdRefKeys: reloadOptions?.forceColdRefKeys,
|
||||
canPublishFailureAsDegraded: () =>
|
||||
getActiveSecretsRuntimeSnapshotRevisionState() === previousRevision,
|
||||
});
|
||||
const plan = buildReloadPlan(diffConfigPaths(previousSnapshot.config, prepared.config));
|
||||
const nextGeneration = params.resolveSharedGatewaySessionGenerationForConfig(
|
||||
prepared.config,
|
||||
);
|
||||
// File diagnostics have channel-owned lifetimes; capture each CAS attempt
|
||||
// immediately before publication so a superseded attempt cannot reuse owners.
|
||||
const credentialOwners = listActiveCredentialDegradedOwners();
|
||||
let publishedSnapshotRevision: number | null = null;
|
||||
let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null;
|
||||
const claimGeneration = () => {
|
||||
publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState();
|
||||
generationOwnership = claimSharedGatewaySessionGenerationIfOwned(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousOwnership,
|
||||
nextGeneration,
|
||||
);
|
||||
};
|
||||
const ownsPreviousGeneration = () =>
|
||||
isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
previousOwnership,
|
||||
);
|
||||
const activateIfCurrent = params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent;
|
||||
if (activateIfCurrent) {
|
||||
const activated = await activateIfCurrent(
|
||||
prepared,
|
||||
previousRevision,
|
||||
{ reason: "reload", activate: true },
|
||||
claimGeneration,
|
||||
ownsPreviousGeneration,
|
||||
);
|
||||
if (!activated) {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
publishedSnapshotRevision = await activateSnapshotIfCurrent(
|
||||
prepared,
|
||||
previousRevision,
|
||||
{
|
||||
canActivate: ownsPreviousGeneration,
|
||||
onActivated: claimGeneration,
|
||||
},
|
||||
);
|
||||
if (publishedSnapshotRevision === null) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (publishedSnapshotRevision === null || generationOwnership === null) {
|
||||
throw new Error("Secrets runtime activation did not publish ownership.");
|
||||
}
|
||||
transaction = {
|
||||
previousSnapshot,
|
||||
previousGeneration,
|
||||
previousRequiredGeneration,
|
||||
prepared,
|
||||
plan,
|
||||
credentialOwners,
|
||||
nextGeneration,
|
||||
generationChanged: previousGeneration !== nextGeneration,
|
||||
generationOwnership,
|
||||
publishedSnapshotRevision,
|
||||
};
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
const { prepared, plan, credentialOwners, generationOwnership, nextGeneration } =
|
||||
transaction;
|
||||
if (transaction.generationChanged) {
|
||||
disconnectStaleSharedGatewayAuthClients({
|
||||
clients: params.clients,
|
||||
expectedGeneration: nextGeneration,
|
||||
});
|
||||
}
|
||||
const targets: ReloadChannelTarget[] = [...plan.restartChannels].map((channel) => ({
|
||||
channel,
|
||||
}));
|
||||
const accountTargets = new Map<string, ReloadChannelTarget>();
|
||||
for (const [channel, accountIds] of plan.restartChannelAccounts ?? []) {
|
||||
if (plan.restartChannels.has(channel)) {
|
||||
continue;
|
||||
}
|
||||
for (const accountId of accountIds) {
|
||||
const target = { channel, accountId };
|
||||
accountTargets.set(`${channel}\0${accountId}`, target);
|
||||
targets.push(target);
|
||||
}
|
||||
}
|
||||
for (const owner of credentialOwners) {
|
||||
if (owner.ownerKind !== "account") {
|
||||
continue;
|
||||
}
|
||||
const separator = owner.ownerId.indexOf(":");
|
||||
if (separator < 0) {
|
||||
continue;
|
||||
}
|
||||
const channel: ChannelKind = owner.ownerId.slice(0, separator);
|
||||
if (plan.restartChannels.has(channel)) {
|
||||
continue;
|
||||
}
|
||||
const accountId = manager.resolveRuntimeAccountId(
|
||||
channel,
|
||||
owner.ownerId.slice(separator + 1),
|
||||
);
|
||||
if (!accountId || manager.isManuallyStopped(channel, accountId)) {
|
||||
continue;
|
||||
}
|
||||
const key = `${channel}\0${accountId}`;
|
||||
const existing = accountTargets.get(key);
|
||||
if (existing) {
|
||||
existing.credentialOwnerId = owner.ownerId;
|
||||
continue;
|
||||
}
|
||||
const target = {
|
||||
channel,
|
||||
accountId,
|
||||
credentialOwnerId: owner.ownerId,
|
||||
inspectOnly: true,
|
||||
};
|
||||
accountTargets.set(key, target);
|
||||
targets.push(target);
|
||||
}
|
||||
const restartTargets = targets.filter(
|
||||
({ channel, accountId }) => !accountId || !manager.isManuallyStopped(channel, accountId),
|
||||
);
|
||||
if (restartTargets.length > 0) {
|
||||
const restartChannels = [...new Set(restartTargets.map(({ channel }) => channel))];
|
||||
if (
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) ||
|
||||
isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS)
|
||||
) {
|
||||
throw new Error(
|
||||
`secrets.reload requires restarting channels: ${restartChannels.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (params.getChannelAutostartSuppression?.()) {
|
||||
throw new Error(
|
||||
`secrets.reload requires restarting channels but channel autostart is suppressed by crash-loop breaker: ${restartChannels.join(", ")}`,
|
||||
);
|
||||
}
|
||||
const failures: string[] = [];
|
||||
for (const target of restartTargets) {
|
||||
const { channel, accountId, credentialOwnerId, inspectOnly } = target;
|
||||
const label = accountId ? `${channel} account ${accountId}` : `${channel} channel`;
|
||||
const assertGenerationOwned = () => {
|
||||
if (
|
||||
!isSharedGatewaySessionGenerationOwnershipCurrent(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
};
|
||||
assertGenerationOwned();
|
||||
params.logChannels.info(
|
||||
`${inspectOnly ? "reinspecting" : "restarting"} ${label} after secrets reload`,
|
||||
);
|
||||
// A rejecting hook may have already changed its exact account lifetime.
|
||||
const touched = { target, restarted: false };
|
||||
touchedTargets.push(touched);
|
||||
try {
|
||||
if (!inspectOnly) {
|
||||
await stopTarget(target);
|
||||
assertGenerationOwned();
|
||||
}
|
||||
await startTarget(target);
|
||||
touched.restarted = true;
|
||||
assertGenerationOwned();
|
||||
} catch (error) {
|
||||
if (
|
||||
credentialOwnerId &&
|
||||
isTrustedSecretSurfaceUnavailableError(error) &&
|
||||
error.ownerKind === "account" &&
|
||||
error.ownerId === credentialOwnerId &&
|
||||
listActiveCredentialDegradedOwners().some(
|
||||
(owner) => owner.ownerKind === "account" && owner.ownerId === credentialOwnerId,
|
||||
)
|
||||
) {
|
||||
touchedTargets.pop();
|
||||
continue;
|
||||
}
|
||||
params.logChannels.info(`failed to restart ${label} after secrets reload`);
|
||||
failures.push(accountId ? `${channel}:${accountId}` : channel);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
throw new Error(
|
||||
`failed to restart channels after secrets reload: ${failures.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (
|
||||
!finalizeOwnedSharedGatewaySessionGeneration(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
generationOwnership,
|
||||
)
|
||||
) {
|
||||
throw new Error("secrets.reload was superseded by a newer config write");
|
||||
}
|
||||
return { warningCount: prepared.warnings.length };
|
||||
} catch (error) {
|
||||
let generationRestored = false;
|
||||
if (transaction) {
|
||||
const failedTransaction = transaction;
|
||||
await restoreSnapshotIfCurrent(
|
||||
failedTransaction.previousSnapshot,
|
||||
failedTransaction.publishedSnapshotRevision,
|
||||
failedTransaction.prepared,
|
||||
() => {
|
||||
generationRestored = replaceOwnedSharedGatewaySessionGenerationState(
|
||||
params.sharedGatewaySessionGenerationState,
|
||||
failedTransaction.generationOwnership,
|
||||
{
|
||||
current: failedTransaction.previousGeneration,
|
||||
required: failedTransaction.previousRequiredGeneration,
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
if (generationRestored && transaction?.generationChanged) {
|
||||
disconnectStaleSharedGatewayAuthClients({
|
||||
clients: params.clients,
|
||||
expectedGeneration: transaction.previousGeneration,
|
||||
});
|
||||
}
|
||||
// Generation fences snapshot rollback, never exact-account liveness recovery.
|
||||
for (const { target, restarted } of touchedTargets) {
|
||||
const { channel, accountId, inspectOnly } = target;
|
||||
const label = accountId ? `${channel} account ${accountId}` : `${channel} channel`;
|
||||
params.logChannels.info(`rolling back ${label} after secrets reload failure`);
|
||||
try {
|
||||
if (restarted || inspectOnly) {
|
||||
await stopTarget(target);
|
||||
}
|
||||
if (!inspectOnly) {
|
||||
await startTarget(target);
|
||||
}
|
||||
} catch {
|
||||
params.logChannels.info(`failed to roll back ${label} after secrets reload`);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}, reloadOptions);
|
||||
}
|
||||
@@ -10,7 +10,10 @@ import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
|
||||
import { resolveMemorySearchConfig } from "../agents/memory-search.js";
|
||||
import { resolveApiKeyForProviderCore } from "../agents/model-auth.js";
|
||||
import { resolveSandboxContext } from "../agents/sandbox/context.js";
|
||||
import type { ChannelGatewayContext } from "../channels/plugins/types.adapters.js";
|
||||
import type { ChannelAccountSnapshot, ChannelPlugin } from "../channels/plugins/types.public.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { tryReadSecretFileSync } from "../infra/secret-file.js";
|
||||
import { selectAgentSystemEvents } from "../infra/system-event-ownership.js";
|
||||
import {
|
||||
peekSystemEventEntries,
|
||||
@@ -18,14 +21,19 @@ import {
|
||||
resetSystemEventsForTest,
|
||||
} from "../infra/system-events.js";
|
||||
import { resolveAuthProfileSecretOwnerId } from "../secrets/runtime-auth-profile-owner.js";
|
||||
import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js";
|
||||
import {
|
||||
listActiveDegradedSecretOwners,
|
||||
setActiveDegradedSecretOwners,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import { getActiveSecretsRuntimeSnapshot } from "../secrets/runtime.js";
|
||||
import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js";
|
||||
import { deleteTestEnvValue, withEnvAsync } from "../test-utils/env.js";
|
||||
import {
|
||||
connectWebchatClient,
|
||||
getGatewayTestPort,
|
||||
installGatewayTestHooks,
|
||||
rpcReq,
|
||||
setTestPluginRegistry,
|
||||
startTestGatewayServer,
|
||||
testState,
|
||||
} from "./test-helpers.js";
|
||||
@@ -182,6 +190,192 @@ describe("Gateway startup SecretRef owner isolation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("recovers only a repaired credential-file account through secrets.reload without restarting sibling accounts", async () => {
|
||||
await withEnvAsync(
|
||||
{ OPENCLAW_SKIP_CHANNELS: undefined, OPENCLAW_SKIP_PROVIDERS: undefined },
|
||||
async () => {
|
||||
const credentialPath = path.join(tempDirs.make("openclaw-gateway-credential-"), "token");
|
||||
const credentialConfigPath = "channels.telegram.accounts.broken.tokenFile";
|
||||
const repairedToken = "repaired-test-token-never-public";
|
||||
type TestAccount = {
|
||||
accountId: string;
|
||||
token?: string;
|
||||
credentialDiagnostics?: Array<{
|
||||
code: "CREDENTIAL_FILE_UNAVAILABLE";
|
||||
path: string;
|
||||
reason: string;
|
||||
}>;
|
||||
};
|
||||
const startAccount = vi.fn(
|
||||
async ({ abortSignal }: ChannelGatewayContext<TestAccount>) =>
|
||||
await new Promise<void>((resolve) => {
|
||||
abortSignal.addEventListener("abort", () => resolve(), { once: true });
|
||||
}),
|
||||
);
|
||||
const plugin: ChannelPlugin<TestAccount> = {
|
||||
...createChannelTestPluginBase({ id: "telegram" }),
|
||||
config: {
|
||||
listAccountIds: (config) => Object.keys(config.channels?.telegram?.accounts ?? {}),
|
||||
resolveAccount: (config, accountId) => {
|
||||
if (!accountId) {
|
||||
throw new Error("Missing Telegram test account id");
|
||||
}
|
||||
const configured = config.channels?.telegram?.accounts?.[accountId];
|
||||
if (!configured) {
|
||||
throw new Error(`Missing Telegram test account ${accountId}`);
|
||||
}
|
||||
const credential = configured.tokenFile
|
||||
? tryReadSecretFileSync(
|
||||
configured.tokenFile,
|
||||
"Telegram bot token",
|
||||
{},
|
||||
{
|
||||
configPath: credentialConfigPath,
|
||||
},
|
||||
)
|
||||
: { status: "available" as const, value: configured.botToken };
|
||||
return {
|
||||
accountId,
|
||||
...(credential.status === "configured_unavailable"
|
||||
? { credentialDiagnostics: [credential.diagnostic] }
|
||||
: { token: typeof credential.value === "string" ? credential.value : undefined }),
|
||||
};
|
||||
},
|
||||
},
|
||||
gateway: { startAccount },
|
||||
};
|
||||
setTestPluginRegistry(
|
||||
createTestRegistry([{ pluginId: "telegram", source: "test", plugin }]),
|
||||
);
|
||||
await writeConfig({
|
||||
...baseConfig(),
|
||||
gateway: { ...baseConfig().gateway, reload: { mode: "off" } },
|
||||
channels: {
|
||||
telegram: {
|
||||
enabled: true,
|
||||
healthMonitor: { enabled: false },
|
||||
accounts: {
|
||||
broken: { tokenFile: credentialPath },
|
||||
healthy: { botToken: "healthy-test-token" },
|
||||
stopped: { botToken: "stopped-test-token" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
if (!configPath) {
|
||||
throw new Error("Gateway test did not configure a config file path");
|
||||
}
|
||||
const originalConfig = readFileSync(configPath);
|
||||
const port = await getGatewayTestPort();
|
||||
server = await startTestGatewayServer(port, { auth: { mode: "none" } });
|
||||
const ws = await connectWebchatClient({ port, scopes: ["operator.admin"] });
|
||||
try {
|
||||
const brokenStart = await rpcReq(ws, "channels.start", {
|
||||
channel: "telegram",
|
||||
accountId: "broken",
|
||||
});
|
||||
expect(brokenStart.ok).toBe(false);
|
||||
for (const accountId of ["healthy", "stopped"]) {
|
||||
const started = await rpcReq<{ accountId: string; started: boolean }>(
|
||||
ws,
|
||||
"channels.start",
|
||||
{ channel: "telegram", accountId },
|
||||
);
|
||||
expect(started.ok, JSON.stringify(started)).toBe(true);
|
||||
expect(started.payload).toMatchObject({ accountId, started: true });
|
||||
}
|
||||
expect(startAccount).toHaveBeenCalledTimes(2);
|
||||
expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([
|
||||
"healthy",
|
||||
"stopped",
|
||||
]);
|
||||
const healthyLifetime = startAccount.mock.calls[0]?.[0].abortSignal;
|
||||
const stoppedLifetime = startAccount.mock.calls[1]?.[0].abortSignal;
|
||||
expect(healthyLifetime?.aborted).toBe(false);
|
||||
expect(stoppedLifetime?.aborted).toBe(false);
|
||||
|
||||
const before = await rpcReq<{
|
||||
channelAccounts: Record<string, ChannelAccountSnapshot[]>;
|
||||
}>(ws, "channels.status", { probe: false });
|
||||
expect(before.ok, JSON.stringify(before)).toBe(true);
|
||||
const initialAccounts = before.payload?.channelAccounts.telegram ?? [];
|
||||
expect(initialAccounts).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ accountId: "broken", configured: true, running: false }),
|
||||
expect.objectContaining({ accountId: "healthy", running: true }),
|
||||
expect.objectContaining({ accountId: "stopped", running: true }),
|
||||
]),
|
||||
);
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ownerId: "telegram:broken",
|
||||
paths: [credentialConfigPath],
|
||||
reason: "credential file is unavailable",
|
||||
}),
|
||||
);
|
||||
expect(brokenStart.error).toMatchObject({ code: "UNAVAILABLE" });
|
||||
const publicDiagnostics = JSON.stringify({
|
||||
error: brokenStart.error,
|
||||
accounts: initialAccounts,
|
||||
});
|
||||
expect(publicDiagnostics).not.toContain(credentialPath);
|
||||
expect(publicDiagnostics).not.toContain(repairedToken);
|
||||
|
||||
const stopped = await rpcReq<{ accountId: string; stopped: boolean }>(
|
||||
ws,
|
||||
"channels.stop",
|
||||
{
|
||||
channel: "telegram",
|
||||
accountId: "stopped",
|
||||
},
|
||||
);
|
||||
expect(stopped.ok, JSON.stringify(stopped)).toBe(true);
|
||||
expect(stopped.payload).toMatchObject({ accountId: "stopped", stopped: true });
|
||||
expect(stoppedLifetime?.aborted).toBe(true);
|
||||
expect(healthyLifetime?.aborted).toBe(false);
|
||||
|
||||
writeFileSync(credentialPath, repairedToken, { mode: 0o600 });
|
||||
expect(readFileSync(configPath)).toEqual(originalConfig);
|
||||
|
||||
const reload = await rpcReq<{ warningCount: number }>(ws, "secrets.reload", {});
|
||||
expect(reload.ok, JSON.stringify(reload)).toBe(true);
|
||||
expect(reload.payload).toMatchObject({ warningCount: 0 });
|
||||
expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([
|
||||
"healthy",
|
||||
"stopped",
|
||||
"broken",
|
||||
]);
|
||||
expect(startAccount.mock.calls[2]?.[0].account.token).toBe(repairedToken);
|
||||
expect(healthyLifetime?.aborted).toBe(false);
|
||||
expect(stoppedLifetime?.aborted).toBe(true);
|
||||
expect(listActiveDegradedSecretOwners()).not.toContainEqual(
|
||||
expect.objectContaining({ ownerId: "telegram:broken" }),
|
||||
);
|
||||
|
||||
const after = await rpcReq<{ channelAccounts: Record<string, ChannelAccountSnapshot[]> }>(
|
||||
ws,
|
||||
"channels.status",
|
||||
{ probe: false },
|
||||
);
|
||||
expect(after.ok, JSON.stringify(after)).toBe(true);
|
||||
expect(after.payload?.channelAccounts.telegram).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ accountId: "broken", running: true }),
|
||||
expect.objectContaining({ accountId: "healthy", running: true }),
|
||||
expect.objectContaining({ accountId: "stopped", running: false }),
|
||||
]),
|
||||
);
|
||||
expect(JSON.stringify(after.payload)).not.toContain(repairedToken);
|
||||
expect(JSON.stringify(after.payload)).not.toContain(credentialPath);
|
||||
expect(readFileSync(configPath)).toEqual(originalConfig);
|
||||
} finally {
|
||||
ws.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("reaches /readyz while isolating every optional owner family", async () => {
|
||||
await withEnvAsync(
|
||||
{
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
} from "./runtime-degraded-state.js";
|
||||
|
||||
afterEach(() => {
|
||||
clearActiveCredentialDegradedOwner("account", "telegram:work");
|
||||
setActiveDegradedSecretOwners([]);
|
||||
});
|
||||
|
||||
@@ -131,6 +132,27 @@ describe("runtime degraded SecretRef owners", () => {
|
||||
"telegram:work",
|
||||
]);
|
||||
|
||||
setActiveDegradedSecretOwners([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
state: "unavailable",
|
||||
degradationState: "stale",
|
||||
paths: ["models.providers.openai.apiKey"],
|
||||
refKeys: ["env:default:OPENAI_API_KEY"],
|
||||
reason: "secret provider failed",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([
|
||||
"openai",
|
||||
"telegram:work",
|
||||
]);
|
||||
expect(() => assertSecretOwnerAvailable("provider", "openai")).not.toThrow();
|
||||
expect(() => assertSecretOwnerAvailable("account", "telegram:work")).toThrow(
|
||||
SecretSurfaceUnavailableError,
|
||||
);
|
||||
|
||||
clearActiveCredentialDegradedOwner("account", "telegram:work");
|
||||
|
||||
expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual(["openai"]);
|
||||
|
||||
@@ -182,14 +182,23 @@ function cloneResolutionErrorOwner(owner: SecretResolutionErrorOwner): SecretRes
|
||||
/** Publishes the degraded-owner snapshot at the same edge as runtime config activation. */
|
||||
export function setActiveDegradedSecretOwners(owners: readonly DegradedSecretOwner[]): void {
|
||||
activeDegradedOwners = owners.map(cloneOwner);
|
||||
activeCredentialDegradedOwners.clear();
|
||||
}
|
||||
|
||||
/** Publishes or clears one runtime-discovered channel credential owner. */
|
||||
/** Publishes one runtime-discovered channel credential owner. */
|
||||
export function setActiveCredentialDegradedOwner(owner: DegradedSecretOwner): void {
|
||||
activeCredentialDegradedOwners.set(ownerKey(owner.ownerKind, owner.ownerId), cloneOwner(owner));
|
||||
}
|
||||
|
||||
/** Lists credential owners independently of the replaceable SecretRef snapshot. */
|
||||
export function listActiveCredentialDegradedOwners(): DegradedSecretOwner[] {
|
||||
return Array.from(activeCredentialDegradedOwners.values(), cloneOwner);
|
||||
}
|
||||
|
||||
/** Clears credential-owner state only when its owning secrets runtime is torn down. */
|
||||
export function clearActiveCredentialDegradedOwners(): void {
|
||||
activeCredentialDegradedOwners.clear();
|
||||
}
|
||||
|
||||
/** Clears one runtime-discovered channel credential owner before re-inspection. */
|
||||
export function clearActiveCredentialDegradedOwner(
|
||||
ownerKind: DegradedSecretOwner["ownerKind"],
|
||||
@@ -200,10 +209,7 @@ export function clearActiveCredentialDegradedOwner(
|
||||
|
||||
/** Returns the active degraded-owner snapshot without exposing mutable registry state. */
|
||||
export function listActiveDegradedSecretOwners(): DegradedSecretOwner[] {
|
||||
return [
|
||||
...activeDegradedOwners.map(cloneOwner),
|
||||
...Array.from(activeCredentialDegradedOwners.values(), cloneOwner),
|
||||
];
|
||||
return [...activeDegradedOwners.map(cloneOwner), ...listActiveCredentialDegradedOwners()];
|
||||
}
|
||||
|
||||
/** Associates a strict activation failure with the owners it prevented from refreshing. */
|
||||
|
||||
@@ -7,6 +7,10 @@ import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { createEmptyPluginRegistry } from "../plugins/registry-empty.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import {
|
||||
listActiveDegradedSecretOwners,
|
||||
setActiveCredentialDegradedOwner,
|
||||
} from "./runtime-degraded-state.js";
|
||||
import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.ts";
|
||||
|
||||
function createOpenAiFileModelsConfig(): NonNullable<OpenClawConfig["models"]> {
|
||||
@@ -208,6 +212,14 @@ describe("secrets runtime provider and media surfaces", () => {
|
||||
const { getRuntimeConfigSourceSnapshot, getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } =
|
||||
await import("../config/runtime-snapshot.js");
|
||||
activateSecretsRuntimeSnapshot(initial);
|
||||
setActiveCredentialDegradedOwner({
|
||||
ownerKind: "account",
|
||||
ownerId: "telegram:work",
|
||||
state: "unavailable",
|
||||
paths: ["channels.telegram.accounts.work.tokenFile"],
|
||||
refKeys: [],
|
||||
reason: "credential file is unavailable",
|
||||
});
|
||||
const runtimeSourceConfig: OpenClawConfig = {
|
||||
...initial.sourceConfig,
|
||||
logging: { level: "debug" },
|
||||
@@ -241,6 +253,9 @@ describe("secrets runtime provider and media surfaces", () => {
|
||||
expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new");
|
||||
expect(getRuntimeConfigSnapshot()).toEqual(active?.config);
|
||||
expect(getRuntimeConfigSourceSnapshot()).toEqual(runtimeSourceConfig);
|
||||
expect(listActiveDegradedSecretOwners()).toContainEqual(
|
||||
expect.objectContaining({ ownerKind: "account", ownerId: "telegram:work" }),
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -26,6 +26,10 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { SecretRef } from "../config/types.secrets.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import {
|
||||
listActiveDegradedSecretOwners,
|
||||
setActiveCredentialDegradedOwner,
|
||||
} from "./runtime-degraded-state.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshotState,
|
||||
activateSecretsRuntimeSnapshotStateIfCurrent,
|
||||
@@ -216,6 +220,47 @@ describe("secrets runtime state", () => {
|
||||
expect(configSnapshot?.sourceConfig).toEqual(snapshot.sourceConfig);
|
||||
});
|
||||
|
||||
it("preserves independent credential owners through snapshot replacement and rollback until teardown", () => {
|
||||
const previous = preparedSnapshot({
|
||||
degradedOwners: [
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
state: "unavailable",
|
||||
degradationState: "stale",
|
||||
paths: ["models.providers.openai.apiKey"],
|
||||
refKeys: ["env:default:OPENAI_API_KEY"],
|
||||
reason: "secret provider failed",
|
||||
},
|
||||
],
|
||||
});
|
||||
activateSnapshot(previous);
|
||||
setActiveCredentialDegradedOwner({
|
||||
ownerKind: "account",
|
||||
ownerId: "telegram:work",
|
||||
state: "unavailable",
|
||||
paths: ["channels.telegram.accounts.work.tokenFile"],
|
||||
refKeys: [],
|
||||
reason: "credential file is unavailable",
|
||||
});
|
||||
const candidate = preparedSnapshot({ config: { gateway: { port: 19_041 } } });
|
||||
|
||||
activateSnapshot(candidate);
|
||||
|
||||
expect(listActiveDegradedSecretOwners()).toMatchObject([
|
||||
{ ownerKind: "account", ownerId: "telegram:work" },
|
||||
]);
|
||||
expect(restoreSnapshotIfCurrent(previous, candidate)).toBe(true);
|
||||
expect(listActiveDegradedSecretOwners()).toMatchObject([
|
||||
{ ownerKind: "provider", ownerId: "openai", degradationState: "stale" },
|
||||
{ ownerKind: "account", ownerId: "telegram:work" },
|
||||
]);
|
||||
|
||||
clearSecretsRuntimeSnapshotState();
|
||||
|
||||
expect(listActiveDegradedSecretOwners()).toEqual([]);
|
||||
});
|
||||
|
||||
it("publishes distinct raw and overlay source snapshots without changing runtime auth", () => {
|
||||
const secretRef = {
|
||||
source: "env" as const,
|
||||
|
||||
@@ -33,6 +33,7 @@ import type { PluginOrigin } from "../plugins/plugin-origin.types.js";
|
||||
import { isRecord } from "../utils.js";
|
||||
import { secretRefKey } from "./ref-contract.js";
|
||||
import {
|
||||
clearActiveCredentialDegradedOwners,
|
||||
setActiveDegradedSecretOwners,
|
||||
type DegradedSecretOwner,
|
||||
type SecretOwnerRefState,
|
||||
@@ -1172,6 +1173,7 @@ export function clearSecretsRuntimeSnapshotState(): void {
|
||||
activeRefreshContext = null;
|
||||
clearActiveRuntimeWebToolsMetadata();
|
||||
setActiveDegradedSecretOwners([]);
|
||||
clearActiveCredentialDegradedOwners();
|
||||
setRuntimeConfigSnapshotRefreshHandler(null);
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearRuntimeAuthProfileStoreSnapshots();
|
||||
|
||||
Reference in New Issue
Block a user