diff --git a/src/commands/status.summary.test.ts b/src/commands/status.summary.test.ts index e3e2d354efb2..5965d1781cc6 100644 --- a/src/commands/status.summary.test.ts +++ b/src/commands/status.summary.test.ts @@ -2,7 +2,11 @@ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { SESSION_TOTAL_TOKENS_VERSION } from "../config/sessions/types.js"; import { setActiveDegradedPlugins } from "../plugins/runtime-degraded-state.js"; -import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js"; +import { + clearActiveCredentialDegradedOwner, + setActiveCredentialDegradedOwner, + setActiveDegradedSecretOwners, +} from "../secrets/runtime-degraded-state.js"; import type { TaskAuditFinding } from "../tasks/task-registry.audit.js"; import type { TaskRecord, TaskRegistrySummary } from "../tasks/task-registry.types.js"; import { normalizeSessionDeliveryState } from "../utils/delivery-context.shared.js"; @@ -219,6 +223,7 @@ describe("getStatusSummary", () => { beforeEach(() => { vi.clearAllMocks(); setActiveDegradedPlugins([]); + clearActiveCredentialDegradedOwner("account", "telegram:work"); setActiveDegradedSecretOwners([]); statusSummaryMocks.taskRegistrySummary = { total: 0, @@ -406,28 +411,44 @@ describe("getStatusSummary", () => { }); }); - it("reports degraded SecretRef owners without exposing ref identifiers", async () => { + it("reports stale snapshot and cold credential owners without exposing ref identifiers", async () => { setActiveDegradedSecretOwners([ { - ownerKind: "account", - ownerId: "discord:ops", + ownerKind: "provider", + ownerId: "openai", state: "unavailable", - degradationState: "cold", - paths: ["channels.discord.accounts.ops.token"], + degradationState: "stale", + paths: ["models.providers.openai.apiKey"], refKeys: ["env:default:PRIVATE_REF_ID"], reason: "provider SecretRef is unresolved (env:default:PRIVATE_REF_ID)", }, ]); + setActiveCredentialDegradedOwner({ + ownerKind: "account", + ownerId: "telegram:work", + state: "unavailable", + paths: ["channels.telegram.accounts.work.tokenFile"], + refKeys: [], + reason: "credential failure includes PRIVATE_REF_ID", + }); const summary = await getStatusSummary(); expect(summary.degradedSecretOwners).toEqual([ + { + ownerKind: "provider", + ownerId: "openai", + state: "unavailable", + degradationState: "stale", + paths: ["models.providers.openai.apiKey"], + reason: "secret resolution failed", + }, { ownerKind: "account", - ownerId: "discord:ops", + ownerId: "telegram:work", state: "unavailable", degradationState: "cold", - paths: ["channels.discord.accounts.ops.token"], + paths: ["channels.telegram.accounts.work.tokenFile"], reason: "secret resolution failed", }, ]); diff --git a/src/gateway/server-aux-handlers.test.ts b/src/gateway/server-aux-handlers.test.ts index 892414419bc9..5b21216d7f9f 100644 --- a/src/gateway/server-aux-handlers.test.ts +++ b/src/gateway/server-aux-handlers.test.ts @@ -33,6 +33,10 @@ import { claimAgentRunDelegatedAuthority, releaseAgentRunDelegatedAuthority, } from "../infra/agent-run-registry.js"; +import { + setActiveCredentialDegradedOwner, + type DegradedSecretOwner, +} from "../secrets/runtime-degraded-state.js"; import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot, @@ -47,6 +51,10 @@ import { import { createAgentRuntimeApprovalAuthorityValidator } from "./agent-runtime-identity-token.js"; import type { GatewayReloadPlan } from "./config-reload.js"; import { createGatewayAuxHandlers } from "./server-aux-handlers.js"; +import { + registerGatewaySecretCredentialReloadCases, + type CredentialReloadHarnessOptions, +} from "./server-secrets-reload.test-support.js"; import { enforceSharedGatewaySessionGenerationForConfigWrite } from "./server-shared-auth-generation.js"; import { createWorkerSessionPlacementStore } from "./worker-environments/placement-store.js"; @@ -185,7 +193,8 @@ async function invokeSecretStoreSet(params: { type RespondCall = [boolean, unknown, { message?: string } | undefined]; type GatewayAuxHandlerParams = Parameters[0]; -type ChannelName = Parameters[0]; +type GatewayChannelManager = GatewayAuxHandlerParams["channelManager"]; +type ChannelName = Parameters[0]; function firstRespondCall(respond: ReturnType): RespondCall { const call = respond.mock.calls[0]; @@ -208,8 +217,10 @@ type SecretsReloadHarnessParams = { sharedGatewaySessionGenerationState?: GatewayAuxHandlerParams["sharedGatewaySessionGenerationState"]; resolveSharedGatewaySessionGenerationForConfig?: GatewayAuxHandlerParams["resolveSharedGatewaySessionGenerationForConfig"]; clients?: GatewayAuxHandlerParams["clients"]; - startChannel?: GatewayAuxHandlerParams["startChannel"]; - stopChannel?: GatewayAuxHandlerParams["stopChannel"]; + startChannel?: GatewayChannelManager["startChannel"]; + stopChannel?: GatewayChannelManager["stopChannel"]; + isManuallyStopped?: (channel: ChannelName, accountId: string) => boolean; + resolveRuntimeAccountId?: (channel: ChannelName, accountId: string) => string | undefined; getChannelAutostartSuppression?: GatewayAuxHandlerParams["getChannelAutostartSuppression"]; logChannelsInfo?: GatewayAuxHandlerParams["logChannels"]["info"]; respond?: ReturnType; @@ -232,8 +243,13 @@ function createSecretsReloadHarness(params: SecretsReloadHarnessParams) { resolveSharedGatewaySessionGenerationForConfig: params.resolveSharedGatewaySessionGenerationForConfig ?? (() => undefined), clients: params.clients ?? [], - startChannel: params.startChannel ?? (async () => {}), - stopChannel: params.stopChannel ?? (async () => {}), + channelManager: { + startChannel: params.startChannel ?? (async () => {}), + stopChannel: params.stopChannel ?? (async () => {}), + isManuallyStopped: params.isManuallyStopped ?? (() => false), + resolveRuntimeAccountId: + params.resolveRuntimeAccountId ?? ((_channel, accountId) => accountId), + }, getChannelAutostartSuppression: params.getChannelAutostartSuppression, logChannels: { info: params.logChannelsInfo ?? vi.fn() }, onApprovalLifecycle: params.onApprovalLifecycle, @@ -267,6 +283,42 @@ function createSecretsReloadHarnessWithChannelMocks( }; } +function createCredentialReloadHarness(options: CredentialReloadHarnessOptions = {}) { + const ownerAccountId = options.ownerAccountId ?? "ops"; + const owner: DegradedSecretOwner = { + ownerKind: "account", + ownerId: `slack:${ownerAccountId}`, + state: "unavailable", + paths: ["env.SERVICE_ACCOUNT_FILE"], + refKeys: [], + reason: "credential file is unavailable", + }; + const config = slackConfig("unchanged-secret"); + activateSnapshot(config); + setActiveCredentialDegradedOwner(owner); + const startChannel = vi.fn().mockImplementation(async () => { + if (options.createFailure) { + throw options.createFailure(owner); + } + }); + const stopChannel = vi.fn().mockResolvedValue(undefined); + const isManuallyStopped = vi.fn(() => options.manualStop ?? false); + return { + ...createSecretsReloadHarness({ + activateRuntimeSecrets: mockResolvedSecrets(config), + buildReloadPlan: () => createReloadPlan(), + startChannel, + stopChannel, + isManuallyStopped, + resolveRuntimeAccountId: () => options.runtimeAccountId ?? ownerAccountId, + }), + owner, + startChannel, + stopChannel, + isManuallyStopped, + }; +} + // Other gateway test helpers (e.g. test-helpers.mocks.ts, test-helpers.server.ts) // set OPENCLAW_SKIP_CHANNELS / OPENCLAW_SKIP_PROVIDERS at module load. When a // shared vitest worker imports those helpers before this file's tests run, @@ -551,18 +603,13 @@ describe("gateway aux handlers", () => { expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 }); }); - it("restarts the whole channel when a secret change is scoped to one account", async () => { - // secrets.reload has no per-account restart path — account-scoped plan - // entries must still produce a channel restart so rotated credentials - // are applied. + it("restarts only the changed account when a secret change is account-scoped", async () => { const buildReloadPlan = () => createReloadPlan({ - restartChannels: new Set(), restartChannelAccounts: new Map([["slack", new Set(["ops"])]]), }); activateSnapshot(slackConfig("old-slack-secret")); - const prepared = createSnapshot(slackConfig("new-slack-secret")); - const activateRuntimeSecrets = vi.fn().mockResolvedValue(prepared); + const activateRuntimeSecrets = mockResolvedSecrets(slackConfig("new-slack-secret")); const { reload, respond, startChannel, stopChannel } = createSecretsReloadHarnessWithChannelMocks({ activateRuntimeSecrets, @@ -571,11 +618,30 @@ describe("gateway aux handlers", () => { await reload(); - expect(stopChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]); - expect(startChannel.mock.calls.map(([ch]) => ch)).toEqual(["slack"]); + expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]); + expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]); expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 }); }); + registerGatewaySecretCredentialReloadCases(createCredentialReloadHarness); + + it("does not restart account targets already covered by a whole-channel target", async () => { + activateSnapshot(slackConfig("old-secret")); + const { reload, startChannel, stopChannel } = createSecretsReloadHarnessWithChannelMocks({ + activateRuntimeSecrets: mockResolvedSecrets(slackConfig("new-secret")), + buildReloadPlan: () => + createReloadPlan({ + restartChannels: new Set(["slack"]), + restartChannelAccounts: new Map([["slack", new Set(["ops"])]]), + }), + }); + + await reload(); + + expect(stopChannel.mock.calls).toEqual([["slack"]]); + expect(startChannel.mock.calls).toEqual([["slack"]]); + }); + it("coalesces concurrent secrets.reload calls so channels are not restarted twice", async () => { const buildReloadPlan = buildRestartChannelsPlan("slack"); activateSnapshot(slackConfig("old-slack-secret")); @@ -731,9 +797,15 @@ describe("gateway aux handlers", () => { expect(firstRespondCall(respond)[0]).toBe(true); }); - it("rolls back stopped channels when a later restart fails", async () => { + it("rolls back only exact stopped accounts when a later account restart fails", async () => { const authAgentDir = "/tmp/openclaw-secrets-reload-concurrent-oauth"; - const buildReloadPlan = buildRestartChannelsPlan("slack", "zalo"); + const buildReloadPlan = () => + createReloadPlan({ + restartChannelAccounts: new Map([ + ["slack", new Set(["ops"])], + ["zalo", new Set(["alerts"])], + ]), + }); activateSnapshot(slackZaloConfig("old-slack-secret", "old-zalo-secret")); const activateRuntimeSecrets = mockResolvedSecrets( slackZaloConfig("new-slack-secret", "new-zalo-secret"), @@ -779,21 +851,30 @@ describe("gateway aux handlers", () => { await reload(); - expect(stopChannel.mock.calls).toEqual([["slack"], ["zalo"], ["slack"]]); - expect(startChannel.mock.calls).toEqual([["slack"], ["zalo"], ["slack"], ["zalo"]]); + expect(stopChannel.mock.calls).toEqual([ + ["slack", "ops", { manual: false }], + ["zalo", "alerts", { manual: false }], + ["slack", "ops", { manual: false }], + ]); + expect(startChannel.mock.calls).toEqual([ + ["slack", "ops", { preserveManualStop: true }], + ["zalo", "alerts", { preserveManualStop: true }], + ["slack", "ops", { preserveManualStop: true }], + ["zalo", "alerts", { preserveManualStop: true }], + ]); expect( logChannelsInfo.mock.calls.some(([msg]) => - String(msg).startsWith("failed to restart zalo channel after secrets reload"), + String(msg).startsWith("failed to restart zalo account alerts after secrets reload"), ), ).toBe(true); expect( logChannelsInfo.mock.calls.some(([msg]) => - String(msg).startsWith("rolling back slack channel after secrets reload failure"), + String(msg).startsWith("rolling back slack account ops after secrets reload failure"), ), ).toBe(true); expect( logChannelsInfo.mock.calls.some(([msg]) => - String(msg).startsWith("rolling back zalo channel after secrets reload failure"), + String(msg).startsWith("rolling back zalo account alerts after secrets reload failure"), ), ).toBe(true); // The handler surfaces the partial-failure so the caller can retry/alert @@ -815,19 +896,13 @@ describe("gateway aux handlers", () => { ).toMatchObject({ access: "access-new", refresh: "refresh-new" }); }); - it("does not roll back over a snapshot published after secrets.reload activation", async () => { - const buildReloadPlan = buildRestartChannelsPlan("slack"); + it("fences account-scoped rollback when a newer snapshot and generation supersede reload", async () => { + const buildReloadPlan = () => + createReloadPlan({ restartChannelAccounts: new Map([["slack", new Set(["ops"])]]) }); activateSnapshot(slackConfig("old-slack-secret")); const prepared = createSnapshot(slackConfig("reload-secret")); const concurrent = createSnapshot(slackConfig("concurrent-secret")); - const activateRuntimeSecrets = vi.fn( - async ( - _config: OpenClawConfig, - _activationParams: Parameters[1], - ) => { - return prepared; - }, - ); + const activateRuntimeSecrets = vi.fn(async () => prepared); const sharedGatewaySessionGenerationState = { current: "gen-old" as string | undefined, required: "gen-old" as string | undefined | null, @@ -841,19 +916,24 @@ describe("gateway aux handlers", () => { }) .mockResolvedValue(undefined); + const stopChannel = vi.fn().mockResolvedValue(undefined); const { reload, respond } = createSecretsReloadHarness({ activateRuntimeSecrets, buildReloadPlan, sharedGatewaySessionGenerationState, resolveSharedGatewaySessionGenerationForConfig: () => "gen-reload", startChannel, - stopChannel: vi.fn().mockResolvedValue(undefined), + stopChannel, }); await reload(); expect(firstRespondCall(respond)[0]).toBe(false); - expect(startChannel).toHaveBeenCalledTimes(2); + expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]); + expect(startChannel.mock.calls).toEqual([ + ["slack", "ops", { preserveManualStop: true }], + ["slack", "ops", { preserveManualStop: true }], + ]); expect(getActiveSecretsRuntimeSnapshot()?.config).toEqual(slackConfig("concurrent-secret")); expect(sharedGatewaySessionGenerationState).toEqual({ current: "gen-concurrent", diff --git a/src/gateway/server-aux-handlers.ts b/src/gateway/server-aux-handlers.ts index c43587f06cac..fdbf5e7bf0f8 100644 --- a/src/gateway/server-aux-handlers.ts +++ b/src/gateway/server-aux-handlers.ts @@ -1,13 +1,11 @@ // Gateway auxiliary method handlers. // Wires reload, secrets, exec approval, and plugin approval RPC handlers. import { randomUUID } from "node:crypto"; -import type { OpenClawConfig } from "../config/types.openclaw.js"; import { type AgentRunDelegatedAuthority, registerAgentRunDelegatedAuthorityClosedHandler, } from "../infra/agent-run-registry.js"; import type { ChannelApprovalKind } from "../infra/approval-types.js"; -import { isTruthyEnvValue } from "../infra/env.js"; import { createExecApprovalForwarder } from "../infra/exec-approval-forwarder.js"; import { type ExecApprovalDecision, @@ -24,21 +22,10 @@ import { resolveCommandSecretsFromActiveRuntimeSnapshot, type CommandSecretAssignment, } from "../secrets/runtime-command-secrets.js"; -import { - getActiveSecretsRuntimeSnapshotState, - getActiveSecretsRuntimeSnapshotRevisionState, - type PreparedSecretsRuntimeSnapshot, -} from "../secrets/runtime-state.js"; import { createLazyPromise } from "../shared/lazy-runtime.js"; import type { AgentRuntimeDelegatedAuthority } from "./agent-runtime-identity-token.js"; import { resolveApprovalSessionAudienceWithFallback } from "./approval-session-audience.js"; import type { ChatAbortControllerEntry } from "./chat-abort.js"; -import { diffConfigPaths } from "./config-diff.js"; -import { - buildGatewayReloadPlan, - type ChannelKind, - type GatewayReloadPlan, -} from "./config-reload-plan.js"; import { createExecApprovalIosPushDelivery, createPluginApprovalIosPushDelivery, @@ -53,7 +40,6 @@ import { pruneTerminalOperatorApprovals, } from "./operator-approval-store.js"; import { QuestionManager } from "./question-manager.js"; -import type { ChannelAutostartSuppression } from "./server-channels.js"; import { publishAppliedApprovalResolution } from "./server-methods/approval-publication.js"; import { cancelAgentRuntimeBoundApprovals, @@ -62,17 +48,9 @@ import { } from "./server-methods/approval-run-cancellation.js"; import type { GatewayRequestContext } from "./server-methods/types.js"; import { - captureSharedGatewaySessionGenerationOwnership, - claimSharedGatewaySessionGenerationIfOwned, - disconnectStaleSharedGatewayAuthClients, - finalizeOwnedSharedGatewaySessionGeneration, - isSharedGatewaySessionGenerationOwnershipCurrent, - replaceOwnedSharedGatewaySessionGenerationState, - type SharedGatewayAuthClient, - type SharedGatewaySessionGenerationOwnership, - type SharedGatewaySessionGenerationState, -} from "./server-shared-auth-generation.js"; -import type { ActivateRuntimeSecrets } from "./server-startup-config.js"; + createGatewaySecretsReloader, + type GatewaySecretsReloaderParams, +} from "./server-secrets-reload.js"; import type { WorkerSessionTurnClaim } from "./worker-environments/placement-record.js"; type GatewayAuxHandlerLogger = { @@ -81,63 +59,19 @@ type GatewayAuxHandlerLogger = { debug?: (message: string) => void; }; -type ReloadSecretsResult = { - warningCount: number; -}; - -async function activateSecretsRuntimeSnapshotIfCurrent( - snapshot: PreparedSecretsRuntimeSnapshot, - expectedRevision: number, - options?: { - canActivate?: () => boolean; - onActivated?: () => void; - }, -): Promise { - const runtime = await import("../secrets/runtime.js"); - if (options?.canActivate && !options.canActivate()) { - return null; - } - if (!runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision)) { - return null; - } - options?.onActivated?.(); - return runtime.getActiveSecretsRuntimeSnapshotRevision(); -} - -async function restoreSecretsRuntimeSnapshotIfCurrent( - snapshot: PreparedSecretsRuntimeSnapshot, - expectedRevision: number, - ownedSnapshot: PreparedSecretsRuntimeSnapshot, - options?: { onActivated?: () => void }, -): Promise { - const runtime = await import("../secrets/runtime.js"); - if (!runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) { - return null; - } - options?.onActivated?.(); - return runtime.getActiveSecretsRuntimeSnapshotRevision(); -} - /** Create auxiliary gateway handlers that are not part of the core descriptor set. */ -export function createGatewayAuxHandlers(params: { - log: GatewayAuxHandlerLogger; - activateRuntimeSecrets: ActivateRuntimeSecrets; - buildReloadPlan?: (changedPaths: string[]) => GatewayReloadPlan; - sharedGatewaySessionGenerationState: SharedGatewaySessionGenerationState; - resolveSharedGatewaySessionGenerationForConfig: (config: OpenClawConfig) => string | undefined; - clients: Iterable; - startChannel: (name: ChannelKind) => Promise; - stopChannel: (name: ChannelKind) => Promise; - getChannelAutostartSuppression?: () => ChannelAutostartSuppression | null; - logChannels: { info: (msg: string) => void }; - onApprovalLifecycle?: (event: OperatorApprovalLifecycleEvent) => void; - onAgentRunAuthorityClosed?: (authority: AgentRunDelegatedAuthority) => void; - validateAgentRuntimeDelegatedAuthority?: (authority: AgentRuntimeDelegatedAuthority) => boolean; - chatAbortControllers?: Map; - registerWorkerTurnClaimClosedHandler?: ( - handler: (claim: WorkerSessionTurnClaim) => void, - ) => () => void; -}) { +export function createGatewayAuxHandlers( + params: GatewaySecretsReloaderParams & { + log: GatewayAuxHandlerLogger; + onApprovalLifecycle?: (event: OperatorApprovalLifecycleEvent) => void; + onAgentRunAuthorityClosed?: (authority: AgentRunDelegatedAuthority) => void; + validateAgentRuntimeDelegatedAuthority?: (authority: AgentRuntimeDelegatedAuthority) => boolean; + chatAbortControllers?: Map; + registerWorkerTurnClaimClosedHandler?: ( + handler: (claim: WorkerSessionTurnClaim) => void, + ) => () => void; + }, +) { // Both approval kinds share one durable first-answer-wins registry and // Gateway-lifetime epoch while retaining separate in-process waiter maps. // A newly constructed Gateway cannot resume the prior lifetime's waiters. @@ -198,7 +132,6 @@ export function createGatewayAuxHandlers(params: { ), { cacheRejections: true }, ); - const buildReloadPlan = params.buildReloadPlan ?? buildGatewayReloadPlan; const pluginApprovalManager = createApprovalManager( "plugin", resolveCanonicalPluginApprovalRequestAllowedDecisions, @@ -344,305 +277,11 @@ export function createGatewayAuxHandlers(params: { ), { cacheRejections: true }, ); - // Serialize the entire `secrets.reload` path (activation + channel restart) - // so concurrent callers cannot overlap the stop/start loop and so the - // "before" snapshot used for the reload-plan diff is always the snapshot - // replaced by this call's activation, not one captured by a prior caller. - let reloadInFlight: Promise | null = null; - const runExclusiveReload = ( - fn: () => Promise, - options: { joinInFlight?: boolean } = {}, - ): Promise => { - if (reloadInFlight) { - if (options.joinInFlight !== false) { - return reloadInFlight; - } - const precedingReload = reloadInFlight; - return precedingReload.catch(() => undefined).then(() => runExclusiveReload(fn, options)); - } - const run = (async () => { - try { - return await fn(); - } finally { - reloadInFlight = null; - } - })(); - reloadInFlight = run; - return run; - }; const loadSecretsHandlers = createLazyPromise( () => import("./server-methods/secrets.js").then(({ createSecretsHandlers }) => createSecretsHandlers({ - reloadSecrets: (reloadOptions) => - runExclusiveReload(async () => { - let transaction: - | { - previousSnapshot: PreparedSecretsRuntimeSnapshot; - previousSharedGatewaySessionGeneration: string | undefined; - previousSharedGatewaySessionGenerationRequired: string | undefined | null; - prepared: PreparedSecretsRuntimeSnapshot; - plan: GatewayReloadPlan; - nextSharedGatewaySessionGeneration: string | undefined; - sharedGatewaySessionGenerationChanged: boolean; - generationOwnership: SharedGatewaySessionGenerationOwnership; - publishedSnapshotRevision: number; - } - | undefined; - const stoppedChannels: ChannelKind[] = []; - const restartedChannels = new Set(); - try { - for (;;) { - const previousSnapshot = getActiveSecretsRuntimeSnapshotState(); - if (!previousSnapshot) { - throw new Error("Secrets runtime snapshot is not active."); - } - const previousSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState(); - const previousGenerationOwnership = - captureSharedGatewaySessionGenerationOwnership( - params.sharedGatewaySessionGenerationState, - ); - // Snapshot both generation fields with the candidate revision. - // A stale preparation retries all three owners together. - const previousSharedGatewaySessionGeneration = - previousGenerationOwnership.generation; - const previousSharedGatewaySessionGenerationRequired = - params.sharedGatewaySessionGenerationState.required; - const prepared = await params.activateRuntimeSecrets( - previousSnapshot.sourceConfig, - { - reason: "reload", - activate: false, - publishFailureAsDegraded: true, - forceColdRefKeys: reloadOptions?.forceColdRefKeys, - canPublishFailureAsDegraded: () => - getActiveSecretsRuntimeSnapshotRevisionState() === previousSnapshotRevision, - }, - ); - const plan = buildReloadPlan( - diffConfigPaths(previousSnapshot.config, prepared.config), - ); - const nextSharedGatewaySessionGeneration = - params.resolveSharedGatewaySessionGenerationForConfig(prepared.config); - let publishedSnapshotRevision: number | null = null; - let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null; - const activateIfCurrent = - params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent; - if (activateIfCurrent) { - const activated = await activateIfCurrent( - prepared, - previousSnapshotRevision, - { - reason: "reload", - activate: true, - }, - async () => { - publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState(); - generationOwnership = claimSharedGatewaySessionGenerationIfOwned( - params.sharedGatewaySessionGenerationState, - previousGenerationOwnership, - nextSharedGatewaySessionGeneration, - ); - }, - () => - isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - previousGenerationOwnership, - ), - ); - if (!activated) { - continue; - } - } else { - publishedSnapshotRevision = await activateSecretsRuntimeSnapshotIfCurrent( - prepared, - previousSnapshotRevision, - { - canActivate: () => - isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - previousGenerationOwnership, - ), - onActivated: () => { - generationOwnership = claimSharedGatewaySessionGenerationIfOwned( - params.sharedGatewaySessionGenerationState, - previousGenerationOwnership, - nextSharedGatewaySessionGeneration, - ); - }, - }, - ); - if (publishedSnapshotRevision === null) { - continue; - } - } - if (publishedSnapshotRevision === null || generationOwnership === null) { - throw new Error("Secrets runtime activation did not publish ownership."); - } - transaction = { - previousSnapshot, - previousSharedGatewaySessionGeneration, - previousSharedGatewaySessionGenerationRequired, - prepared, - plan, - nextSharedGatewaySessionGeneration, - sharedGatewaySessionGenerationChanged: - previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration, - generationOwnership, - publishedSnapshotRevision, - }; - if ( - !isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - generationOwnership, - ) - ) { - throw new Error("secrets.reload was superseded by a newer config write"); - } - break; - } - const { - prepared, - plan, - generationOwnership, - nextSharedGatewaySessionGeneration, - sharedGatewaySessionGenerationChanged, - } = transaction; - if (sharedGatewaySessionGenerationChanged) { - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: nextSharedGatewaySessionGeneration, - }); - } - // Account-scoped changes restart their whole channel here: - // secrets.reload has no per-account restart path, and a missed - // restart would leave rotated credentials unapplied. - const channelsToRestart = new Set([ - ...plan.restartChannels, - ...(plan.restartChannelAccounts?.keys() ?? []), - ]); - if (channelsToRestart.size > 0) { - const restartChannels = [...channelsToRestart]; - if ( - isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || - isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS) - ) { - throw new Error( - `secrets.reload requires restarting channels: ${restartChannels.join(", ")}`, - ); - } - if (params.getChannelAutostartSuppression?.()) { - throw new Error( - `secrets.reload requires restarting channels but channel autostart is suppressed by crash-loop breaker: ${restartChannels.join(", ")}`, - ); - } - const restartFailures: ChannelKind[] = []; - for (const channel of restartChannels) { - if ( - !isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - generationOwnership, - ) - ) { - throw new Error("secrets.reload was superseded by a newer config write"); - } - params.logChannels.info(`restarting ${channel} channel after secrets reload`); - // Track for rollback before awaiting stopChannel: if stopChannel - // throws after partially stopping the channel, still attempt recovery. - stoppedChannels.push(channel); - try { - await params.stopChannel(channel); - if ( - !isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - generationOwnership, - ) - ) { - throw new Error("secrets.reload was superseded by a newer config write"); - } - await params.startChannel(channel); - restartedChannels.add(channel); - if ( - !isSharedGatewaySessionGenerationOwnershipCurrent( - params.sharedGatewaySessionGenerationState, - generationOwnership, - ) - ) { - throw new Error("secrets.reload was superseded by a newer config write"); - } - } catch { - params.logChannels.info( - `failed to restart ${channel} channel after secrets reload`, - ); - restartFailures.push(channel); - } - } - if (restartFailures.length > 0) { - throw new Error( - `failed to restart channels after secrets reload: ${restartFailures.join(", ")}`, - ); - } - } - if ( - !finalizeOwnedSharedGatewaySessionGeneration( - params.sharedGatewaySessionGenerationState, - generationOwnership, - ) - ) { - throw new Error("secrets.reload was superseded by a newer config write"); - } - return { warningCount: prepared.warnings.length }; - } catch (err) { - let generationRestored = false; - if (transaction) { - const failedTransaction = transaction; - await restoreSecretsRuntimeSnapshotIfCurrent( - failedTransaction.previousSnapshot, - failedTransaction.publishedSnapshotRevision, - failedTransaction.prepared, - { - onActivated: () => { - generationRestored = replaceOwnedSharedGatewaySessionGenerationState( - params.sharedGatewaySessionGenerationState, - failedTransaction.generationOwnership, - { - current: failedTransaction.previousSharedGatewaySessionGeneration, - required: - failedTransaction.previousSharedGatewaySessionGenerationRequired, - }, - ); - }, - }, - ); - } - if (generationRestored && transaction) { - if (transaction.sharedGatewaySessionGenerationChanged) { - disconnectStaleSharedGatewayAuthClients({ - clients: params.clients, - expectedGeneration: transaction.previousSharedGatewaySessionGeneration, - }); - } - } - // Generation ownership fences state rollback, not liveness. - // Restart stopped channels against whichever runtime is current now. - for (const channel of stoppedChannels) { - params.logChannels.info( - `rolling back ${channel} channel after secrets reload failure`, - ); - try { - if (restartedChannels.has(channel)) { - await params.stopChannel(channel); - } - await params.startChannel(channel); - } catch { - params.logChannels.info( - `failed to roll back ${channel} channel after secrets reload`, - ); - } - } - throw err; - } - }, reloadOptions), + reloadSecrets: createGatewaySecretsReloader(params), log: params.log, resolveSecrets: async ({ allowedPaths, diff --git a/src/gateway/server-channels.test.ts b/src/gateway/server-channels.test.ts index b94b7404539a..ae23251e738b 100644 --- a/src/gateway/server-channels.test.ts +++ b/src/gateway/server-channels.test.ts @@ -1,8 +1,11 @@ /** * Server channel lifecycle tests. */ +import fs from "node:fs"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createDeferred } from "../../test/helpers/promise.js"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { ChannelIngressUnavailableError } from "../channels/message/ingress-unavailable.js"; import type { ChannelAccountLinkState, @@ -15,6 +18,7 @@ import type { } from "../channels/plugins/types.public.js"; import { formatGatewayChannelsStatusLines } from "../commands/channels/status.runtime.js"; import type { GatewayNativeApprovalRuntime } from "../infra/approval-gateway-runtime.types.js"; +import { tryReadSecretFileSync } from "../infra/secret-file.js"; import { createSubsystemLogger, type SubsystemLogger, @@ -34,6 +38,7 @@ import { import { DEFAULT_ACCOUNT_ID } from "../routing/session-key.js"; import type { RuntimeEnv } from "../runtime.js"; import { + clearActiveCredentialDegradedOwner, listActiveDegradedSecretOwners, setActiveDegradedSecretOwners, } from "../secrets/runtime-degraded-state.js"; @@ -102,6 +107,7 @@ const CHANNEL_APPROVAL_GATEWAY_RUNTIME_CONTEXT_CAPABILITY = "approval.gateway"; type ApprovalGatewayRequestRuntime = Pick; const createdManagers: Array<{ manager: ChannelManager; channelIds: ChannelId[] }> = []; +const channelTempDirs = useAutoCleanupTempDirTracker(afterEach); function healthOf(account: ChannelAccountSnapshot | undefined) { return evaluateChannelHealth(account ?? {}, { @@ -315,6 +321,9 @@ describe("server-channels auto restart", () => { hoisted.sleepWithAbort.mockClear(); hoisted.startChannelApprovalHandlerBootstrap.mockReset(); hoisted.startChannelApprovalHandlerBootstrap.mockResolvedValue(async () => {}); + for (const owner of listActiveDegradedSecretOwners()) { + clearActiveCredentialDegradedOwner(owner.ownerKind, owner.ownerId); + } setActiveDegradedSecretOwners([]); }); @@ -330,6 +339,9 @@ describe("server-channels auto restart", () => { vi.clearAllTimers(); vi.useRealTimers(); resetGatewayWorkAdmission(); + for (const owner of listActiveDegradedSecretOwners()) { + clearActiveCredentialDegradedOwner(owner.ownerKind, owner.ownerId); + } setActiveDegradedSecretOwners([]); setActivePluginRegistry(previousRegistry ?? createEmptyPluginRegistry()); }); @@ -2724,7 +2736,8 @@ describe("server-channels auto restart", () => { }); it("keeps one file-credential account cold and recovers it without restarting siblings", async () => { - let broken = true; + const credentialPath = path.join(channelTempDirs.make("openclaw-channel-credential-"), "token"); + const credentialConfigPath = "channels.telegram.accounts.broken.tokenFile"; const startAccount = vi.fn( async ({ abortSignal }: ChannelGatewayContext) => await new Promise((resolve) => { @@ -2733,56 +2746,70 @@ describe("server-channels auto restart", () => { ); installTestRegistry( createTestPlugin({ - id: "discord", + id: "telegram", listAccountIds: () => ["broken", "healthy"], - resolveAccount: (_cfg, accountId) => ({ - enabled: true, - configured: true, - ...(accountId === "broken" && broken - ? { - credentialDiagnostics: [ + resolveAccount: (_cfg, accountId) => { + const credential = + accountId === "broken" + ? tryReadSecretFileSync( + credentialPath, + "Telegram bot token", + {}, { - code: "CREDENTIAL_FILE_UNAVAILABLE" as const, - path: "channels.discord.accounts.broken.tokenFile", - reason: "not-found", + configPath: credentialConfigPath, }, - ], - } - : {}), - }), + ) + : { status: "available" as const, value: "healthy-token" }; + return { + enabled: true, + configured: true, + ...(credential.status === "configured_unavailable" + ? { credentialDiagnostics: [credential.diagnostic] } + : {}), + }; + }, startAccount, }), ); - const manager = createManager({ channelIds: ["discord"] }); + const manager = createManager({ channelIds: ["telegram"] }); await expect(manager.startChannels()).resolves.toBeUndefined(); expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual(["healthy"]); - expect(manager.getRuntimeSnapshot().channelAccounts.discord?.broken).toMatchObject({ + expect(manager.getRuntimeSnapshot().channelAccounts.telegram?.broken).toMatchObject({ configured: true, running: false, lastError: - "Secret owner account:discord:broken is configured but unavailable (credential file is unavailable).", + "Secret owner account:telegram:broken is configured but unavailable (credential file is unavailable).", }); expect(listActiveDegradedSecretOwners()).toContainEqual( expect.objectContaining({ - ownerId: "discord:broken", - paths: ["channels.discord.accounts.broken.tokenFile"], + ownerId: "telegram:broken", + paths: [credentialConfigPath], refKeys: [], }), ); - broken = false; - await manager.startChannel("discord", "broken"); + await expect(manager.startChannel("telegram", "broken")).rejects.toMatchObject({ + code: "SECRET_SURFACE_UNAVAILABLE", + ownerId: "telegram:broken", + }); + expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual(["healthy"]); + expect(listActiveDegradedSecretOwners()).toContainEqual( + expect.objectContaining({ ownerId: "telegram:broken" }), + ); + + fs.writeFileSync(credentialPath, "repaired-token", { mode: 0o600 }); + await manager.startChannel("telegram", "broken"); expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([ "healthy", "broken", ]); expect(listActiveDegradedSecretOwners()).not.toContainEqual( - expect.objectContaining({ ownerId: "discord:broken" }), + expect.objectContaining({ ownerId: "telegram:broken" }), ); - await manager.stopChannel("discord"); + await manager.stopChannel("telegram"); }); it("uses fallback logger and runtime when a channel is missing startup wiring", async () => { @@ -3023,6 +3050,76 @@ describe("server-channels auto restart", () => { await manager.stopChannel("discord"); }); + it("retires only the credential owner for an evicted channel account", async () => { + let accountIds = ["removed", "retained"]; + installTestRegistry( + createTestPlugin({ + listAccountIds: () => accountIds, + startAccount: async () => {}, + resolveAccount: (_cfg, accountId) => ({ + enabled: true, + configured: true, + credentialDiagnostics: [ + { + code: "CREDENTIAL_FILE_UNAVAILABLE" as const, + path: `channels.discord.accounts.${accountId}.tokenFile`, + reason: "not-found", + }, + ], + }), + }), + ); + const manager = createManager(); + + await manager.startChannels(); + expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([ + "discord:removed", + "discord:retained", + ]); + + accountIds = ["retained"]; + await expect(manager.startChannel("discord")).rejects.toMatchObject({ + ownerId: "discord:retained", + }); + + expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([ + "discord:retained", + ]); + }); + + it("resolves only an unambiguous authoritative runtime account for a normalized owner", async () => { + let accountIds = ["Ops Team"]; + installTestRegistry( + createTestPlugin({ + id: "line", + listAccountIds: () => accountIds, + startAccount: async () => {}, + resolveAccount: (_cfg, accountId) => ({ + enabled: true, + configured: true, + credentialDiagnostics: [ + { + code: "CREDENTIAL_FILE_UNAVAILABLE" as const, + path: `channels.line.accounts.${accountId}.channelAccessTokenFile`, + reason: "not-found", + }, + ], + }), + }), + ); + const manager = createManager({ channelIds: ["line"] }); + + await manager.startChannels(); + + expect(manager.resolveRuntimeAccountId("line", "ops-team")).toBe("Ops Team"); + expect(manager.resolveRuntimeAccountId("line", "missing")).toBeUndefined(); + + accountIds = ["Ops Team", "ops-team"]; + await manager.startChannels(); + + expect(manager.resolveRuntimeAccountId("line", "ops-team")).toBeUndefined(); + }); + it("reuses plugin account resolution for health monitor overrides", () => { installTestRegistry( createTestPlugin({ diff --git a/src/gateway/server-channels.ts b/src/gateway/server-channels.ts index 444c3e2f2cab..f759edfa1096 100644 --- a/src/gateway/server-channels.ts +++ b/src/gateway/server-channels.ts @@ -283,7 +283,9 @@ export type ChannelManager = { }; // Channel docking: lifecycle hooks (`plugin.gateway`) flow through this manager. -export function createChannelManager(opts: ChannelManagerOptions): ChannelManager { +export function createChannelManager(opts: ChannelManagerOptions): ChannelManager & { + resolveRuntimeAccountId: (channelId: ChannelId, accountId: string) => string | undefined; +} { const { getRuntimeConfig, channelLogs, @@ -490,6 +492,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage continue; } store.runtimes.delete(id); + clearActiveCredentialDegradedOwner("account", restartKey(channelId, normalizeAccountId(id))); store.pluginCommandCatalogOwners.delete(id); restarts.delete(restartKey(channelId, id)); manuallyStopped.delete(restartKey(channelId, id)); @@ -1429,6 +1432,12 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage ambientAutostartSuppressedChannelIds.has(channelId), markChannelLoggedOut, isManuallyStopped: isManuallyStoppedFlag, + resolveRuntimeAccountId: (channelId, accountId) => { + const matches = [...(channelStores.get(channelId)?.runtimes.keys() ?? [])].filter( + (id) => normalizeAccountId(id) === accountId, + ); + return matches.length === 1 ? matches[0] : undefined; + }, isAutoRestartScheduled, resetRestartAttempts, isHealthMonitorEnabled, diff --git a/src/gateway/server-core-runtime.ts b/src/gateway/server-core-runtime.ts index 6806e6539774..15ec2b188c74 100644 --- a/src/gateway/server-core-runtime.ts +++ b/src/gateway/server-core-runtime.ts @@ -127,8 +127,6 @@ export async function startGatewayCoreRuntime(input: { readinessEventLoopHealth, workerDispatchAuthority, clients, - startChannel, - stopChannel, sharedGatewaySessionGenerationState, resolveSharedGatewaySessionGenerationForConfig, sessionMessageSubscribers, @@ -339,8 +337,7 @@ export async function startGatewayCoreRuntime(input: { sharedGatewaySessionGenerationState, resolveSharedGatewaySessionGenerationForConfig, clients, - startChannel, - stopChannel, + channelManager, getChannelAutostartSuppression: channelManager.getAutostartSuppression, logChannels, registerWorkerTurnClaimClosedHandler: workerEnvironmentStartup?.placementStore diff --git a/src/gateway/server-secrets-reload.test-support.ts b/src/gateway/server-secrets-reload.test-support.ts new file mode 100644 index 000000000000..3a15118b91f6 --- /dev/null +++ b/src/gateway/server-secrets-reload.test-support.ts @@ -0,0 +1,103 @@ +// Additional credential-owner lifecycle cases registered in the existing auxiliary-handler suite. +import { expect, it, vi } from "vitest"; +import { + listActiveDegradedSecretOwners, + SecretSurfaceUnavailableError, + type DegradedSecretOwner, +} from "../secrets/runtime-degraded-state.js"; + +export type CredentialReloadHarnessOptions = { + ownerAccountId?: string; + runtimeAccountId?: string; + manualStop?: boolean; + createFailure?: (owner: DegradedSecretOwner) => Error; +}; + +type CredentialReloadHarness = { + owner: DegradedSecretOwner; + reload: () => Promise; + respond: ReturnType; + startChannel: ReturnType; + stopChannel: ReturnType; + isManuallyStopped: ReturnType; +}; + +/** Registers focused account recovery cases against the existing RPC owner and shared fixture. */ +export function registerGatewaySecretCredentialReloadCases( + createHarness: (options?: CredentialReloadHarnessOptions) => CredentialReloadHarness, +): void { + it("reinspects a previously degraded account without stopping healthy siblings when config is unchanged", async () => { + const { reload, respond, startChannel, stopChannel } = createHarness(); + + await reload(); + + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]); + expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 }); + }); + + it("accepts only the same trusted credential owner remaining degraded after reinspection", async () => { + const { owner, reload, respond, startChannel, stopChannel } = createHarness({ + createFailure: (degradedOwner) => new SecretSurfaceUnavailableError(degradedOwner), + }); + + await reload(); + + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel.mock.calls).toEqual([["slack", "ops", { preserveManualStop: true }]]); + expect(listActiveDegradedSecretOwners()).toContainEqual(expect.objectContaining(owner)); + expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 }); + }); + + it.each(["forged", "different-owner"] as const)( + "rejects a %s unavailable error instead of treating account recovery as degraded success", + async (failureKind) => { + const { owner, reload, respond, stopChannel } = createHarness({ + createFailure: (degradedOwner) => + failureKind === "different-owner" + ? new SecretSurfaceUnavailableError({ ...degradedOwner, ownerId: "slack:other" }) + : Object.setPrototypeOf( + Object.assign(new Error("forged unavailable error"), { + name: "SecretSurfaceUnavailableError", + code: "SECRET_SURFACE_UNAVAILABLE", + ownerKind: "account", + ownerId: degradedOwner.ownerId, + }), + SecretSurfaceUnavailableError.prototype, + ), + }); + + await reload(); + + expect(respond.mock.calls[0]?.[0]).toBe(false); + expect(stopChannel.mock.calls).toEqual([["slack", "ops", { manual: false }]]); + expect(listActiveDegradedSecretOwners()).toContainEqual(expect.objectContaining(owner)); + }, + ); + + it("does not reinspect a degraded account that an operator manually stopped", async () => { + const { reload, respond, startChannel, stopChannel } = createHarness({ manualStop: true }); + + await reload(); + + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel).not.toHaveBeenCalled(); + expect(listActiveDegradedSecretOwners()).toContainEqual( + expect.objectContaining({ ownerId: "slack:ops" }), + ); + expect(respond).toHaveBeenCalledWith(true, { ok: true, warningCount: 0 }); + }); + + it("recovers the authoritative raw account identity instead of its normalized owner suffix", async () => { + const { reload, startChannel, stopChannel, isManuallyStopped } = createHarness({ + ownerAccountId: "ops-team", + runtimeAccountId: "Ops Team", + }); + + await reload(); + + expect(isManuallyStopped).toHaveBeenCalledWith("slack", "Ops Team"); + expect(stopChannel).not.toHaveBeenCalled(); + expect(startChannel.mock.calls).toEqual([["slack", "Ops Team", { preserveManualStop: true }]]); + }); +} diff --git a/src/gateway/server-secrets-reload.ts b/src/gateway/server-secrets-reload.ts new file mode 100644 index 000000000000..1a6d0a623727 --- /dev/null +++ b/src/gateway/server-secrets-reload.ts @@ -0,0 +1,409 @@ +// Owns serialized secrets snapshot replacement and exact channel-account lifecycle recovery. +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isTruthyEnvValue } from "../infra/env.js"; +import { + isTrustedSecretSurfaceUnavailableError, + listActiveCredentialDegradedOwners, + type DegradedSecretOwner, +} from "../secrets/runtime-degraded-state.js"; +import { + getActiveSecretsRuntimeSnapshotRevisionState, + getActiveSecretsRuntimeSnapshotState, + type PreparedSecretsRuntimeSnapshot, +} from "../secrets/runtime-state.js"; +import { diffConfigPaths } from "./config-diff.js"; +import { + buildGatewayReloadPlan, + type ChannelKind, + type GatewayReloadPlan, +} from "./config-reload-plan.js"; +import type { ChannelAutostartSuppression, createChannelManager } from "./server-channels.js"; +import { + captureSharedGatewaySessionGenerationOwnership, + claimSharedGatewaySessionGenerationIfOwned, + disconnectStaleSharedGatewayAuthClients, + finalizeOwnedSharedGatewaySessionGeneration, + isSharedGatewaySessionGenerationOwnershipCurrent, + replaceOwnedSharedGatewaySessionGenerationState, + type SharedGatewayAuthClient, + type SharedGatewaySessionGenerationOwnership, + type SharedGatewaySessionGenerationState, +} from "./server-shared-auth-generation.js"; +import type { ActivateRuntimeSecrets } from "./server-startup-config.js"; + +type ReloadSecretsResult = { warningCount: number }; +type ReloadSecretsOptions = { forceColdRefKeys?: ReadonlySet; joinInFlight?: boolean }; +type ReloadChannelTarget = { + channel: ChannelKind; + accountId?: string; + credentialOwnerId?: string; + inspectOnly?: boolean; +}; + +export type GatewaySecretsReloaderParams = { + activateRuntimeSecrets: ActivateRuntimeSecrets; + buildReloadPlan?: (changedPaths: string[]) => GatewayReloadPlan; + sharedGatewaySessionGenerationState: SharedGatewaySessionGenerationState; + resolveSharedGatewaySessionGenerationForConfig: (config: OpenClawConfig) => string | undefined; + clients: Iterable; + channelManager: Pick< + ReturnType, + "startChannel" | "stopChannel" | "isManuallyStopped" | "resolveRuntimeAccountId" + >; + getChannelAutostartSuppression?: () => ChannelAutostartSuppression | null; + logChannels: { info: (message: string) => void }; +}; + +async function activateSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + options: { canActivate: () => boolean; onActivated: () => void }, +): Promise { + const runtime = await import("../secrets/runtime.js"); + if ( + !options.canActivate() || + !runtime.activateSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision) + ) { + return null; + } + options.onActivated(); + return runtime.getActiveSecretsRuntimeSnapshotRevision(); +} + +async function restoreSnapshotIfCurrent( + snapshot: PreparedSecretsRuntimeSnapshot, + expectedRevision: number, + ownedSnapshot: PreparedSecretsRuntimeSnapshot, + onActivated: () => void, +): Promise { + const runtime = await import("../secrets/runtime.js"); + if (runtime.restoreSecretsRuntimeSnapshotIfCurrent(snapshot, expectedRevision, ownedSnapshot)) { + onActivated(); + } +} + +/** Keeps snapshot CAS, generation ownership, and exact account recovery in one transaction. */ +export function createGatewaySecretsReloader(params: GatewaySecretsReloaderParams) { + const buildReloadPlan = params.buildReloadPlan ?? buildGatewayReloadPlan; + const manager = params.channelManager; + let reloadInFlight: Promise | null = null; + const runExclusiveReload = ( + fn: () => Promise, + options: ReloadSecretsOptions = {}, + ): Promise => { + if (reloadInFlight) { + return options.joinInFlight === false + ? reloadInFlight.catch(() => undefined).then(() => runExclusiveReload(fn, options)) + : reloadInFlight; + } + const run = (async () => { + try { + return await fn(); + } finally { + reloadInFlight = null; + } + })(); + reloadInFlight = run; + return run; + }; + + return (reloadOptions?: ReloadSecretsOptions) => + runExclusiveReload(async () => { + let transaction: + | { + previousSnapshot: PreparedSecretsRuntimeSnapshot; + previousGeneration: string | undefined; + previousRequiredGeneration: string | undefined | null; + prepared: PreparedSecretsRuntimeSnapshot; + plan: GatewayReloadPlan; + credentialOwners: DegradedSecretOwner[]; + nextGeneration: string | undefined; + generationChanged: boolean; + generationOwnership: SharedGatewaySessionGenerationOwnership; + publishedSnapshotRevision: number; + } + | undefined; + const touchedTargets: Array<{ target: ReloadChannelTarget; restarted: boolean }> = []; + const startTarget = ({ channel, accountId }: ReloadChannelTarget) => + accountId + ? manager.startChannel(channel, accountId, { preserveManualStop: true }) + : manager.startChannel(channel); + const stopTarget = ({ channel, accountId }: ReloadChannelTarget) => + accountId + ? manager.stopChannel(channel, accountId, { manual: false }) + : manager.stopChannel(channel); + + try { + for (;;) { + const previousSnapshot = getActiveSecretsRuntimeSnapshotState(); + if (!previousSnapshot) { + throw new Error("Secrets runtime snapshot is not active."); + } + const previousRevision = getActiveSecretsRuntimeSnapshotRevisionState(); + const previousOwnership = captureSharedGatewaySessionGenerationOwnership( + params.sharedGatewaySessionGenerationState, + ); + const previousGeneration = previousOwnership.generation; + const previousRequiredGeneration = params.sharedGatewaySessionGenerationState.required; + const prepared = await params.activateRuntimeSecrets(previousSnapshot.sourceConfig, { + reason: "reload", + activate: false, + publishFailureAsDegraded: true, + forceColdRefKeys: reloadOptions?.forceColdRefKeys, + canPublishFailureAsDegraded: () => + getActiveSecretsRuntimeSnapshotRevisionState() === previousRevision, + }); + const plan = buildReloadPlan(diffConfigPaths(previousSnapshot.config, prepared.config)); + const nextGeneration = params.resolveSharedGatewaySessionGenerationForConfig( + prepared.config, + ); + // File diagnostics have channel-owned lifetimes; capture each CAS attempt + // immediately before publication so a superseded attempt cannot reuse owners. + const credentialOwners = listActiveCredentialDegradedOwners(); + let publishedSnapshotRevision: number | null = null; + let generationOwnership: SharedGatewaySessionGenerationOwnership | null = null; + const claimGeneration = () => { + publishedSnapshotRevision = getActiveSecretsRuntimeSnapshotRevisionState(); + generationOwnership = claimSharedGatewaySessionGenerationIfOwned( + params.sharedGatewaySessionGenerationState, + previousOwnership, + nextGeneration, + ); + }; + const ownsPreviousGeneration = () => + isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + previousOwnership, + ); + const activateIfCurrent = params.activateRuntimeSecrets.activatePreparedSnapshotIfCurrent; + if (activateIfCurrent) { + const activated = await activateIfCurrent( + prepared, + previousRevision, + { reason: "reload", activate: true }, + claimGeneration, + ownsPreviousGeneration, + ); + if (!activated) { + continue; + } + } else { + publishedSnapshotRevision = await activateSnapshotIfCurrent( + prepared, + previousRevision, + { + canActivate: ownsPreviousGeneration, + onActivated: claimGeneration, + }, + ); + if (publishedSnapshotRevision === null) { + continue; + } + } + if (publishedSnapshotRevision === null || generationOwnership === null) { + throw new Error("Secrets runtime activation did not publish ownership."); + } + transaction = { + previousSnapshot, + previousGeneration, + previousRequiredGeneration, + prepared, + plan, + credentialOwners, + nextGeneration, + generationChanged: previousGeneration !== nextGeneration, + generationOwnership, + publishedSnapshotRevision, + }; + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } + break; + } + + const { prepared, plan, credentialOwners, generationOwnership, nextGeneration } = + transaction; + if (transaction.generationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: nextGeneration, + }); + } + const targets: ReloadChannelTarget[] = [...plan.restartChannels].map((channel) => ({ + channel, + })); + const accountTargets = new Map(); + for (const [channel, accountIds] of plan.restartChannelAccounts ?? []) { + if (plan.restartChannels.has(channel)) { + continue; + } + for (const accountId of accountIds) { + const target = { channel, accountId }; + accountTargets.set(`${channel}\0${accountId}`, target); + targets.push(target); + } + } + for (const owner of credentialOwners) { + if (owner.ownerKind !== "account") { + continue; + } + const separator = owner.ownerId.indexOf(":"); + if (separator < 0) { + continue; + } + const channel: ChannelKind = owner.ownerId.slice(0, separator); + if (plan.restartChannels.has(channel)) { + continue; + } + const accountId = manager.resolveRuntimeAccountId( + channel, + owner.ownerId.slice(separator + 1), + ); + if (!accountId || manager.isManuallyStopped(channel, accountId)) { + continue; + } + const key = `${channel}\0${accountId}`; + const existing = accountTargets.get(key); + if (existing) { + existing.credentialOwnerId = owner.ownerId; + continue; + } + const target = { + channel, + accountId, + credentialOwnerId: owner.ownerId, + inspectOnly: true, + }; + accountTargets.set(key, target); + targets.push(target); + } + const restartTargets = targets.filter( + ({ channel, accountId }) => !accountId || !manager.isManuallyStopped(channel, accountId), + ); + if (restartTargets.length > 0) { + const restartChannels = [...new Set(restartTargets.map(({ channel }) => channel))]; + if ( + isTruthyEnvValue(process.env.OPENCLAW_SKIP_CHANNELS) || + isTruthyEnvValue(process.env.OPENCLAW_SKIP_PROVIDERS) + ) { + throw new Error( + `secrets.reload requires restarting channels: ${restartChannels.join(", ")}`, + ); + } + if (params.getChannelAutostartSuppression?.()) { + throw new Error( + `secrets.reload requires restarting channels but channel autostart is suppressed by crash-loop breaker: ${restartChannels.join(", ")}`, + ); + } + const failures: string[] = []; + for (const target of restartTargets) { + const { channel, accountId, credentialOwnerId, inspectOnly } = target; + const label = accountId ? `${channel} account ${accountId}` : `${channel} channel`; + const assertGenerationOwned = () => { + if ( + !isSharedGatewaySessionGenerationOwnershipCurrent( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } + }; + assertGenerationOwned(); + params.logChannels.info( + `${inspectOnly ? "reinspecting" : "restarting"} ${label} after secrets reload`, + ); + // A rejecting hook may have already changed its exact account lifetime. + const touched = { target, restarted: false }; + touchedTargets.push(touched); + try { + if (!inspectOnly) { + await stopTarget(target); + assertGenerationOwned(); + } + await startTarget(target); + touched.restarted = true; + assertGenerationOwned(); + } catch (error) { + if ( + credentialOwnerId && + isTrustedSecretSurfaceUnavailableError(error) && + error.ownerKind === "account" && + error.ownerId === credentialOwnerId && + listActiveCredentialDegradedOwners().some( + (owner) => owner.ownerKind === "account" && owner.ownerId === credentialOwnerId, + ) + ) { + touchedTargets.pop(); + continue; + } + params.logChannels.info(`failed to restart ${label} after secrets reload`); + failures.push(accountId ? `${channel}:${accountId}` : channel); + } + } + if (failures.length > 0) { + throw new Error( + `failed to restart channels after secrets reload: ${failures.join(", ")}`, + ); + } + } + if ( + !finalizeOwnedSharedGatewaySessionGeneration( + params.sharedGatewaySessionGenerationState, + generationOwnership, + ) + ) { + throw new Error("secrets.reload was superseded by a newer config write"); + } + return { warningCount: prepared.warnings.length }; + } catch (error) { + let generationRestored = false; + if (transaction) { + const failedTransaction = transaction; + await restoreSnapshotIfCurrent( + failedTransaction.previousSnapshot, + failedTransaction.publishedSnapshotRevision, + failedTransaction.prepared, + () => { + generationRestored = replaceOwnedSharedGatewaySessionGenerationState( + params.sharedGatewaySessionGenerationState, + failedTransaction.generationOwnership, + { + current: failedTransaction.previousGeneration, + required: failedTransaction.previousRequiredGeneration, + }, + ); + }, + ); + } + if (generationRestored && transaction?.generationChanged) { + disconnectStaleSharedGatewayAuthClients({ + clients: params.clients, + expectedGeneration: transaction.previousGeneration, + }); + } + // Generation fences snapshot rollback, never exact-account liveness recovery. + for (const { target, restarted } of touchedTargets) { + const { channel, accountId, inspectOnly } = target; + const label = accountId ? `${channel} account ${accountId}` : `${channel} channel`; + params.logChannels.info(`rolling back ${label} after secrets reload failure`); + try { + if (restarted || inspectOnly) { + await stopTarget(target); + } + if (!inspectOnly) { + await startTarget(target); + } + } catch { + params.logChannels.info(`failed to roll back ${label} after secrets reload`); + } + } + throw error; + } + }, reloadOptions); +} diff --git a/src/gateway/server-startup-secret-owner-isolation.test.ts b/src/gateway/server-startup-secret-owner-isolation.test.ts index a7af53e9dcfc..e45683d62353 100644 --- a/src/gateway/server-startup-secret-owner-isolation.test.ts +++ b/src/gateway/server-startup-secret-owner-isolation.test.ts @@ -10,7 +10,10 @@ import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { resolveMemorySearchConfig } from "../agents/memory-search.js"; import { resolveApiKeyForProviderCore } from "../agents/model-auth.js"; import { resolveSandboxContext } from "../agents/sandbox/context.js"; +import type { ChannelGatewayContext } from "../channels/plugins/types.adapters.js"; +import type { ChannelAccountSnapshot, ChannelPlugin } from "../channels/plugins/types.public.js"; import type { OpenClawConfig } from "../config/config.js"; +import { tryReadSecretFileSync } from "../infra/secret-file.js"; import { selectAgentSystemEvents } from "../infra/system-event-ownership.js"; import { peekSystemEventEntries, @@ -18,14 +21,19 @@ import { resetSystemEventsForTest, } from "../infra/system-events.js"; import { resolveAuthProfileSecretOwnerId } from "../secrets/runtime-auth-profile-owner.js"; -import { setActiveDegradedSecretOwners } from "../secrets/runtime-degraded-state.js"; +import { + listActiveDegradedSecretOwners, + setActiveDegradedSecretOwners, +} from "../secrets/runtime-degraded-state.js"; import { getActiveSecretsRuntimeSnapshot } from "../secrets/runtime.js"; +import { createChannelTestPluginBase, createTestRegistry } from "../test-utils/channel-plugins.js"; import { deleteTestEnvValue, withEnvAsync } from "../test-utils/env.js"; import { connectWebchatClient, getGatewayTestPort, installGatewayTestHooks, rpcReq, + setTestPluginRegistry, startTestGatewayServer, testState, } from "./test-helpers.js"; @@ -182,6 +190,192 @@ describe("Gateway startup SecretRef owner isolation", () => { }); }); + it("recovers only a repaired credential-file account through secrets.reload without restarting sibling accounts", async () => { + await withEnvAsync( + { OPENCLAW_SKIP_CHANNELS: undefined, OPENCLAW_SKIP_PROVIDERS: undefined }, + async () => { + const credentialPath = path.join(tempDirs.make("openclaw-gateway-credential-"), "token"); + const credentialConfigPath = "channels.telegram.accounts.broken.tokenFile"; + const repairedToken = "repaired-test-token-never-public"; + type TestAccount = { + accountId: string; + token?: string; + credentialDiagnostics?: Array<{ + code: "CREDENTIAL_FILE_UNAVAILABLE"; + path: string; + reason: string; + }>; + }; + const startAccount = vi.fn( + async ({ abortSignal }: ChannelGatewayContext) => + await new Promise((resolve) => { + abortSignal.addEventListener("abort", () => resolve(), { once: true }); + }), + ); + const plugin: ChannelPlugin = { + ...createChannelTestPluginBase({ id: "telegram" }), + config: { + listAccountIds: (config) => Object.keys(config.channels?.telegram?.accounts ?? {}), + resolveAccount: (config, accountId) => { + if (!accountId) { + throw new Error("Missing Telegram test account id"); + } + const configured = config.channels?.telegram?.accounts?.[accountId]; + if (!configured) { + throw new Error(`Missing Telegram test account ${accountId}`); + } + const credential = configured.tokenFile + ? tryReadSecretFileSync( + configured.tokenFile, + "Telegram bot token", + {}, + { + configPath: credentialConfigPath, + }, + ) + : { status: "available" as const, value: configured.botToken }; + return { + accountId, + ...(credential.status === "configured_unavailable" + ? { credentialDiagnostics: [credential.diagnostic] } + : { token: typeof credential.value === "string" ? credential.value : undefined }), + }; + }, + }, + gateway: { startAccount }, + }; + setTestPluginRegistry( + createTestRegistry([{ pluginId: "telegram", source: "test", plugin }]), + ); + await writeConfig({ + ...baseConfig(), + gateway: { ...baseConfig().gateway, reload: { mode: "off" } }, + channels: { + telegram: { + enabled: true, + healthMonitor: { enabled: false }, + accounts: { + broken: { tokenFile: credentialPath }, + healthy: { botToken: "healthy-test-token" }, + stopped: { botToken: "stopped-test-token" }, + }, + }, + }, + }); + const configPath = process.env.OPENCLAW_CONFIG_PATH; + if (!configPath) { + throw new Error("Gateway test did not configure a config file path"); + } + const originalConfig = readFileSync(configPath); + const port = await getGatewayTestPort(); + server = await startTestGatewayServer(port, { auth: { mode: "none" } }); + const ws = await connectWebchatClient({ port, scopes: ["operator.admin"] }); + try { + const brokenStart = await rpcReq(ws, "channels.start", { + channel: "telegram", + accountId: "broken", + }); + expect(brokenStart.ok).toBe(false); + for (const accountId of ["healthy", "stopped"]) { + const started = await rpcReq<{ accountId: string; started: boolean }>( + ws, + "channels.start", + { channel: "telegram", accountId }, + ); + expect(started.ok, JSON.stringify(started)).toBe(true); + expect(started.payload).toMatchObject({ accountId, started: true }); + } + expect(startAccount).toHaveBeenCalledTimes(2); + expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([ + "healthy", + "stopped", + ]); + const healthyLifetime = startAccount.mock.calls[0]?.[0].abortSignal; + const stoppedLifetime = startAccount.mock.calls[1]?.[0].abortSignal; + expect(healthyLifetime?.aborted).toBe(false); + expect(stoppedLifetime?.aborted).toBe(false); + + const before = await rpcReq<{ + channelAccounts: Record; + }>(ws, "channels.status", { probe: false }); + expect(before.ok, JSON.stringify(before)).toBe(true); + const initialAccounts = before.payload?.channelAccounts.telegram ?? []; + expect(initialAccounts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ accountId: "broken", configured: true, running: false }), + expect.objectContaining({ accountId: "healthy", running: true }), + expect.objectContaining({ accountId: "stopped", running: true }), + ]), + ); + expect(listActiveDegradedSecretOwners()).toContainEqual( + expect.objectContaining({ + ownerId: "telegram:broken", + paths: [credentialConfigPath], + reason: "credential file is unavailable", + }), + ); + expect(brokenStart.error).toMatchObject({ code: "UNAVAILABLE" }); + const publicDiagnostics = JSON.stringify({ + error: brokenStart.error, + accounts: initialAccounts, + }); + expect(publicDiagnostics).not.toContain(credentialPath); + expect(publicDiagnostics).not.toContain(repairedToken); + + const stopped = await rpcReq<{ accountId: string; stopped: boolean }>( + ws, + "channels.stop", + { + channel: "telegram", + accountId: "stopped", + }, + ); + expect(stopped.ok, JSON.stringify(stopped)).toBe(true); + expect(stopped.payload).toMatchObject({ accountId: "stopped", stopped: true }); + expect(stoppedLifetime?.aborted).toBe(true); + expect(healthyLifetime?.aborted).toBe(false); + + writeFileSync(credentialPath, repairedToken, { mode: 0o600 }); + expect(readFileSync(configPath)).toEqual(originalConfig); + + const reload = await rpcReq<{ warningCount: number }>(ws, "secrets.reload", {}); + expect(reload.ok, JSON.stringify(reload)).toBe(true); + expect(reload.payload).toMatchObject({ warningCount: 0 }); + expect(startAccount.mock.calls.map(([context]) => context.accountId)).toEqual([ + "healthy", + "stopped", + "broken", + ]); + expect(startAccount.mock.calls[2]?.[0].account.token).toBe(repairedToken); + expect(healthyLifetime?.aborted).toBe(false); + expect(stoppedLifetime?.aborted).toBe(true); + expect(listActiveDegradedSecretOwners()).not.toContainEqual( + expect.objectContaining({ ownerId: "telegram:broken" }), + ); + + const after = await rpcReq<{ channelAccounts: Record }>( + ws, + "channels.status", + { probe: false }, + ); + expect(after.ok, JSON.stringify(after)).toBe(true); + expect(after.payload?.channelAccounts.telegram).toEqual( + expect.arrayContaining([ + expect.objectContaining({ accountId: "broken", running: true }), + expect.objectContaining({ accountId: "healthy", running: true }), + expect.objectContaining({ accountId: "stopped", running: false }), + ]), + ); + expect(JSON.stringify(after.payload)).not.toContain(repairedToken); + expect(JSON.stringify(after.payload)).not.toContain(credentialPath); + expect(readFileSync(configPath)).toEqual(originalConfig); + } finally { + ws.close(); + } + }, + ); + }); + it("reaches /readyz while isolating every optional owner family", async () => { await withEnvAsync( { diff --git a/src/secrets/runtime-degraded-state.test.ts b/src/secrets/runtime-degraded-state.test.ts index c486554a7428..d036ad3848d3 100644 --- a/src/secrets/runtime-degraded-state.test.ts +++ b/src/secrets/runtime-degraded-state.test.ts @@ -13,6 +13,7 @@ import { } from "./runtime-degraded-state.js"; afterEach(() => { + clearActiveCredentialDegradedOwner("account", "telegram:work"); setActiveDegradedSecretOwners([]); }); @@ -131,6 +132,27 @@ describe("runtime degraded SecretRef owners", () => { "telegram:work", ]); + setActiveDegradedSecretOwners([ + { + ownerKind: "provider", + ownerId: "openai", + state: "unavailable", + degradationState: "stale", + paths: ["models.providers.openai.apiKey"], + refKeys: ["env:default:OPENAI_API_KEY"], + reason: "secret provider failed", + }, + ]); + + expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual([ + "openai", + "telegram:work", + ]); + expect(() => assertSecretOwnerAvailable("provider", "openai")).not.toThrow(); + expect(() => assertSecretOwnerAvailable("account", "telegram:work")).toThrow( + SecretSurfaceUnavailableError, + ); + clearActiveCredentialDegradedOwner("account", "telegram:work"); expect(listActiveDegradedSecretOwners().map((owner) => owner.ownerId)).toEqual(["openai"]); diff --git a/src/secrets/runtime-degraded-state.ts b/src/secrets/runtime-degraded-state.ts index 3be0b3139c19..f3edf3bfad0d 100644 --- a/src/secrets/runtime-degraded-state.ts +++ b/src/secrets/runtime-degraded-state.ts @@ -182,14 +182,23 @@ function cloneResolutionErrorOwner(owner: SecretResolutionErrorOwner): SecretRes /** Publishes the degraded-owner snapshot at the same edge as runtime config activation. */ export function setActiveDegradedSecretOwners(owners: readonly DegradedSecretOwner[]): void { activeDegradedOwners = owners.map(cloneOwner); - activeCredentialDegradedOwners.clear(); } -/** Publishes or clears one runtime-discovered channel credential owner. */ +/** Publishes one runtime-discovered channel credential owner. */ export function setActiveCredentialDegradedOwner(owner: DegradedSecretOwner): void { activeCredentialDegradedOwners.set(ownerKey(owner.ownerKind, owner.ownerId), cloneOwner(owner)); } +/** Lists credential owners independently of the replaceable SecretRef snapshot. */ +export function listActiveCredentialDegradedOwners(): DegradedSecretOwner[] { + return Array.from(activeCredentialDegradedOwners.values(), cloneOwner); +} + +/** Clears credential-owner state only when its owning secrets runtime is torn down. */ +export function clearActiveCredentialDegradedOwners(): void { + activeCredentialDegradedOwners.clear(); +} + /** Clears one runtime-discovered channel credential owner before re-inspection. */ export function clearActiveCredentialDegradedOwner( ownerKind: DegradedSecretOwner["ownerKind"], @@ -200,10 +209,7 @@ export function clearActiveCredentialDegradedOwner( /** Returns the active degraded-owner snapshot without exposing mutable registry state. */ export function listActiveDegradedSecretOwners(): DegradedSecretOwner[] { - return [ - ...activeDegradedOwners.map(cloneOwner), - ...Array.from(activeCredentialDegradedOwners.values(), cloneOwner), - ]; + return [...activeDegradedOwners.map(cloneOwner), ...listActiveCredentialDegradedOwners()]; } /** Associates a strict activation failure with the owners it prevented from refreshing. */ diff --git a/src/secrets/runtime-provider-and-media-surfaces.test.ts b/src/secrets/runtime-provider-and-media-surfaces.test.ts index 3c19be8552cb..1d240d8aa638 100644 --- a/src/secrets/runtime-provider-and-media-surfaces.test.ts +++ b/src/secrets/runtime-provider-and-media-surfaces.test.ts @@ -7,6 +7,10 @@ import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { OpenClawConfig } from "../config/config.js"; import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; +import { + listActiveDegradedSecretOwners, + setActiveCredentialDegradedOwner, +} from "./runtime-degraded-state.js"; import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.ts"; function createOpenAiFileModelsConfig(): NonNullable { @@ -208,6 +212,14 @@ describe("secrets runtime provider and media surfaces", () => { const { getRuntimeConfigSourceSnapshot, getRuntimeConfigSnapshot, setRuntimeConfigSnapshot } = await import("../config/runtime-snapshot.js"); activateSecretsRuntimeSnapshot(initial); + setActiveCredentialDegradedOwner({ + ownerKind: "account", + ownerId: "telegram:work", + state: "unavailable", + paths: ["channels.telegram.accounts.work.tokenFile"], + refKeys: [], + reason: "credential file is unavailable", + }); const runtimeSourceConfig: OpenClawConfig = { ...initial.sourceConfig, logging: { level: "debug" }, @@ -241,6 +253,9 @@ describe("secrets runtime provider and media surfaces", () => { expect(active?.config.models?.providers?.openai?.apiKey).toBe("model-new"); expect(getRuntimeConfigSnapshot()).toEqual(active?.config); expect(getRuntimeConfigSourceSnapshot()).toEqual(runtimeSourceConfig); + expect(listActiveDegradedSecretOwners()).toContainEqual( + expect.objectContaining({ ownerKind: "account", ownerId: "telegram:work" }), + ); } finally { await fs.rm(root, { recursive: true, force: true }); } diff --git a/src/secrets/runtime-state.test.ts b/src/secrets/runtime-state.test.ts index dc17d959056e..571ffb07d6a9 100644 --- a/src/secrets/runtime-state.test.ts +++ b/src/secrets/runtime-state.test.ts @@ -26,6 +26,10 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { SecretRef } from "../config/types.secrets.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { captureEnv } from "../test-utils/env.js"; +import { + listActiveDegradedSecretOwners, + setActiveCredentialDegradedOwner, +} from "./runtime-degraded-state.js"; import { activateSecretsRuntimeSnapshotState, activateSecretsRuntimeSnapshotStateIfCurrent, @@ -216,6 +220,47 @@ describe("secrets runtime state", () => { expect(configSnapshot?.sourceConfig).toEqual(snapshot.sourceConfig); }); + it("preserves independent credential owners through snapshot replacement and rollback until teardown", () => { + const previous = preparedSnapshot({ + degradedOwners: [ + { + ownerKind: "provider", + ownerId: "openai", + state: "unavailable", + degradationState: "stale", + paths: ["models.providers.openai.apiKey"], + refKeys: ["env:default:OPENAI_API_KEY"], + reason: "secret provider failed", + }, + ], + }); + activateSnapshot(previous); + setActiveCredentialDegradedOwner({ + ownerKind: "account", + ownerId: "telegram:work", + state: "unavailable", + paths: ["channels.telegram.accounts.work.tokenFile"], + refKeys: [], + reason: "credential file is unavailable", + }); + const candidate = preparedSnapshot({ config: { gateway: { port: 19_041 } } }); + + activateSnapshot(candidate); + + expect(listActiveDegradedSecretOwners()).toMatchObject([ + { ownerKind: "account", ownerId: "telegram:work" }, + ]); + expect(restoreSnapshotIfCurrent(previous, candidate)).toBe(true); + expect(listActiveDegradedSecretOwners()).toMatchObject([ + { ownerKind: "provider", ownerId: "openai", degradationState: "stale" }, + { ownerKind: "account", ownerId: "telegram:work" }, + ]); + + clearSecretsRuntimeSnapshotState(); + + expect(listActiveDegradedSecretOwners()).toEqual([]); + }); + it("publishes distinct raw and overlay source snapshots without changing runtime auth", () => { const secretRef = { source: "env" as const, diff --git a/src/secrets/runtime-state.ts b/src/secrets/runtime-state.ts index c62cd8989b11..18ff361182b6 100644 --- a/src/secrets/runtime-state.ts +++ b/src/secrets/runtime-state.ts @@ -33,6 +33,7 @@ import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; import { isRecord } from "../utils.js"; import { secretRefKey } from "./ref-contract.js"; import { + clearActiveCredentialDegradedOwners, setActiveDegradedSecretOwners, type DegradedSecretOwner, type SecretOwnerRefState, @@ -1172,6 +1173,7 @@ export function clearSecretsRuntimeSnapshotState(): void { activeRefreshContext = null; clearActiveRuntimeWebToolsMetadata(); setActiveDegradedSecretOwners([]); + clearActiveCredentialDegradedOwners(); setRuntimeConfigSnapshotRefreshHandler(null); clearRuntimeConfigSnapshot(); clearRuntimeAuthProfileStoreSnapshots();